01-spooldir

-------------------------------------- spooldir -----------------------------------------
1、创建cnf

vim flume_spooldir_file_hdfs.cnf

2、配置信息

# 组件说明
a1.sources 	= s1
a1.channels = c1
a1.sinks    = k1

# 初始化数据源
a1.sources.s1.type = spooldir
a1.sources.s1.spoolDir = /root/data/flume
a1.sources.s1.ignorePattern = ^(.)*\\.bak$
a1.sources.s1.fileSuffix = .bak

# 初始化通道
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /opt/software/flume190/mydata/checkpoint
a1.channels.c1.dataDirs = /opt/software/flume190/mydata/data 
a1.channels.c1.capacity = 100000
a1.channels.c1.transactionCapacity = 10000

# 初始化数据槽
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.19.130:9820/flume/events/fakeorder/%Y-%m-%d/%H
a1.sinks.k1.hdfs.round = true		# 时间舍弃
a1.sinks.k1.hdfs.roundValue = 10
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.filePrefix = log_%Y%m%d_%H
a1.sinks.k1.hdfs.fileSuffix = .log
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 134217728
a1.sinks.k1.hdfs.bacthSize = 1000
a1.sinks.k1.hdfs.threadsPoolSize = 4
a1.sinks.k1.hdfs.idleTimeout = 0
a1.sinks.k1.hdfs.minBlockReplicas = 1

# 关联组件
a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

3、启动

flume-ng agent -n a1 -c conf/ -f flume-conf-files/flume_spooldir_file_hdfs.cnf -Dflume.root.logger=INFO,console