01-spooldir

-------------------------------------- spooldir -----------------------------------------
1、 establish cnf

vim flume_spooldir_file_hdfs.cnf

2、 Configuration information

#  Component description 
a1.sources 	= s1
a1.channels = c1
a1.sinks    = k1

#  Initialize the data source 
a1.sources.s1.type = spooldir
a1.sources.s1.spoolDir = /root/data/flume
a1.sources.s1.ignorePattern = ^(.)*\\.bak$
a1.sources.s1.fileSuffix = .bak

#  Initialize channel 
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /opt/software/flume190/mydata/checkpoint
a1.channels.c1.dataDirs = /opt/software/flume190/mydata/data 
a1.channels.c1.capacity = 100000
a1.channels.c1.transactionCapacity = 10000

#  Initialize data slot 
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.19.130:9820/flume/events/fakeorder/%Y-%m-%d/%H
a1.sinks.k1.hdfs.round = true		#  Time abandonment 
a1.sinks.k1.hdfs.roundValue = 10
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.filePrefix = log_%Y%m%d_%H
a1.sinks.k1.hdfs.fileSuffix = .log
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 134217728
a1.sinks.k1.hdfs.bacthSize = 1000
a1.sinks.k1.hdfs.threadsPoolSize = 4
a1.sinks.k1.hdfs.idleTimeout = 0
a1.sinks.k1.hdfs.minBlockReplicas = 1

#  Associated components 
a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

3、 start-up

flume-ng agent -n a1 -c conf/ -f flume-conf-files/flume_spooldir_file_hdfs.cnf -Dflume.root.logger=INFO,console