02-taildir source

------------------------------------taildir source ------------------------------------
1、 Content query

====================================
[root@singlelucky flume]# head -10000 prolog.log>tail01/prologhead10000.log
[root@singlelucky flume]# ls tail01
prologhead10000.log
[root@singlelucky flume]# wc tail01/prologhead10000.log 
  10000   50000   1688953 tail01/prologhead10000.log
   Row number 	   Number of words 	 size 
====================================

2、 establish cnf

vim test_taildir_hffs.cnf

3、 Configuration information

#  Component description 
a1.sources 	= s1
a1.channels = c1
a1.sinks    = k1

#  Initialize the data source 
a1.sources.s1.type = taildir
a1.sources.s1.filegroups = f1 f2
a1.sources.s1.filegroups.f1 = /root/data/flume/tail01/prolog.*\\.log
a1.sources.s1.filegroups.f2 = /root/data/flume/tail02/prolog.*\\.log
a1.sources.s1.positionFile = /opt/software/flume190/data/taildir/taildir_position.conf

#  Initialize channel 
a1.channels.c1.type = file
a1.channels.c1.checkpointDir = /opt/software/flume190/mydata/checkpoint
a1.channels.c1.dataDirs = /opt/software/flume190/mydata/data 
a1.channels.c1.capacity = 10000
a1.channels.c1.transactionCapacity = 1000

#  Initialize data slot 
a1.sinks.k1.type = hdfs
a1.sinks.k1.hdfs.path = hdfs://192.168.19.130:9820/flume/events/tailevent/%Y-%m-%d/%H
a1.sinks.k1.hdfs.round = true		#  Time abandonment 
a1.sinks.k1.hdfs.roundValue = 10
a1.sinks.k1.hdfs.roundUnit = minute
a1.sinks.k1.hdfs.filePrefix = log_%Y%m%d_%H
a1.sinks.k1.hdfs.fileSuffix = .log
a1.sinks.k1.hdfs.useLocalTimeStamp = true
a1.sinks.k1.hdfs.writeFormat = Text
a1.sinks.k1.hdfs.rollCount = 0
a1.sinks.k1.hdfs.rollSize = 134217728
a1.sinks.k1.hdfs.bacthSize = 1000
a1.sinks.k1.hdfs.threadsPoolSize = 4
a1.sinks.k1.hdfs.idleTimeout = 0
a1.sinks.k1.hdfs.minBlockReplicas = 1

#  Associated components 
a1.sources.s1.channels = c1
a1.sinks.k1.channel = c1

4、 start-up

flume-ng agent -n a1 -c conf/ -f flume-conf-files/test_taildir_hffs.cnf -Dflume.root.logger=INFO,console