使用kafka作为生产者生产数据到hdfs(单节点)

标签：timeout   time   nbsp   memory   mat   生产者   ext   prefix   hdfs   

查看kafka官网的userguide

agent.sources = kafkaSource

agent.channels = memoryChannel

agent.sinks = hdfsSink

agent.sources.kafkaSource.type = org.apache.flume.source.kafka.KafkaSource

agent.sources.kafksSource.zookeeperConnect = 192.168.57.11:2181

agent.sources.kafkaSource.topic = tets_zhang

agent.sources.kafkaSource.groupld = test-consumer-group

agent.sources.kafkaSource.kafka.consumer.timeout.ms = 100

agent.channels.memoryChannel.type = memory

agent.channels.memoryChannel.capacity = 100

agent.channels.memoryChannel.transactionCapacity = 100

agent.sinks.hdfsSink.type = hdfs

agent.sinks..hdfsSink.hdfs.path = hdfs://beicai/test/zhang

agent.sinks.hdfsSink.hdfs.writeFormat = Text

agent.sinks.hdfsSink.hdfs.fileType = DataStream

agent.sinks.hdfsSink.hdfs..rollSize = 1024

agent.sinks.hdfsSink.hdfs.rollCount = 0

agent.sinks.hdfsSink.hdfs.rollinterval = 60

agent.sinks.hdfsSink.hdfs.filePrefix = test

agent.sinks.hdfsSink.hdfs.fileSuffix = .data

agent.sinks.hdfsSink.hdfs.inUserPrefix = _

agent.sinks.hdfsSink.hdfs.inUserSuffix=

agent.sinks.hdfsSink.hdfs.fileType = DataStream

agent.sinks.hdfsSink.hdfs.writeFormat = TEXT

agent.sinks.hdfsSink.hdfs.rollnterval = 1

agent.sinks.sink1.hdfs.filePrefix = A

agent.sources.kafkaSource.channels = memoryChannel

agent.sinks.hdfsSink.channel = memoryChanne

使用kafka作为生产者生产数据到hdfs(单节点)

标签：timeout   time   nbsp   memory   mat   生产者   ext   prefix   hdfs   

原文地址：https://www.cnblogs.com/zdy0521/p/11847948.html