flume从kafka取数据，并发往另外一台kafka配置

flume从kafka取数据，并发往另外一台kafka配置

 

agent.sources = s1
agent.sinks = k1
agent.channels = c1

agent.sources.s1.type = org.apache.flume.source.kafka.KafkaSource
agent.sources.s1.kafka.bootstrap.servers = 192.168.133.137:9092
agent.sources.s1.kafka.topics = test1
agent.sources.s1.kafka.consumer.group.id = test1

agent.sources.s1.interceptors = i1                         #注意：这个拦截器需要配置，不然flume会把sink的topic当成source的topic，导致sink的topic不生效
agent.sources.s1.interceptors.i1.type = static
agent.sources.s1.interceptors.i1.key = topic
agent.sources.s1.interceptors.i1.preserveExisting = false
agent.sources.s1.interceptors.i1.value =test2    # 配置sink的topic

agent.channels.c1.type = memory
agent.channels.c1.capacity = 100000
agent.channels.c1.transactionCapacity = 5000

agent.sinks.k1.type = org.apache.flume.sink.kafka.KafkaSink
agent.sinks.k1.kafka.topic = test2
agent.sinks.k1.kafka.bootstrap.servers = 192.168.133.137:9092
agent.sinks.k1.flumeBatchSize = 200
agent.sinks.k1.producer.acks = 1
# agent.sinks.k1.producer.linger.ms = 1
# agent.sinks.k1.producer.compression.type = snappy

agent.sources.s1.channels = c1
agent.sinks.k1.channel = c1