1 下载tar包
- Download binary distro: https://github.com/zendesk/maxwell/releases/download/v1.19.4/maxwell-1.19.4.tar.gz
2 解压
tar -zvxf maxwell-1.19.4.tar.gz
3 修改配置
cd maxwell-1.19.4
cp config.properties.example config.properties
根据自己mysql以及kafka配置
# tl;dr config
log_level=info
producer=kafka
kafka.bootstrap.servers=hdp1:9092,hdp2:9092,hdp3:9092,hdp4:9092
# mysql login info
host=127.0.0.1
user=maxwell
password=pwd123456
4 修改mysql的binlog配置 根据自己机器配置
vi /etc/my.cnf
$ vi my.cnf
[mysqld]
server_id=1
log-bin=master
binlog_format=row
5 创建同步用户
mysql> GRANT ALL on maxwell.* to 'maxwell'@'%' identified by 'XXXXXX';
mysql> GRANT SELECT, REPLICATION CLIENT, REPLICATION SLAVE on *.* to 'maxwell'@'%';
mysql>flush privileges;
6 重启mysql查看bin log是否修改成功
service mysqld restart
show variables like '%binlog%'
7 测试一下maxwell
[root@hdp1 /mnt/software/maxwell-1.19.4]#bin/maxwell --user='maxwell' --password='pwd123456' --host='127.0.0.1' --producer=stdout
#插入数据 根据自己mysql情况测试
mysql> INSERT INTO `cmcc_offset` (`topics`, `partId`, `offsets`, `groupId`, `create_time`, `updater`, `update_time`, `is_delete`)
成功收到数据
8 单独的kafka producer配置
单独copy指定版本kafka的client的jar包到lib目录下， 我自己的kafka版本是1.1.0的
启动kafka的producer命令，需要单独制定kafka的version
bin/maxwell --user='maxwell' --password='pwd123456' --host='127.0.0.1' --producer=kafka --kafka_version=1.1.0 --kafka.bootstrap.servers=hdp1:9092,hdp2:9092,hdp3:9092,hdp4:9092 --kafka_topic=maxwell
遇到问题 Kafka的K大写了，改成小写就OK了
java.lang.RuntimeException: Unknown producer type: Kafka
at com.zendesk.maxwell.MaxwellContext.getProducer(MaxwellContext.java:372) ~[maxwell-1.19.4.jar:1.19.4]
at com.zendesk.maxwell.Maxwell.startInner(Maxwell.java:186) ~[maxwell-1.19.4.jar:1.19.4]
at com.zendesk.maxwell.Maxwell.start(Maxwell.java:156) ~[maxwell-1.19.4.jar:1.19.4]
at com.zendesk.maxwell.Maxwell.main(Maxwell.java:246) ~[maxwell-1.19.4.jar:1.19.4]
01:08:02,127 INFO TaskManager - Stopped all tasks
java.lang.RuntimeException: Unknown producer type: Kafka
at com.zendesk.maxwell.MaxwellContext.getProducer(MaxwellContext.java:372)
at com.zendesk.maxwell.Maxwell.startInner(Maxwell.java:186)
at com.zendesk.maxwell.Maxwell.start(Maxwell.java:156)
at com.zendesk.maxwell.Maxwell.main(Maxwell.java:246)
[root@hdp1 /mnt/software/maxwell-1.19.4]#xcall.sh jps
其他细节详情请看官网http://maxwells-daemon.io/quickstart/