mysql-canal-rabbitmq installation and deployment super detailed tutorial

# binlog log storage path log-bin=D:\env\mysql-5.7.28-winx64\binlog
# The log records the format in which each row of data is modified binlog-format=ROW
# The service ID of the current machine. If it is a cluster, it cannot be repeated server_id=1

mysql> show variables like '%log_bin%';
+---------------------------------+----------------------+
| Variable_name | Value |
+---------------------------------+----------------------+
| log_bin | ON |
| log_bin_basename | D:\env\mysql-5 |
| log_bin_index | D:\env\mysql-5.index |
| log_bin_trust_function_creators | OFF |
| log_bin_use_v1_row_events | OFF |
| sql_log_bin | ON |
+---------------------------------+----------------------+
6 rows in set, 1 warning (0.00 sec)

CREATE USER canal IDENTIFIED BY 'canal';
GRANT SELECT, REPLICATION SLAVE, REPLICATION CLIENT ON *.* TO 'canal'@'%';
FLUSH PRIVILEGES;

sudo wget https://github.com/alibaba/canal/releases/download/canal-1.1.4/canal.deployer-1.1.4.tar.gz
sudo tar -zxvf canal.deployer-1.1.4.tar.gz

sudo wget https://github.com/alibaba/canal/releases/download/canal-1.1.5-alpha-1/canal.deployer-1.1.5-SNAPSHOT.tar.gz
sudo tar -zxvf canal.deployer-1.1.5-SNAPSHOT.tar.gz

# tcp bind ip, the IP address of the current node canal.ip = 192.168.2.108
# register ip to zookeeper, register to ZK's IP address, as shown in Figure 1.
canal.register.ip = 192.168.2.108

canal.zkServers = zk cluster # tcp, kafka, RocketMQ, the latest version 1.1.5 can directly connect to rabbitmq
canal.serverMode = rabbitmq

# destinations, the instance list deployed on the current server, corresponding to the name of each instance folder (../conf/<instance_name>) canal.destinations = example2

# Set the mq server address, here is the rabbitmq server address #!! The default configuration after downloading here is to configure the IP: port # rabbitmq does not need to configure the port here canal.mq.servers = 192.168.208.100

# The following items are all new versions 1.1.5 that support rabbitmq configuration canal.mq.vhost=/
canal.mq.exchange=example2-ex # Specify the exchange name on rabbitmq, the name created in the "Create `Exchange`" step canal.mq.username=admin # Username for connecting to rabbitmq canal.mq.password=**** # Password for connecting to rabbitmq canal.mq.aliyunuid= 

# position info, database connection information canal.instance.master.address=192.168.2.108:3306
# The following two configurations require the values ​​of the fields `File` and `Position` obtained by executing `SHOW MASTER STATUS` in the database of the address configured above canal.instance.master.journal.name=mysql-5.7
canal.instance.master.position=674996

# table meta tsdb info, disable tsdb to record the time series version of table meta canal.instance.tsdb.enable=false

# username/password, the username and password for the instance connection data canal.instance.dbUsername=canal
canal.instance.dbPassword=canal

# table regex, regular matching database table to be monitored canal.instance.filter.regex=ysb\\.useropcosttimes_prod

# mq config, specify the routing to which rabbitmq is bound, see the third step of "Configure rabbitmq" for details.
canal.mq.topic=example2-routingkey

λ .\startup.bat
start cmd : java -Xms128m -Xmx512m -XX:PermSize=128m -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n -DappName=otter-canal -Dlogback.configurationFile="d:\env\green\canal-1.1.5\bin\\..\conf\logback.xml" -Dcanal.conf="d:\env\green\canal-1.1.5\bin\\..\conf\canal.properties" -classpath "d:\env\green\canal-1.1.5\bin\\..\conf\..\lib\*;d:\env\green\canal-1.1.5\bin\\..\conf" java -Xms128m -Xmx512m -XX:PermSize=128m -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 -server -Xdebug -Xnoagent -Djava.compiler=NONE -Xrunjdwp:transport=dt_socket,address=9099,server=y,suspend=n -DappName=otter-canal -Dlogback.configurationFile="d:\env\green\canal-1.1.5\bin\\..\conf\logback.xml" -Dcanal.conf="d:\env\green\canal-1.1.5\bin\\..\conf\canal.properties" -classpath "d:\env\green\canal-1.1.5\bin\\..\conf\..\lib\*;d:\env\green\canal-1.1.5\bin\\..\conf" com.alibaba.otter.canal.deployer.CanalLauncher
Java HotSpot(TM) Server VM warning: ignoring option PermSize=128m; support was removed in 8.0
Listening for transport dt_socket at address: 9099

sudo wget https://github.com/alibaba/canal/releases/download/canal-1.1.5-alpha-1/canal.admin-1.1.5-SNAPSHOT.tar.gz
sudo tar -zxvf canal.admin-1.1.5-SNAPSHOT.tar.gz

server:
 port: 8089
spring:
 jackson:
  date-format: yyyy-MM-dd HH:mm:ss
  time-zone: GMT+8

spring.datasource:
 address: 192.168.2.108:3306
 database: canal_manager
 Username: canal
 password: canal
 driver-class-name: com.mysql.jdbc.Driver
 # `serverTimezone=UTC` needs to be added to the end of the database connection string, otherwise a time zone exception will be reported during startup;
 url: jdbc:mysql://${spring.datasource.address}/${spring.datasource.database}?useUnicode=true&characterEncoding=UTF-8&useSSL=false&serverTimezone=UTC
 Hikari:
  maximum-pool-size: 30
  minimum-idle: 1

canal:
 # Configure the administrator account and password of canal-adminUser: admin
 adminPasswd: 123456

λ .\startup.bat
start cmd : java -Xms128m -Xmx512m -Djava.awt.headless=true -Djava.net.preferIPv4Stack=true -Dapplication.codeset=UTF-8 -Dfile.encoding=UTF-8 -DappName=canal-admin -classpath "D:\env\green\canal-1.1.5-admin\bin\\..\conf\..\lib\*;D:\env\green\canal-1.1.5-admin\bin\\..\conf" com.alibaba.otter.canal.admin.CanalAdminApplication
2020-04-13 20:01:39.495 [main] INFO com.alibaba.otter.canal.admin.CanalAdminApplication - Starting CanalAdminApplication on Memento-PC with PID 50696 (D:\env\green\canal-1.1.5-admin\lib\canal-admin-server-1.1.5-SNAPSHOT.jar started by Memento in D:\env\green\canal-1.1.5-admin\bin)
2020-04-13 20:01:39.527 [main] INFO com.alibaba.otter.canal.admin.CanalAdminApplication - No active profile set, falling back to default profiles: default
2020-04-13 20:01:39.566 [main] INFO osbwscAnnotationConfigServletWebServerApplicationContext - Refreshing org.springframework.boot.web.servlet.context.AnnotationConfigServletWebServerApplicationContext@13a5bf6: startup date [Mon Apr 13 20:01:39 CST 2020]; root of context hierarchy
2020-04-13 20:01:41.149 [main] INFO osboot.web.embedded.tomcat.TomcatWebServer - Tomcat initialized with port(s): 8089 (http)
2020-04-13 20:01:41.166 [main] INFO org.apache.coyote.http11.Http11NioProtocol - Initializing ProtocolHandler ["http-nio-8089"]
2020-04-13 20:01:41.176 [main] INFO org.apache.catalina.core.StandardService - Starting service [Tomcat]
2020-04-13 20:01:41.177 [main] INFO org.apache.catalina.core.StandardEngine - Starting Servlet Engine: Apache Tomcat/8.5.29
...
2020-04-13 20:01:42.996 [main] INFO org.apache.coyote.http11.Http11NioProtocol - Starting ProtocolHandler ["http-nio-8089"]
2020-04-13 20:01:43.007 [main] INFO org.apache.tomcat.util.net.NioSelectorPool - Using a shared selector for servlet write/read
2020-04-13 20:01:43.019 [main] INFO osboot.web.embedded.tomcat.TomcatWebServer - Tomcat started on port(s): 8089 (http) with context path ''
2020-04-13 20:01:43.024 [main] INFO com.alibaba.otter.canal.admin.CanalAdminApplication - Started CanalAdminApplication in 3.919 seconds (JVM running for 5.241)

GRANT ALL PRIVILEGES ON *.* TO 'canal'@'%';

# canal admin config
canal.admin.manager = 192.168.2.108:8089

canal.instance.global.mode = manager

@rem set canal_conf=...
set canal_conf=%conf_dir%\canal_local.properties

# 2. position info, specify the binlog position information where MySQL starts to synchronize canal.instance.master.address=192.168.0.25:63306
canal.instance.master.journal.name=mysql-bin.001349
canal.instance.master.position=198213313

# 3. Username/password, set the database username and password for syncing MySQL canal.instance.dbUsername=xxxx
canal.instance.dbPassword=xxx

# 4. table regex, regular expression matching the data table to be synchronized canal.instance.filter.regex=xxxx

# 5. mq config, specify the routing binding on mysql, see `1.2.3`
canal.mq.topic=example2-routingkey

canal.instance.tsdb.enable = true
canal.instance.tsdb.dir = ${canal.file.data.dir:../conf}/${canal.instance.destination:}
canal.instance.tsdb.url = jdbc:h2:${canal.instance.tsdb.dir}/h2;CACHE_SIZE=1000;MODE=MYSQL;
canal.instance.tsdb.dbUsername = canal
canal.instance.tsdb.dbPassword = canal

canal.destinations # List of instances deployed on the current server canal.conf.dir # Path to the conf directory canal.auto.scan # Enable automatic instance scanning # If configured to true, instance configuration changes under the canal.conf.dir directory will be automatically triggered # 1. Added instance directory: triggers instance configuration loading, and automatically starts when lazy is true;
# 2. Instance directory deletion: Uninstall the corresponding instance configuration, and shut down if it has been started;
# 3. Changes in instance.properties file: reload instance configuration, and automatically restart if it has been started;
canal.auto.scan.interval # instance automatic scan interval, unit s
canal.instance.global.mode # Global configuration loading modecanal.instance.global.lazy # Global lazy modecanal.instance.global.manager.address # Link information of global manager configuration modecanal.instance.global.spring.xml # Component file of global spring configuration modecanal.instance.example.mode
canal.instance.example.lazy
canal.instance.example.spring.xml
# Instance-level configuration definition. If configured, it will automatically overwrite the global configuration definition modecanal.instance.tsdb.enable # Whether to enable the time series version recording function of table metacanal.instance.tsdb.dir # Local storage path of the time series version. The default is the instance directorycanal.instance.tsdb.url # Database connection address of the time series version. The default is the local embedded databasecanal.instance.tsdb.dbUsername # Database connection account of the time series versioncanal.instance.tsdb.dbPassword # Database connection password of the time series version

canal.id # Unique identifier for each canal server instancecanal.ip # Local IP information bound to the canal server. If not configured, a local IP will be selected by default to start the servicecanal.port # Port for the canal server to provide socket servicecanal.zkServers # Connection address of the canal server to connect to the zookeeper cluster, for example: 10.20.144.22:2181,10.20.144.23:2181
canal.zookeeper.flush.period # The update frequency of canal persistent data to zookeeper, in ms
canal.instance.memory.batch.mode # Data cache mode in canal memory store# 1. ITEMSIZE: limit according to buffer.size, only limit the number of records# 2. MEMSIZE: limit the size of cached records according to the size of buffer.size * buffer.memunit;
canal.instance.memory.buffer.size # The number of buffer records that can be cached in the canal memory store, needs to be an exponent of 2canal.instance.memory.buffer.memunit # The unit size of the memory record, defaults to 1KB, and combines with buffer.size to determine the final memory usage sizecanal.instance.transactions.size # The maximum transaction complete resolution length supported. After exceeding this length, a transaction may be split into multiple submissions to the canal store, and the complete visibility of the transaction cannot be guaranteedcanal.instance.fallbackIntervalInSeconds # When canal switches to MySQL, the time it needs to look forward when looking for binlog on the new MySQL database, unit s
# Description: There may be parsing delays or clock inconsistencies in the MySQL master and slave databases. It is necessary to roll back for a period of time to ensure that data is not lost canal.instance.detecting.enable # Whether to enable heartbeat check canal.instance.detecting.sql # Heartbeat check sql, insert into retl.xdual values(1,now()) on duplicate key update x=now()
canal.instance.detecting.interval.time # Heartbeat check frequency, unit s
canal.instance.detecting.retry.threshold # Heartbeat check failed retry timescanal.instance.detecting.heatbeatHaEnable # After the heartbeat check fails, whether to enable MySQL automatic switching# Note: For example, if the heartbeat check fails more than the threshold, if this configuration is true, canal will automatically connect to the MySQL standby database to obtain binlog datacanal.instance.network.receiveBufferSize # Network connection parameters, SocketOptions.SO_RCVBUF
canal.instance.network.sendBufferSize # Network connection parameters, SocketOptions.SO_SNDBUF
canal.instance.network.soTimeout # Network connection parameters, SocketOptions.SO_TIMEOUT