Canal Dts
canal mysql 到mysql 数据同步
使用canal 的tcp 模式达到mysql 到mysql 的数据同步
canal-server 安装
apiVersion: v1
kind: ConfigMap
metadata:
name: canal-server
namespace: iot
data:
admin_manager: "canal-admin:8089"
admin_port: "11110"
admin_user: "admin"
admin_password: "6BB4837EB74329105EE4568DDA7DC67ED2CA2AD9" # 123456加密
#admin_register_cluster: "local" ##集群名称
admin_register_auto: "true"
admin_register_name: "iot-canal-server" ##canal-server 自动注册名称
---
kind: StatefulSet ##canal-server 采用StatefulSet 方式部署
apiVersion: apps/v1
metadata:
name: canal-server
namespace: iot
labels:
app.kubernetes.io/name: canal-server
app: canal-server
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: canal-server
app: canal-server
template:
metadata:
name: canal-server
labels:
app.kubernetes.io/name: canal-server
app: canal-server
spec:
imagePullSecrets:
- name: secret
containers:
- name: canal-server
image: 'canal/canal-server:v1.1.6'
imagePullPolicy: Always
ports:
- name: tcp
containerPort: 11111
protocol: TCP
env:
- name: POD_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.name
- name: SERVICE_NAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: 'metadata.labels[''app'']'
- name: STS_NAMESPACE
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: metadata.namespace
- name: canal.register.ip
value: $(POD_NAME).$(SERVICE_NAME).$(STS_NAMESPACE) ## canal-server采用实例名，防止每次重启导致的IP不同，Instance实例选择时的变化
- name: canal.admin.manager
valueFrom:
configMapKeyRef:
name: canal-server
key: admin_manager
- name: canal.admin.port
valueFrom:
configMapKeyRef:
name: canal-server
key: admin_port
- name: canal.admin.user
valueFrom:
configMapKeyRef:
name: canal-server
key: admin_user
- name: canal.admin.passwd
valueFrom:
configMapKeyRef:
name: canal-server
key: admin_password
#- name: canal.admin.register.cluster
# valueFrom:
# configMapKeyRef:
# name: canal-server
# key: admin_register_cluster
- name: canal.admin.register.auto
valueFrom:
configMapKeyRef:
name: canal-server
key: admin_register_auto
- name: canal.admin.register.name
valueFrom:
configMapKeyRef:
name: canal-server
key: admin_register_name
resources:
requests:
cpu: 250m
memory: 1024Mi
livenessProbe:
tcpSocket:
port: 11112
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 30
readinessProbe:
tcpSocket:
port: 11112
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 30
restartPolicy: Always
nodeSelector: {}
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/name: canal-server
namespaces:
- iot
topologyKey: kubernetes.io/hostname
serviceName: canal-server
revisionHistoryLimit: 10
---
kind: Service
apiVersion: v1
metadata:
name: canal-server
namespace: iot
spec:
ports:
- name: http
nodePort: 31608
port: 11110
protocol: TCP
targetPort: 11110
- name: tcp11111
nodePort: 31749
port: 11111
protocol: TCP
targetPort: 11111
- name: tcp11112
nodePort: 30822
port: 11112
protocol: TCP
targetPort: 11112
type: NodePort
selector:
app.kubernetes.io/name: canal-server
app: canal-server
canal-admin
apiVersion: v1
kind: ConfigMap
metadata:
name: canal-admin
namespace: iot
data:
admin_user: "admin"
admin_password: "123456"
datasource_address: "mysql:3306"
datasource_database: "canal_manager"
datasource_username: "canal"
datasource_password: "canal"
---
kind: Deployment
apiVersion: apps/v1
metadata:
name: canal-admin
namespace: iot
labels:
app.kubernetes.io/name: canal-admin
spec:
replicas: 1
selector:
matchLabels:
app.kubernetes.io/name: canal-admin
template:
metadata:
name: canal-admin
creationTimestamp: null
labels:
app.kubernetes.io/name: canal-admin
spec:
containers:
- name: canal-admin
image: 'canal/canal-admin:v1.1.6'
imagePullPolicy: IfNotPresent
ports:
- name: web
containerPort: 8089
protocol: TCP
env:
- name: server.port
value: '8089'
- name: canal.adminUser
valueFrom:
configMapKeyRef:
name: canal-admin
key: admin_user
- name: canal.adminPasswd
valueFrom:
configMapKeyRef:
name: canal-admin
key: admin_password
- name: spring.datasource.address
valueFrom:
configMapKeyRef:
name: canal-admin
key: datasource_address
- name: spring.datasource.database
valueFrom:
configMapKeyRef:
name: canal-admin
key: datasource_database
- name: spring.datasource.username
valueFrom:
configMapKeyRef:
name: canal-admin
key: datasource_username
- name: spring.datasource.password
valueFrom:
configMapKeyRef:
name: canal-admin
key: datasource_password
resources:
requests:
cpu: 250m
memory: 256Mi
livenessProbe:
httpGet:
path: /
port: 8089
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 30
readinessProbe:
httpGet:
path: /
port: 8089
initialDelaySeconds: 10
timeoutSeconds: 5
periodSeconds: 30
restartPolicy: Always
nodeSelector: {}
affinity:
podAntiAffinity:
preferredDuringSchedulingIgnoredDuringExecution:
- weight: 1
podAffinityTerm:
labelSelector:
matchLabels:
app.kubernetes.io/name: canal-admin
namespaces:
- iot
topologyKey: kubernetes.io/hostname
revisionHistoryLimit: 10
---
kind: Service
apiVersion: v1
metadata:
name: canal-admin
namespace: iot
spec:
type: NodePort
ports:
- protocol: TCP
port: 8089
targetPort: 8089
selector:
app.kubernetes.io/name: canal-admin
表同步配置
创建Instance，这里需要注意Instance 的名字要和在配置文件的名字一致，否则会同步不了数据
创建一个名为dc-canal_dic_dt_data 的Instance
##需要修改部分
# position info
canal.instance.master.address=172.19.0.31:32060
canal.instance.master.journal.name=
canal.instance.master.position=
canal.instance.master.timestamp=
canal.instance.master.gtid=
# username/password
canal.instance.dbUsername=canal
canal.instance.dbPassword=canal
canal.instance.connectionCharset = UTF-8
# enable druid Decrypt database password
canal.instance.enableDruid=false
canal.instance.filter.regex=dc-cana.*
#可以搜索同步库下的所有表，只需要做映射即可
备注
# 如果是同步一个表的话，就是db.table;如果是多个表同步可以显示写出来或者用正则匹配
# db.table1,db.table2,db.table3
#mysql 数据解析关注的表，Perl正则表达式.
#多个正则之间以逗号(,)分隔，转义符需要双斜杠(\\)
#常见例子：
#1. 所有表：.* or .*\\..*
#2. canal schema下所有表： canal\\..*
#3. canal下的以canal打头的表：canal\\.canal.*
#4. canal schema下的一张表：canal\\.test1
#5. 多个规则组合使用：canal\\..*,mysql.test1,mysql.test2 (逗号分隔)
canal.adapter
Client-adapter 分为适配器与启动器两部分，