Initial commit

app/infra/canal/cmd/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "canal.tidb.toml",
+        "canal-test.toml",
+    ],
+    importpath = "go-common/app/infra/canal/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/http:go_default_library",
+        "//app/infra/canal/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/cmd/canal-test.toml

@@ -0,0 +1,25 @@
+[monitor]
+user = ""
+token = ""
+secret = ""
+
+[httpClient]
+key    = "654af11b5df0c9d3"
+secret = "a7512b8b243b82f4bdb72cf2824b3f8e"
+dial      = "1s"
+timeout   = "1s"
+keepAlive = "60s"
+
+[log]
+dir = "/data/log/canal/"
+Stdout = true
+
+[db]
+#    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_canal?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    dsn = "test:test@tcp(127.0.0.1:3306)/bilibili_canal?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4&allowNativePasswords=True"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "1s"
+    execTimeout = "1s"
+    tranTimeout = "1s"

app/infra/canal/cmd/canal.tidb.toml

@@ -0,0 +1,11 @@
+[instance]
+  name = "thumbup"
+ 	ClusterID     = "1"
+ 	Addrs         = [""]
+ 	Offset        = 0
+ 	CommitTS      = 0
+ 	monitor_period = "1s"
+  [[instance.db]]
+    schema = "bilibili_likes"
+    [[instance.db.table]]
+      name = "counts"

app/infra/canal/cmd/database.sql

@@ -0,0 +1,61 @@
+create database bilibili_canal;
+
+ CREATE TABLE `master_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `addr` varchar(64) NOT NULL COMMENT 'db addr hostname:port',
+  `bin_name` varchar(20) NOT NULL DEFAULT '' COMMENT 'binlog name',
+  `bin_pos` int(11) NOT NULL DEFAULT '0' COMMENT 'binlog position',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `cluster` varchar(50) NOT NULL DEFAULT '' COMMENT 'cluster',
+  `leader` varchar(20) NOT NULL DEFAULT '' COMMENT 'leader',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  `is_delete` tinyint(4) NOT NULL DEFAULT '0' COMMENT '是否删除 0-否 1-是',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_addr` (`addr`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=46585559 DEFAULT CHARSET=utf8 COMMENT='canal位置信息记录'
+
+CREATE TABLE `canal_apply` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增ID',
+  `addr` varchar(64) NOT NULL COMMENT 'db addr hostname:port',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT 'remark',
+  `cluster` varchar(50) NOT NULL DEFAULT '' COMMENT '集群',
+  `leader` varchar(20) NOT NULL DEFAULT '' COMMENT 'leader',
+  `comment` text NOT NULL COMMENT 'comment',
+  `state` tinyint(4) NOT NULL DEFAULT '1' COMMENT 'state',
+  `operator` varchar(32) NOT NULL DEFAULT '' COMMENT 'operator',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  `conf_id` int(11) NOT NULL DEFAULT '0' COMMENT '配置id',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_addr` (`addr`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8 COMMENT='canal申请信息'
+
+CREATE TABLE `hbase_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `cluster_name` varchar(20) NOT NULL DEFAULT '' COMMENT '集群名称',
+  `table_name` varchar(60) NOT NULL DEFAULT '' COMMENT '表名',
+  `latest_ts` int(11) unsigned NOT NULL DEFAULT '0' COMMENT 'lastest ts',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name_table` (`cluster_name`,`table_name`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COMMENT='hbase latest_ts表'
+
+CREATE TABLE `tidb_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `name` varchar(20) NOT NULL DEFAULT '' COMMENT 'name',
+  `cluster_id` varchar(40) NOT NULL DEFAULT '' COMMENT 'cluster id',
+  `offset` bigint(20) NOT NULL DEFAULT 0 COMMENT 'offset',
+  `tso` bigint(20) NOT NULL DEFAULT '0' COMMENT '全局时间戳',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name` (`name`),
+  KEY `ix_mtime` (`mtime`)
+) COMMENT='tidb info';

app/infra/canal/cmd/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/http"
+	"go-common/app/infra/canal/service"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("canal start")
+	canal := service.NewCanal(conf.Conf)
+	http.Init(conf.Conf, canal)
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-ch
+		log.Info("canal get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			canal.Close()
+			log.Info("canal exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}