Initial commit

app/infra/BUILD

@@ -0,0 +1,29 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/canal:all-srcs",
+        "//app/infra/config/cmd:all-srcs",
+        "//app/infra/config/conf:all-srcs",
+        "//app/infra/config/dao/v1:all-srcs",
+        "//app/infra/config/dao/v2:all-srcs",
+        "//app/infra/config/http:all-srcs",
+        "//app/infra/config/model:all-srcs",
+        "//app/infra/config/rpc/client:all-srcs",
+        "//app/infra/config/rpc/server:all-srcs",
+        "//app/infra/config/service/v1:all-srcs",
+        "//app/infra/config/service/v2:all-srcs",
+        "//app/infra/databus:all-srcs",
+        "//app/infra/discovery:all-srcs",
+        "//app/infra/notify:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/OWNERS

@@ -0,0 +1,10 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- haoguanwei
+- maojian
+labels:
+- infra
+- new-project
+options:
+  no_parent_owners: true

app/infra/canal/BUILD

@@ -0,0 +1,22 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/canal/cmd:all-srcs",
+        "//app/infra/canal/conf:all-srcs",
+        "//app/infra/canal/dao:all-srcs",
+        "//app/infra/canal/http:all-srcs",
+        "//app/infra/canal/infoc:all-srcs",
+        "//app/infra/canal/model:all-srcs",
+        "//app/infra/canal/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/CHANGELOG.md

@@ -0,0 +1,78 @@
+### Canal
+#### Version 3.3.4
+> 1.修复tidb start reload阻塞
+
+#### Version 3.3.3
+> 1. 修复tidb sync问题
+
+#### Version 3.3.2
+> 1. 修复tidb重复关闭的问题
+
+#### Version 3.3.1
+> 1.更新企业微信报警
+
+#### Vesion 3.3.0
+1. 增加tidb的支持
+
+#### Version 3.3.2
+> 1.修复canal  reload阻塞
+
+#### Version 3.2.2
+> 1.backoff 策略报警  
+
+#### Version 3.2.1
+> 1.添加告警开关  
+
+#### Vesion 3.2.0
+1. 迁移infra
+
+#### Version 3.1.5
+1. 增加错误检查并发送企业微信告警
+
+#### Version 3.1.4
+1. 修复重置instance泄露target连接问题
+
+#### Version 3.1.3
+1. 检查master权限改为post接口rsion
+
+#### Version 3.1.2
+1. 添加error日志
+
+#### Version 3.1.1
+1. 增加告警时间周期配置
+2. 增加hbase同步延迟告警
+
+#### Version 3.1.0
+1. 增加同步Hbase功能
+2. 增加企业微信告警
+
+#### Version 3.0.9
+1. 修复不报警问题
+
+#### Version 3.0.8
+1. 升级基础库
+
+#### Version 3.0.7
+1. 升级基础库
+
+#### Version 3.0.6
+1. 增加测试环境强制同步pos接口
+
+#### Version 3.0.5 
+1. 增加本地canal测试配置  
+2. 增加同步最新bin_pos接口
+
+#### Version 3.0.4
+1. 修复异常close导致的panic
+
+#### Version 3.0.3
+1. discovery统一
+
+#### Version 3.0.2
+1. 修复err为nil panic
+
+#### Version 3.0.1
+1. 检测实例返回binname和binpos
+
+#### Version 3.0.0
+1. 合并大仓库,完全引用siddontang的go-mysql作为binlog replication

app/infra/canal/CONTRIBUTORS.md

@@ -0,0 +1,12 @@
+# Owner
+haoguanwei
+lintanghui
+
+# Author
+lintanghui
+guhao
+chenshangqiang
+
+# Reviewer
+maojian
+haoguanwei

app/infra/canal/OWNERS

@@ -0,0 +1,18 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- chenshangqiang
+- guhao
+- haoguanwei
+- lintanghui
+labels:
+- infra
+- infra/canal
+options:
+  no_parent_owners: true
+reviewers:
+- chenshangqiang
+- guhao
+- haoguanwei
+- lintanghui
+- maojian

app/infra/canal/README.md

@@ -0,0 +1,16 @@
+# go-common/app/infra/canal
+
+##### 项目简介
+> 1. 实现MySQL slave协议，充当伪从库
+> 2. 一个canal节点可同时同步一个或多个MySQL database,可指定开始binlog位置
+
+##### 编译环境
+> 1. 请只用golang v1.7.x以上版本编译执行。
+
+##### 依赖包
+> 1. 公共依赖
+
+##### 编译执行
+> 1. 启动执行
+> 2. 项目文档http://info.bilibili.co/pages/viewpage.action?pageId=4547253
+

app/infra/canal/cmd/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = [
+        "canal.tidb.toml",
+        "canal-test.toml",
+    ],
+    importpath = "go-common/app/infra/canal/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/http:go_default_library",
+        "//app/infra/canal/service:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/cmd/canal-test.toml

@@ -0,0 +1,25 @@
+[monitor]
+user = ""
+token = ""
+secret = ""
+
+[httpClient]
+key    = "654af11b5df0c9d3"
+secret = "a7512b8b243b82f4bdb72cf2824b3f8e"
+dial      = "1s"
+timeout   = "1s"
+keepAlive = "60s"
+
+[log]
+dir = "/data/log/canal/"
+Stdout = true
+
+[db]
+#    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_canal?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    dsn = "test:test@tcp(127.0.0.1:3306)/bilibili_canal?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4&allowNativePasswords=True"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "1s"
+    execTimeout = "1s"
+    tranTimeout = "1s"

app/infra/canal/cmd/canal.tidb.toml

@@ -0,0 +1,11 @@
+[instance]
+  name = "thumbup"
+ 	ClusterID     = "1"
+ 	Addrs         = [""]
+ 	Offset        = 0
+ 	CommitTS      = 0
+ 	monitor_period = "1s"
+  [[instance.db]]
+    schema = "bilibili_likes"
+    [[instance.db.table]]
+      name = "counts"

app/infra/canal/cmd/database.sql

@@ -0,0 +1,61 @@
+create database bilibili_canal;
+
+ CREATE TABLE `master_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `addr` varchar(64) NOT NULL COMMENT 'db addr hostname:port',
+  `bin_name` varchar(20) NOT NULL DEFAULT '' COMMENT 'binlog name',
+  `bin_pos` int(11) NOT NULL DEFAULT '0' COMMENT 'binlog position',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `cluster` varchar(50) NOT NULL DEFAULT '' COMMENT 'cluster',
+  `leader` varchar(20) NOT NULL DEFAULT '' COMMENT 'leader',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  `is_delete` tinyint(4) NOT NULL DEFAULT '0' COMMENT '是否删除 0-否 1-是',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_addr` (`addr`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=46585559 DEFAULT CHARSET=utf8 COMMENT='canal位置信息记录'
+
+CREATE TABLE `canal_apply` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT '自增ID',
+  `addr` varchar(64) NOT NULL COMMENT 'db addr hostname:port',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT 'remark',
+  `cluster` varchar(50) NOT NULL DEFAULT '' COMMENT '集群',
+  `leader` varchar(20) NOT NULL DEFAULT '' COMMENT 'leader',
+  `comment` text NOT NULL COMMENT 'comment',
+  `state` tinyint(4) NOT NULL DEFAULT '1' COMMENT 'state',
+  `operator` varchar(32) NOT NULL DEFAULT '' COMMENT 'operator',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  `conf_id` int(11) NOT NULL DEFAULT '0' COMMENT '配置id',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_addr` (`addr`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=17 DEFAULT CHARSET=utf8 COMMENT='canal申请信息'
+
+CREATE TABLE `hbase_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `cluster_name` varchar(20) NOT NULL DEFAULT '' COMMENT '集群名称',
+  `table_name` varchar(60) NOT NULL DEFAULT '' COMMENT '表名',
+  `latest_ts` int(11) unsigned NOT NULL DEFAULT '0' COMMENT 'lastest ts',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name_table` (`cluster_name`,`table_name`),
+  KEY `ix_mtime` (`mtime`)
+) ENGINE=InnoDB AUTO_INCREMENT=2 DEFAULT CHARSET=utf8 COMMENT='hbase latest_ts表'
+
+CREATE TABLE `tidb_info` (
+  `id` int(11) unsigned NOT NULL AUTO_INCREMENT COMMENT 'ID',
+  `name` varchar(20) NOT NULL DEFAULT '' COMMENT 'name',
+  `cluster_id` varchar(40) NOT NULL DEFAULT '' COMMENT 'cluster id',
+  `offset` bigint(20) NOT NULL DEFAULT 0 COMMENT 'offset',
+  `tso` bigint(20) NOT NULL DEFAULT '0' COMMENT '全局时间戳',
+  `remark` varchar(100) NOT NULL DEFAULT '' COMMENT '备注',
+  `ctime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP COMMENT '创建时间',
+  `mtime` timestamp NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP COMMENT '修改时间',
+  PRIMARY KEY (`id`),
+  UNIQUE KEY `uk_name` (`name`),
+  KEY `ix_mtime` (`mtime`)
+) COMMENT='tidb info';

app/infra/canal/cmd/main.go

@@ -0,0 +1,42 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/http"
+	"go-common/app/infra/canal/service"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("canal start")
+	canal := service.NewCanal(conf.Conf)
+	http.Init(conf.Conf, canal)
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-ch
+		log.Info("canal get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			canal.Close()
+			log.Info("canal exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/infra/canal/conf/BUILD

@@ -0,0 +1,46 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal_conf.go",
+        "conf.go",
+        "tidb_conf.go",
+    ],
+    importpath = "go-common/app/infra/canal/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/infoc:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/client:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/conf/canal_conf.go

@@ -0,0 +1,310 @@
+package conf
+
+import (
+	"fmt"
+	"io/ioutil"
+	"net"
+	"regexp"
+	"strings"
+
+	"go-common/app/infra/canal/infoc"
+	"go-common/library/conf"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+	"github.com/siddontang/go-mysql/canal"
+	"github.com/siddontang/go-mysql/client"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+var (
+	// config change event
+	event      = make(chan *InsConf, 1)
+	hbaseEvent = make(chan *HBaseInsConf, 1)
+	tidbEvent  = make(chan *TiDBInsConf, 1)
+	canalPath  string
+)
+
+// Addition addition attrbute of canal.
+type Addition struct {
+	PrimaryKey []string `toml:"primarykey"` // kafka msg key
+	OmitField  []string `toml:"omitfield"`  // field will be ignored in table
+}
+
+// CTable canal table.
+type CTable struct {
+	PrimaryKey []string `toml:"primarykey"` // kafka msg key
+	OmitField  []string `toml:"omitfield"`  // field will be ignored in table
+	OmitAction []string `toml:"omitaction"` // action will be ignored in table
+	Name       string   `toml:"name"`       // table name support regular expression
+	Tables     []string
+}
+
+// Database represent mysql db
+type Database struct {
+	Schema   string          `toml:"schema"`
+	Databus  *databus.Config `toml:"databus"`
+	Infoc    *infoc.Config   `toml:"infoc"`
+	CTables  []*CTable       `toml:"table"`
+	TableMap map[string]*Addition
+}
+
+// CheckTable check database tables.
+func (db *Database) CheckTable(addr, user, passwd string) (err error) {
+	var (
+		conn  *client.Conn
+		res   *mysql.Result
+		regex *regexp.Regexp
+		table string
+	)
+	db.TableMap = make(map[string]*Addition)
+	if conn, err = client.Connect(addr, user, passwd, db.Schema); err != nil {
+		return
+	}
+	defer conn.Close()
+	if res, err = conn.Execute(fmt.Sprintf("SHOW TABLES FROM `%s`", db.Schema)); err != nil {
+		log.Error("conn.Execute() error(%v)", err)
+		return
+	}
+	for _, ctable := range db.CTables {
+		if regex, err = regexp.Compile(ctable.Name); err != nil {
+			log.Error("regexp.Compile(%s) error(%v)", ctable.Name, err)
+			return
+		}
+		for _, value := range res.Values {
+			table = fmt.Sprintf("%s", value[0])
+			if regex.MatchString(table) {
+				db.TableMap[table] = &Addition{
+					PrimaryKey: ctable.PrimaryKey,
+					OmitField:  ctable.OmitField,
+				}
+				ctable.Tables = append(ctable.Tables, table)
+			}
+		}
+		if len(ctable.Tables) == 0 {
+			return fmt.Errorf("addr(%s) db(%s) subscribles nothing,table(%s) is empty", addr, db.Schema, ctable.Name)
+		}
+	}
+	return
+}
+
+// InsConf instance config
+type InsConf struct {
+	*canal.Config
+
+	MonitorPeriod xtime.Duration    `toml:"monitor_period"`
+	MonitorOff    bool              `toml:"monitor_off"`
+	Databases     []*Database       `toml:"db"`
+	MasterInfo    *MasterInfoConfig `toml:"masterinfo"`
+}
+
+// HBaseTable hbase canal table.
+type HBaseTable struct {
+	Name      string   `toml:"name"`      // table name
+	OmitField []string `toml:"omitfield"` // field will be ignored in table
+}
+
+// HBaseDatabase hbase database.
+type HBaseDatabase struct {
+	Tables  []*HBaseTable   `toml:"table"`
+	Databus *databus.Config `toml:"databus"`
+}
+
+// HBaseInsConf hbase instance config.
+type HBaseInsConf struct {
+	Cluster string
+	Root    string
+	Addrs   []string
+
+	MonitorPeriod xtime.Duration    `toml:"monitor_period"`
+	MonitorOff    bool              `toml:"monitor_off"`
+	Databases     []*HBaseDatabase  `toml:"db"`
+	MasterInfo    *MasterInfoConfig `toml:"masterinfo"`
+}
+
+// CanalConfig config struct
+type CanalConfig struct {
+	Instances      []*InsConf      `toml:"instance"`
+	HBaseInstances []*HBaseInsConf `toml:"hbase_instance"`
+	TiDBInstances  []*TiDBInsConf  `toml:"tidb_instance"`
+}
+
+func newInsConf(fn, fc string) (c *InsConf, err error) {
+	var ic struct {
+		InsConf *InsConf `toml:"instance"`
+	}
+	ipPort := strings.TrimSuffix(fn, ".toml")
+	if _, _, err = net.SplitHostPort(ipPort); err != nil {
+		return
+	}
+	if _, err = toml.Decode(fc, &ic); err != nil {
+		return
+	}
+	if ic.InsConf == nil {
+		err = fmt.Errorf("file(%s) cannot decode toml", fn)
+		return
+	}
+	if ic.InsConf.Addr != ipPort {
+		err = fmt.Errorf("file(%s) name not equal addr(%s)", fn, ic.InsConf.Addr)
+		return
+	}
+	if ic.InsConf.MasterInfo == nil {
+		ic.InsConf.MasterInfo = Conf.MasterInfo
+	}
+	return ic.InsConf, nil
+}
+
+func newHBaseConf(fn, fc string) (c *HBaseInsConf, err error) {
+	var ic struct {
+		InsConf *HBaseInsConf `toml:"instance"`
+	}
+	if _, err = toml.Decode(fc, &ic); err != nil {
+		return
+	}
+	if ic.InsConf == nil {
+		err = fmt.Errorf("file(%s) cannot decode toml", fn)
+		return
+	}
+	cluster := strings.TrimSuffix(fn, ".hbase.toml")
+	if ic.InsConf.Cluster != cluster {
+		err = fmt.Errorf("file(%s) name not equal name(%s)", cluster, ic.InsConf.Cluster)
+		return
+	}
+	if ic.InsConf.MasterInfo == nil {
+		ic.InsConf.MasterInfo = Conf.MasterInfo
+	}
+	return ic.InsConf, nil
+}
+
+// LoadCanalConf load canal config.
+func LoadCanalConf() (c *CanalConfig, err error) {
+	var (
+		result []*conf.Value
+		ok     bool
+	)
+	if canalPath != "" {
+		result, err = localCanal()
+	} else {
+		result, ok = ConfClient.Configs()
+		if !ok {
+			panic("no canal-config")
+		}
+	}
+	c = new(CanalConfig)
+	im := map[string]struct{}{}
+	for _, ns := range result {
+		if ns.Name == "canal.toml" || ns.Name == "common.toml" {
+			continue
+		}
+		if strings.HasSuffix(ns.Name, ".hbase.toml") {
+			var ic *HBaseInsConf
+			if ic, err = newHBaseConf(ns.Name, ns.Config); err != nil {
+				err = fmt.Errorf("file(%s) decode error(%v)", ns.Name, err)
+				return
+			}
+			c.HBaseInstances = append(c.HBaseInstances, ic)
+		} else if strings.HasSuffix(ns.Name, ".tidb.toml") {
+			var ic *TiDBInsConf
+			if ic, err = newTiDBConf(ns.Name, ns.Config); err != nil {
+				err = fmt.Errorf("file(%s) decode error(%v)", ns.Name, err)
+				return
+			}
+			c.TiDBInstances = append(c.TiDBInstances, ic)
+		} else {
+			var ic *InsConf
+			if !strings.HasSuffix(ns.Name, ".toml") {
+				err = fmt.Errorf("file(%s) name is not a toml", ns.Name)
+				continue
+			}
+			if ic, err = newInsConf(ns.Name, ns.Config); err != nil {
+				err = fmt.Errorf("file(%s) decode error(%v)", ns.Name, err)
+				return
+			}
+			if _, ok := im[ic.Addr]; ok {
+				err = fmt.Errorf("file(%s) repeat with other toml", ns.Name)
+				return
+			}
+			im[ic.Addr] = struct{}{}
+			c.Instances = append(c.Instances, ic)
+		}
+	}
+	if canalPath == "" {
+		go func() {
+			for name := range ConfClient.Event() {
+				log.Info("config(%s) reload", name)
+				reloadConfig(name)
+			}
+		}()
+	}
+	return
+}
+
+func localCanal() (vs []*conf.Value, ok error) {
+	fs, err := ioutil.ReadDir(canalPath)
+	if err != nil {
+		panic(err)
+	}
+	for _, f := range fs {
+		if !strings.HasSuffix(f.Name(), ".toml") {
+			continue
+		}
+		ct, err := ioutil.ReadFile(canalPath + f.Name())
+		if err != nil {
+			continue
+		}
+		vs = append(vs, &conf.Value{
+			Name:   f.Name(),
+			Config: string(ct),
+		})
+	}
+	return
+}
+
+// Event returns config change event chan,
+func Event() chan *InsConf {
+	return event
+}
+
+// HBaseEvent returns config change event chan,
+func HBaseEvent() chan *HBaseInsConf {
+	return hbaseEvent
+}
+
+func reloadConfig(name string) {
+	var (
+		cf string
+		ok bool
+	)
+	if name == "canal.toml" || name == "common.toml" {
+		LoadCanal()
+		return
+	}
+	if !strings.HasSuffix(name, ".toml") {
+		return
+	}
+	if cf, ok = ConfClient.Value(name); !ok {
+		// TODO(felix): auto reload? or restart hard?
+		return
+	}
+	if strings.HasSuffix(name, ".hbase.toml") {
+		ic, err := newHBaseConf(name, cf)
+		if err != nil {
+			return
+		}
+		hbaseEvent <- ic
+	} else if strings.HasSuffix(name, ".tidb.toml") {
+		ic, err := newTiDBConf(name, cf)
+		if err != nil {
+			return
+		}
+		tidbEvent <- ic
+	} else {
+		ic, err := newInsConf(name, cf)
+		if err != nil {
+			return
+		}
+		event <- ic
+	}
+}

app/infra/canal/conf/conf.go

@@ -0,0 +1,96 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+	"time"
+
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/naming/discovery"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	// ConfClient get config client
+	ConfClient *conf.Client
+	// Conf canal config variable
+	Conf = &Config{}
+)
+
+// Config canal config struct
+type Config struct {
+	Monitor *Monitor
+	// xlog
+	Log *log.Config
+	// http client
+	HTTPClient *bm.ClientConfig
+	// http server
+	BM *bm.ServerConfig
+	// master info
+	MasterInfo *MasterInfoConfig
+	// discovery
+	Discovery *discovery.Config
+	// db
+	DB *sql.Config
+}
+
+// Monitor wechat monitor
+type Monitor struct {
+	User   string
+	Token  string
+	Secret string
+}
+
+// MasterInfoConfig save pos of binlog in file or db
+type MasterInfoConfig struct {
+	Addr     string        `toml:"addr"`
+	DBName   string        `toml:"dbName"`
+	User     string        `toml:"user"`
+	Password string        `toml:"password"`
+	Timeout  time.Duration `toml:"timeout"`
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+	flag.StringVar(&canalPath, "canal", "", "canal instance path")
+}
+
+//Init int config
+func Init() (err error) {
+	if confPath != "" {
+		_, err = toml.DecodeFile(confPath, &Conf)
+		return
+	}
+	return remote()
+}
+
+func remote() (err error) {
+	if ConfClient, err = conf.New(); err != nil {
+		return
+	}
+	ConfClient.WatchAll()
+	err = LoadCanal()
+	return
+}
+
+// LoadCanal canal config
+func LoadCanal() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = ConfClient.Value("canal.toml"); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/infra/canal/conf/tidb_conf.go

@@ -0,0 +1,39 @@
+package conf
+
+import (
+	"fmt"
+
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+// TiDBInsConf tidb instance config
+type TiDBInsConf struct {
+	Name          string
+	ClusterID     string
+	Addrs         []string
+	Offset        int64
+	CommitTS      int64
+	MonitorPeriod xtime.Duration `toml:"monitor_period"`
+	Databases     []*Database    `toml:"db"`
+}
+
+func newTiDBConf(fn, fc string) (c *TiDBInsConf, err error) {
+	var ic struct {
+		InsConf *TiDBInsConf `toml:"instance"`
+	}
+	if _, err = toml.Decode(fc, &ic); err != nil {
+		return
+	}
+	if ic.InsConf == nil {
+		err = fmt.Errorf("file(%s) cannot decode toml", fn)
+		return
+	}
+	return ic.InsConf, nil
+}
+
+// TiDBEvent .
+func TiDBEvent() chan *TiDBInsConf {
+	return tidbEvent
+}

app/infra/canal/dao/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+    ],
+    importpath = "go-common/app/infra/canal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "mysql_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/infra/canal/dao/dao.go

@@ -0,0 +1,23 @@
+package dao
+
+import (
+	"go-common/app/infra/canal/conf"
+	"go-common/library/database/sql"
+)
+
+// Dao dao
+type Dao struct {
+	// config
+	c *conf.Config
+	// db
+	db *sql.DB
+}
+
+// New dao new
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:  c,
+		db: sql.NewMySQL(c.DB),
+	}
+	return
+}

app/infra/canal/dao/dao_test.go

@@ -0,0 +1,34 @@
+package dao
+
+import (
+	"flag"
+	"os"
+	"testing"
+
+	"go-common/app/infra/canal/conf"
+)
+
+var d *Dao
+
+func TestMain(m *testing.M) {
+	if os.Getenv("DEPLOY_ENV") != "" {
+		flag.Set("app_id", "")
+		flag.Set("conf_token", "")
+		flag.Set("tree_id", "")
+		flag.Set("conf_version", "docker-1")
+		flag.Set("deploy_env", "uat")
+		flag.Set("conf_host", "config.bilibili.co")
+		flag.Set("conf_path", "/tmp")
+		flag.Set("region", "sh")
+		flag.Set("zone", "sh001")
+	} else {
+		flag.Set("conf", "../cmd/canal-test.toml")
+	}
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/infra/canal/dao/mysql.go

@@ -0,0 +1,40 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/infra/canal/model"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_tidbPositonSQL       = "SELECT name, cluster_id, offset, tso FROM  tidb_info WHERE name = ?"
+	_updateTidbPositonSQL = "INSERT INTO tidb_info(name, cluster_id, offset, tso) VALUES(?,?,?,?) ON DUPLICATE KEY UPDATE offset = ?, tso = ?"
+)
+
+// TiDBPosition get tidb positon
+func (d *Dao) TiDBPosition(c context.Context, name string) (res *model.TiDBInfo, err error) {
+	res = &model.TiDBInfo{}
+	if err = d.db.QueryRow(c, _tidbPositonSQL, name).Scan(&res.Name, &res.ClusterID, &res.Offset, &res.CommitTS); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("db.TidbPosition.Query error(%v,%v,%v)", _tidbPositonSQL, name, err)
+		return
+	}
+	return
+}
+
+// UpdateTiDBPosition update tidb position
+func (d *Dao) UpdateTiDBPosition(c context.Context, info *model.TiDBInfo) (err error) {
+	if info == nil {
+		return
+	}
+	if _, err = d.db.Exec(c, _updateTidbPositonSQL, info.Name, info.ClusterID, info.Offset, info.CommitTS, info.Offset, info.CommitTS); err != nil {
+		log.Error("db.UpdateTiDBPosition.Exec error(%v,%+v,%v)", _updateTidbPositonSQL, info, err)
+	}
+	return
+}

app/infra/canal/dao/mysql_test.go

@@ -0,0 +1,28 @@
+package dao
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/infra/canal/model"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestDao_TiDBPosition(t *testing.T) {
+	info := &model.TiDBInfo{
+		Name:      "test",
+		ClusterID: "1",
+		Offset:    2,
+		CommitTS:  403845808070328359,
+	}
+	convey.Convey("add position", t, func(ctx convey.C) {
+		err := d.UpdateTiDBPosition(context.Background(), info)
+		ctx.So(err, convey.ShouldBeNil)
+		ctx.Convey("get position", func(ctx convey.C) {
+			gotRes, err := d.TiDBPosition(context.Background(), info.Name)
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(gotRes, convey.ShouldResemble, info)
+		})
+	})
+}

app/infra/canal/http/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal.go",
+        "http.go",
+        "infoc.go",
+    ],
+    importpath = "go-common/app/infra/canal/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/infoc:go_default_library",
+        "//app/infra/canal/service:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/http/canal.go

@@ -0,0 +1,48 @@
+package http
+
+import (
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+func errors(c *bm.Context) {
+	type result struct {
+		Error         string            `json:"error"`
+		InstanceError map[string]string `json:"instance_error"`
+	}
+	res := result{
+		Error:         cs.Error(),
+		InstanceError: cs.Errors(),
+	}
+	c.JSON(res, nil)
+}
+
+func checkMaster(c *bm.Context) {
+	arg := new(struct {
+		Addr     string `form:"addr" validate:"required"`
+		User     string `form:"user" validate:"required"`
+		Password string `form:"password" validate:"required"`
+	})
+	if err := c.Bind(arg); err != nil {
+		return
+	}
+	name, pos, err := cs.CheckMaster(arg.Addr, arg.User, arg.Password)
+	if err != nil {
+		c.JSON(nil, ecode.AccessDenied)
+		return
+	}
+	res := map[string]interface{}{"name": name, "pos:": pos}
+	c.JSON(res, nil)
+}
+
+func syncPos(c *bm.Context) {
+	arg := new(struct {
+		Addr string `form:"addr" validate:"required"`
+	})
+	if err := c.Bind(arg); err != nil {
+		log.Error("syncpos params err %v", err)
+		return
+	}
+	c.JSON(nil, cs.PosSync(arg.Addr))
+}

app/infra/canal/http/http.go

@@ -0,0 +1,46 @@
+package http
+
+import (
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+var (
+	cs *service.Canal
+)
+
+// Init int http service
+func Init(c *conf.Config, cs *service.Canal) {
+	initService(cs)
+	// init router
+	eg := bm.DefaultServer(c.BM)
+	initRouter(eg)
+	// init Outer serve
+	if err := eg.Start(); err != nil {
+		log.Error("bm.DefaultServer error(%v)", err)
+		panic(err)
+	}
+}
+
+func initService(canal *service.Canal) {
+	cs = canal
+}
+
+// initRouter init outer router api path.
+func initRouter(e *bm.Engine) {
+	// init api
+	e.Ping(ping)
+	group := e.Group("/x/internal/canal")
+	{
+		group.GET("/infoc/post", infocPost)
+		group.GET("/infoc/current", infocCurrent)
+		group.GET("/errors", errors)
+		group.POST("/master/check", checkMaster)
+		group.POST("/test/sync", syncPos)
+	}
+}
+
+func ping(c *bm.Context) {
+}

app/infra/canal/http/infoc.go

@@ -0,0 +1,178 @@
+package http
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"hash/crc32"
+	"io/ioutil"
+	"net/http"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	config "go-common/library/conf"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/BurntSushi/toml"
+	"github.com/siddontang/go-mysql/canal"
+)
+
+const (
+	_heartHeat   = 60
+	_readTimeout = 90
+	_flavor      = "mysql"
+	_updateUser  = "canal"
+	_updateMark  = "infoc"
+)
+
+// InfocConf .
+type infocConf struct {
+	Addr     string     `json:"db_addr"`
+	User     string     `json:"user"`
+	Pass     string     `json:"pass"`
+	InfocDBs []*infocDB `json:"databases"`
+}
+
+// InfocDB .
+type infocDB struct {
+	Schema           string       `json:"schema"`
+	Tables           []*infoTable `json:"tables"`
+	LancerAddr       string       `json:"lancer_addr"`
+	LancerTaskID     string       `json:"lancer_task_id"`
+	LancerReportAddr string       `json:"lancer_report_addr"`
+	Proto            string       `json:"proto"`
+}
+
+// InfoTable .
+type infoTable struct {
+	Name       string   `json:"name"`
+	OmitFlied  []string `json:"omit_field"`
+	OmitAction []string `json:"omit_action"`
+}
+
+func infocPost(c *bm.Context) {
+	var (
+		ics []*infocConf
+		bs  []byte
+		err error
+		buf *bytes.Buffer
+	)
+	content := make(map[string]string)
+	if bs, err = ioutil.ReadAll(c.Request.Body); err != nil {
+		c.AbortWithStatus(http.StatusInternalServerError)
+		return
+	}
+	if err = json.Unmarshal(bs, &ics); err != nil {
+		c.AbortWithStatus(http.StatusInternalServerError)
+		return
+	}
+	for _, ifc := range ics {
+		databases := make([]*conf.Database, len(ifc.InfocDBs))
+		for idx, infocDB := range ifc.InfocDBs {
+			tables := make([]*conf.CTable, len(infocDB.Tables))
+			for ix, table := range infocDB.Tables {
+				tables[ix] = &conf.CTable{
+					Name:       table.Name,
+					OmitAction: table.OmitAction,
+					OmitField:  table.OmitFlied,
+				}
+			}
+			databases[idx] = &conf.Database{
+				Schema: infocDB.Schema,
+				Infoc: &infoc.Config{
+					TaskID:       infocDB.LancerTaskID,
+					Addr:         infocDB.LancerAddr,
+					ReporterAddr: infocDB.LancerReportAddr,
+					Proto:        infocDB.Proto,
+				},
+				CTables: tables,
+			}
+		}
+		ic := &conf.InsConf{
+			Databases: databases,
+			Config: &canal.Config{
+				Addr:            ifc.Addr,
+				User:            ifc.User,
+				Password:        ifc.Pass,
+				ServerID:        crc32.ChecksumIEEE([]byte(ifc.Addr)),
+				Flavor:          _flavor,
+				HeartbeatPeriod: _heartHeat,
+				ReadTimeout:     _readTimeout,
+			},
+		}
+		var isc = &struct {
+			InsConf *conf.InsConf `toml:"instance"`
+		}{
+			InsConf: ic,
+		}
+		buf = new(bytes.Buffer)
+		if err = toml.NewEncoder(buf).Encode(isc); err != nil {
+			c.AbortWithStatus(http.StatusInternalServerError)
+			return
+		}
+		content[fmt.Sprintf("%v.toml", ifc.Addr)] = buf.String()
+	}
+	for cn, cv := range content {
+		value, err := conf.ConfClient.ConfIng(cn)
+		if err == nil {
+			err = conf.ConfClient.Update(value.CID, cv, _updateUser, _updateMark)
+		} else if err == ecode.NothingFound {
+			err = conf.ConfClient.Create(cn, cv, _updateUser, _updateMark)
+		}
+		if err != nil {
+			c.AbortWithStatus(http.StatusInternalServerError)
+			return
+		}
+	}
+}
+
+func infocCurrent(c *bm.Context) {
+	var (
+		ok     bool
+		result []*config.Value
+	)
+	if result, ok = conf.ConfClient.Configs(); !ok {
+		c.Status(http.StatusInternalServerError)
+		return
+	}
+	ics := make([]*infocConf, 0, len(result))
+	for _, ns := range result {
+		var ic struct {
+			InsConf *conf.InsConf `toml:"instance"`
+		}
+		if _, err := toml.Decode(ns.Config, &ic); err != nil {
+			c.AbortWithStatus(http.StatusInternalServerError)
+			return
+		}
+		if ic.InsConf == nil {
+			continue
+		}
+		icf := &infocConf{
+			Addr: ic.InsConf.Addr,
+			User: ic.InsConf.User,
+			Pass: ic.InsConf.Password,
+		}
+		for _, icdb := range ic.InsConf.Databases {
+			if icdb.Infoc == nil {
+				continue
+			}
+			tables := make([]*infoTable, len(icdb.CTables))
+			for idx, ctable := range icdb.CTables {
+				tables[idx] = &infoTable{
+					Name:       ctable.Name,
+					OmitFlied:  ctable.OmitField,
+					OmitAction: ctable.OmitAction,
+				}
+			}
+			icf.InfocDBs = append(icf.InfocDBs, &infocDB{
+				Schema:       icdb.Schema,
+				Tables:       tables,
+				LancerAddr:   icdb.Infoc.Addr,
+				LancerTaskID: icdb.Infoc.TaskID,
+			})
+		}
+		ics = append(ics, icf)
+	}
+	c.JSON(ics, nil)
+}

app/infra/canal/infoc/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "infoc.go",
+        "reporter.go",
+    ],
+    importpath = "go-common/app/infra/canal/infoc",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/log:go_default_library",
+        "//library/net/ip:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/infoc/infoc.go

@@ -0,0 +1,144 @@
+package infoc
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"encoding/json"
+	"net"
+	"strconv"
+	"sync"
+	"time"
+
+	"go-common/library/log"
+)
+
+var (
+	_infoc2Magic = []byte{172, 190} // NOTE: magic 0xAC0xBE
+	_infoc2Type  = []byte{0, 0}     // NOTE: type 0
+
+	_infocTimeout = 500 * time.Millisecond
+)
+
+// Config is infoc config.
+type Config struct {
+	TaskID string
+	// udp or tcp
+	Proto string
+	Addr  string
+	// reporter
+	ReporterAddr string
+}
+
+// Infoc infoc struct.
+type Infoc struct {
+	c      *Config
+	header []byte
+	// udp or tcp
+	conn net.Conn
+	lock sync.Mutex
+	// reporter
+	reporter *reporter
+}
+
+// New new infoc2 logger.
+func New(c *Config) (i *Infoc) {
+	i = &Infoc{
+		c:      c,
+		header: []byte(c.TaskID),
+	}
+	var err error
+	if i.conn, err = net.Dial(i.c.Proto, i.c.Addr); err != nil {
+		log.Error("infoc net dial error(%v)", err)
+	}
+	if c.ReporterAddr != "" {
+		i.reporter = newReporter(c.TaskID, c.ReporterAddr)
+		go i.reporter.reportproc()
+	}
+	return
+}
+
+// Rows the affected by binlog enent.
+func (i *Infoc) Rows(rows int64) {
+	if i.reporter != nil {
+		i.reporter.receiveIncr(rows)
+	}
+}
+
+// Send send message.
+func (i *Infoc) Send(ctx context.Context, key string, v interface{}) (err error) {
+	var b []byte
+	if b, err = json.Marshal(v); err != nil {
+		log.Error("json.Marshal(%v) error(%v)", v, err)
+		return
+	}
+	var (
+		res bytes.Buffer
+		buf bytes.Buffer
+	)
+	res.Write(_infoc2Magic)
+	// type and body buf, for calc length.
+	buf.Write(_infoc2Type)
+	buf.Write(i.header)
+	buf.WriteString(strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10))
+	// // append first arg
+	if _, err = buf.WriteString(string(b)); err != nil {
+		return
+	}
+	// put length
+	var ls [4]byte
+	binary.BigEndian.PutUint32(ls[:], uint32(buf.Len()))
+	res.Write(ls[:])       // NOTE: write length
+	res.Write(buf.Bytes()) // NOTE：write type and body
+	// write
+	if err = i.write(res.Bytes()); err != nil {
+		log.Error("infoc write error(%v)", err)
+		return
+	}
+	if i.reporter != nil {
+		i.reporter.sendIncr(1)
+	}
+	return
+}
+
+// write write data into connection.
+func (i *Infoc) write(bs []byte) (err error) {
+	defer func() {
+		if err != nil {
+			if i.conn != nil {
+				i.conn.Close()
+			}
+			i.conn = nil
+		}
+		i.lock.Unlock()
+	}()
+	i.lock.Lock()
+	// connection and write
+	if i.conn == nil {
+		if i.conn, err = net.DialTimeout(i.c.Proto, i.c.Addr, _infocTimeout); err != nil {
+			log.Error("infoc net dial error(%v)", err)
+			return
+		}
+	}
+	if i.c.Proto == "tcp" {
+		i.conn.SetDeadline(time.Now().Add(_infocTimeout))
+	}
+	if _, err = i.conn.Write(bs); err != nil {
+		log.Error("infoc net write error(%v)", err)
+	}
+	return
+}
+
+// Flush flush reporter count.
+func (i *Infoc) Flush() {
+	if i.reporter != nil {
+		i.reporter.flush()
+	}
+}
+
+// Close close resource.
+func (i *Infoc) Close() {
+	if i.conn != nil {
+		i.conn.Close()
+	}
+}

app/infra/canal/infoc/reporter.go

@@ -0,0 +1,84 @@
+package infoc
+
+import (
+	"fmt"
+	"net"
+	"sync/atomic"
+	"time"
+
+	"go-common/library/log"
+	"go-common/library/net/ip"
+)
+
+type reporter struct {
+	taskID string
+	addr   string
+	iip    string
+
+	receiveCount int64
+	sendCount    int64
+
+	fails []string
+}
+
+func newReporter(taskID, addr string) (r *reporter) {
+	r = &reporter{
+		taskID: taskID,
+		addr:   addr,
+		iip:    ip.InternalIP(),
+	}
+	return
+}
+
+func (r *reporter) receiveIncr(delta int64) {
+	atomic.AddInt64(&r.receiveCount, delta)
+}
+
+func (r *reporter) sendIncr(delta int64) {
+	atomic.AddInt64(&r.sendCount, delta)
+}
+
+func (r *reporter) reportproc() {
+	tick := time.NewTicker(1 * time.Minute)
+	for {
+		<-tick.C
+		r.reporter()
+	}
+}
+
+func (r *reporter) flush() {
+	r.reporter()
+}
+
+func (r *reporter) reporter() {
+	const _timeout = time.Second
+	conn, err := net.DialTimeout("tcp", r.addr, _timeout)
+	if err != nil {
+		log.Error("infoc reporter flush dial error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	conn.SetDeadline(time.Now().Add(_timeout))
+	var fails []string
+	for _, fail := range r.fails {
+		if _, err = conn.Write([]byte(fail)); err != nil {
+			log.Error("infoc reporter write fail error(%v)", err)
+			fails = append(fails, fail)
+		}
+	}
+	for _, rc := range r.record(time.Now()) {
+		if _, err = conn.Write([]byte(rc)); err != nil {
+			log.Error("infoc reporter write error(%v)", err)
+			fails = append(fails, rc)
+		}
+	}
+	r.fails = fails
+}
+
+func (r *reporter) record(now time.Time) []string {
+	rc := atomic.SwapInt64(&r.receiveCount, 0)
+	sc := atomic.SwapInt64(&r.sendCount, 0)
+	rcW := fmt.Sprintf("agent.receive.count\001%d\001%s\001%d\001%s\001\001", rc, r.iip, now.UnixNano()/int64(time.Millisecond), r.taskID)
+	scW := fmt.Sprintf("agent.send.success.count\001%d\001%s\001%d\001%s\001\001", sc, r.iip, now.UnixNano()/int64(time.Millisecond), r.taskID)
+	return []string{rcW, scW}
+}

app/infra/canal/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/infra/canal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/model/model.go

@@ -0,0 +1,19 @@
+package model
+
+// Data msg will be push to databus
+type Data struct {
+	Action string `json:"action"`
+	Table  string `json:"table"`
+	// kafka key
+	Key string                 `json:"-"`
+	Old map[string]interface{} `json:"old,omitempty"`
+	New map[string]interface{} `json:"new,omitempty"`
+}
+
+// TiDBInfo tidb db model
+type TiDBInfo struct {
+	Name      string
+	ClusterID string
+	Offset    int64
+	CommitTS  int64
+}

app/infra/canal/service/BUILD

@@ -0,0 +1,76 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "canal.go",
+        "hbase.go",
+        "instance.go",
+        "master.go",
+        "target.go",
+        "tidb_check.go",
+        "tidb_data.go",
+        "tidb_instance.go",
+        "tidb_proc.go",
+    ],
+    importpath = "go-common/app/infra/canal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/canal/conf:go_default_library",
+        "//app/infra/canal/dao:go_default_library",
+        "//app/infra/canal/infoc:go_default_library",
+        "//app/infra/canal/model:go_default_library",
+        "//app/infra/canal/service/reader:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/hbase.v2:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/netutil:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/stat/prom:go_default_library",
+        "//vendor/github.com/juju/errors:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/canal:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/client:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/mysql:go_default_library",
+        "//vendor/github.com/siddontang/go-mysql/replication:go_default_library",
+        "//vendor/github.com/tsuna/gohbase/hrpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/infra/canal/service/reader:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["tidb_data_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/canal/model:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+    ],
+)

app/infra/canal/service/canal.go

@@ -0,0 +1,364 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/http"
+	"net/url"
+	"regexp"
+	"strings"
+	"sync"
+	"sync/atomic"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/dao"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	xhttp "go-common/library/net/http/blademaster"
+	"go-common/library/net/netutil"
+	"go-common/library/stat/prom"
+
+	"github.com/siddontang/go-mysql/client"
+)
+
+var (
+	stats       = prom.New().WithState("go_canal_counter", []string{"type", "addr", "scheme", "table", "action"})
+	tblReplacer = regexp.MustCompile("[0-9]+") // NOTE: replace number of sub-table name to space
+)
+
+// Canal is canal.
+type Canal struct {
+	dao            *dao.Dao
+	instances      map[string]*Instance
+	hbaseInstances map[string]*HBaseInstance
+	tidbInstances  map[string]*tidbInstance
+	insl           sync.RWMutex
+	tidbInsl       sync.RWMutex
+	err            error
+	backoff        netutil.Backoff
+	errCount       int64
+	lastErr        time.Time
+}
+
+// NewCanal load config and start canal instance.
+func NewCanal(config *conf.Config) (c *Canal) {
+	c = &Canal{
+		dao: dao.New(config),
+	}
+	cfg, err := conf.LoadCanalConf()
+	if err != nil {
+		panic(err)
+	}
+	c.instances = make(map[string]*Instance, len(cfg.Instances))
+	c.hbaseInstances = make(map[string]*HBaseInstance, len(cfg.HBaseInstances))
+	c.tidbInstances = make(map[string]*tidbInstance, len(cfg.TiDBInstances))
+	for _, insc := range cfg.Instances {
+		ins, err := NewInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%+v)", err)
+		}
+		c.insl.Lock()
+		c.instances[insc.Addr] = ins
+		c.insl.Unlock()
+		if err == nil {
+			go ins.Start()
+			log.Info("start canal instance(%s) success", ins.String())
+		}
+	}
+	c.backoff = &netutil.BackoffConfig{
+		MaxDelay:  120 * time.Second,
+		BaseDelay: 1.0 * time.Second,
+		Factor:    1.6,
+		Jitter:    0.2,
+	}
+	c.lastErr = time.Now()
+	go c.eventproc()
+	// hbase
+	for _, insc := range cfg.HBaseInstances {
+		ins, err := NewHBaseInstance(insc)
+		if err != nil {
+			log.Error("new hbase instance error(%+v)", err)
+		}
+		c.insl.Lock()
+		c.hbaseInstances[insc.Cluster] = ins
+		c.insl.Unlock()
+		if err == nil {
+			ins.Start()
+			log.Info("start hbase instance(%s) success", ins.String())
+		}
+	}
+	go c.hbaseeventproc()
+
+	// tidb
+	for _, insc := range cfg.TiDBInstances {
+		ins, err := newTiDBInstance(c, insc)
+		if err != nil {
+			log.Error("new instance error(%+v)", err)
+			c.sendWx(fmt.Sprintf("new tidb canal instance(%s) failed error(%v)", ins.String(), err))
+			continue
+		}
+		c.tidbInsl.Lock()
+		c.tidbInstances[insc.Name] = ins
+		c.tidbInsl.Unlock()
+		if err == nil {
+			go ins.start()
+			log.Info("start tidb canal instance(%s) success", ins.String())
+		}
+	}
+	go c.tidbEventproc()
+	go c.monitorproc()
+	// errproc
+	go c.errproc()
+	return
+}
+
+// CheckMaster check master status.
+func (c *Canal) CheckMaster(addr, user, pwd string) (name string, pos int64, err error) {
+	conn, err := client.Connect(addr, user, pwd, "")
+	if err != nil {
+		return
+	}
+	rr, err := conn.Execute("SHOW MASTER STATUS")
+	if err != nil {
+		return
+	}
+	name, _ = rr.GetString(0, 0)
+	pos, _ = rr.GetInt(0, 1)
+	if name != "" && pos > 0 {
+		return
+	}
+	return "", 0, fmt.Errorf("check master no name|pos")
+}
+
+// PosSync sync newewst bin_pos.
+func (c *Canal) PosSync(addr string) (err error) {
+	c.insl.Lock()
+	old, ok := c.instances[addr]
+	c.insl.Unlock()
+	if !ok {
+		return
+	}
+	pos, err := old.GetMasterPos()
+	if err != nil {
+		return
+	}
+	old.Close()
+	old.OnPosSynced(pos, true)
+
+	ins, _ := NewInstance(old.config)
+	c.insl.Lock()
+	c.instances[addr] = ins
+	c.insl.Unlock()
+	ins.Start()
+	return
+}
+
+// Errors returns instance errors.
+func (c *Canal) Errors() (ies map[string]string) {
+	ies = map[string]string{}
+	c.insl.RLock()
+	for _, i := range c.instances {
+		ies[i.String()] = i.Error()
+	}
+	for _, i := range c.hbaseInstances {
+		ies[i.String()] = i.Error()
+	}
+	c.insl.RUnlock()
+	return
+}
+
+// Error returns canal error.
+func (c *Canal) Error() string {
+	if c.err == nil {
+		return ""
+	}
+	return c.err.Error()
+}
+
+// errproc check errors.
+func (c *Canal) errproc() {
+	for {
+		time.Sleep(10 * time.Second)
+		es := c.Error()
+		if es != "" {
+			c.sendWx(fmt.Sprintf("canal occur error(%s)", es))
+		}
+		ies := c.Errors()
+		for k, v := range ies {
+			if v != "" {
+				c.sendWx(fmt.Sprintf("canal instance(%s) occur error(%s)", k, v))
+			}
+		}
+	}
+}
+
+// Close close canal instance
+func (c *Canal) Close() {
+	c.insl.RLock()
+	defer c.insl.RUnlock()
+	for _, ins := range c.instances {
+		ins.Close()
+		log.Info("close canal instance(%s) success", ins.String())
+	}
+	for _, ins := range c.hbaseInstances {
+		ins.Close()
+		log.Info("close hbase instance(%s) success", ins.String())
+	}
+	for _, ins := range c.tidbInstances {
+		ins.close()
+		log.Info("close tidb instance(%s) success", ins.String())
+	}
+}
+
+func (c *Canal) eventproc() {
+	ech := conf.Event()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		ins, err := NewInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.sendWx(fmt.Sprintf("reload canal instance(%s) failed error(%v)", ins.String(), err))
+			c.insl.Lock()
+			if old, ok := c.instances[insc.Addr]; ok {
+				old.Close()
+			}
+			c.instances[insc.Addr] = ins
+			c.insl.Unlock()
+			continue
+		}
+		c.insl.Lock()
+		if old, ok := c.instances[insc.Addr]; ok {
+			old.Close()
+		}
+		c.instances[insc.Addr] = ins
+		c.insl.Unlock()
+		go ins.Start()
+		log.Info("reload canal instance(%s) success", ins.String())
+		//	c.sendWx(fmt.Sprintf("reload canal instance(%s) success", ins.String()))
+	}
+}
+
+func (c *Canal) hbaseeventproc() {
+	ech := conf.HBaseEvent()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		ins, err := NewHBaseInstance(insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.insl.Lock()
+			if old, ok := c.hbaseInstances[insc.Cluster]; ok {
+				old.Close()
+			}
+			c.hbaseInstances[insc.Cluster] = ins
+			c.insl.Unlock()
+			continue
+		}
+		c.insl.Lock()
+		if old, ok := c.hbaseInstances[insc.Cluster]; ok {
+			old.Close()
+		}
+		c.hbaseInstances[insc.Cluster] = ins
+		c.insl.Unlock()
+		ins.Start()
+		log.Info("reload hbase instance(%s) success", ins.String())
+	}
+}
+
+// monitorproc monitor instance delay.
+func (c *Canal) monitorproc() {
+	if env.DeployEnv != env.DeployEnvProd {
+		return
+	}
+	const delay = 2 * time.Minute
+	for {
+		time.Sleep(delay)
+		c.insl.RLock()
+		for _, ins := range c.instances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && !ins.config.MonitorOff && dt > threshold {
+				for _, db := range ins.config.Databases {
+					c.sendWx(fmt.Sprintf("canal env(%s) 数据库(%s)地址(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, db.Schema, ins.config.Addr, threshold, dt))
+				}
+			}
+		}
+		for _, ins := range c.hbaseInstances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && !ins.config.MonitorOff && dt > threshold {
+				c.sendWx(fmt.Sprintf("canal env(%s) hbase集群(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, ins.config.Cluster, threshold, dt))
+			}
+		}
+		c.insl.RUnlock()
+		c.tidbInsl.RLock()
+		for _, ins := range c.tidbInstances {
+			if ins.closed {
+				continue
+			}
+			threshold := int64(time.Duration(ins.config.MonitorPeriod) / time.Second)
+			if threshold <= 0 {
+				threshold = int64(delay / time.Second)
+			}
+			dt := ins.delay()
+			if ins.config != nil && dt > threshold {
+				for _, db := range ins.config.Databases {
+					c.sendWx(fmt.Sprintf("tidb canal env(%s) 数据库(%s)地址(%s) 同步延迟时间超过阈值:%d秒 当前超过:%d秒", env.DeployEnv, db.Schema, ins.config.Name, threshold, dt))
+				}
+			}
+		}
+		c.tidbInsl.RUnlock()
+	}
+}
+
+func (c *Canal) sendWx(msg string) {
+	count := atomic.LoadInt64(&c.errCount)
+	atomic.AddInt64(&c.errCount, 1)
+	duration := c.backoff.Backoff(int(count))
+	if time.Since(c.lastErr) < duration {
+		return
+	}
+	c.lastErr = time.Now()
+	sendWechat(msg, conf.Conf.Monitor.User)
+}
+
+func sendWechat(msg string, user string) {
+	params := url.Values{}
+	params.Set("Action", "CreateWechatMessage")
+	params.Set("PublicKey", "9c178e51a7d4dc8aa1dbef0c790b06e7574c4d0etracehubtuhui@bilibili.com")
+	params.Set("UserName", user)
+	params.Set("Content", msg)
+	params.Set("TreeId", "bilibili.main.common-arch.canal")
+	params.Set("Title", "canal 监控报警")
+	params.Set("Signature", "1")
+	req, _ := http.NewRequest("POST", "http://merak.bilibili.co", strings.NewReader(params.Encode()))
+	req.Header.Add("content-type", "application/x-www-form-urlencoded; charset=UTF-8")
+	var v struct {
+		ReqID    string `json:"reqId"`
+		Status   int64  `json:"status"`
+		Response struct {
+			status int
+		} `json:"Response"`
+	}
+	if err := xhttp.NewClient(conf.Conf.HTTPClient).Do(context.TODO(), req, &v); err != nil {
+		log.Error("send wechat monitor status(%d) msg(%v,%v) error(%v)", v.Status, v.Response, v.Response.status, err)
+	}
+}

app/infra/canal/service/hbase.go

@@ -0,0 +1,153 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	hbase "go-common/library/database/hbase.v2"
+
+	"github.com/pkg/errors"
+	"github.com/tsuna/gohbase/hrpc"
+)
+
+// HBaseInstance hbase canal instance
+type HBaseInstance struct {
+	config *conf.HBaseInsConf
+	client *hbase.Client
+	// one instance can have lots of target different by schema and table
+	targets []*databus.Databus
+	hinfo   *hbaseInfo
+
+	// scan latest timestamp, be used check delay
+	latestTimestamp int64
+
+	err    error
+	closed bool
+}
+
+// NewHBaseInstance new canal instance
+func NewHBaseInstance(c *conf.HBaseInsConf) (ins *HBaseInstance, err error) {
+	// new instance
+	ins = &HBaseInstance{config: c}
+	// check and modify config
+	if c.MasterInfo == nil {
+		err = errors.New("no masterinfo config")
+		ins.err = err
+		return
+	}
+	// new hbase info
+	if ins.hinfo, err = newHBaseInfo(ins.config.Cluster, ins.config.MasterInfo); err != nil {
+		log.Error("init hbase info error(%v)", err)
+		ins.err = errors.Wrap(err, "init hbase info")
+		return
+	}
+	// new hbase client
+	ins.client = hbase.NewClient(&hbase.Config{Zookeeper: &hbase.ZKConfig{
+		Root:  c.Root,
+		Addrs: c.Addrs,
+	}})
+	return
+}
+
+// Start start binlog receive
+func (ins *HBaseInstance) Start() {
+	for _, db := range ins.config.Databases {
+		databus := databus.New(db.Databus)
+		ins.targets = append(ins.targets, databus)
+		for _, tb := range db.Tables {
+			go ins.scanproc(db, databus, tb)
+		}
+	}
+}
+
+func (ins *HBaseInstance) scanproc(c *conf.HBaseDatabase, databus *databus.Databus, table *conf.HBaseTable) {
+	latestTs := ins.hinfo.LatestTs(table.Name)
+	if latestTs == 0 {
+		latestTs = uint64(time.Now().UnixNano() / 1e6)
+	}
+AGAIN:
+	for {
+		time.Sleep(300 * time.Millisecond)
+		nowTs := uint64(time.Now().UnixNano() / 1e6)
+		// scan
+		scanner, err := ins.client.Scan(context.TODO(), []byte(table.Name), hrpc.TimeRangeUint64(latestTs, nowTs))
+		if err != nil {
+			time.Sleep(time.Second)
+			continue
+		}
+		ins.latestTimestamp = time.Now().Unix()
+		for {
+			res, err := scanner.Next()
+			if err == io.EOF {
+				latestTs = nowTs
+				ins.hinfo.Save(table.Name, latestTs)
+				continue AGAIN
+			} else if err != nil {
+				time.Sleep(time.Second)
+				continue AGAIN
+			}
+			var key string
+			data := map[string]interface{}{}
+			for _, cell := range res.Cells {
+				row := string(cell.Row)
+				if key == "" {
+					key = row
+				}
+				if _, ok := data[row]; !ok {
+					data[row] = map[string]string{}
+				}
+				column := string(cell.Qualifier)
+				omit := false
+				for _, field := range table.OmitField {
+					if field == column {
+						omit = true
+						break
+					}
+				}
+				if !omit {
+					cm := data[row].(map[string]string)
+					cm[column] = string(cell.Value)
+				}
+			}
+			real := &model.Data{Table: table.Name, New: data}
+			databus.Send(context.TODO(), key, real)
+			log.Info("hbase databus:group(%s)topic(%s) pub(key:%s, value:%+v) succeed", c.Databus.Group, c.Databus.Topic, key, real)
+		}
+	}
+}
+
+// Close close instance
+func (ins *HBaseInstance) Close() {
+	if ins.err != nil {
+		return
+	}
+	ins.client.Close()
+	for _, t := range ins.targets {
+		t.Close()
+	}
+	ins.closed = true
+	ins.err = errors.New("canal hbase instance closed")
+}
+
+func (ins *HBaseInstance) String() string {
+	return ins.config.Cluster
+}
+
+// Error returns instance error.
+func (ins *HBaseInstance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *HBaseInstance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}

app/infra/canal/service/instance.go

@@ -0,0 +1,172 @@
+package service
+
+import (
+	"fmt"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+	"github.com/siddontang/go-mysql/canal"
+	"github.com/siddontang/go-mysql/mysql"
+	"github.com/siddontang/go-mysql/replication"
+)
+
+// Instance canal instance
+type Instance struct {
+	*canal.Canal
+	config *conf.InsConf
+	// one instance can have lots of target different by schema and table
+	targets []*Target
+	master  *dbMasterInfo
+
+	// binlog latest timestamp, be used check delay
+	latestTimestamp int64
+	err             error
+	closed          bool
+}
+
+// NewInstance new canal instance
+func NewInstance(c *conf.InsConf) (ins *Instance, err error) {
+	// new instance
+	ins = &Instance{config: c}
+	// check and modify config
+	if c.MasterInfo == nil {
+		err = errors.New("no masterinfo config")
+		ins.err = err
+		return
+	}
+	if c.ReadTimeout == 0 {
+		c.ReadTimeout = 90
+	}
+	c.ReadTimeout = c.ReadTimeout * time.Second
+	c.HeartbeatPeriod = c.HeartbeatPeriod * time.Second
+
+	if ins.master, err = newDBMasterInfo(ins.config.Addr, ins.config.MasterInfo); err != nil {
+		log.Error("init master info error(%v)", err)
+		ins.err = errors.Wrap(err, "init master info")
+		return
+	}
+	if ins.targets, err = newTargets(c); err != nil {
+		log.Error("db.init() error(%v)", err)
+		ins.err = errors.Wrap(err, "db init")
+		return
+	}
+	ins.latestTimestamp = time.Now().Unix()
+	// new canal
+	if ins.Canal, err = canal.NewCanal(c.Config); err != nil {
+		log.Error("canal.NewCanal(%v) error(%v)", c.Config, err)
+		ins.err = errors.Wrapf(err, "canal NewCanal(%v)", c.Config)
+		return
+	}
+	// implement self as canal's event handler
+	ins.Canal.SetEventHandler(ins)
+
+	return
+}
+
+func newTargets(c *conf.InsConf) (targets []*Target, err error) {
+	targets = make([]*Target, 0, len(c.Databases))
+	for _, db := range c.Databases {
+		if err = db.CheckTable(c.Addr, c.User, c.Password); err != nil {
+			log.Error("db.CheckTable() error(%v)", err)
+			return
+		}
+		targets = append(targets, NewTarget(db))
+	}
+	return
+}
+
+// Start start binlog receive
+func (ins *Instance) Start() {
+	pos := ins.master.Pos()
+	if pos.Name == "" || pos.Pos == 0 {
+		var err error
+		if pos, err = ins.Canal.GetMasterPos(); err != nil {
+			log.Error("c.MasterPos error(%v)", err)
+			ins.err = errors.Wrap(err, "canal get master pos when start")
+			return
+		}
+	}
+	ins.err = ins.Canal.RunFrom(pos)
+}
+
+// Close close instance
+func (ins *Instance) Close() {
+	if ins.err != nil {
+		return
+	}
+	ins.Canal.Close()
+	for _, t := range ins.targets {
+		t.close()
+	}
+	ins.closed = true
+	ins.err = errors.New("canal closed")
+}
+
+// Check filter row event
+func (ins *Instance) Check(ev *canal.RowsEvent) (ts []*Target) {
+	for _, t := range ins.targets {
+		if t.compare(ev.Table.Schema, ev.Table.Name, ev.Action) {
+			ts = append(ts, t)
+		}
+	}
+	return
+}
+
+func (ins *Instance) String() string {
+	return ins.config.Addr
+}
+
+// OnRotate OnRotate
+func (ins *Instance) OnRotate(re *replication.RotateEvent) error {
+	log.Info("OnRotate binlog addr(%s) rotate binname(%s) pos(%d)", ins.config.Addr, re.NextLogName, re.Position)
+	return nil
+}
+
+// OnDDL OnDDL
+func (ins *Instance) OnDDL(pos mysql.Position, qe *replication.QueryEvent) error {
+	log.Info("OnDDL binlog addr(%s) ddl binname(%s) pos(%d)", ins.config.Addr, pos.Name, pos.Pos)
+	return nil
+}
+
+// OnXID OnXID
+func (ins *Instance) OnXID(mysql.Position) error {
+	return nil
+}
+
+//OnGTID OnGTID
+func (ins *Instance) OnGTID(mysql.GTIDSet) error {
+	return nil
+}
+
+// OnPosSynced OnPosSynced
+func (ins *Instance) OnPosSynced(pos mysql.Position, force bool) error {
+	return ins.master.Save(pos, force)
+}
+
+// OnRow send the envent to table
+func (ins *Instance) OnRow(ev *canal.RowsEvent) error {
+	for _, t := range ins.Check(ev) {
+		t.send(ev)
+	}
+	if stats != nil {
+		stats.Incr("syncer_counter", ins.String(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+		stats.State("delay_syncer", ins.delay(), ins.String(), ev.Table.Schema, "", "")
+	}
+	ins.latestTimestamp = time.Now().Unix()
+	return nil
+}
+
+// Error returns instance error.
+func (ins *Instance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *Instance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}

app/infra/canal/service/master.go

@@ -0,0 +1,145 @@
+package service
+
+import (
+	"sync"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+
+	"github.com/juju/errors"
+	"github.com/siddontang/go-mysql/client"
+	"github.com/siddontang/go-mysql/mysql"
+)
+
+type dbMasterInfo struct {
+	c *conf.MasterInfoConfig
+
+	addr    string
+	binName string
+	binPos  uint32
+
+	l sync.RWMutex
+
+	lastSaveTime time.Time
+}
+
+func newDBMasterInfo(addr string, c *conf.MasterInfoConfig) (*dbMasterInfo, error) {
+	m := &dbMasterInfo{c: c, addr: addr}
+	conn, err := client.Connect(c.Addr, c.User, c.Password, c.DBName)
+	if err != nil {
+		log.Error("db master info client error(%v)", err)
+		return nil, errors.Trace(err)
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	r, err := conn.Execute("SELECT addr,bin_name,bin_pos FROM master_info WHERE addr=?", addr)
+	if err != nil {
+		log.Error("new db load master.info error(%v)", err)
+		return nil, errors.Trace(err)
+	}
+	if r.RowNumber() == 0 {
+		if m.c.Timeout > 0 {
+			conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+		}
+		if _, err = conn.Execute("INSERT INTO master_info (addr,bin_name,bin_pos) VALUE (?,'',0)", addr); err != nil {
+			log.Error("insert master.info error(%v)", err)
+			return nil, errors.Trace(err)
+		}
+	} else {
+		m.addr, _ = r.GetStringByName(0, "addr")
+		m.binName, _ = r.GetStringByName(0, "bin_name")
+		bpos, _ := r.GetIntByName(0, "bin_pos")
+		m.binPos = uint32(bpos)
+	}
+	return m, nil
+}
+
+func (m *dbMasterInfo) Save(pos mysql.Position, force bool) error {
+	n := time.Now()
+	if !force && n.Sub(m.lastSaveTime) < 2*time.Second {
+		return nil
+	}
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("db master info client error(%v)", err)
+		return errors.Trace(err)
+	}
+	defer conn.Close()
+
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	if _, err = conn.Execute("UPDATE master_info SET bin_name=?,bin_pos=? WHERE addr=?", pos.Name, pos.Pos, m.addr); err != nil {
+		log.Error("db save master info error(%v)", err)
+		return errors.Trace(err)
+	}
+	m.lastSaveTime = n
+	return nil
+}
+
+func (m *dbMasterInfo) Pos() mysql.Position {
+	var pos mysql.Position
+	m.l.RLock()
+	pos.Name = m.binName
+	pos.Pos = m.binPos
+	m.l.RUnlock()
+	return pos
+}
+
+type hbaseInfo struct {
+	c *conf.MasterInfoConfig
+
+	name string
+}
+
+func newHBaseInfo(name string, c *conf.MasterInfoConfig) (*hbaseInfo, error) {
+	m := &hbaseInfo{c: c, name: name}
+	return m, nil
+}
+
+func (m *hbaseInfo) LatestTs(table string) (lts uint64) {
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("db hbase info client error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	r, err := conn.Execute("SELECT latest_ts FROM hbase_info WHERE cluster_name=? AND table_name=?", m.name, table)
+	if err != nil {
+		log.Error("new db load hbase.info error(%v)", err)
+		return
+	}
+	if r.RowNumber() == 0 {
+		if m.c.Timeout > 0 {
+			conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+		}
+		if _, err = conn.Execute("INSERT INTO hbase_info (cluster_name,table_name,latest_ts) VALUE (?,?,0)", m.name, table); err != nil {
+			log.Error("insert hbase.info error(%v)", err)
+			return
+		}
+	} else {
+		lts, _ = r.GetUintByName(0, "latest_ts")
+	}
+	return 0
+}
+
+func (m *hbaseInfo) Save(table string, latestTs uint64) {
+	conn, err := client.Connect(m.c.Addr, m.c.User, m.c.Password, m.c.DBName)
+	if err != nil {
+		log.Error("save hbase info client error(%v)", err)
+		return
+	}
+	defer conn.Close()
+	if m.c.Timeout > 0 {
+		conn.SetDeadline(time.Now().Add(m.c.Timeout * time.Second))
+	}
+	if _, err = conn.Execute("UPDATE hbase_info SET latest_ts=? WHERE cluster_name=? AND table_name=?", latestTs, m.name, table); err != nil {
+		log.Error("save hbase info error(%v)", err)
+	}
+}

app/infra/canal/service/reader/BUILD

@@ -0,0 +1,37 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "offset.go",
+        "reader.go",
+    ],
+    importpath = "go-common/app/infra/canal/service/reader",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/log:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/canal/service/reader/offset.go

@@ -0,0 +1,173 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+	pkgerr "github.com/pkg/errors"
+)
+
+// KafkaSeeker seeks offset in kafka topics by given condition
+type KafkaSeeker struct {
+	consumer sarama.Consumer
+	client   sarama.Client
+}
+
+// NewKafkaSeeker creates an instance of KafkaSeeker
+func NewKafkaSeeker(addr []string, config *sarama.Config) (*KafkaSeeker, error) {
+	client, err := sarama.NewClient(addr, config)
+	if err != nil {
+		return nil, pkgerr.WithStack(err)
+	}
+
+	consumer, err := sarama.NewConsumerFromClient(client)
+	if err != nil {
+		return nil, pkgerr.WithStack(err)
+	}
+
+	s := &KafkaSeeker{
+		client:   client,
+		consumer: consumer,
+	}
+
+	return s, nil
+}
+
+// Close releases resources of KafkaSeeker
+func (ks *KafkaSeeker) Close() {
+	ks.consumer.Close()
+	ks.client.Close()
+}
+
+// Seek seeks the first offset which binlog CommitTs bigger than ts
+func (ks *KafkaSeeker) Seek(topic string, ts int64, partitions []int32) (offsets []int64, err error) {
+	if len(partitions) == 0 {
+		partitions, err = ks.consumer.Partitions(topic)
+		if err != nil {
+			log.Error("tidb get partitions from topic %s error %v", topic, err)
+			return nil, pkgerr.WithStack(err)
+		}
+	}
+
+	offsets, err = ks.seekOffsets(topic, partitions, ts)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		log.Error("tidb seek offsets error %v", err)
+	}
+	return
+}
+
+func (ks *KafkaSeeker) getTSFromMSG(msg *sarama.ConsumerMessage) (ts int64, err error) {
+	binlog := new(pb.Binlog)
+	err = binlog.Unmarshal(msg.Value)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	return binlog.CommitTs, nil
+}
+
+// seekOffsets returns all valid offsets in partitions
+func (ks *KafkaSeeker) seekOffsets(topic string, partitions []int32, pos int64) ([]int64, error) {
+	offsets := make([]int64, len(partitions))
+	for _, partition := range partitions {
+		start, err := ks.client.GetOffset(topic, partition, sarama.OffsetOldest)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+
+		end, err := ks.client.GetOffset(topic, partition, sarama.OffsetNewest)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+
+		offset, err := ks.seekOffset(topic, partition, start, end-1, pos)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return nil, err
+		}
+		offsets[partition] = offset
+	}
+
+	return offsets, nil
+}
+
+func (ks *KafkaSeeker) seekOffset(topic string, partition int32, start int64, end int64, ts int64) (offset int64, err error) {
+	startTS, err := ks.getTSAtOffset(topic, partition, start)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	if ts < startTS {
+		log.Warn("given ts %v is smaller than oldest message's ts %v, some binlogs may lose", ts, startTS)
+		offset = start
+		return
+	} else if ts == startTS {
+		offset = start + 1
+		return
+	}
+
+	for start < end {
+		mid := (end-start)/2 + start
+		var midTS int64
+		midTS, err = ks.getTSAtOffset(topic, partition, mid)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			return
+		}
+
+		if midTS <= ts {
+			start = mid + 1
+		} else {
+			end = mid
+		}
+	}
+
+	var endTS int64
+	endTS, err = ks.getTSAtOffset(topic, partition, end)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	if endTS <= ts {
+		return sarama.OffsetNewest, nil
+	}
+
+	return end, nil
+}
+
+func (ks *KafkaSeeker) getTSAtOffset(topic string, partition int32, offset int64) (ts int64, err error) {
+	pc, err := ks.consumer.ConsumePartition(topic, partition, offset)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+	defer pc.Close()
+
+	for msg := range pc.Messages() {
+		ts, err = ks.getTSFromMSG(msg)
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	panic("unreachable")
+}

app/infra/canal/service/reader/reader.go

@@ -0,0 +1,174 @@
+// Copyright 2018 PingCAP, Inc.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package reader
+
+import (
+	"fmt"
+
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+	pkgerr "github.com/pkg/errors"
+)
+
+func init() {
+	// log.SetLevel(log.LOG_LEVEL_NONE)
+	sarama.MaxResponseSize = 1 << 30
+}
+
+// Config for Reader
+type Config struct {
+	KafkaAddr []string
+	// the CommitTs of binlog return by reader will bigger than the config CommitTs
+	CommitTS  int64
+	Offset    int64 // start at kafka offset
+	ClusterID string
+	Name      string
+}
+
+// Message read from reader
+type Message struct {
+	Binlog *pb.Binlog
+	Offset int64 // kafka offset
+}
+
+// Reader to read binlog from kafka
+type Reader struct {
+	cfg    *Config
+	client sarama.Client
+
+	msgs      chan *Message
+	stop      chan struct{}
+	clusterID string
+}
+
+func (r *Reader) getTopic() (string, int32) {
+	return r.cfg.ClusterID + "_obinlog", 0
+}
+
+func (r *Reader) name() string {
+	return fmt.Sprintf("%s-%s", r.cfg.Name, r.cfg.ClusterID)
+}
+
+// NewReader creates an instance of Reader
+func NewReader(cfg *Config) (r *Reader, err error) {
+	r = &Reader{
+		cfg:       cfg,
+		stop:      make(chan struct{}),
+		msgs:      make(chan *Message, 1024),
+		clusterID: cfg.ClusterID,
+	}
+
+	r.client, err = sarama.NewClient(r.cfg.KafkaAddr, nil)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		r = nil
+		return
+	}
+	if (r.cfg.Offset == 0) && (r.cfg.CommitTS > 0) {
+		r.cfg.Offset, err = r.getOffsetByTS(r.cfg.CommitTS)
+		if err != nil {
+			err = pkgerr.WithStack(err)
+			r = nil
+			return
+		}
+		log.Info("tidb %s: set offset to: %v", r.name(), r.cfg.Offset)
+	}
+	return
+}
+
+// Close shuts down the reader
+func (r *Reader) Close() {
+	close(r.stop)
+
+	r.client.Close()
+}
+
+// Messages returns a chan that contains unread buffered message
+func (r *Reader) Messages() (msgs <-chan *Message) {
+	return r.msgs
+}
+
+func (r *Reader) getOffsetByTS(ts int64) (offset int64, err error) {
+	seeker, err := NewKafkaSeeker(r.cfg.KafkaAddr, nil)
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	topic, partition := r.getTopic()
+	offsets, err := seeker.Seek(topic, ts, []int32{partition})
+	if err != nil {
+		err = pkgerr.WithStack(err)
+		return
+	}
+
+	offset = offsets[0]
+
+	return
+}
+
+// Run start consume msg
+func (r *Reader) Run() {
+	offset := r.cfg.Offset
+	log.Info("tidb %s start at offset: %v", r.name(), offset)
+
+	consumer, err := sarama.NewConsumerFromClient(r.client)
+	if err != nil {
+		log.Error("tidb %s NewConsumerFromClient err: %v", r.name(), err)
+		return
+	}
+	defer consumer.Close()
+	topic, partition := r.getTopic()
+	partitionConsumer, err := consumer.ConsumePartition(topic, partition, offset)
+	if err != nil {
+		log.Error("tidb %s ConsumePartition err: %v", r.name(), err)
+		return
+	}
+	defer partitionConsumer.Close()
+	for {
+		select {
+		case <-r.stop:
+			partitionConsumer.Close()
+			close(r.msgs)
+			log.Info("tidb %s reader stop to run", r.name())
+			return
+		case kmsg, ok := <-partitionConsumer.Messages():
+			if !ok {
+				close(r.msgs)
+				log.Info("tidb %s reader stop to run because partitionConsumer close", r.name())
+				return
+			}
+			if kmsg == nil {
+				continue
+			}
+			log.Info("tidb %s get kmsg offset: %v", r.name(), kmsg.Offset)
+			binlog := new(pb.Binlog)
+			err := binlog.Unmarshal(kmsg.Value)
+			if err != nil {
+				log.Warn("%s unmarshal err %+v", r.name(), err)
+				continue
+			}
+			if r.cfg.CommitTS > 0 && binlog.CommitTs <= r.cfg.CommitTS {
+				log.Warn("%s skip binlog CommitTs: ", r.name(), binlog.CommitTs)
+				continue
+			}
+			r.msgs <- &Message{
+				Binlog: binlog,
+				Offset: kmsg.Offset,
+			}
+		}
+	}
+}

app/infra/canal/service/target.go

@@ -0,0 +1,329 @@
+package service
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"hash/crc32"
+	"strings"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	"go-common/app/infra/canal/model"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	"github.com/pkg/errors"
+	"github.com/siddontang/go-mysql/canal"
+)
+
+var (
+	errInvalidAction = errors.New("invalid rows action")
+	errInvalidUpdate = errors.New("invalid update rows event")
+	errBinlogFormat  = errors.New("binlog format failed")
+)
+
+type producer interface {
+	Rows(int64)
+	Send(context.Context, string, interface{}) error
+	Close()
+	Name() string
+}
+
+type databusP struct {
+	group, topic string
+	*databus.Databus
+}
+
+func (d *databusP) Rows(b int64) {
+	// ignore
+}
+
+func (d *databusP) Send(c context.Context, key string, data interface{}) error {
+	return d.Databus.Send(c, key, data)
+}
+
+func (d *databusP) Name() string {
+	return fmt.Sprintf("databus:group(%s)topic(%s)", d.group, d.topic)
+}
+
+func (d *databusP) Close() {
+	d.Databus.Close()
+}
+
+// infocP infoc producer
+type infocP struct {
+	taskID string
+	*infoc.Infoc
+}
+
+// Rows rows
+func (i *infocP) Rows(b int64) {
+	i.Infoc.Rows(b)
+}
+
+// Send send msg
+func (i *infocP) Send(c context.Context, key string, data interface{}) error {
+	return i.Infoc.Send(c, key, data)
+}
+
+// Name infoc name
+func (i *infocP) Name() string {
+	return fmt.Sprintf("infoc(%s)", i.taskID)
+}
+
+// Close close infoc
+func (i *infocP) Close() {
+	i.Infoc.Flush()
+	i.Infoc.Close()
+}
+
+// Target databus target
+type Target struct {
+	producers []producer
+	eventLen  uint32
+	events    []chan *canal.RowsEvent
+	db        *conf.Database
+
+	closed bool
+}
+
+// NewTarget new databus target
+func NewTarget(db *conf.Database) (t *Target) {
+	t = &Target{
+		db:       db,
+		eventLen: uint32(len(db.CTables)),
+	}
+	t.events = make([]chan *canal.RowsEvent, t.eventLen)
+	if db.Databus != nil {
+		t.producers = append(t.producers, &databusP{group: db.Databus.Group, topic: db.Databus.Topic, Databus: databus.New(db.Databus)})
+	}
+	if db.Infoc != nil {
+		t.producers = append(t.producers, &infocP{taskID: db.Infoc.TaskID, Infoc: infoc.New(db.Infoc)})
+	}
+	for i := 0; i < int(t.eventLen); i++ {
+		ch := make(chan *canal.RowsEvent, 1024)
+		t.events[i] = ch
+		go t.proc(ch)
+	}
+	return
+}
+
+// compare check if the binlog event is needed
+// check the table name and schame
+func (t *Target) compare(schame, table, action string) bool {
+	if t.db.Schema == schame {
+		for _, ctb := range t.db.CTables {
+			for _, tb := range ctb.Tables {
+				if table == tb {
+					for _, act := range ctb.OmitAction {
+						if act == action { // NOTE: omit action
+							return false
+						}
+					}
+					return true
+				}
+			}
+		}
+	}
+	return false
+}
+
+// send send rows event into event chans
+// and hash by table%concurrency.
+func (t *Target) send(ev *canal.RowsEvent) {
+	yu := crc32.ChecksumIEEE([]byte(ev.Table.Name))
+	t.events[yu%t.eventLen] <- ev
+}
+
+func (t *Target) close() {
+	for _, p := range t.producers {
+		p.Close()
+	}
+	t.closed = true
+}
+
+// proc aync method for transfer the binlog data
+// when connection is bad, just refresh it with retry
+func (t *Target) proc(ch chan *canal.RowsEvent) {
+	type pData struct {
+		datas    []*model.Data
+		producer producer
+	}
+	var (
+		err         error
+		normalDatas []*pData
+		errorDatas  []*pData
+		ev          *canal.RowsEvent
+	)
+	for {
+		if t.closed {
+			return
+		}
+		if len(errorDatas) != 0 {
+			normalDatas = errorDatas
+			errorDatas = errorDatas[0:0]
+			time.Sleep(time.Second)
+		} else {
+			ev = <-ch
+			var datas []*model.Data
+			if datas, err = makeDatas(ev, t.db.TableMap); err != nil {
+				log.Error("makeData(%v) error(%v)", ev, err)
+				continue
+			}
+			normalDatas = normalDatas[0:0]
+			for _, p := range t.producers {
+				p.Rows(int64(len(datas)))
+				normalDatas = append(normalDatas, &pData{datas: datas, producer: p})
+				if stats != nil {
+					stats.Incr("send_counter", p.Name(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+				}
+			}
+		}
+		for _, pd := range normalDatas {
+			var eDatas []*model.Data
+			for _, data := range pd.datas {
+				if err = pd.producer.Send(context.TODO(), data.Key, data); err != nil {
+					// retry pub error data
+					eDatas = append(eDatas, data)
+					continue
+				}
+				log.Info("%s pub(key:%s, value:%+v) succeed", pd.producer.Name(), data.Key, data)
+			}
+			if len(eDatas) > 0 {
+				errorDatas = append(errorDatas, &pData{datas: eDatas, producer: pd.producer})
+				if stats != nil && ev != nil {
+					stats.Incr("retry_counter", pd.producer.Name(), ev.Table.Schema, tblReplacer.ReplaceAllString(ev.Table.Name, ""), ev.Action)
+				}
+				log.Error("%s scheme(%s) pub fail,add to retry", pd.producer.Name(), ev.Table.Schema)
+			}
+		}
+	}
+}
+
+// makeDatas parse the binlog event and return the model.Data struct
+// a little bit cautious about the binlog type
+// if the type is update:
+//     the old value and new value will alternate appearing in the event.Rows
+func makeDatas(e *canal.RowsEvent, tbMap map[string]*conf.Addition) (datas []*model.Data, err error) {
+	var (
+		rowsLen     = len(e.Rows)
+		firstRowLen = len(e.Rows[0])
+		lenCol      = len(e.Table.Columns)
+	)
+	if rowsLen == 0 || firstRowLen == 0 || firstRowLen != lenCol {
+		log.Error("rows length(%d) first row length(%d) columns length(%d)", rowsLen, firstRowLen, lenCol)
+		err = errBinlogFormat
+		return
+	}
+	datas = make([]*model.Data, 0, rowsLen)
+	switch e.Action {
+	case canal.InsertAction, canal.DeleteAction:
+		for _, values := range e.Rows {
+			var keys []string
+			data := &model.Data{
+				Action: e.Action,
+				Table:  e.Table.Name,
+				// the first primary key as the kafka key
+				Key: fmt.Sprint(values[0]),
+				New: make(map[string]interface{}, lenCol),
+			}
+			for i, c := range e.Table.Columns {
+				if c.IsUnsigned {
+					values[i] = unsignIntCase(values[i])
+				}
+				if strings.Contains(c.RawType, "binary") {
+					if bs, ok := values[i].(string); ok {
+						values[i] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+				}
+				data.New[c.Name] = values[i]
+			}
+			// set kafka key and remove omit columns data
+			addition, ok := tbMap[e.Table.Name]
+			if ok {
+				for _, omit := range addition.OmitField {
+					delete(data.New, omit)
+				}
+				for _, primary := range addition.PrimaryKey {
+					if _, ok := data.New[primary]; ok {
+						keys = append(keys, fmt.Sprint(data.New[primary]))
+					}
+				}
+			}
+			if len(keys) != 0 {
+				data.Key = strings.Join(keys, ",")
+			}
+			datas = append(datas, data)
+		}
+	case canal.UpdateAction:
+		if rowsLen%2 != 0 {
+			err = errInvalidUpdate
+			return
+		}
+		for i := 0; i < rowsLen; i += 2 {
+			var keys []string
+			data := &model.Data{
+				Action: e.Action,
+				Table:  e.Table.Name,
+				// the first primary key as the kafka key
+				Key: fmt.Sprint(e.Rows[i][0]),
+				Old: make(map[string]interface{}, lenCol),
+				New: make(map[string]interface{}, lenCol),
+			}
+			for j, c := range e.Table.Columns {
+				if c.IsUnsigned {
+					e.Rows[i][j] = unsignIntCase(e.Rows[i][j])
+					e.Rows[i+1][j] = unsignIntCase(e.Rows[i+1][j])
+				}
+				if strings.Contains(c.RawType, "binary") {
+					if bs, ok := e.Rows[i][j].(string); ok {
+						e.Rows[i][j] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+					if bs, ok := e.Rows[i+1][j].(string); ok {
+						e.Rows[i+1][j] = base64.StdEncoding.EncodeToString([]byte(bs))
+					}
+				}
+				data.Old[c.Name] = e.Rows[i][j]
+				data.New[c.Name] = e.Rows[i+1][j]
+			}
+			// set kafka key and remove omit columns data
+			addition, ok := tbMap[e.Table.Name]
+			if ok {
+				for _, omit := range addition.OmitField {
+					delete(data.New, omit)
+					delete(data.Old, omit)
+				}
+				for _, primary := range addition.PrimaryKey {
+					if _, ok := data.New[primary]; ok {
+						keys = append(keys, fmt.Sprint(data.New[primary]))
+					}
+				}
+			}
+			if len(keys) != 0 {
+				data.Key = strings.Join(keys, ",")
+			}
+			datas = append(datas, data)
+		}
+	default:
+		err = errInvalidAction
+	}
+	return
+}
+
+func unsignIntCase(i interface{}) (v interface{}) {
+	switch si := i.(type) {
+	case int8:
+		v = uint8(si)
+	case int16:
+		v = uint16(si)
+	case int32:
+		v = uint32(si)
+	case int64:
+		v = uint64(si)
+	default:
+		v = i
+	}
+	return
+}

app/infra/canal/service/tidb_check.go

@@ -0,0 +1,65 @@
+package service
+
+import (
+	"regexp"
+
+	"go-common/library/log"
+)
+
+func (ins *tidbInstance) check() (err error) {
+	for _, db := range ins.config.Databases {
+		for _, ctable := range db.CTables {
+			if _, err = regexp.Compile(ctable.Name); err != nil {
+				log.Error("regexp.Compile(%s) error(%v)", ctable.Name, err)
+				return
+			}
+		}
+	}
+	return
+}
+
+func (ins *tidbInstance) getTable(dbName, table string) *Table {
+	if ins.ignoreTables[dbName] != nil && ins.ignoreTables[dbName][table] {
+		return nil
+	}
+	if ins.tables[dbName] != nil && ins.tables[dbName][table] != nil {
+		return ins.tables[dbName][table]
+	}
+	var regex *regexp.Regexp
+	for _, db := range ins.config.Databases {
+		if db.Schema != dbName {
+			continue
+		}
+		for _, ctable := range db.CTables {
+			regex, _ = regexp.Compile(ctable.Name)
+			if !regex.MatchString(table) {
+				continue
+			}
+			if ins.tables[dbName] == nil {
+				ins.tables[dbName] = make(map[string]*Table)
+			}
+			t := &Table{
+				PrimaryKey: ctable.PrimaryKey,
+				OmitField:  make(map[string]bool),
+				OmitAction: make(map[string]bool),
+				name:       ctable.Name,
+				ch:         make(chan *msg, 1024),
+			}
+			for _, action := range ctable.OmitAction {
+				t.OmitAction[action] = true
+			}
+			for _, field := range ctable.OmitField {
+				t.OmitField[field] = true
+			}
+			ins.waitTable.Add(1)
+			go ins.proc(t.ch)
+			ins.tables[dbName][table] = t
+			return t
+		}
+	}
+	if ins.ignoreTables[dbName] == nil {
+		ins.ignoreTables[dbName] = make(map[string]bool)
+	}
+	ins.ignoreTables[dbName][table] = true
+	return nil
+}

app/infra/canal/service/tidb_data.go

@@ -0,0 +1,123 @@
+package service
+
+import (
+	"encoding/base64"
+	"fmt"
+	"strings"
+
+	"go-common/app/infra/canal/model"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+// lower case column field type in mysql
+// https://dev.mysql.com/doc/refman/8.0/en/data-types.html
+// for numeric type: int bigint smallint tinyint float double decimal bit
+// for string type: text longtext mediumtext char tinytext varchar
+// blob longblog mediumblog binary tinyblob varbinary
+// enum set
+// for json type: json
+
+// for text and char type, string_value is set
+// for blob and binary type, bytes_value is set
+// for enum, set, uint64_value is set
+// for json, bytes_value is set
+
+func tidbMakeData(m *msg) (data *model.Data, err error) {
+	action := m.mu.GetType()
+	if (action != pb.MutationType_Insert) && (action != pb.MutationType_Delete) && (action != pb.MutationType_Update) {
+		err = errInvalidAction
+		return
+	}
+	data = &model.Data{
+		Action: strings.ToLower(action.String()),
+		Table:  m.table,
+	}
+	var keys []string
+	switch action {
+	case pb.MutationType_Insert, pb.MutationType_Delete:
+		var values = m.mu.GetRow().GetColumns()
+		for i, c := range m.columns {
+			for _, key := range m.keys {
+				if c.Name == key {
+					keys = append(keys, columnToString(values[i]))
+					break
+				}
+			}
+			if m.ignore[c.Name] {
+				continue
+			}
+			if data.New == nil {
+				data.New = make(map[string]interface{}, len(m.columns))
+			}
+			if strings.Contains(c.GetMysqlType(), "binary") {
+				data.New[c.Name] = base64.StdEncoding.EncodeToString(values[i].GetBytesValue())
+				continue
+			}
+			data.New[c.Name] = columnToValue(values[i])
+		}
+	case pb.MutationType_Update:
+		if m.mu.Row == nil || m.mu.ChangeRow == nil {
+			err = errInvalidUpdate
+			return
+		}
+		var oldValues = m.mu.GetChangeRow().GetColumns()
+		var newValues = m.mu.GetRow().GetColumns()
+		for i, c := range m.columns {
+			for _, key := range m.keys {
+				if c.Name == key {
+					keys = append(keys, columnToString(newValues[i]))
+					break
+				}
+			}
+			if m.ignore[c.Name] {
+				continue
+			}
+			if data.New == nil {
+				data.New = make(map[string]interface{}, len(m.columns))
+			}
+			if data.Old == nil {
+				data.Old = make(map[string]interface{}, len(m.columns))
+			}
+			if strings.Contains(c.GetMysqlType(), "binary") {
+				data.Old[c.Name] = base64.StdEncoding.EncodeToString(oldValues[i].GetBytesValue())
+				data.New[c.Name] = base64.StdEncoding.EncodeToString(newValues[i].GetBytesValue())
+				continue
+			}
+			data.Old[c.Name] = columnToValue(oldValues[i])
+			data.New[c.Name] = columnToValue(newValues[i])
+		}
+	}
+	if len(keys) == 0 {
+		data.Key = columnToString(m.mu.GetRow().GetColumns()[0])
+	} else {
+		data.Key = strings.Join(keys, ",")
+	}
+	if data.New == nil && data.Old == nil {
+		data = nil
+	}
+	return
+}
+
+func columnToValue(c *pb.Column) interface{} {
+	if c.GetIsNull() {
+		return nil
+	}
+	if c.Int64Value != nil {
+		return c.GetInt64Value()
+	}
+	if c.Uint64Value != nil {
+		return c.GetUint64Value()
+	}
+	if c.DoubleValue != nil {
+		return c.GetDoubleValue()
+	}
+	if c.StringValue != nil {
+		return c.GetStringValue()
+	}
+	return c.GetBytesValue()
+}
+
+func columnToString(c *pb.Column) string {
+	return fmt.Sprint(columnToValue(c))
+}

app/infra/canal/service/tidb_data_test.go

@@ -0,0 +1,188 @@
+package service
+
+import (
+	"encoding/json"
+	"reflect"
+	"testing"
+
+	"go-common/app/infra/canal/model"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+func Test_tidbMakeData(t *testing.T) {
+	insertMsg, insertData := prepareInsertData()
+	delMsg, delData := prepareDeleteData()
+	updateMsg, updateData := prepareUpdateData()
+	updateMsg2, updateData2 := prepareUpdateData2()
+	type args struct {
+		m *msg
+	}
+	tests := []struct {
+		name     string
+		args     args
+		wantData *model.Data
+		wantErr  bool
+	}{
+		{name: "insert", args: args{m: insertMsg}, wantData: insertData, wantErr: false},
+		{name: "delete", args: args{m: delMsg}, wantData: delData, wantErr: false},
+		{name: "update", args: args{m: updateMsg}, wantData: updateData, wantErr: false},
+		{name: "update2", args: args{m: updateMsg2}, wantData: updateData2, wantErr: false},
+	}
+	for _, tt := range tests {
+		t.Run(tt.name, func(t *testing.T) {
+			gotData, err := tidbMakeData(tt.args.m)
+			if (err != nil) != tt.wantErr {
+				t.Errorf("tidbMakeData() error = %v, wantErr %v", err, tt.wantErr)
+				return
+			}
+			gotjson, _ := json.Marshal(gotData)
+			wantjson, _ := json.Marshal(tt.wantData)
+			if !reflect.DeepEqual(gotjson, wantjson) {
+				t.Errorf("tidbMakeData() = %v, want %v", gotData, tt.wantData)
+			}
+		})
+	}
+}
+
+func prepareInsertData() (*msg, *model.Data) {
+	insertPb := &pb.Binlog{}
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846216359608325,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":0,"row":{"columns":[{"uint64_value":1},{"string_value":"2018-10-26 18:50:57"},{"string_value":"2018-10-26 18:50:57"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":1},{"uint64_value":8167601},{"uint64_value":1}]}}]}]}}`), insertPb)
+	insertMsg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          insertPb.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"id", "mid"},
+		columns:     insertPb.DmlData.Tables[0].ColumnInfo,
+	}
+	insertData := &model.Data{
+		Action: "insert",
+		Table:  "counts",
+		Key:    "1,8167601",
+		New: map[string]interface{}{
+			"id":          1,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  1,
+			"mid":         8167601,
+			"type":        1,
+			"mtime":       "2018-10-26 18:50:57",
+		},
+	}
+	return insertMsg, insertData
+}
+
+func prepareDeleteData() (*msg, *model.Data) {
+	pbData := &pb.Binlog{}
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846189135953921,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":2,"row":{"columns":[{"uint64_value":7},{"string_value":"2018-01-11 12:19:10"},{"string_value":"2018-01-11 12:19:10"},{"uint64_value":2},{"uint64_value":0},{"uint64_value":897},{"uint64_value":27515233},{"uint64_value":1}]}}]}]}}`), pbData)
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          pbData.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"message_id"},
+		columns:     pbData.DmlData.Tables[0].ColumnInfo,
+	}
+	data := &model.Data{
+		Action: "delete",
+		Table:  "counts",
+		Key:    "897",
+		New: map[string]interface{}{
+			"id":          7,
+			"business_id": 2,
+			"origin_id":   0,
+			"message_id":  897,
+			"mid":         27515233,
+			"type":        1,
+			"mtime":       "2018-01-11 12:19:10",
+		},
+	}
+	return msg, data
+}
+
+func prepareUpdateData() (*msg, *model.Data) {
+	pbData := &pb.Binlog{}
+	// update likes type from 1 to 0
+	json.Unmarshal([]byte(`{"type":0,"commit_ts":403846165844459523,"dml_data":{"tables":[{"schema_name":"bilibili_likes","table_name":"likes","column_info":[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"mid","mysql_type":"int","is_primary_key":false},{"name":"type","mysql_type":"tinyint","is_primary_key":false}],"mutations":[{"type":1,"row":{"columns":[{"uint64_value":4},{"string_value":"2018-10-26 18:47:44"},{"string_value":"2017-12-22 15:05:29"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":46997},{"uint64_value":88895031},{"uint64_value":0}]},"change_row":{"columns":[{"uint64_value":4},{"string_value":"2017-12-22 15:55:52"},{"string_value":"2017-12-22 15:05:29"},{"uint64_value":5},{"uint64_value":0},{"uint64_value":46997},{"uint64_value":88895031},{"uint64_value":1}]}}]}]}}`), pbData)
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		mu:          pbData.DmlData.Tables[0].Mutations[0],
+		ignore:      map[string]bool{"ctime": true},
+		keys:        []string{"mid"},
+		columns:     pbData.DmlData.Tables[0].ColumnInfo,
+	}
+	data := &model.Data{
+		Action: "update",
+		Table:  "counts",
+		Key:    "88895031",
+		Old: map[string]interface{}{
+			"id":          4,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  46997,
+			"mid":         88895031,
+			"type":        1,
+			"mtime":       "2017-12-22 15:55:52",
+		},
+		New: map[string]interface{}{
+			"id":          4,
+			"business_id": 5,
+			"origin_id":   0,
+			"message_id":  46997,
+			"mid":         88895031,
+			"type":        0,
+			"mtime":       "2018-10-26 18:47:44",
+		},
+	}
+	return msg, data
+}
+
+func prepareUpdateData2() (*msg, *model.Data) {
+	muJson := `{"type":1,"row":{"columns":[{"uint64_value":0},{"string_value":"2018-11-03 17:07:44"},{"string_value":"2018-11-03 14:55:38"},{"uint64_value":3},{"uint64_value":0},{"uint64_value":88889},{"uint64_value":3},{"uint64_value":0},{"int64_value":0},{"int64_value":0},{"uint64_value":8167601}]},"change_row":{"columns":[{"uint64_value":0},{"string_value":"2018-11-03 16:36:39"},{"string_value":"2018-11-03 14:55:38"},{"uint64_value":3},{"uint64_value":0},{"uint64_value":88889},{"uint64_value":2},{"uint64_value":0},{"int64_value":0},{"int64_value":0},{"uint64_value":8167601}]}}`
+	columnJson := `[{"name":"id","mysql_type":"bigint","is_primary_key":false},{"name":"mtime","mysql_type":"timestamp","is_primary_key":false},{"name":"ctime","mysql_type":"timestamp","is_primary_key":false},{"name":"business_id","mysql_type":"int","is_primary_key":false},{"name":"origin_id","mysql_type":"bigint","is_primary_key":false},{"name":"message_id","mysql_type":"bigint","is_primary_key":false},{"name":"likes_count","mysql_type":"int","is_primary_key":false},{"name":"dislikes_count","mysql_type":"int","is_primary_key":false},{"name":"likes_change","mysql_type":"bigint","is_primary_key":false},{"name":"dislikes_change","mysql_type":"bigint","is_primary_key":false},{"name":"up_mid","mysql_type":"int","is_primary_key":false}]`
+	msg := &msg{
+		db:          "bilibili_likes",
+		table:       "counts",
+		tableRegexp: "counts",
+		keys:        []string{"message_id"},
+	}
+	json.Unmarshal([]byte(columnJson), &msg.columns)
+	json.Unmarshal([]byte(muJson), &msg.mu)
+	data := &model.Data{
+		Action: "update",
+		Table:  "counts",
+		Key:    "88889",
+		Old: map[string]interface{}{
+			"ctime":           "2018-11-03 14:55:38",
+			"origin_id":       0,
+			"dislikes_count":  0,
+			"up_mid":          8167601,
+			"id":              0,
+			"mtime":           "2018-11-03 16:36:39",
+			"likes_count":     2,
+			"likes_change":    0,
+			"dislikes_change": 0,
+			"business_id":     3,
+			"message_id":      88889,
+		},
+		New: map[string]interface{}{
+			"likes_count":     3,
+			"dislikes_count":  0,
+			"likes_change":    0,
+			"id":              0,
+			"mtime":           "2018-11-03 17:07:44",
+			"ctime":           "2018-11-03 14:55:38",
+			"origin_id":       0,
+			"message_id":      88889,
+			"business_id":     3,
+			"dislikes_change": 0,
+			"up_mid":          8167601,
+		},
+	}
+	return msg, data
+}

app/infra/canal/service/tidb_instance.go

@@ -0,0 +1,186 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/app/infra/canal/infoc"
+	"go-common/app/infra/canal/model"
+	"go-common/app/infra/canal/service/reader"
+	"go-common/library/log"
+	"go-common/library/queue/databus"
+
+	pb "github.com/pingcap/tidb-tools/tidb_binlog/slave_binlog_proto/go-binlog"
+)
+
+type tidbInstance struct {
+	canal  *Canal
+	config *conf.TiDBInsConf
+	// one instance can have lots of target different by schema and table
+	targets      map[string][]producer
+	latestOffset int64
+	// scan latest timestamp, be used check delay
+	latestTimestamp int64
+	reader          *reader.Reader
+	err             error
+	closed          bool
+	tables          map[string]map[string]*Table
+	ignoreTables    map[string]map[string]bool
+	waitConsume     sync.WaitGroup
+	waitTable       sync.WaitGroup
+}
+
+// Table db table
+type Table struct {
+	PrimaryKey []string        // kafka msg key
+	OmitField  map[string]bool // field will be ignored in table
+	OmitAction map[string]bool // action will be ignored in table
+	name       string
+	ch         chan *msg
+}
+
+type msg struct {
+	db          string
+	table       string
+	tableRegexp string
+	mu          *pb.TableMutation
+	ignore      map[string]bool
+	keys        []string
+	columns     []*pb.ColumnInfo
+}
+
+// newTiDBInstance new canal instance
+func newTiDBInstance(cl *Canal, c *conf.TiDBInsConf) (ins *tidbInstance, err error) {
+	// new instance
+	ins = &tidbInstance{
+		config:       c,
+		canal:        cl,
+		targets:      make(map[string][]producer, len(c.Databases)),
+		tables:       make(map[string]map[string]*Table),
+		ignoreTables: make(map[string]map[string]bool),
+	}
+	cfg := &reader.Config{
+		Name:      c.Name,
+		KafkaAddr: c.Addrs,
+		Offset:    c.Offset,
+		CommitTS:  c.CommitTS,
+		ClusterID: c.ClusterID,
+	}
+	if err = ins.check(); err != nil {
+		return
+	}
+	position, err := cl.dao.TiDBPosition(context.Background(), c.Name)
+	if err == nil && position != nil {
+		cfg.Offset = position.Offset
+		cfg.CommitTS = position.CommitTS
+	}
+	ins.latestOffset = cfg.Offset
+	ins.latestTimestamp = cfg.CommitTS
+	for _, db := range c.Databases {
+		if db.Databus != nil {
+			if ins.targets == nil {
+				ins.targets = make(map[string][]producer)
+			}
+			ins.targets[db.Schema] = append(ins.targets[db.Schema], &databusP{group: db.Databus.Group, topic: db.Databus.Topic, Databus: databus.New(db.Databus)})
+		}
+		if db.Infoc != nil {
+			ins.targets[db.Schema] = append(ins.targets[db.Schema], &infocP{taskID: db.Infoc.TaskID, Infoc: infoc.New(db.Infoc)})
+		}
+	}
+	ins.reader, ins.err = reader.NewReader(cfg)
+	return ins, ins.err
+}
+
+// start start binlog receive
+func (ins *tidbInstance) start() {
+	defer ins.waitConsume.Done()
+	ins.waitConsume.Add(2)
+	go ins.reader.Run()
+	go ins.syncproc()
+	for msg := range ins.reader.Messages() {
+		ins.process(msg)
+	}
+}
+
+// close close instance
+func (ins *tidbInstance) close() {
+	if ins.closed {
+		return
+	}
+	ins.closed = true
+	ins.reader.Close()
+	ins.waitConsume.Wait()
+	for _, tables := range ins.tables {
+		for _, table := range tables {
+			close(table.ch)
+		}
+	}
+	ins.waitTable.Wait()
+	ins.sync()
+}
+
+// String .
+func (ins *tidbInstance) String() string {
+	return fmt.Sprintf("%s-%s", ins.config.Name, ins.config.ClusterID)
+}
+
+func (ins *tidbInstance) process(m *reader.Message) (err error) {
+	if m.Binlog.Type == pb.BinlogType_DDL {
+		log.Info("tidb %s got ddl: %s", ins.String(), m.Binlog.DdlData.String())
+		return
+	}
+	for _, table := range m.Binlog.DmlData.Tables {
+		tb := ins.getTable(table.GetSchemaName(), table.GetTableName())
+		if tb == nil {
+			continue
+		}
+		for _, mu := range table.Mutations {
+			action := strings.ToLower(mu.GetType().String())
+			if tb.OmitAction[action] {
+				continue
+			}
+			tb.ch <- &msg{
+				db:          table.GetSchemaName(),
+				table:       table.GetTableName(),
+				mu:          mu,
+				ignore:      tb.OmitField,
+				keys:        tb.PrimaryKey,
+				columns:     table.ColumnInfo,
+				tableRegexp: tb.name,
+			}
+			if stats != nil {
+				stats.Incr("syncer_counter", ins.String(), table.GetSchemaName(), tb.name, action)
+				stats.State("delay_syncer", ins.delay(), ins.String(), tb.name, "", "")
+			}
+		}
+	}
+	ins.latestOffset = m.Offset
+	ins.latestTimestamp = m.Binlog.CommitTs
+	return nil
+}
+
+// Error returns instance error.
+func (ins *tidbInstance) Error() string {
+	if ins.err == nil {
+		return ""
+	}
+	return fmt.Sprintf("+%v", ins.err)
+}
+
+func (ins *tidbInstance) delay() int64 {
+	return time.Now().Unix() - ins.latestTimestamp
+}
+
+func (ins *tidbInstance) sync() {
+	info := &model.TiDBInfo{
+		Name:      ins.config.Name,
+		ClusterID: ins.config.ClusterID,
+		Offset:    ins.latestOffset,
+		CommitTS:  ins.latestTimestamp,
+	}
+	ins.canal.dao.UpdateTiDBPosition(context.Background(), info)
+}

app/infra/canal/service/tidb_proc.go

@@ -0,0 +1,78 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"time"
+
+	"go-common/app/infra/canal/conf"
+	"go-common/library/log"
+)
+
+func (ins *tidbInstance) proc(ch chan *msg) {
+	defer ins.waitTable.Done()
+	for {
+		msg, ok := <-ch
+		if !ok {
+			return
+		}
+		data, err := tidbMakeData(msg)
+		if err != nil {
+			log.Error("tidb MakeData(%+v) err: %+v", msg, err)
+			continue
+		}
+		if data == nil {
+			continue
+		}
+		for _, target := range ins.targets[msg.db] {
+			for {
+				if err = target.Send(context.TODO(), data.Key, data); err == nil {
+					stats.Incr("send_counter", target.Name(), msg.db, msg.tableRegexp, data.Action)
+					break
+				}
+				stats.Incr("retry_counter", target.Name(), msg.db, msg.tableRegexp, data.Action)
+				log.Error("tidb %s scheme(%s) pub fail,add to retry", target.Name(), msg.db)
+				time.Sleep(time.Second)
+			}
+			log.Info("tidb %s pub(key:%s, value:%+v) succeed", target.Name(), data.Key, data)
+		}
+	}
+}
+
+func (ins *tidbInstance) syncproc() {
+	defer ins.waitConsume.Done()
+	for {
+		if ins.closed {
+			return
+		}
+		time.Sleep(time.Second * 5)
+		ins.sync()
+	}
+}
+
+func (c *Canal) tidbEventproc() {
+	ech := conf.TiDBEvent()
+	for {
+		insc := <-ech
+		if insc == nil {
+			continue
+		}
+		c.tidbInsl.Lock()
+		if old, ok := c.tidbInstances[insc.Name]; ok {
+			old.close()
+		}
+		c.tidbInsl.Unlock()
+		ins, err := newTiDBInstance(c, insc)
+		if err != nil {
+			log.Error("new instance error(%v)", err)
+			c.sendWx(fmt.Sprintf("reload tidb canal instance(%s) failed error(%v)", ins.String(), err))
+			continue
+		}
+		c.tidbInsl.Lock()
+		c.tidbInstances[insc.Name] = ins
+		c.tidbInsl.Unlock()
+		go ins.start()
+		log.Info("reload tidb canal instance(%s) success", ins.String())
+		c.sendWx(fmt.Sprintf("reload tidb canal instance(%s) success", ins.String()))
+	}
+}

app/infra/config/CHANGELOG.md

@@ -0,0 +1,138 @@
+### config-service
+
+#### Version 2.3.5
+>1. 增加一个http接口，直接获取当前最新发布的内容
+
+#### Version 2.3.4
+>1. log日志修改
+
+#### Version 2.3.3
+>1. sql语句中force是关键字要加``
+
+#### Version 2.3.2
+>1. 配置中心最新规则：
+规则1：当前版本大于0时优先级 单机强制 > 全局强制(当前版本小于等于最近的一次强制版本号时才会拉取,主要为了castr发版时能拉到最新配置) > 指定版本 > 当前发布最新版本
+规则2：当前版本小于等于0时没有单机强制和全局面强制逻辑，优先级为  指定版本 > 当前最新发布版本
+
+#### Version 2.3.1
+>1. ut补全
+
+#### Version 2.2.1
+>1. 忽视appiont,执行强制更新功能
+
+#### Version 2.1.4
+>1. sdk 多处连接泄露修复
+
+#### Version 2.1.3
+>1. sdk 连接泄露bug修复
+
+#### Version 2.1.2
+>1. discovery注册
+
+#### Version 2.1.1
+>1. 限流
+>2. 去tree_id并兼容老tree_id接口
+
+#### Version 2.1.0
+>1. bm架构
+
+#### Version 2.0.4
+>1. 增加register
+
+#### Version 2.0.3
+>1. fix sh001 limit and mv file to main
+
+#### Version 2.0.2
+>1. 移除statsd 模块
+
+#### Version 2.0.2
+>1. 版本推送不依赖redis
+
+#### Version 2.0.1
+>1. 修复file.so 接口，兼容新的和老的file.so逻辑
+
+#### Version 2.0.0
+>1. 配置中心版本v4,走新库新表
+>2. 支持客户端增量更新
+
+#### Version 1.6.2
+>1. push 接口推送时候如果数据库内容为空，返回失败
+
+#### Version 1.6.1
+>1. check接口增加自定义参数上报
+
+#### Version 1.6.0
+
+>1. 增加批量添加配置接口
+>2. 增加配置拷贝接口
+>3. 增加修改版本下的所有配置接口，没有就新加，有就覆盖
+>4. 增加返回未配置完成版本ID 列表
+
+#### Version 1.5.0
+
+>1. 增加配置文件命名空间，支持公共配置
+
+#### Version 1.4.3
+
+>1. 再次修复数据库连接bad connection 问题
+
+#### Version 1.4.2
+
+>1. 修复数据库连接bad connection 问题
+
+#### Version 1.4.1
+
+>1. bugfix 修复缓存文件key 错误
+
+#### Version 1.4.0
+
+>1. 接入普罗米修斯监控
+>2. 更新go-common和go-business 为最新的
+>3. file.so 接口添加日志返回，配置内容本地缓存
+
+#### Version 1.3.0
+
+>1. 增加builds，versions接口,分别获取所有构建版本和版本id。
+>2. check接口支持appoint参数，返回appoint作为版本id。
+
+#### Version 1.2.4
+
+>1. 更改主机历史记录保存时间，超过3小时没有在线。在查询的时候删除。
+>2. 主机超时时间延长5秒
+
+#### Version 1.2.3
+
+>1. file接口去掉hostName参数，version改为可选参数。更改获取逻辑，直接从数据库获取单个配置文件。
+
+#### Version 1.2.2
+
+>1. 将返回值由json改为文件内容
+
+#### Version 1.2.1
+
+>1. 增加获取单个配置接口file
+
+
+#### Version 1.2.0
+>1. 启动参数增加token字段，应用注册生成，用于应用和环境权限限制
+>2. 应用启动配置文件本地map缓存一份，同时写入配置文件
+
+
+#### Version 1.0.1
+>1. APM 添加build version映射表，对应于config版本号
+>2. 发布版本，只需要修改这个build version映射表的config版本号。
+>3. 目前先去掉推送主机和主机标记功能，下个版本考虑。
+>4. 程序编译时自己带build version，可以直接通过映射表，获取使用的配置版本。
+
+
+#### Version 1.0.0
+>1. 支持推APM推服务主机配置，生成本地缓存版本。
+>2. 支持长轮询监听新配置文件更新，并推送版本号到客户端。
+>3. 支持按版本号获取配置文件。
+>4. php以进程方式启动，并监听配置文件更新。
+>5. go以零配置启动，下载配置文件，并监听配置更新。
+
+
+
+
+

app/infra/config/CONTRIBUTORS.md

@@ -0,0 +1,16 @@
+# Owner
+maojian
+haoguanwei
+lintanghui
+chenzhihui
+
+# Author 
+zhoujixiang
+chenshangqiang
+linli
+
+# Reviewer
+maojian
+chenzhihui
+lintanghui
+haoguanwei

app/infra/config/OWNERS

@@ -0,0 +1,26 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- chenshangqiang
+- chenzhihui
+- haoguanwei
+- linli
+- lintanghui
+- maojian
+- zhoujixiang
+labels:
+- infra
+- infra/config
+- main
+- service
+- service/main/config
+options:
+  no_parent_owners: true
+reviewers:
+- chenshangqiang
+- chenzhihui
+- haoguanwei
+- linli
+- lintanghui
+- maojian
+- zhoujixiang

app/infra/config/README.md

@@ -0,0 +1,21 @@
+# go-common/business/main/service/config
+
+##### 项目简介
+> 1. 配置中心服务端，提供配置文件的管理和拉取
+> 2. 支持心跳检测，配置动态下发
+
+##### 编译环境
+> 1. 请只用golang v1.7.x以上版本编译执行。
+
+##### 依赖包
+> 1. 公共依赖
+
+##### 编译执行
+> 1. 启动执行
+> 2. 项目文档http://info.bilibili.co/pages/viewpage.action?pageId=1741193
+
+##### 测试
+> 1. 执行当前目录下所有测试文件，测试所有功能
+
+##### 特别说明
+> 2. 管理平台地址 http://apm-monitor.bilibili.co

app/infra/config/cmd/BUILD

@@ -0,0 +1,48 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["config-service-example.toml"],
+    importpath = "go-common/app/infra/config/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/http:go_default_library",
+        "//app/infra/config/rpc/server:go_default_library",
+        "//app/infra/config/service/v1:go_default_library",
+        "//app/infra/config/service/v2:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/ip:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/cmd/config-service-example.toml

@@ -0,0 +1,60 @@
+# This is a TOML document. Boom.
+
+pollTimeout = "30s"
+pathCache = "/tmp/"
+
+[log]
+stdout = true
+
+[antispam]
+    on=true
+    second=60
+    n=10
+    hour=12
+    m=500
+    [antispam.redis]
+        name = "config-service/hosts"
+        proto = "tcp"
+        addr = "172.18.33.61:6811"
+        idle = 100
+        active = 100
+        dialTimeout = "1s"
+        readTimeout = "1s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+        expire = "24h"
+        
+[ecode]
+    domain = "172.16.33.248:6401"
+
+[db]
+    name = "[config]tcp@172.16.33.205:3308"
+    dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_apm?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 5
+    queryTimeout = "500ms"
+    execTimeout = "500ms"
+    tranTimeout = "500ms"
+    [db.breaker]
+    window = "3s"
+    sleep = "100ms"
+    bucket = 10
+    ratio = 0.5
+    request = 100
+
+[redis]
+    name = "config-service/hosts"
+    proto = "tcp"
+    addr = "172.18.33.61:6811"
+    idle = 100
+    active = 100
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"
+
+[orm]
+dsn = "test:test@tcp(172.16.33.205:3308)/bilibili_config?timeout=5s&readTimeout=5s&writeTimeout=5s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+active = 5
+idle = 5
+idleTimeout = "4h"

app/infra/config/cmd/main.go

@@ -0,0 +1,82 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/infra/config/conf"
+	"go-common/app/infra/config/http"
+	"go-common/app/infra/config/rpc/server"
+	"go-common/app/infra/config/service/v1"
+	"go-common/app/infra/config/service/v2"
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/naming"
+	"go-common/library/naming/discovery"
+	xip "go-common/library/net/ip"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		log.Error("conf.Init() error(%v)", err)
+		panic(err)
+	}
+	// init log
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	// service init
+	svr2 := v2.New(conf.Conf)
+	svr := v1.New(conf.Conf)
+	rpcSvr := rpc.New(conf.Conf, svr, svr2)
+	http.Init(conf.Conf, svr, svr2, rpcSvr)
+	// start discovery register
+	var (
+		err    error
+		cancel context.CancelFunc
+	)
+	if env.IP == "" {
+		ip := xip.InternalIP()
+		hn, _ := os.Hostname()
+		dis := discovery.New(nil)
+		ins := &naming.Instance{
+			Zone:     env.Zone,
+			Env:      env.DeployEnv,
+			AppID:    "config.service",
+			Hostname: hn,
+			Addrs: []string{
+				"http://" + ip + ":" + env.HTTPPort,
+				"gorpc://" + ip + ":" + env.GORPCPort,
+			},
+		}
+		if cancel, err = dis.Register(context.Background(), ins); err != nil {
+			panic(err)
+		}
+	}
+	// end discovery register
+
+	// init signal
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("config-service get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			if cancel != nil {
+				cancel()
+			}
+			rpcSvr.Close()
+			svr.Close()
+			log.Info("config-service exit")
+			return
+		case syscall.SIGHUP:
+		// TODO reload
+		default:
+			return
+		}
+	}
+}

app/infra/config/conf/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/infra/config/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/antispam:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/conf/conf.go

@@ -0,0 +1,56 @@
+package conf
+
+import (
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/database/orm"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/antispam"
+	v "go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/rpc"
+	"go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	// Conf init config
+	Conf *Config
+)
+
+// Config config.
+type Config struct {
+	// log
+	Log *log.Config
+	//rpc server2
+	RPCServer *rpc.ServerConfig
+	// db
+	DB *sql.Config
+	// redis
+	Redis *redis.Config
+	// timeout
+	PollTimeout time.Duration
+	// local cache
+	PathCache string
+	// orm
+	ORM *orm.Config
+	//BM
+	BM *bm.ServerConfig
+	// Antispam
+	Antispam *antispam.Config
+	Verify   *v.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "./config-service-example.toml", "config path")
+}
+
+// Init init.
+func Init() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}

app/infra/config/dao/v1/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "file.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/infra/config/dao/v1",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/dao/v1/dao.go

@@ -0,0 +1,58 @@
+package v1
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/infra/config/conf"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+)
+
+// Dao dao.
+type Dao struct {
+	// mysql
+	db *sql.DB
+	// redis
+	redis  *redis.Pool
+	expire time.Duration
+	// cache
+	pathCache string
+}
+
+// New new a dao.
+func New(c *conf.Config) *Dao {
+	d := &Dao{
+		// db
+		db: sql.NewMySQL(c.DB),
+		// redis
+		redis:  redis.NewPool(c.Redis),
+		expire: time.Duration(c.PollTimeout),
+		// cache
+		pathCache: c.PathCache,
+	}
+	return d
+}
+
+// BeginTran begin transcation.
+func (d *Dao) BeginTran(c context.Context) (tx *sql.Tx, err error) {
+	return d.db.Begin(c)
+}
+
+// Close close resuouces.
+func (d *Dao) Close() {
+	if d.db != nil {
+		d.db.Close()
+	}
+	if d.redis != nil {
+		d.redis.Close()
+	}
+}
+
+// Ping ping is ok.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.pingRedis(c); err != nil {
+		return
+	}
+	return d.db.Ping(c)
+}

app/infra/config/dao/v1/file.go

@@ -0,0 +1,70 @@
+package v1
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/log"
+)
+
+// SetFile set config file.
+func (d *Dao) SetFile(name string, conf *model.Content) (err error) {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		log.Error("json.Marshal(%v) error(%v)", conf, err)
+		return
+	}
+	p := path.Join(d.pathCache, name)
+	if err = ioutil.WriteFile(p, b, 0644); err != nil {
+		log.Error("ioutil.WriteFile(%s) error(%v)", p, err)
+	}
+	return
+}
+
+// File return config file.
+func (d *Dao) File(name string) (res *model.Content, err error) {
+	p := path.Join(d.pathCache, name)
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		log.Error("ioutil.ReadFile(%s) error(%v)", p, err)
+		return
+	}
+	res = &model.Content{}
+	if err = json.Unmarshal(b, &res); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", b, err)
+	}
+	return
+}
+
+// DelFile delete file cache.
+func (d *Dao) DelFile(name string) (err error) {
+	p := path.Join(d.pathCache, name)
+	if err = os.Remove(p); err != nil {
+		log.Error("os.Remove(%s) error(%v)", p, err)
+	}
+	return
+}
+
+// SetFileStr save string file.
+func (d *Dao) SetFileStr(name string, val string) (err error) {
+	p := path.Join(d.pathCache, name)
+	if err = ioutil.WriteFile(p, []byte(val), 0644); err != nil {
+		log.Error("ioutil.WriteFile(%s) error(%v)", p, err)
+	}
+	return
+}
+
+// FileStr get string file.
+func (d *Dao) FileStr(name string) (file string, err error) {
+	p := path.Join(d.pathCache, name)
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		log.Error("ioutil.ReadFile(%s) error(%v)", p, err)
+		return
+	}
+	file = string(b)
+	return
+}

app/infra/config/dao/v1/mysql.go

@@ -0,0 +1,224 @@
+package v1
+
+import (
+	"bytes"
+	"context"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+const (
+	_getToken        = "SELECT token FROM service_name WHERE name=? AND environment=?"
+	_getBuildVersion = "SELECT b.config_id FROM service_name s, build_version b WHERE b.version=? AND s.name=? And s.environment=? AND s.id=b.service_id AND b.state=2"
+	_getVersions     = "SELECT c.id,c.remark FROM service_config c, service_name s WHERE s.name=? AND s.environment=? AND s.id =c.service_id AND c.state=? ORDER BY c.id DESC"
+	_getNamespace    = "SELECT id,namespace FROM service_namespace WHERE config_id=?"
+	_getValue        = "SELECT config_id,namespace_id,name,config FROM service_config_value WHERE config_id=?"
+	_getFile         = "SELECT config FROM service_config_value WHERE config_id=? AND name =?"
+	_getBuilds       = "SELECT b.version FROM service_name s ,build_version b WHERE s.name=? AND s.environment=? AND s.id=b.service_id AND b.state=2 ORDER BY b.id DESC"
+	_getServiceID    = "SELECT id FROM service_name where name=? AND environment =?"
+	_insertVersion   = "INSERT INTO service_config(service_id,state,operator) VALUES (?,?,?)"
+	_insertConfigs   = "INSERT INTO service_config_value(config_id,name,config,operator) VALUES "
+	_updateConfigs   = "UPDATE service_config_value SET config=?,operator=? WHERE config_id = ? AND name = ?"
+	_insertLog       = "INSERT INTO log(username,business,info) VALUES (?,?,?)"
+)
+
+// Token return a Secret from mysql.
+func (d *Dao) Token(c context.Context, svr, env string) (token string, err error) {
+	row := d.db.QueryRow(c, _getToken, svr, env)
+	if err = row.Scan(&token); err != nil {
+		log.Error("row.Scan error(%v) svrName(%v)", err, svr)
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+	}
+	return
+}
+
+// BuildVersion return service build version from mysql.
+func (d *Dao) BuildVersion(c context.Context, svr, bver, env string) (version int64, err error) {
+	row := d.db.QueryRow(c, _getBuildVersion, bver, svr, env)
+	if err = row.Scan(&version); err != nil {
+		if err == sql.ErrNoRows {
+			version = model.UnknownVersion
+			err = nil
+			return
+		}
+		log.Error("row.Scan error(%v)", err)
+	}
+	return
+}
+
+// Values return values from mysql.
+func (d *Dao) Values(c context.Context, ver int64) (rs []*model.NSValue, err error) {
+	rows, err := d.db.Query(c, _getValue, ver)
+	if err != nil {
+		log.Error("db.Query(%d) error(%v)", ver, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r model.NSValue
+		if err = rows.Scan(&r.ConfigID, &r.NamespaceID, &r.Name, &r.Config); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		rs = append(rs, &r)
+	}
+	return
+}
+
+// Namespaces return namespaces from mysql
+func (d *Dao) Namespaces(c context.Context, ver int64) (rs map[int64]string, err error) {
+	rows, err := d.db.Query(c, _getNamespace, ver)
+	if err != nil {
+		log.Error("db.Query(%d) error(%v)", err)
+		return
+	}
+	rs = make(map[int64]string)
+	defer rows.Close()
+	for rows.Next() {
+		var id int64
+		var name string
+		if err = rows.Scan(&id, &name); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		rs[id] = name
+	}
+	return
+}
+
+//Versions return versions from mysql
+func (d *Dao) Versions(c context.Context, svr, env string, state int8) (rs []*model.ReVer, err error) {
+	rows, err := d.db.Query(c, _getVersions, svr, env, state)
+	if err != nil {
+		log.Error("db.Query(%s) error(%v)", svr, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r model.ReVer
+		if err = rows.Scan(&r.Version, &r.Remark); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		rs = append(rs, &r)
+	}
+	return
+}
+
+// Value return values from mysql.
+func (d *Dao) Value(c context.Context, fname string, ver int64) (file string, err error) {
+	row := d.db.QueryRow(c, _getFile, ver, fname)
+	if err = row.Scan(&file); err != nil {
+		log.Error("row.Scan error(%v)", err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+			return
+		}
+	}
+	return
+}
+
+// Builds get service builds.
+func (d *Dao) Builds(c context.Context, svr, env string) (rs []string, err error) {
+	rows, err := d.db.Query(c, _getBuilds, svr, env)
+	if err != nil {
+		log.Error("db.Query(%s) error(%v)", svr, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var r string
+		if err = rows.Scan(&r); err != nil {
+			log.Error("rows.Scan error(%v)", err)
+			return
+		}
+		rs = append(rs, r)
+	}
+	return
+}
+
+// ServiceID get ServiceID.
+func (d *Dao) ServiceID(c context.Context, svr, env string) (ID int64, err error) {
+	row := d.db.QueryRow(c, _getServiceID, svr, env)
+	if err != nil {
+		log.Error("db.Query(%s) error(%v)", svr, err)
+		return
+	}
+	if err = row.Scan(&ID); err != nil {
+		log.Error("row.Scan error(%v)", err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+			return
+		}
+	}
+	return
+}
+
+// TxInsertVer insert version.
+func (d *Dao) TxInsertVer(tx *sql.Tx, svrID int64, user string) (verID int64, err error) {
+	row, err := tx.Exec(_insertVersion, svrID, model.ConfigIng, user)
+	if err != nil {
+		log.Error("db.insert(%d) error(%v)", svrID, err)
+		return
+	}
+	return row.LastInsertId()
+}
+
+// TxInsertValues insert config values.
+func (d *Dao) TxInsertValues(c context.Context, tx *sql.Tx, verID int64, user string, data map[string]string) (err error) {
+	var (
+		buffer   bytes.Buffer
+		insertTp string
+		stmt     *sql.Stmt
+		is       []interface{}
+	)
+	buffer.WriteString(_insertConfigs)
+	insertTp = "(?,?,?,?),"
+	for key, val := range data {
+		buffer.WriteString(insertTp)
+		is = append(is, verID)
+		is = append(is, key)
+		is = append(is, val)
+		is = append(is, user)
+	}
+	buffer.Truncate(buffer.Len() - 1)
+	if stmt, err = tx.Prepare(buffer.String()); err != nil {
+		log.Error("d.insert() error(%v)", err)
+		return
+	}
+	_, err = stmt.Exec(c, is...)
+	if err != nil {
+		log.Error("d.insert() error(%v)", err)
+	}
+	return
+}
+
+// TxUpdateValues update config values.
+func (d *Dao) TxUpdateValues(tx *sql.Tx, verID int64, user string, data map[string]string) (err error) {
+	for key, val := range data {
+		if _, err = tx.Exec(_updateConfigs, val, user, verID, key); err != nil {
+			log.Error("db.UpdateValues(%d) error(%v)", user, err)
+			break
+		}
+	}
+	if err != nil {
+		log.Error("d.insert() error(%v)", err)
+	}
+	return
+}
+
+// InsertLog insert log.
+func (d *Dao) InsertLog(c context.Context, user, business, info string) (err error) {
+	_, err = d.db.Exec(c, _insertLog, user, business, info)
+	if err != nil {
+		log.Error("db.InsertLog(%d) error(%v)", user, err)
+		return
+	}
+	return
+}

app/infra/config/dao/v1/redis.go

@@ -0,0 +1,100 @@
+package v1
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	expireDuration = 3 * time.Hour
+	_hostKey       = "%s_%s"
+)
+
+// Hostkey host cache key
+func hostKey(svr, env string) string {
+	return fmt.Sprintf(_hostKey, svr, env)
+}
+
+// Hosts return service hosts from redis.
+func (d *Dao) Hosts(c context.Context, svr, env string) (hosts []*model.Host, err error) {
+	var (
+		dels    []string
+		now     = time.Now()
+		hostkey = hostKey(svr, env)
+		conn    = d.redis.Get(c)
+	)
+	defer conn.Close()
+	res, err := redis.Strings(conn.Do("HGETALL", hostkey))
+	if err != nil {
+		log.Error("conn.Do(HGETALL, %s) error(%v)", hostkey, err)
+		return
+	}
+	for i, r := range res {
+		if i%2 == 0 {
+			continue
+		}
+		h := &model.Host{}
+		if err = json.Unmarshal([]byte(r), h); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", r, err)
+			return
+		}
+		if now.Sub(h.HeartbeatTime.Time()) <= d.expire+5 {
+			h.State = model.HostOnline
+			hosts = append(hosts, h)
+		} else if now.Sub(h.HeartbeatTime.Time()) >= expireDuration {
+			dels = append(dels, h.Name)
+		} else {
+			h.State = model.HostOffline
+			hosts = append(hosts, h)
+		}
+	}
+	if len(dels) > 0 {
+		if _, err1 := conn.Do("HDEL", hostkey, dels); err1 != nil {
+			log.Error("conn.Do(HDEL, %s, %v) error(%v)", hostkey, dels, err1)
+		}
+	}
+	return
+}
+
+// SetHost add service host to redis.
+func (d *Dao) SetHost(c context.Context, host *model.Host, svr, env string) (err error) {
+	hostkey := hostKey(svr, env)
+	b, err := json.Marshal(host)
+	if err != nil {
+		log.Error("json.Marshal(%s) error(%v)", host, err)
+		return
+	}
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("HSET", hostkey, host.Name, string(b)); err != nil {
+		log.Error("conn.Do(SET, %s, %s, %v) error(%v)", hostkey, host.Name, host, err)
+	}
+	return
+}
+
+// ClearHost clear all hosts.
+func (d *Dao) ClearHost(c context.Context, svr, env string) (err error) {
+	var (
+		hostkey = hostKey(svr, env)
+		conn    = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", hostkey); err != nil {
+		log.Error("conn.Do(DEL, %s) error(%v)", hostkey, err)
+	}
+	return
+}
+
+// Ping check Redis connection
+func (d *Dao) pingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	_, err = conn.Do("SET", "PING", "PONG")
+	conn.Close()
+	return
+}

app/infra/config/dao/v2/BUILD

@@ -0,0 +1,70 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "app_test.go",
+        "build_test.go",
+        "config_test.go",
+        "dao_test.go",
+        "file_test.go",
+        "force_test.go",
+        "redis_test.go",
+        "tag_test.go",
+    ],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "app.go",
+        "build.go",
+        "config.go",
+        "dao.go",
+        "file.go",
+        "force.go",
+        "redis.go",
+        "tag.go",
+    ],
+    importpath = "go-common/app/infra/config/dao/v2",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/orm:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/xstr:go_default_library",
+        "//vendor/github.com/jinzhu/gorm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/dao/v2/app.go

@@ -0,0 +1,46 @@
+package v2
+
+import (
+	"go-common/app/infra/config/model"
+	"go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// AppByTree get token by Name.
+func (d *Dao) AppByTree(zone, env string, treeID int64) (app *model.App, err error) {
+	app = &model.App{}
+	row := d.DB.Select("id,token").Where("tree_id = ? AND env=? AND zone=?", treeID, env, zone).Model(&model.DBApp{}).Row()
+	if err = row.Scan(&app.ID, &app.Token); err != nil {
+		log.Error("AppByTree(%v) error(%v)", treeID, err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+		}
+	}
+	return
+}
+
+// AppsByNameEnv get token by Name.
+func (d *Dao) AppsByNameEnv(name, env string) (apps []*model.DBApp, err error) {
+	if err = d.DB.Where("env = ? and  name like ?", env, "%"+name).Find(&apps).Error; err != nil {
+		log.Error("AppsByNameEnv(%v) error(%v)", name, env)
+		return
+	}
+	if len(apps) == 0 {
+		err = ecode.NothingFound
+	}
+	return
+}
+
+// AppGet ...
+func (d *Dao) AppGet(zone, env, token string) (app *model.App, err error) {
+	app = &model.App{}
+	row := d.DB.Select("id,token,env,zone,tree_id").Where("token = ? AND env= ? AND zone= ?", token, env, zone).Model(&model.DBApp{}).Row()
+	if err = row.Scan(&app.ID, &app.Token, &app.Env, &app.Zone, &app.TreeID); err != nil {
+		log.Error("AppGet zone(%v) env(%v) token(%v) error(%v)", zone, env, token, err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+		}
+	}
+	return
+}

app/infra/config/dao/v2/app_test.go

@@ -0,0 +1,51 @@
+package v2
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestV2AppByTree(t *testing.T) {
+	var (
+		zone   = ""
+		env    = ""
+		treeID = int64(0)
+	)
+	convey.Convey("AppByTree", t, func(ctx convey.C) {
+		app, err := d.AppByTree(zone, env, treeID)
+		ctx.Convey("Then err should be nil.app should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(app, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2AppsByNameEnv(t *testing.T) {
+	var (
+		name = "main.common-arch.apm-admin"
+		env  = "fat1"
+	)
+	convey.Convey("AppsByNameEnv", t, func(ctx convey.C) {
+		apps, err := d.AppsByNameEnv(name, env)
+		ctx.Convey("Then err should be nil.apps should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(apps, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2AppGet(t *testing.T) {
+	var (
+		zone  = "sh001"
+		env   = "fat1"
+		token = "a882c5530bcc11e8ab68522233017188"
+	)
+	convey.Convey("AppGet", t, func(ctx convey.C) {
+		app, err := d.AppGet(zone, env, token)
+		ctx.Convey("Then err should be nil.app should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(app, convey.ShouldNotBeNil)
+		})
+	})
+}

app/infra/config/dao/v2/build.go

@@ -0,0 +1,77 @@
+package v2
+
+import (
+	"database/sql"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// BuildsByAppID get builds by app id.
+func (d *Dao) BuildsByAppID(appID int64) (builds []string, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Select("name").Model(&model.Build{}).Where("app_id = ? ", appID).Rows(); err != nil {
+		log.Error("BuildsByAppID(%v) error(%v)", appID, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var build string
+		if err = rows.Scan(&build); err != nil {
+			log.Error("BuildsByAppID(%v) error(%v)", appID, err)
+			return
+		}
+		builds = append(builds, build)
+	}
+	if len(builds) == 0 {
+		err = ecode.NothingFound
+	}
+	return
+}
+
+// BuildsByAppIDs get builds by app id.
+func (d *Dao) BuildsByAppIDs(appIDs []int64) (builds []string, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Select("name").Model(&model.Build{}).Where("app_id in (?) ", appIDs).Rows(); err != nil {
+		log.Error("BuildsByAppIDs(%v) error(%v)", appIDs, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var build string
+		if err = rows.Scan(&build); err != nil {
+			log.Error("BuildsByAppIDs(%v) error(%v)", appIDs, err)
+			return
+		}
+		builds = append(builds, build)
+	}
+	if len(builds) == 0 {
+		err = ecode.NothingFound
+	}
+	return
+}
+
+// TagID get TagID by ID.
+func (d *Dao) TagID(appID int64, build string) (tagID int64, err error) {
+	row := d.DB.Select("tag_id").Where("app_id =? and name= ?", appID, build).Model(&model.Build{}).Row()
+	if err = row.Scan(&tagID); err != nil {
+		log.Error("TagID(%v) error(%v)", build, err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+		}
+	}
+	return
+}
+
+// BuildID get build by ID.
+func (d *Dao) BuildID(appID int64, build string) (buildID int64, err error) {
+	row := d.DB.Select("id").Where("app_id =? and name= ?", appID, build).Model(&model.Build{}).Row()
+	if err = row.Scan(&buildID); err != nil {
+		log.Error("buildID(%v) error(%v)", buildID, err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+		}
+	}
+	return
+}

app/infra/config/dao/v2/build_test.go

@@ -0,0 +1,47 @@
+package v2
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestV2BuildsByAppID(t *testing.T) {
+	var (
+		appID = int64(24)
+	)
+	convey.Convey("BuildsByAppID", t, func(ctx convey.C) {
+		builds, err := d.BuildsByAppID(appID)
+		ctx.Convey("Then err should be nil.builds should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(builds, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2BuildsByAppIDs(t *testing.T) {
+	var (
+		appIDs = []int64{24}
+	)
+	convey.Convey("BuildsByAppIDs", t, func(ctx convey.C) {
+		builds, err := d.BuildsByAppIDs(appIDs)
+		ctx.Convey("Then err should be nil.builds should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(builds, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2TagID(t *testing.T) {
+	var (
+		appID = int64(24)
+		build = "docker-1"
+	)
+	convey.Convey("TagID", t, func(ctx convey.C) {
+		tagID, err := d.TagID(appID, build)
+		ctx.Convey("Then err should be nil.tagID should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(tagID, convey.ShouldNotBeNil)
+		})
+	})
+}

app/infra/config/dao/v2/config.go

@@ -0,0 +1,43 @@
+package v2
+
+import (
+	"database/sql"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+)
+
+// UpdateConfValue update config state/
+func (d *Dao) UpdateConfValue(ID int64, value string) (err error) {
+	err = d.DB.Model(&model.Config{ID: ID}).Where("state=?", model.ConfigIng).Update("comment", value).Error
+	return
+}
+
+// UpdateConfState update config state/
+func (d *Dao) UpdateConfState(ID int64, state int8) (err error) {
+	err = d.DB.Model(&model.Config{ID: ID}).Update("state", state).Error
+	return
+}
+
+// ConfigsByIDs get Config by IDs.
+func (d *Dao) ConfigsByIDs(ids []int64) (confs []*model.Value, err error) {
+	var rows *sql.Rows
+	if rows, err = d.DB.Where(ids).Select("id,name,comment").Where("state = ?", model.ConfigEnd).Model(&model.Config{}).Rows(); err != nil {
+		log.Error("ConfigsByIDs(%v) error(%v)", ids, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		v := new(model.Value)
+		if err = rows.Scan(&v.ConfigID, &v.Name, &v.Config); err != nil {
+			log.Error("ConfigsByIDs(%v) error(%v)", ids, err)
+			return
+		}
+		confs = append(confs, v)
+	}
+	if len(confs) == 0 {
+		err = ecode.NothingFound
+	}
+	return
+}

app/infra/config/dao/v2/config_test.go

@@ -0,0 +1,53 @@
+package v2
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestV2UpdateConfValue(t *testing.T) {
+	var (
+		ID    = int64(0)
+		value = ""
+	)
+	convey.Convey("UpdateConfValue", t, func(ctx convey.C) {
+		err := d.UpdateConfValue(ID, value)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2UpdateConfState(t *testing.T) {
+	var (
+		ID     = int64(855)
+		state  = int8(0)
+		state2 = int8(2)
+	)
+	convey.Convey("UpdateConfState", t, func(ctx convey.C) {
+		err := d.UpdateConfState(ID, state)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+	convey.Convey("UpdateConfState restoration", t, func(ctx convey.C) {
+		err := d.UpdateConfState(ID, state2)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2ConfigsByIDs(t *testing.T) {
+	var (
+		ids = []int64{855, 788}
+	)
+	convey.Convey("ConfigsByIDs", t, func(ctx convey.C) {
+		confs, err := d.ConfigsByIDs(ids)
+		ctx.Convey("Then err should be nil.confs should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(confs, convey.ShouldNotBeNil)
+		})
+	})
+}

app/infra/config/dao/v2/dao.go

@@ -0,0 +1,55 @@
+package v2
+
+import (
+	"context"
+	"time"
+
+	"go-common/app/infra/config/conf"
+	"go-common/library/cache/redis"
+	"go-common/library/database/orm"
+
+	"github.com/jinzhu/gorm"
+)
+
+// Dao dao.
+type Dao struct {
+	// redis
+	redis  *redis.Pool
+	expire time.Duration
+	// cache
+	pathCache string
+	//DB
+	DB *gorm.DB
+}
+
+// New new a dao.
+func New(c *conf.Config) *Dao {
+	d := &Dao{
+		// redis
+		redis:  redis.NewPool(c.Redis),
+		expire: time.Duration(c.PollTimeout),
+		// cache
+		pathCache: c.PathCache,
+		// orm
+		DB: orm.NewMySQL(c.ORM),
+	}
+	return d
+}
+
+// Ping ping is ok.
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.pingRedis(c); err != nil {
+		return
+	}
+	return d.DB.DB().PingContext(c)
+}
+
+// Close close resuouces.
+func (d *Dao) Close() {
+	if d.DB != nil {
+		d.DB.Close()
+	}
+	if d.redis != nil {
+		d.redis.Close()
+	}
+}

app/infra/config/dao/v2/dao_test.go

@@ -0,0 +1,36 @@
+package v2
+
+import (
+	"flag"
+	"go-common/app/infra/config/conf"
+	"os"
+	"testing"
+)
+
+var (
+	d *Dao
+)
+
+func TestMain(m *testing.M) {
+	// if os.Getenv("DEPLOY_ENV") != "" {
+	// 	// flag.Set("app_id", "")
+	// 	// flag.Set("conf_token", "")
+	// 	// flag.Set("tree_id", "")
+	// 	// flag.Set("conf_version", "docker-1")
+	// 	// flag.Set("deploy_env", "fat1")
+	// 	// flag.Set("conf_host", "config.bilibili.co")
+	// 	// flag.Set("conf_path", "/tmp")
+	// 	// flag.Set("region", "sh")
+	// 	// flag.Set("zone", "sh001")
+	// } else {
+	// 	flag.Set("conf", "../../cmd/config-service-example.toml")
+	// }
+	flag.Set("conf", "../../cmd/config-service-example.toml")
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	d = New(conf.Conf)
+	m.Run()
+	os.Exit(0)
+}

app/infra/config/dao/v2/file.go

@@ -0,0 +1,70 @@
+package v2
+
+import (
+	"encoding/json"
+	"io/ioutil"
+	"os"
+	"path"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/log"
+)
+
+// SetFile set config file.
+func (d *Dao) SetFile(name string, conf *model.Content) (err error) {
+	b, err := json.Marshal(conf)
+	if err != nil {
+		log.Error("json.Marshal(%v) error(%v)", conf, err)
+		return
+	}
+	p := path.Join(d.pathCache, name)
+	if err = ioutil.WriteFile(p, b, 0644); err != nil {
+		log.Error("ioutil.WriteFile(%s) error(%v)", p, err)
+	}
+	return
+}
+
+// File return config file.
+func (d *Dao) File(name string) (res *model.Content, err error) {
+	p := path.Join(d.pathCache, name)
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		log.Error("ioutil.ReadFile(%s) error(%v)", p, err)
+		return
+	}
+	res = &model.Content{}
+	if err = json.Unmarshal(b, &res); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", b, err)
+	}
+	return
+}
+
+// DelFile delete file cache.
+func (d *Dao) DelFile(name string) (err error) {
+	p := path.Join(d.pathCache, name)
+	if err = os.Remove(p); err != nil {
+		log.Error("os.Remove(%s) error(%v)", p, err)
+	}
+	return
+}
+
+// SetFileStr save string file.
+func (d *Dao) SetFileStr(name string, val string) (err error) {
+	p := path.Join(d.pathCache, name)
+	if err = ioutil.WriteFile(p, []byte(val), 0644); err != nil {
+		log.Error("ioutil.WriteFile(%s) error(%v)", p, err)
+	}
+	return
+}
+
+// FileStr get string file.
+func (d *Dao) FileStr(name string) (file string, err error) {
+	p := path.Join(d.pathCache, name)
+	b, err := ioutil.ReadFile(p)
+	if err != nil {
+		log.Error("ioutil.ReadFile(%s) error(%v)", p, err)
+		return
+	}
+	file = string(b)
+	return
+}

app/infra/config/dao/v2/file_test.go

@@ -0,0 +1,72 @@
+package v2
+
+import (
+	"go-common/app/infra/config/model"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestV2SetFile(t *testing.T) {
+	var (
+		name = "main.common-arch.apm-admin_460"
+		conf = &model.Content{}
+	)
+	convey.Convey("SetFile", t, func(ctx convey.C) {
+		err := d.SetFile(name, conf)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2File(t *testing.T) {
+	var (
+		name = "main.common-arch.apm-admin_460"
+	)
+	convey.Convey("File", t, func(ctx convey.C) {
+		res, err := d.File(name)
+		ctx.Convey("Then err should be nil.res should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(res, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2DelFile(t *testing.T) {
+	var (
+		name = "main.common-arch.apm-admin_460"
+	)
+	convey.Convey("DelFile", t, func(ctx convey.C) {
+		err := d.DelFile(name)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2SetFileStr(t *testing.T) {
+	var (
+		name = "main.common-arch.apm-admin_460"
+		val  = "test"
+	)
+	convey.Convey("SetFileStr", t, func(ctx convey.C) {
+		err := d.SetFileStr(name, val)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2FileStr(t *testing.T) {
+	var (
+		name = "main.common-arch.apm-admin_460"
+	)
+	convey.Convey("FileStr", t, func(ctx convey.C) {
+		file, err := d.FileStr(name)
+		ctx.Convey("Then err should be nil.file should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(file, convey.ShouldNotBeNil)
+		})
+	})
+}

app/infra/config/dao/v2/force.go

@@ -0,0 +1,22 @@
+package v2
+
+import (
+	"database/sql"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/log"
+)
+
+// Force get force by ID.
+func (d *Dao) Force(appID int64, hostname string) (version int64, err error) {
+	row := d.DB.Select("version").Where("app_id = ? and hostname = ?", appID, hostname).Model(&model.Force{}).Row()
+	if err = row.Scan(&version); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			version = 0
+		} else {
+			log.Error("version(%v) error(%v)", version, err)
+		}
+	}
+	return
+}

app/infra/config/dao/v2/force_test.go

@@ -0,0 +1,17 @@
+package v2
+
+// func TestV2Force(t *testing.T) {
+// 	convey.SkipConvey("Force", t, func(ctx convey.C) {
+// 		var (
+// 			appID    = int64(24)
+// 			hostname = "apm-admin-31733-7c776f4d86-rxv6z"
+// 		)
+// 		ctx.Convey("When everything goes positive", func(ctx convey.C) {
+// 			version, err := d.Force(appID, hostname)
+// 			ctx.Convey("Then err should be nil.version should not be nil.", func(ctx convey.C) {
+// 				ctx.So(err, convey.ShouldBeNil)
+// 				ctx.So(version, convey.ShouldNotBeNil)
+// 			})
+// 		})
+// 	})
+// }

app/infra/config/dao/v2/redis.go

@@ -0,0 +1,90 @@
+package v2
+
+import (
+	"context"
+	"encoding/json"
+	"time"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	expireDuration = 3 * time.Hour
+)
+
+// Hosts return service hosts from redis.
+func (d *Dao) Hosts(c context.Context, svr string) (hosts []*model.Host, err error) {
+	var (
+		dels []string
+		now  = time.Now()
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	res, err := redis.Strings(conn.Do("HGETALL", svr))
+	if err != nil {
+		log.Error("conn.Do(HGETALL, %s) error(%v)", svr, err)
+		return
+	}
+	for i, r := range res {
+		if i%2 == 0 {
+			continue
+		}
+		h := &model.Host{}
+		if err = json.Unmarshal([]byte(r), h); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", r, err)
+			return
+		}
+		if now.Sub(h.HeartbeatTime.Time()) <= d.expire+5 {
+			h.State = model.HostOnline
+			hosts = append(hosts, h)
+		} else if now.Sub(h.HeartbeatTime.Time()) >= expireDuration {
+			dels = append(dels, h.Name)
+		} else {
+			h.State = model.HostOffline
+			hosts = append(hosts, h)
+		}
+	}
+	if len(dels) > 0 {
+		if _, err1 := conn.Do("HDEL", svr, dels); err1 != nil {
+			log.Error("conn.Do(HDEL, %s, %v) error(%v)", svr, dels, err1)
+		}
+	}
+	return
+}
+
+// SetHost add service host to redis.
+func (d *Dao) SetHost(c context.Context, host *model.Host, svr string) (err error) {
+	b, err := json.Marshal(host)
+	if err != nil {
+		log.Error("json.Marshal(%s) error(%v)", host, err)
+		return
+	}
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("HSET", svr, host.Name, string(b)); err != nil {
+		log.Error("conn.Do(SET, %s, %s, %v) error(%v)", svr, host.Name, host, err)
+	}
+	return
+}
+
+// ClearHost clear all hosts.
+func (d *Dao) ClearHost(c context.Context, svr string) (err error) {
+	var (
+		conn = d.redis.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("DEL", svr); err != nil {
+		log.Error("conn.Do(DEL, %s) error(%v)", svr, err)
+	}
+	return
+}
+
+// Ping check Redis connection
+func (d *Dao) pingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	_, err = conn.Do("SET", "PING", "PONG")
+	conn.Close()
+	return
+}

app/infra/config/dao/v2/redis_test.go

@@ -0,0 +1,62 @@
+package v2
+
+import (
+	"context"
+	"go-common/app/infra/config/model"
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestV2Hosts(t *testing.T) {
+	var (
+		c   = context.TODO()
+		svr = "11133_fat1_sh001"
+	)
+	convey.Convey("Hosts", t, func(ctx convey.C) {
+		hosts, err := d.Hosts(c, svr)
+		ctx.Convey("Then err should be nil.hosts should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(hosts, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2SetHost(t *testing.T) {
+	var (
+		c    = context.TODO()
+		host = &model.Host{}
+		svr  = "11133_fat1_sh001"
+	)
+	convey.Convey("SetHost", t, func(ctx convey.C) {
+		err := d.SetHost(c, host, svr)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2ClearHost(t *testing.T) {
+	var (
+		c   = context.TODO()
+		svr = "11133_fat1_sh001"
+	)
+	convey.Convey("ClearHost", t, func(ctx convey.C) {
+		err := d.ClearHost(c, svr)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}
+
+func TestV2pingRedis(t *testing.T) {
+	var (
+		c = context.TODO()
+	)
+	convey.Convey("pingRedis", t, func(ctx convey.C) {
+		err := d.pingRedis(c)
+		ctx.Convey("Then err should be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+		})
+	})
+}

app/infra/config/dao/v2/tag.go

@@ -0,0 +1,79 @@
+package v2
+
+import (
+	"database/sql"
+	"go-common/app/infra/config/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+//Tags get builds by app id.
+func (d *Dao) Tags(appID int64) (tags []*model.ReVer, err error) {
+	rows, err := d.DB.Select("id,mark").Where("app_id = ? ", appID).Model(&model.DBTag{}).Rows()
+	if err != nil {
+		log.Error("Tags(%v) error(%v)", appID, err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		reVer := &model.ReVer{}
+		if err = rows.Scan(&reVer.Version, &reVer.Remark); err != nil {
+			log.Error("Tags(%v) error(%v)", appID, err)
+			return
+		}
+		tags = append(tags, reVer)
+	}
+	if len(tags) == 0 {
+		err = ecode.NothingFound
+	}
+	return
+}
+
+//ConfIDs get tag by id.
+func (d *Dao) ConfIDs(ID int64) (ids []int64, err error) {
+	tag := &model.DBTag{}
+	if err = d.DB.First(tag, ID).Error; err != nil {
+		log.Error("ConfIDs(%v) error(%v)", ID, err)
+		return
+	}
+	ids, _ = xstr.SplitInts(tag.ConfigIDs)
+	if len(ids) == 0 {
+		err = ecode.NothingFound
+	}
+	return
+}
+
+// TagForce get force by tag.
+func (d *Dao) TagForce(ID int64) (force int8, err error) {
+	row := d.DB.Select("`force`").Where("id = ?", ID).Model(&model.DBTag{}).Row()
+	if err = row.Scan(&force); err != nil {
+		log.Error("tagID(%v) error(%v)", ID, err)
+		if err == sql.ErrNoRows {
+			err = ecode.NothingFound
+		}
+	}
+	return
+}
+
+// LastForce ...
+func (d *Dao) LastForce(appID, buildID int64) (lastForce int64, err error) {
+	row := d.DB.Select("id").Where("`app_id` = ? and `build_id` = ? and `force` = 1", appID, buildID).Model(&model.DBTag{}).Row()
+	if err = row.Scan(&lastForce); err != nil {
+		log.Error("lastForce(%v) error(%v)", lastForce, err)
+		if err == sql.ErrNoRows {
+			err = nil
+			lastForce = 0
+		}
+	}
+	return
+}
+
+//TagAll ...
+func (d *Dao) TagAll(tagID int64) (tag *model.DBTag, err error) {
+	tag = &model.DBTag{}
+	if err = d.DB.First(tag, tagID).Error; err != nil {
+		log.Error("tagID(%v) error(%v)", tagID, err)
+	}
+	return
+}

app/infra/config/dao/v2/tag_test.go

@@ -0,0 +1,33 @@
+package v2
+
+import (
+	"testing"
+
+	"github.com/smartystreets/goconvey/convey"
+)
+
+func TestV2Tags(t *testing.T) {
+	var (
+		appID = int64(24)
+	)
+	convey.Convey("Tags", t, func(ctx convey.C) {
+		tags, err := d.Tags(appID)
+		ctx.Convey("Then err should be nil.tags should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(tags, convey.ShouldNotBeNil)
+		})
+	})
+}
+
+func TestV2ConfIDs(t *testing.T) {
+	var (
+		ID = int64(460)
+	)
+	convey.Convey("ConfIDs", t, func(ctx convey.C) {
+		ids, err := d.ConfIDs(ID)
+		ctx.Convey("Then err should be nil.ids should not be nil.", func(ctx convey.C) {
+			ctx.So(err, convey.ShouldBeNil)
+			ctx.So(ids, convey.ShouldNotBeNil)
+		})
+	})
+}

app/infra/config/http/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "config2.go",
+        "http.go",
+        "local.go",
+    ],
+    importpath = "go-common/app/infra/config/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//app/infra/config/service/v1:go_default_library",
+        "//app/infra/config/service/v2:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/antispam:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//vendor/github.com/dgryski/go-farm:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/http/config.go

@@ -0,0 +1,500 @@
+package http
+
+import (
+	"encoding/json"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// push config update
+func push(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		buildVer string
+		ver      int64
+		env      string
+	)
+	query := c.Request.Form
+	verStr := query.Get("version")
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if buildVer = query.Get("build_ver"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if ver, err = strconv.ParseInt(verStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	service := &model.Service{Name: svr, BuildVersion: buildVer, Version: ver, Env: env}
+	// update & write cache
+	c.JSON(nil, confSvc.Push(c, service))
+}
+
+// hosts client hosts
+func hosts(c *bm.Context) {
+	var (
+		err  error
+		svr  string
+		data []*model.Host
+		env  string
+	)
+	query := c.Request.URL.Query()
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if data, err = confSvc.Hosts(c, svr, env); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// versions client versions which the configuration is complete
+func versions(c *bm.Context) {
+	var (
+		err  error
+		svr  string
+		data *model.Versions
+		env  string
+		bver string
+	)
+	query := c.Request.URL.Query()
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if bver = query.Get("build"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if data, err = confSvc.VersionSuccess(c, svr, env, bver); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// versions client versions which the configuration is complete
+func versionIng(c *bm.Context) {
+	var (
+		err  error
+		svr  string
+		data []int64
+		env  string
+	)
+	query := c.Request.URL.Query()
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if data, err = confSvc.VersionIng(c, svr, env); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// config get config file
+func config(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		host     string
+		buildVer string
+		version  int64
+		env      string
+		token    string
+	)
+
+	query := c.Request.URL.Query()
+	verStr := query.Get("version")
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if token = query.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if host = query.Get("hostname"); host == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if version, err = strconv.ParseInt(verStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	service := &model.Service{Name: svr, BuildVersion: buildVer, Env: env, Token: token, Version: version, Host: host}
+	data, err := confSvc.Config(c, service)
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// file get one file value
+func file(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		buildVer string
+		env      string
+		token    string
+		file     string
+		ver      int64
+		data     string
+	)
+
+	query := c.Request.URL.Query()
+	// params
+	if buildVer = query.Get("build"); buildVer == "" {
+		data = "build is null"
+	}
+	if !strings.HasPrefix(buildVer, "shsb") && !strings.HasPrefix(buildVer, "shylf") &&
+		query.Get("zone") != "" && query.Get("env") != "" && query.Get("treeid") != "" {
+		file2(c)
+		return
+	}
+	if verStr := query.Get("version"); verStr == "" {
+		ver = model.UnknownVersion
+	} else {
+		if ver, err = strconv.ParseInt(verStr, 10, 64); err != nil {
+			data = "version must be num"
+		}
+	}
+	if svr = query.Get("service"); svr == "" {
+		data = "service is null"
+	}
+	if env = query.Get("environment"); env == "" {
+		data = "environment is null"
+	}
+	if token = query.Get("token"); token == "" {
+		data = "token is null"
+	}
+	if file = query.Get("fileName"); file == "" {
+		data = "fileName is null"
+	}
+	service := &model.Service{Name: svr, BuildVersion: buildVer, Env: env, File: file, Token: token, Version: ver}
+	if data == "" {
+		if data, err = confSvc.File(c, service); err != nil {
+			data = err.Error()
+			c.AbortWithStatus(http.StatusInternalServerError)
+		}
+	} else {
+		c.AbortWithStatus(http.StatusBadRequest)
+	}
+
+	if _, err = c.Writer.Write([]byte(data)); err != nil {
+		log.Error("Response().Write(%v) error(%v)", data, err)
+	}
+}
+
+// check check config version
+func check(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		host     string
+		buildVer string
+		ip       string
+		ver      int64
+		env      string
+		token    string
+		appoint  int64
+		query    = c.Request.URL.Query()
+	)
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if token = query.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if ip = query.Get("ip"); ip == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if host = query.Get("hostname"); host == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if ver, err = strconv.ParseInt(query.Get("version"), 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	appoint, _ = strconv.ParseInt(query.Get("appoint"), 10, 64)
+	// check config version
+	rhost := &model.Host{Service: svr, Name: host, BuildVersion: buildVer, IP: ip, ConfigVersion: ver, Appoint: appoint, Customize: query.Get("customize")}
+	evt, err := confSvc.CheckVersion(c, rhost, env, token)
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	// wait for version change
+	select {
+	case e := <-evt:
+		c.JSON(e, nil)
+	case <-time.After(time.Duration(cnf.PollTimeout)):
+		c.JSON(nil, ecode.NotModified)
+	case <-c.Writer.(http.CloseNotifier).CloseNotify():
+		c.JSON(nil, ecode.NotModified)
+	}
+	confSvc.Unsub(svr, host, env)
+}
+
+//clear  host in redis
+func clearhost(c *bm.Context) {
+	var (
+		svr string
+		env string
+	)
+	query := c.Request.Form
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(nil, confSvc.ClearHost(c, svr, env))
+}
+
+// versions client versions which the configuration is complete
+func builds(c *bm.Context) {
+	var (
+		svr     string
+		bs, bs2 []string
+		env     string
+	)
+	query := c.Request.URL.Query()
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	bs, _ = confSvc.Builds(c, svr, env)
+	bs2, _ = confSvc2.TmpBuilds(svr, env)
+	bs = append(bs, bs2...)
+	if len(bs) == 0 {
+		c.JSON(nil, ecode.NothingFound)
+		return
+	}
+	c.JSON(bs, nil)
+}
+
+func addConfigs(c *bm.Context) {
+	var (
+		svr    string
+		env    string
+		token  string
+		user   string
+		data   map[string]string
+		err    error
+		values = c.Request.PostForm
+	)
+	// params
+	if svr = values.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = values.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if token = values.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if user = values.Get("user"); user == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if err = json.Unmarshal([]byte(values.Get("data")), &data); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(nil, confSvc.AddConfigs(c, svr, env, token, user, data))
+}
+
+func copyConfigs(c *bm.Context) {
+	var (
+		svr    string
+		env    string
+		token  string
+		build  string
+		user   string
+		err    error
+		ver    int64
+		values = c.Request.PostForm
+	)
+	// params
+	if svr = values.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = values.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if token = values.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if user = values.Get("user"); user == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if build = values.Get("build"); build == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if ver, err = confSvc.CopyConfigs(c, svr, env, token, user, build); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(ver, nil)
+}
+
+func updateConfigs(c *bm.Context) {
+	var (
+		svr    string
+		env    string
+		token  string
+		ver    int64
+		user   string
+		data   map[string]string
+		err    error
+		values = c.Request.PostForm
+	)
+	// params
+	if svr = values.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = values.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if ver, err = strconv.ParseInt(values.Get("version"), 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if token = values.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if user = values.Get("user"); user == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if err = json.Unmarshal([]byte(values.Get("data")), &data); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(nil, confSvc.UpdateConfigs(c, svr, env, token, user, ver, data))
+}
+
+// configN get config namespace file
+func configN(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		host     string
+		buildVer string
+		version  int64
+		env      string
+		token    string
+	)
+
+	query := c.Request.URL.Query()
+	verStr := query.Get("version")
+	// params
+	if svr = query.Get("service"); svr == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if env = query.Get("environment"); env == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if token = query.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if host = query.Get("hostname"); host == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if version, err = strconv.ParseInt(verStr, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	service := &model.Service{Name: svr, BuildVersion: buildVer, Env: env, Token: token, Version: version, Host: host}
+	data, err := confSvc.Config2(c, service)
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}

app/infra/config/http/config2.go

@@ -0,0 +1,352 @@
+package http
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"strconv"
+	"time"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// versions client versions which the configuration is complete
+func versions2(c *bm.Context) {
+	var (
+		err   error
+		svr   string
+		data  *model.Versions
+		bver  string
+		token string
+		env   string
+		zone  string
+	)
+	query := c.Request.URL.Query()
+	// params
+	svr = query.Get("service")
+	if svr == "" {
+		token = query.Get("token")
+		zone = query.Get("zone")
+		env = query.Get("env")
+		if zone == "" || env == "" || token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	if bver = query.Get("build"); bver == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if data, err = confSvc2.VersionSuccess(c, svr, bver); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// config get config file
+func config2(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		buildVer string
+		version  int64
+		token    string
+		ids      []int64
+		zone     string
+		env      string
+	)
+	query := c.Request.URL.Query()
+	// params
+	if token = query.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	svr = query.Get("service")
+	if svr == "" {
+		zone = query.Get("zone")
+		env = query.Get("env")
+		if zone == "" || env == "" || token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if version, err = strconv.ParseInt(query.Get("version"), 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if idsStr := query.Get("ids"); len(idsStr) != 0 {
+		if err = json.Unmarshal([]byte(query.Get("ids")), &ids); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	data, err := confSvc2.Config(c, svr, token, version, ids)
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// file get one file value
+func file2(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		buildVer string
+		token    string
+		file     string
+		ver      int64
+		treeID   string
+		env      string
+		zone     string
+		data     string
+	)
+
+	query := c.Request.URL.Query()
+	// params
+	if verStr := query.Get("version"); verStr == "" {
+		ver = model.UnknownVersion
+	} else {
+		if ver, err = strconv.ParseInt(verStr, 10, 64); err != nil {
+			data = "version must be num"
+		}
+	}
+	if env = query.Get("env"); env == "" {
+		data = "env is null"
+	}
+	if zone = query.Get("zone"); zone == "" {
+		data = "zone is null"
+	}
+	if token = query.Get("token"); token == "" {
+		data = "token is null"
+	}
+	if treeID = query.Get("treeid"); treeID == "" {
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			data = "appid is null"
+		}
+	} else {
+		svr = fmt.Sprintf("%s_%s_%s", treeID, env, zone)
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		data = "build is null"
+	}
+	if file = query.Get("fileName"); file == "" {
+		data = "fileName is null"
+	}
+	service := &model.Service{Name: svr, BuildVersion: buildVer, File: file, Token: token, Version: ver}
+	if data == "" {
+		if data, err = confSvc2.File(c, service); err != nil {
+			data = err.Error()
+			c.AbortWithStatus(http.StatusInternalServerError)
+		}
+	} else {
+		c.AbortWithStatus(http.StatusBadRequest)
+	}
+	if _, err = c.Writer.Write([]byte(data)); err != nil {
+		log.Error("Response().Write(%v) error(%v)", data, err)
+	}
+}
+
+// check check config version
+func check2(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		host     string
+		buildVer string
+		ip       string
+		ver      int64
+		token    string
+		appoint  int64
+		zone     string
+		env      string
+		query    = c.Request.URL.Query()
+	)
+	// params
+	if token = query.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	svr = query.Get("service")
+	if svr == "" {
+		zone = query.Get("zone")
+		env = query.Get("env")
+		if zone == "" || env == "" || token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	if ip = query.Get("ip"); ip == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if host = query.Get("hostname"); host == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	if ver, err = strconv.ParseInt(query.Get("version"), 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	appoint, _ = strconv.ParseInt(query.Get("appoint"), 10, 64)
+	// check config version
+	rhost := &model.Host{Service: svr, Name: host, BuildVersion: buildVer, IP: ip, ConfigVersion: ver, Appoint: appoint, Customize: query.Get("customize")}
+	evt, err := confSvc2.CheckVersion(c, rhost, token)
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	// wait for version change
+	select {
+	case e := <-evt:
+		c.JSON(e, nil)
+	case <-time.After(time.Duration(cnf.PollTimeout)):
+		c.JSON(nil, ecode.NotModified)
+	case <-c.Writer.(http.CloseNotifier).CloseNotify():
+		c.JSON(nil, ecode.NotModified)
+	}
+	confSvc2.Unsub(svr, host)
+}
+
+//clear  host in redis
+func clearhost2(c *bm.Context) {
+	var (
+		svr   string
+		zone  string
+		env   string
+		token string
+		err   error
+	)
+	query := c.Request.Form
+	svr = query.Get("service")
+	if svr == "" {
+		token = query.Get("token")
+		zone = query.Get("zone")
+		env = query.Get("env")
+		if zone == "" || env == "" || token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	c.JSON(nil, confSvc2.ClearHost(c, svr))
+}
+
+// versions client versions which the configuration is complete
+func builds2(c *bm.Context) {
+	var (
+		err   error
+		svr   string
+		data  []string
+		token string
+		env   string
+		zone  string
+	)
+	query := c.Request.URL.Query()
+	// params
+	svr = query.Get("service")
+	if svr == "" {
+		token = query.Get("token")
+		zone = query.Get("zone")
+		env = query.Get("env")
+		if zone == "" || env == "" || token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	if data, err = confSvc2.Builds(c, svr); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}
+
+// config get config file
+func latest(c *bm.Context) {
+	var (
+		err      error
+		svr      string
+		buildVer string
+		version  int64
+		token    string
+		ids      []int64
+		zone     string
+		env      string
+		verStr   string
+	)
+	query := c.Request.URL.Query()
+	// params
+	if token = query.Get("token"); token == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	svr = query.Get("service")
+	if svr == "" {
+		zone = query.Get("zone")
+		env = query.Get("env")
+		if zone == "" || env == "" || token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+		if svr, err = confSvc2.AppService(zone, env, token); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	}
+	if buildVer = query.Get("build"); buildVer == "" {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	verStr = query.Get("version")
+	if len(verStr) > 0 {
+		if version, err = strconv.ParseInt(verStr, 10, 64); err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			return
+		}
+	} else {
+		rhost := &model.Host{Service: svr, BuildVersion: buildVer}
+		version, err = confSvc2.CheckLatest(c, rhost, token)
+		if err != nil {
+			c.JSON(nil, err)
+			return
+		}
+	}
+	data, err := confSvc2.ConfigCheck(c, svr, token, version, ids)
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(data, nil)
+}

app/infra/config/http/http.go

@@ -0,0 +1,117 @@
+package http
+
+import (
+	"io"
+	"strconv"
+	"strings"
+
+	"go-common/app/infra/config/conf"
+	"go-common/app/infra/config/service/v1"
+	"go-common/app/infra/config/service/v2"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/antispam"
+	v "go-common/library/net/http/blademaster/middleware/verify"
+
+	"github.com/dgryski/go-farm"
+)
+
+var (
+	cnf      *conf.Config
+	verify   *v.Verify
+	confSvc  *v1.Service
+	confSvc2 *v2.Service
+	anti     *antispam.Antispam
+)
+
+// Init init.
+func Init(c *conf.Config, s *v1.Service, s2 *v2.Service, rpcCloser io.Closer) {
+	initService(c)
+	verify = v.New(c.Verify)
+	cnf = c
+	confSvc = s
+	confSvc2 = s2
+	engine := bm.DefaultServer(c.BM)
+	innerRouter(engine)
+	if err := engine.Start(); err != nil {
+		log.Error("engine.Start() error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+	b := e.Group("/", verify.Verify)
+	noAuth := e.Group("/")
+	{
+		v1 := b.Group("v1/config/")
+		{
+			v1.GET("host/infos", hosts)
+			v1.POST("host/clear", clearhost)
+			v1.POST("push", push)
+		}
+		{
+			noAuth.GET("v1/config/versions", versions)
+			noAuth.GET("v1/config/builds", builds)
+			noAuth.GET("v1/config/check", check)
+			noAuth.GET("v1/config/get", config)
+			noAuth.GET("v1/config/get2", configN)
+			noAuth.GET("v1/config/file.so", file)
+			noAuth.GET("v1/config/version/ing", versionIng)
+			noAuth.POST("v1/config/config/add", addConfigs)
+			noAuth.POST("v1/config/config/copy", copyConfigs)
+			noAuth.POST("v1/config/config/update", updateConfigs)
+
+			noAuth.GET("config/v2/versions", versions2)
+			noAuth.GET("config/v2/builds", builds2)
+			noAuth.GET("config/v2/check", check2)
+			noAuth.GET("config/v2/get", setMid, anti.ServeHTTP, config2)
+			noAuth.GET("config/v2/file.so", file2)
+			noAuth.GET("config/v2/latest", latest)
+		}
+		v2 := b.Group("config/v2/")
+		{
+			v2.POST("host/clear", clearhost2)
+		}
+	}
+}
+func setMid(c *bm.Context) {
+	var (
+		token   string
+		service string
+		query   = c.Request.URL.Query()
+		hash    uint64
+	)
+	service = query.Get("service")
+	if service == "" {
+		token = query.Get("token")
+		if token == "" {
+			c.JSON(nil, ecode.RequestErr)
+			c.Abort()
+			return
+		}
+		hash = farm.Hash64([]byte(token))
+	} else {
+		arrs := strings.Split(service, "_")
+		if len(arrs) != 3 {
+			c.JSON(nil, ecode.RequestErr)
+			c.Abort()
+			return
+		}
+		_, err := strconv.ParseInt(arrs[0], 10, 64)
+		if err != nil {
+			c.JSON(nil, ecode.RequestErr)
+			c.Abort()
+			return
+		}
+		hash = farm.Hash64([]byte(service))
+	}
+	c.Set("mid", int64(hash))
+}
+
+func initService(c *conf.Config) {
+	anti = antispam.New(c.Antispam)
+}

app/infra/config/http/local.go

@@ -0,0 +1,25 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+)
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	var (
+		err error
+	)
+	if err = confSvc.Ping(c); err != nil {
+		log.Error("config service ping error(%v)", err)
+		c.JSON(nil, err)
+		http.Error(c.Writer, "", http.StatusServiceUnavailable)
+	}
+}
+
+// register check server ok.
+func register(c *bm.Context) {
+	c.JSON(map[string]struct{}{}, nil)
+}

app/infra/config/model/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "app.go",
+        "build.go",
+        "client.go",
+        "common_config.go",
+        "config.go",
+        "force.go",
+        "rpc.go",
+        "tag.go",
+    ],
+    importpath = "go-common/app/infra/config/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/model/app.go

@@ -0,0 +1,30 @@
+package model
+
+import "go-common/library/time"
+
+// DBApp mysql app DB.
+type DBApp struct {
+	ID     int64     `json:"id" gorm:"primary_key"`
+	Name   string    `json:"name"`
+	Token  string    `json:"token"`
+	Env    string    `json:"env"`
+	Zone   string    `json:"zone"`
+	TreeID int       `json:"tree_id"`
+	Ctime  time.Time `json:"ctime"`
+	Mtime  time.Time `json:"mtime"`
+}
+
+// TableName app
+func (DBApp) TableName() string {
+	return "app"
+}
+
+// App app local cache.
+type App struct {
+	ID     int64  `json:"id"`
+	Name   string `json:"name"`
+	Token  string `json:"token"`
+	Env    string `json:"env"`
+	Zone   string `json:"zone"`
+	TreeID int    `json:"tree_id"`
+}

app/infra/config/model/build.go

@@ -0,0 +1,19 @@
+package model
+
+import "go-common/library/time"
+
+// Build build.
+type Build struct {
+	ID       int64     `json:"id"`
+	AppID    int64     `json:"app_id"`
+	Name     string    `json:"name"`
+	TagID    int64     `json:"tag_id"`
+	Operator string    `json:"operator"`
+	Ctime    time.Time `json:"ctime"`
+	Mtime    time.Time `json:"mtime"`
+}
+
+// TableName build.
+func (Build) TableName() string {
+	return "build"
+}

app/infra/config/model/client.go

@@ -0,0 +1,95 @@
+package model
+
+import (
+	"go-common/library/time"
+)
+
+const (
+	// HostOffline host offline state.
+	HostOffline = 0
+	// HostOnline host online state.
+	HostOnline = 1
+	// HostStateOK host state ok.
+	HostStateOK = 2
+	// UnknownVersion unknown version.
+	UnknownVersion = -1
+)
+
+// Diff return to client.
+type Diff struct {
+	Version int64   `json:"version"`
+	Diffs   []int64 `json:"diffs"`
+}
+
+// Version return to client.
+type Version struct {
+	Version int64 `json:"version"`
+}
+
+// ReVer reVer
+type ReVer struct {
+	Version int64  `json:"version"`
+	Remark  string `json:"remark"`
+}
+
+// Versions versions
+type Versions struct {
+	Version []*ReVer `json:"version"`
+	DefVer  int64    `json:"defver"`
+}
+
+// Content return to client.
+type Content struct {
+	Version int64  `json:"version"`
+	Md5     string `json:"md5"`
+	Content string `json:"content"`
+}
+
+// Namespace the key-value config object.
+type Namespace struct {
+	Name string            `json:"name"`
+	Data map[string]string `json:"data"`
+}
+
+// Service service
+type Service struct {
+	Name         string
+	BuildVersion string
+	Env          string
+	Token        string
+	File         string
+	Version      int64
+	Host         string
+	IP           string
+	Appoint      int64
+}
+
+// NSValue config value.
+type NSValue struct {
+	ConfigID    int64  `json:"cid"`
+	NamespaceID int64  `json:"nsid"`
+	Name        string `json:"name"`
+	Config      string `json:"config"`
+}
+
+// Value config value.
+type Value struct {
+	ConfigID int64  `json:"cid"`
+	Name     string `json:"name"`
+	Config   string `json:"config"`
+}
+
+// Host host.
+type Host struct {
+	Name          string    `json:"hostname"`
+	Service       string    `json:"service"`
+	BuildVersion  string    `json:"build"`
+	IP            string    `json:"ip"`
+	ConfigVersion int64     `json:"version"`
+	HeartbeatTime time.Time `json:"heartbeat_time"`
+	State         int       `json:"state"`
+	Appoint       int64     `json:"appoint"`
+	Customize     string    `json:"customize"`
+	Force         int8      `json:"force"`
+	ForceVersion  int64     `json:"force_version"`
+}

app/infra/config/model/common_config.go

@@ -0,0 +1,21 @@
+package model
+
+import "go-common/library/time"
+
+// CommonConf common config.
+type CommonConf struct {
+	ID       int64     `json:"id" gorm:"primary_key"`
+	TeamID   int64     `json:"team_id"`
+	Name     string    `json:"name"`
+	Comment  string    `json:"comment"`
+	State    int8      `json:"state"`
+	Mark     string    `json:"mark"`
+	Operator string    `json:"operator"`
+	Ctime    time.Time `json:"ctime"`
+	Mtime    time.Time `json:"mtime"`
+}
+
+// TableName commonConfig.
+func (CommonConf) TableName() string {
+	return "common_config"
+}

app/infra/config/model/config.go

@@ -0,0 +1,29 @@
+package model
+
+import "go-common/library/time"
+
+var (
+	//ConfigIng config ing.
+	ConfigIng = int8(1)
+	//ConfigEnd config ing.
+	ConfigEnd = int8(2)
+)
+
+// Config config.
+type Config struct {
+	ID       int64     `json:"id" gorm:"primary_key"`
+	AppID    int64     `json:"app_id"`
+	Name     string    `json:"name"`
+	Comment  string    `json:"comment"`
+	From     int64     `json:"from"`
+	State    int8      `json:"state"`
+	Mark     string    `json:"mark"`
+	Operator string    `json:"operator"`
+	Ctime    time.Time `json:"ctime"`
+	Mtime    time.Time `json:"mtime"`
+}
+
+// TableName config
+func (Config) TableName() string {
+	return "config"
+}

app/infra/config/model/force.go

@@ -0,0 +1,20 @@
+package model
+
+import "go-common/library/time"
+
+// Force ...
+type Force struct {
+	ID       int64     `json:"id"`
+	AppID    int64     `json:"app_id"`
+	HostName string    `json:"hostname"`
+	IP       string    `json:"ip"`
+	Version  int64     `json:"version"`
+	Operator string    `json:"operator"`
+	Ctime    time.Time `json:"ctime"`
+	Mtime    time.Time `json:"mtime"`
+}
+
+// TableName build.
+func (Force) TableName() string {
+	return "force"
+}

app/infra/config/model/rpc.go

@@ -0,0 +1,18 @@
+package model
+
+// ArgConf config param.
+type ArgConf struct {
+	App      string
+	BuildVer string
+	Ver      int64
+	Env      string
+	Hosts    map[string]string
+	SType    int8
+}
+
+// ArgToken token param.
+type ArgToken struct {
+	App   string
+	Token string
+	Env   string
+}

app/infra/config/model/tag.go

@@ -0,0 +1,20 @@
+package model
+
+import "go-common/library/time"
+
+// DBTag tag table in mysql.
+type DBTag struct {
+	ID        int64     `json:"id" gorm:"primary_key"`
+	AppID     int64     `json:"app_id"`
+	ConfigIDs string    `json:"config_ids"`
+	Mark      string    `json:"mark"`
+	Force     int8      `json:"force"`
+	Operator  string    `json:"operator"`
+	Ctime     time.Time `json:"ctime"`
+	Mtime     time.Time `json:"mtime"`
+}
+
+// TableName tag.
+func (DBTag) TableName() string {
+	return "tag"
+}

app/infra/config/rpc/client/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["config_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = ["//app/infra/config/model:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importpath = "go-common/app/infra/config/rpc/client",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/model:go_default_library",
+        "//library/net/rpc:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/rpc/client/config.go

@@ -0,0 +1,78 @@
+package config
+
+import (
+	"context"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/net/rpc"
+)
+
+const (
+	_appid = "config.service"
+
+	_push       = "RPC.Push"
+	_setToken   = "RPC.SetToken"
+	_pushV4     = "RPC.PushV4"
+	_force      = "RPC.Force"
+	_setTokenV4 = "RPC.SetTokenV4"
+	_hosts      = "RPC.Hosts"
+	_clearHost  = "RPC.ClearHost"
+)
+
+var (
+	_noArg = &struct{}{}
+)
+
+//Service2 service.
+type Service2 struct {
+	client *rpc.Client2
+}
+
+// New2 new a config service.
+func New2(c *rpc.ClientConfig) (s *Service2) {
+	s = &Service2{}
+	s.client = rpc.NewDiscoveryCli(_appid, c)
+	return
+}
+
+// Push push new ver to config-service
+func (s *Service2) Push(c context.Context, arg *model.ArgConf) (err error) {
+	err = s.client.Boardcast(c, _push, arg, _noArg)
+	return
+}
+
+// SetToken update token in config-service
+func (s *Service2) SetToken(c context.Context, arg *model.ArgToken) (err error) {
+	err = s.client.Boardcast(c, _setToken, arg, _noArg)
+	return
+}
+
+// PushV4 push new ver to config-service
+func (s *Service2) PushV4(c context.Context, arg *model.ArgConf) (err error) {
+	err = s.client.Boardcast(c, _pushV4, arg, _noArg)
+	return
+}
+
+// SetTokenV4 update token in config-service
+func (s *Service2) SetTokenV4(c context.Context, arg *model.ArgToken) (err error) {
+	err = s.client.Boardcast(c, _setTokenV4, arg, _noArg)
+	return
+}
+
+//Hosts get host list.
+func (s *Service2) Hosts(c context.Context, svr string) (hosts []*model.Host, err error) {
+	err = s.client.Call(c, _hosts, svr, &hosts)
+	return
+}
+
+// ClearHost update token in config-service
+func (s *Service2) ClearHost(c context.Context, svr string) (err error) {
+	err = s.client.Call(c, _clearHost, svr, _noArg)
+	return
+}
+
+// Force push new host ver to config-service
+func (s *Service2) Force(c context.Context, arg *model.ArgConf) (err error) {
+	err = s.client.Boardcast(c, _force, arg, _noArg)
+	return
+}

app/infra/config/rpc/client/config_test.go

@@ -0,0 +1,61 @@
+package config
+
+import (
+	"context"
+	"fmt"
+	"testing"
+	"time"
+
+	"go-common/app/infra/config/model"
+)
+
+func TestConf(t *testing.T) {
+	s := New2(nil)
+	time.Sleep(1 * time.Second)
+
+	// coin
+	testPush(t, s)
+	testSetToken(t, s)
+	testHosts(t, s)
+	testClearHost(t, s)
+}
+
+func testPush(t *testing.T, s *Service2) {
+	arg := &model.ArgConf{
+		App:      "zjx_test",
+		BuildVer: "1_0_0_0",
+		Ver:      113,
+		Env:      "2",
+	}
+	if err := s.Push(context.TODO(), arg); err != nil {
+		fmt.Println(err)
+		t.FailNow()
+	}
+}
+func testSetToken(t *testing.T, s *Service2) {
+	arg := &model.ArgToken{
+		App:   "zjx_test",
+		Token: "123",
+		Env:   "2",
+	}
+	if err := s.SetToken(context.TODO(), arg); err != nil {
+		fmt.Println(err)
+		t.FailNow()
+	}
+}
+
+func testHosts(t *testing.T, s *Service2) {
+	if hosts, err := s.Hosts(context.TODO(), "testApp4890934756659"); err != nil {
+		t.Log(err)
+		t.FailNow()
+	} else {
+		t.Log(len(hosts))
+	}
+}
+
+func testClearHost(t *testing.T, s *Service2) {
+	if err := s.ClearHost(context.TODO(), "testApp4890934756659"); err != nil {
+		t.Log(err)
+		t.FailNow()
+	}
+}

app/infra/config/rpc/server/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["config.go"],
+    importpath = "go-common/app/infra/config/rpc/server",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//app/infra/config/service/v1:go_default_library",
+        "//app/infra/config/service/v2:go_default_library",
+        "//library/net/rpc:go_default_library",
+        "//library/net/rpc/context:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/rpc/server/config.go

@@ -0,0 +1,76 @@
+package rpc
+
+import (
+	"go-common/app/infra/config/conf"
+	"go-common/app/infra/config/service/v1"
+	"go-common/app/infra/config/service/v2"
+	"go-common/library/net/rpc"
+	"go-common/library/net/rpc/context"
+
+	"go-common/app/infra/config/model"
+)
+
+// RPC export rpc service
+type RPC struct {
+	s  *v1.Service
+	s2 *v2.Service
+}
+
+// New new rpc server.
+func New(c *conf.Config, s *v1.Service, s2 *v2.Service) (svr *rpc.Server) {
+	r := &RPC{s: s, s2: s2}
+	svr = rpc.NewServer(c.RPCServer)
+	if err := svr.Register(r); err != nil {
+		panic(err)
+	}
+	return
+}
+
+// Ping check connection success.
+func (r *RPC) Ping(c context.Context, arg *struct{}, res *struct{}) (err error) {
+	return
+}
+
+// Push push new config change to config-service
+func (r *RPC) Push(c context.Context, a *model.ArgConf, res *struct{}) (err error) {
+	service := &model.Service{Name: a.App, BuildVersion: a.BuildVer, Version: a.Ver, Env: a.Env}
+	err = r.s.Push(c, service)
+	return
+}
+
+//SetToken update Token
+func (r *RPC) SetToken(c context.Context, a *model.ArgToken, res *struct{}) (err error) {
+	r.s.SetToken(c, a.App, a.Env, a.Token)
+	return
+}
+
+// PushV4 push new config change to config-service
+func (r *RPC) PushV4(c context.Context, a *model.ArgConf, res *struct{}) (err error) {
+	service := &model.Service{Name: a.App, BuildVersion: a.BuildVer, Version: a.Ver}
+	err = r.s2.Push(c, service)
+	return
+}
+
+//SetTokenV4 update Token
+func (r *RPC) SetTokenV4(c context.Context, a *model.ArgToken, res *struct{}) (err error) {
+	r.s2.SetToken(a.App, a.Token)
+	return
+}
+
+//Hosts get host list.
+func (r *RPC) Hosts(c context.Context, svr string, res *[]*model.Host) (err error) {
+	*res, err = r.s2.Hosts(c, svr)
+	return
+}
+
+//ClearHost clear host.
+func (r *RPC) ClearHost(c context.Context, svr string, res *struct{}) error {
+	return r.s2.ClearHost(c, svr)
+}
+
+// Force push new host config change to config-service
+func (r *RPC) Force(c context.Context, a *model.ArgConf, res *struct{}) (err error) {
+	service := &model.Service{Name: a.App, BuildVersion: a.BuildVer, Version: a.Ver}
+	err = r.s2.Force(c, service, a.Hosts, a.SType)
+	return
+}

app/infra/config/service/v1/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["client_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "client.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/infra/config/service/v1",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/dao/v1:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/service/v1/client.go

@@ -0,0 +1,570 @@
+package v1
+
+import (
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	"go-common/app/infra/config/model"
+	"go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+const (
+	_buildVerKey = "%s_%s_%s"
+	_pushKey     = "%s_%s_%s"
+	_cacheKey    = "%s_%s_%d_%s"
+	_cacheKey2   = "%s_%s_%d_%s_2"
+	_fileKey     = "%s_%d"
+)
+
+var (
+	addBusiness    = "前端接口api添加配置版本"
+	addInfo        = "添加版本:%d"
+	copyBusiness   = "前端接口api拷贝配置版本"
+	copyInfo       = "拷贝版本:%d，新的版本:%d"
+	updateBusiness = "前端接口api更新配置"
+	updateInfo     = "更新版本:%d"
+)
+
+// PushKey push sub id
+func pushKey(svr, host, env string) string {
+	return fmt.Sprintf(_pushKey, svr, host, env)
+}
+
+// buildVerKey version mapping key
+func buildVerKey(svr, bver, env string) string {
+	return fmt.Sprintf(_buildVerKey, svr, bver, env)
+}
+
+// cacheKey config cache key
+func cacheKey(svr, bver, env string, ver int64) string {
+	return fmt.Sprintf(_cacheKey, svr, bver, ver, env)
+}
+
+// cacheKey config cache key
+func cacheKey2(svr, bver, env string, ver int64) string {
+	return fmt.Sprintf(_cacheKey2, svr, bver, ver, env)
+}
+
+// fileKey
+func fileKey(filename string, ver int64) string {
+	return fmt.Sprintf(_fileKey, filename, ver)
+}
+
+// tokenKey
+func tokenKey(svr, env string) string {
+	return fmt.Sprintf("%s_%s", svr, env)
+}
+
+// genConfig generate config
+func genConfig(ver int64, cs []*model.NSValue) (conf *model.Content, err error) {
+	var b []byte
+	data := make(map[string]string)
+	for _, c := range cs {
+		data[c.Name] = c.Config
+	}
+	if b, err = json.Marshal(data); err != nil {
+		return
+	}
+	mb := md5.Sum(b)
+	conf = &model.Content{
+		Version: ver,
+		Md5:     hex.EncodeToString(mb[:]),
+		Content: string(b),
+	}
+	return
+}
+
+// genConfig2 generate config
+func genConfig2(ver int64, cs []*model.NSValue, ns map[int64]string) (conf *model.Content, err error) {
+	var (
+		b  []byte
+		v  string
+		ok bool
+		s  *model.Namespace
+	)
+	nsc := make(map[string]*model.Namespace)
+	for _, c := range cs {
+		if v, ok = ns[c.NamespaceID]; !ok && c.NamespaceID != 0 {
+			continue
+		}
+		if s, ok = nsc[v]; !ok {
+			s = &model.Namespace{Name: v, Data: map[string]string{}}
+			nsc[v] = s
+		}
+		s.Data[c.Name] = c.Config
+	}
+	if b, err = json.Marshal(nsc); err != nil {
+		return
+	}
+	mb := md5.Sum(b)
+	conf = &model.Content{
+		Version: ver,
+		Md5:     hex.EncodeToString(mb[:]),
+		Content: string(b),
+	}
+	return
+}
+
+// Push version to clients & generate config caches
+func (s *Service) Push(c context.Context, svr *model.Service) (err error) {
+	var (
+		hosts      []*model.Host
+		values     []*model.NSValue
+		conf       *model.Content
+		conf2      *model.Content
+		namespaces map[int64]string
+	)
+	if values, err = s.dao.Values(c, svr.Version); err != nil {
+		return
+	}
+	if namespaces, err = s.dao.Namespaces(c, svr.Version); err != nil {
+		return
+	}
+	if len(values) == 0 {
+		err = fmt.Errorf("config values is empty. svr:%s, host:%s, buildVer:%s, ver:%d", svr.Name, svr.Host, svr.BuildVersion, svr.Version)
+		log.Error("%v", err)
+		return
+	}
+	// compatible old version sdk
+	if conf, err = genConfig(svr.Version, values); err != nil {
+		log.Error("get config value:%s error(%v) ", values, err)
+		return
+	}
+	cacheKey := cacheKey(svr.Name, svr.BuildVersion, svr.Env, svr.Version)
+	if err = s.dao.SetFile(cacheKey, conf); err != nil {
+		log.Error("set confCashe error. svr:%s, buildVer:%s, ver:%d", svr.Name, svr.BuildVersion, svr.Env)
+		err = nil
+	}
+	if conf2, err = genConfig2(svr.Version, values, namespaces); err != nil {
+		log.Error("get config2 value:%s error(%v) ", values, err)
+		return
+	}
+	cacheKey2 := cacheKey2(svr.Name, svr.BuildVersion, svr.Env, svr.Version)
+	if err = s.dao.SetFile(cacheKey2, conf2); err != nil {
+		log.Error("set confCashe2 error. svr:%s, buildVer:%s, ver:%d", svr.Name, svr.BuildVersion, svr.Env)
+		err = nil
+	}
+	s.setVersion(svr.Name, svr.BuildVersion, svr.Env, svr.Version)
+	// push hosts
+	if hosts, err = s.dao.Hosts(c, svr.Name, svr.Env); err != nil {
+		log.Error("get hosts error. svr:%s, buildVer:%s, ver:%d", svr.Name, svr.BuildVersion, svr.Version)
+		err = nil
+		return
+	}
+	for _, h := range hosts {
+		if h.State == model.HostOnline {
+			pushKey := pushKey(h.Service, h.Name, svr.Env)
+			if ok := s.pubEvent(pushKey, &model.Version{Version: conf.Version}); ok {
+				log.Info("s.events.Pub(%s, %d) ok: %t", pushKey, conf.Version, ok)
+			}
+		}
+	}
+	return
+}
+
+// Config return config content.
+func (s *Service) Config(c context.Context, svr *model.Service) (conf *model.Content, err error) {
+	var values []*model.NSValue
+	if err = s.appAuth(c, svr.Name, svr.Env, svr.Token); err != nil {
+		return
+	}
+	cacheName := cacheKey(svr.Name, svr.BuildVersion, svr.Env, svr.Version)
+	if conf, err = s.dao.File(cacheName); err == nil {
+		return
+	}
+	if values, err = s.dao.Values(c, svr.Version); err != nil {
+		return
+	}
+	if len(values) == 0 {
+		err = fmt.Errorf("config values is empty. svr:%s, host:%s, buildVer:%s, ver:%d", svr.Name, svr.Host, svr.BuildVersion, svr.Version)
+		log.Error("%v", err)
+		return
+	}
+	if conf, err = genConfig(svr.Version, values); err != nil {
+		log.Error("get config value:%s error(%v) ", values, err)
+		return
+	}
+	if err = s.dao.SetFile(cacheName, conf); err != nil {
+		err = nil
+	}
+	return
+}
+
+// Config2 return config content.
+func (s *Service) Config2(c context.Context, svr *model.Service) (conf *model.Content, err error) {
+	var (
+		values     []*model.NSValue
+		namespaces map[int64]string
+	)
+	if err = s.appAuth(c, svr.Name, svr.Env, svr.Token); err != nil {
+		return
+	}
+	cacheName := cacheKey2(svr.Name, svr.BuildVersion, svr.Env, svr.Version)
+	if conf, err = s.dao.File(cacheName); err == nil {
+		return
+	}
+	if namespaces, err = s.dao.Namespaces(c, svr.Version); err != nil {
+		return
+	}
+	if values, err = s.dao.Values(c, svr.Version); err != nil {
+		return
+	}
+	if len(values) == 0 {
+		err = fmt.Errorf("config values is empty. svr:%s, host:%s, buildVer:%s, ver:%d", svr.Name, svr.Host, svr.BuildVersion, svr.Version)
+		log.Error("%v", err)
+		return
+	}
+	if conf, err = genConfig2(svr.Version, values, namespaces); err != nil {
+		log.Error("get config value:(%s) error(%v) ", values, err)
+		return
+	}
+	if err = s.dao.SetFile(cacheName, conf); err != nil {
+		err = nil
+	}
+	return
+}
+
+// File get one file content.
+func (s *Service) File(c context.Context, svr *model.Service) (val string, err error) {
+	var (
+		curVer int64
+		ok     bool
+	)
+	if err = s.appAuth(c, svr.Name, svr.Env, svr.Token); err != nil {
+		return
+	}
+	if svr.Version != model.UnknownVersion {
+		curVer = svr.Version
+	} else {
+		curVer, ok = s.version(svr.Name, svr.BuildVersion, svr.Env)
+		if !ok {
+			if curVer, err = s.dao.BuildVersion(c, svr.Name, svr.BuildVersion, svr.Env); err != nil {
+				log.Error("BuildVersion(%v) error(%v)", svr, err)
+				return
+			}
+			s.setVersion(svr.Name, svr.BuildVersion, svr.Env, curVer)
+		}
+	}
+	fKey := fileKey(svr.File, curVer)
+	if val, err = s.dao.FileStr(fKey); err == nil {
+		return
+	}
+	if val, err = s.dao.Value(c, svr.File, curVer); err != nil {
+		log.Error("Value(%v) error(%v)", svr.File, err)
+		return
+	}
+	s.dao.SetFileStr(fKey, val)
+	return
+}
+
+// CheckVersion check client version.
+func (s *Service) CheckVersion(c context.Context, rhost *model.Host, env, token string) (evt chan *model.Version, err error) {
+	var (
+		curVer int64
+	)
+	if err = s.appAuth(c, rhost.Service, env, token); err != nil {
+		return
+	}
+	// set heartbeat
+	rhost.HeartbeatTime = xtime.Time(time.Now().Unix())
+	if err = s.dao.SetHost(c, rhost, rhost.Service, env); err != nil {
+		err = nil
+	}
+	evt = make(chan *model.Version, 1)
+	if rhost.Appoint > 0 {
+		if rhost.Appoint != rhost.ConfigVersion {
+			evt <- &model.Version{Version: rhost.Appoint}
+		}
+		return
+	}
+	// get current version, return if has new config version
+	if curVer, err = s.curVer(c, rhost.Service, rhost.BuildVersion, env); err != nil {
+		return
+	}
+
+	if curVer == model.UnknownVersion {
+		err = ecode.NothingFound
+		return
+	}
+	if curVer != rhost.ConfigVersion {
+		evt <- &model.Version{Version: curVer}
+		return
+	}
+	pushKey := pushKey(rhost.Service, rhost.Name, env)
+	s.eLock.Lock()
+	s.events[pushKey] = evt
+	s.eLock.Unlock()
+	return
+}
+
+// AppAuth check app is auth
+func (s *Service) appAuth(c context.Context, svr, env, token string) (err error) {
+	var (
+		dbToken  string
+		ok       bool
+		tokenKey = tokenKey(svr, env)
+	)
+	s.eLock.RLock()
+	dbToken, ok = s.token[tokenKey]
+	s.eLock.RUnlock()
+	if !ok {
+		if dbToken, err = s.dao.Token(c, svr, env); err != nil {
+			log.Error("Token(%v,%v) error(%v)", svr, env, err)
+			return
+		}
+		s.SetToken(c, svr, env, dbToken)
+	}
+
+	if dbToken != token {
+		err = ecode.AccessDenied
+	}
+	return
+}
+
+// SetToken update Token
+func (s *Service) SetToken(c context.Context, svr, env, token string) {
+	tokenKey := tokenKey(svr, env)
+	s.eLock.Lock()
+	s.token[tokenKey] = token
+	s.eLock.Unlock()
+}
+
+// Hosts return client hosts.
+func (s *Service) Hosts(c context.Context, svr, env string) (hosts []*model.Host, err error) {
+	return s.dao.Hosts(c, svr, env)
+}
+
+// VersionSuccess return client versions which configuration is complete
+func (s *Service) VersionSuccess(c context.Context, svr, env, bver string) (versions *model.Versions, err error) {
+	var (
+		vers []*model.ReVer
+		ver  int64
+	)
+	if vers, err = s.dao.Versions(c, svr, env, model.ConfigEnd); err != nil {
+		log.Error("Versions(%v,%v,%v) error(%v)", svr, env, bver, err)
+		return
+	}
+	if ver, err = s.dao.BuildVersion(c, svr, bver, env); err != nil {
+		log.Error("BuildVersion(%v) error(%v)", svr, err)
+		return
+	}
+	versions = &model.Versions{
+		Version: vers,
+		DefVer:  ver,
+	}
+	return
+}
+
+// VersionIng return client versions which configuration is creating
+func (s *Service) VersionIng(c context.Context, svr, env string) (vers []int64, err error) {
+	var (
+		res []*model.ReVer
+	)
+	if res, err = s.dao.Versions(c, svr, env, model.ConfigIng); err != nil {
+		log.Error("Versions(%v,%v) error(%v)", svr, env, err)
+		return
+	}
+	vers = make([]int64, 0)
+	for _, reVer := range res {
+		vers = append(vers, reVer.Version)
+	}
+	return
+}
+
+// Builds all builds
+func (s *Service) Builds(c context.Context, svr, env string) (builds []string, err error) {
+	return s.dao.Builds(c, svr, env)
+}
+
+// AddConfigs insert config into db.
+func (s *Service) AddConfigs(c context.Context, svr, env, token, user string, data map[string]string) (err error) {
+	var (
+		svrID int64
+		ver   int64
+	)
+	if err = s.appAuth(c, svr, env, token); err != nil {
+		return
+	}
+	if svrID, err = s.dao.ServiceID(c, svr, env); err != nil {
+		return
+	}
+	var tx *sql.Tx
+	if tx, err = s.dao.BeginTran(c); err != nil {
+		log.Error("begin tran error(%v)", err)
+		return
+	}
+	if ver, err = s.dao.TxInsertVer(tx, svrID, user); err != nil {
+		tx.Rollback()
+		return
+	}
+	if len(data) != 0 {
+		if err = s.dao.TxInsertValues(c, tx, ver, user, data); err != nil {
+			tx.Rollback()
+			return
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("tx.Commit error(%v)", err)
+		return
+	}
+	s.dao.InsertLog(c, user, addBusiness, fmt.Sprintf(addInfo, ver))
+	return
+}
+
+// CopyConfigs copy config in newVer.
+func (s *Service) CopyConfigs(c context.Context, svr, env, token, user string, build string) (ver int64, err error) {
+	var (
+		svrID  int64
+		curVer int64
+		values []*model.NSValue
+	)
+	if err = s.appAuth(c, svr, env, token); err != nil {
+		return
+	}
+	if curVer, err = s.curVer(c, svr, build, env); err != nil {
+		return
+	}
+	if values, err = s.dao.Values(c, curVer); err != nil {
+		return
+	}
+	data := make(map[string]string)
+	for _, c := range values {
+		data[c.Name] = c.Config
+	}
+	if svrID, err = s.dao.ServiceID(c, svr, env); err != nil {
+		return
+	}
+	var tx *sql.Tx
+	if tx, err = s.dao.BeginTran(c); err != nil {
+		log.Error("begin tran error(%v)", err)
+		return
+	}
+	if ver, err = s.dao.TxInsertVer(tx, svrID, user); err != nil {
+		tx.Rollback()
+		return
+	}
+	if err = s.dao.TxInsertValues(c, tx, ver, user, data); err != nil {
+		tx.Rollback()
+		return
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("tx.Commit error(%v)", err)
+		return
+	}
+	s.dao.InsertLog(c, user, copyBusiness, fmt.Sprintf(copyInfo, curVer, ver))
+	return
+}
+
+// UpdateConfigs update config.
+func (s *Service) UpdateConfigs(c context.Context, svr, env, token, user string, ver int64, data map[string]string) (err error) {
+	var (
+		values  []*model.NSValue
+		addData = make(map[string]string)
+		udata   = make(map[string]string)
+	)
+	if err = s.appAuth(c, svr, env, token); err != nil {
+		return
+	}
+	if len(data) == 0 {
+		return
+	}
+	if values, err = s.dao.Values(c, ver); err != nil {
+		return
+	}
+	if len(values) == 0 {
+		return ecode.NothingFound
+	}
+	oldData := make(map[string]string)
+	for _, c := range values {
+		oldData[c.Name] = c.Config
+	}
+	for k, v := range data {
+		if _, ok := oldData[k]; ok {
+			udata[k] = v
+		} else {
+			addData[k] = v
+		}
+	}
+	var tx *sql.Tx
+	if tx, err = s.dao.BeginTran(c); err != nil {
+		log.Error("begin tran error(%v)", err)
+		return
+	}
+	if len(addData) != 0 {
+		if err = s.dao.TxInsertValues(c, tx, ver, user, addData); err != nil {
+			tx.Rollback()
+			return
+		}
+	}
+	if len(udata) != 0 {
+		if err = s.dao.TxUpdateValues(tx, ver, user, udata); err != nil {
+			tx.Rollback()
+			return
+		}
+	}
+	if err = tx.Commit(); err != nil {
+		log.Error("tx.Commit error(%v)", err)
+		return
+	}
+	s.dao.InsertLog(c, user, updateBusiness, fmt.Sprintf(updateInfo, ver))
+	return
+}
+
+func (s *Service) version(svr, bver, env string) (ver int64, ok bool) {
+	verKey := buildVerKey(svr, bver, env)
+	s.vLock.RLock()
+	ver, ok = s.versions[verKey]
+	s.vLock.RUnlock()
+	return
+}
+
+func (s *Service) setVersion(svr, bver, env string, ver int64) {
+	verKey := buildVerKey(svr, bver, env)
+	s.vLock.Lock()
+	s.versions[verKey] = ver
+	s.vLock.Unlock()
+}
+
+// ClearHost clear service hosts.
+func (s *Service) ClearHost(c context.Context, svr, env string) (err error) {
+	return s.dao.ClearHost(c, svr, env)
+}
+
+// pubEvent publish a event to chan.
+func (s *Service) pubEvent(key string, evt *model.Version) (ok bool) {
+	s.eLock.RLock()
+	c, ok := s.events[key]
+	s.eLock.RUnlock()
+	if ok {
+		c <- evt
+	}
+	return
+}
+
+// Unsub unsub a event.
+func (s *Service) Unsub(svr, host, env string) {
+	key := pushKey(svr, host, env)
+	s.eLock.Lock()
+	delete(s.events, key)
+	s.eLock.Unlock()
+}
+
+func (s *Service) curVer(c context.Context, svr, build, env string) (ver int64, err error) {
+	var ok bool
+	// get current version, return if has new config version
+	ver, ok = s.version(svr, build, env)
+	if !ok {
+		if ver, err = s.dao.BuildVersion(c, svr, build, env); err != nil {
+			return
+		}
+		s.setVersion(svr, build, env, ver)
+	}
+	return
+}

app/infra/config/service/v1/client_test.go

@@ -0,0 +1,216 @@
+package v1
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/infra/config/conf"
+	"go-common/app/infra/config/model"
+
+	"github.com/BurntSushi/toml"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func svr(t *testing.T) *Service {
+	var (
+		confPath = "../cmd/config-service-example.toml"
+		conf     *conf.Config
+	)
+	Convey("get apply", t, func() {
+		_, err := toml.DecodeFile(confPath, &conf)
+		So(err, ShouldBeNil)
+	})
+	return New(conf)
+}
+
+func TestService_CheckVersionest(t *testing.T) {
+	var (
+		c        = context.TODO()
+		svrName  = "zjx_test"
+		hostname = "test_host"
+		bver     = "v1.0.0"
+		ip       = "123"
+		version  = int64(-1)
+		env      = "10"
+		token    = "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc"
+		appoint  = int64(97)
+	)
+	svr := svr(t)
+	rhost := &model.Host{Service: svrName, Name: hostname, BuildVersion: bver, IP: ip, ConfigVersion: version, Appoint: appoint, Customize: "test"}
+	Convey("get tag id by name", t, func() {
+		event, err := svr.CheckVersion(c, rhost, env, token)
+		So(err, ShouldBeNil)
+		So(event, ShouldNotBeEmpty)
+		Convey("get tag id by name", func() {
+			e := <-event
+			So(e, ShouldNotBeEmpty)
+		})
+	})
+}
+
+func TestService_Hosts(t *testing.T) {
+	svr := svr(t)
+	Convey("should get hosts", t, func() {
+		_, err := svr.Hosts(context.TODO(), "zjx_test", "10")
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestService_Config(t *testing.T) {
+	var (
+		c        = context.TODO()
+		svrName  = "zjx_test"
+		hostname = "test_host"
+		bver     = "v1.0.0"
+		version  = int64(78)
+		env      = "10"
+		token    = "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc"
+	)
+	svr := svr(t)
+	service := &model.Service{Name: svrName, BuildVersion: bver, Env: env, Token: token, Version: version, Host: hostname}
+	Convey("should get hosts", t, func() {
+		conf, err := svr.Config(c, service)
+		So(err, ShouldBeNil)
+		So(conf, ShouldNotBeEmpty)
+	})
+}
+
+func TestService_Config2(t *testing.T) {
+	var (
+		c        = context.TODO()
+		svrName  = "config_test"
+		hostname = "test_host"
+		bver     = "shsb-docker-1"
+		version  = int64(199)
+		env      = "10"
+		token    = "qmVUPwNXnNfcSpuyqbiIBb0H4GcbSZFV"
+	)
+	service := &model.Service{Name: svrName, BuildVersion: bver, Env: env, Token: token, Version: version, Host: hostname}
+	svr := svr(t)
+	Convey("should get Config2", t, func() {
+		conf, err := svr.Config2(c, service)
+		So(err, ShouldBeNil)
+		So(conf, ShouldNotBeEmpty)
+	})
+}
+
+func TestService_Push(t *testing.T) {
+	var (
+		c       = context.TODO()
+		svrName = "zjx_test"
+		bver    = "v1.0.0"
+		version = int64(113)
+		env     = "10"
+	)
+	service := &model.Service{Name: svrName, BuildVersion: bver, Version: version, Env: env}
+	svr := svr(t)
+	Convey("should get Config2", t, func() {
+		err := svr.Push(c, service)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestService_SetToken(t *testing.T) {
+	var (
+		c       = context.TODO()
+		svrName = "zjx_test"
+		env     = "10"
+		token   = "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc"
+	)
+	svr := svr(t)
+	Convey("should get Config2", t, func() {
+		svr.SetToken(c, svrName, env, token)
+	})
+}
+func TestService_ClearHost(t *testing.T) {
+	var (
+		c       = context.TODO()
+		svrName = "zjx_test"
+		env     = "10"
+	)
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		err := svr.ClearHost(c, svrName, env)
+		So(err, ShouldBeNil)
+	})
+}
+func TestService_VersionSuccess(t *testing.T) {
+	var (
+		c       = context.TODO()
+		svrName = "zjx_test"
+		bver    = "v1.0.0"
+		env     = "10"
+	)
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		vers, err := svr.VersionSuccess(c, svrName, env, bver)
+		So(err, ShouldBeNil)
+		So(vers, ShouldNotBeEmpty)
+	})
+}
+func TestService_Builds(t *testing.T) {
+	var (
+		c       = context.TODO()
+		svrName = "zjx_test"
+		env     = "10"
+		err     error
+		builds  []string
+	)
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		builds, err = svr.Builds(c, svrName, env)
+		So(err, ShouldBeNil)
+		So(builds, ShouldNotBeEmpty)
+	})
+}
+
+func TestService_File(t *testing.T) {
+	var (
+		c        = context.TODO()
+		svrName  = "zjx_test"
+		bver     = "v1.0.0"
+		env      = "10"
+		fileName = "test.toml"
+		token    = "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc"
+		ver      = int64(74)
+		err      error
+	)
+	service := &model.Service{Name: svrName, BuildVersion: bver, Env: env, File: fileName, Token: token, Version: ver}
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		_, err = svr.File(c, service)
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestService_AddConfigs(t *testing.T) {
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		err := svr.AddConfigs(context.TODO(), "zjx_test", "10", "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc", "zjx", map[string]string{"aa": "bb"})
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestService_UpdateConfigs(t *testing.T) {
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		err := svr.UpdateConfigs(context.TODO(), "zjx_test", "10", "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc", "zjx", 491, map[string]string{"test": "test123"})
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestService_CopyConfigs(t *testing.T) {
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		_, err := svr.CopyConfigs(context.TODO(), "zjx_test", "10", "AXiLBa3Bww3inhfm6qx7g0zLY6WkLSZc", "zjx", "shsb-docker-1")
+		So(err, ShouldBeNil)
+	})
+}
+
+func TestService_VersionIng(t *testing.T) {
+	svr := svr(t)
+	Convey("should  clear host", t, func() {
+		_, err := svr.VersionIng(context.TODO(), "zjx_test1", "10")
+		So(err, ShouldBeNil)
+	})
+}

app/infra/config/service/v1/service.go

@@ -0,0 +1,43 @@
+package v1
+
+import (
+	"context"
+	"sync"
+
+	"go-common/app/infra/config/conf"
+	"go-common/app/infra/config/dao/v1"
+	"go-common/app/infra/config/model"
+	xtime "go-common/library/time"
+)
+
+// Service service.
+type Service struct {
+	dao         *v1.Dao
+	vLock       sync.RWMutex
+	versions    map[string]int64 // serviceName_buildVersion > configVersion
+	eLock       sync.RWMutex
+	events      map[string]chan *model.Version
+	PollTimeout xtime.Duration
+	token       map[string]string
+}
+
+// New new a service.
+func New(c *conf.Config) (s *Service) {
+	s = new(Service)
+	s.dao = v1.New(c)
+	s.versions = make(map[string]int64)
+	s.events = make(map[string]chan *model.Version)
+	s.PollTimeout = c.PollTimeout
+	s.token = make(map[string]string)
+	return
+}
+
+// Ping check is ok.
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// Close close resources.
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/infra/config/service/v2/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["client_test.go"],
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "cache_tag.go",
+        "client.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/infra/config/service/v2",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/infra/config/conf:go_default_library",
+        "//app/infra/config/dao/v2:go_default_library",
+        "//app/infra/config/model:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/infra/config/service/v2/cache_tag.go

@@ -0,0 +1,45 @@
+package v2
+
+// cacheTag tag.
+type cacheTag struct {
+	Tag     int64
+	ConfIDs []int64
+	Force   int8
+}
+
+// curTag current tag version.
+type curTag struct {
+	O *cacheTag
+	C *cacheTag
+}
+
+func (tag *curTag) diff() (diffs []int64) {
+	if tag.O == nil || tag.C == nil {
+		return nil
+	}
+	oIDs := tag.O.ConfIDs
+	tmp := make(map[int64]struct{}, len(oIDs))
+	for _, oID := range tag.O.ConfIDs {
+		tmp[oID] = struct{}{}
+	}
+	for _, ID := range tag.C.ConfIDs {
+		if _, ok := tmp[ID]; !ok {
+			diffs = append(diffs, ID)
+		}
+	}
+	return
+}
+
+func (tag *curTag) old() int64 {
+	if tag.O == nil {
+		return 0
+	}
+	return tag.O.Tag
+}
+
+func (tag *curTag) cur() int64 {
+	if tag.C == nil {
+		return 0
+	}
+	return tag.C.Tag
+}