Initial commit

app/interface/openplatform/monitor-end/CHANGELOG.md

@@ -0,0 +1,59 @@
+# open-monitor
+
+##### Version 0.2.4
+> 1. regex bfs url
+> 2. 添加product 白名单
+> 3. 修复map并发写问题
+
+##### Version 0.2.3
+> 1. add switch
+
+##### Version 0.2.2
+> 1. clean subevent
+
+##### Version 0.2.1
+> 1. 添加app prometheus
+
+##### Version 0.2.0
+> 1. 添加错误日志收集接口
+> 2. 邮件告警
+> 3. 告警配置
+
+##### Version 0.1.3
+> 1. add regex
+
+##### Version 0.1.2
+> 1. limit alloc memory
+
+##### Version 0.1.1
+> 1. 增加summary抽样
+
+##### Version 0.1.0
+> 1. add prometheus switch
+
+##### Version 0.0.9
+> 1. fix unknow type warning
+
+##### Version 0.0.8
+> 1. fix unencode warning
+
+##### Version 0.0.7
+> 1. add default product
+
+##### Version 0.0.6
+> 1. add stop and pause interface
+
+##### Version 0.0.5
+> 1. split url of web/h5 log
+
+##### Version 0.0.4
+> 1. update codes
+
+##### Version 0.0.3
+> 1. fix encoded data
+
+##### Version 0.0.2
+> 1. fix kafka version
+
+##### Version 0.0.1
+> 1. init

app/interface/openplatform/monitor-end/CONTRIBUTORS.md

@@ -0,0 +1,10 @@
+# Owner
+liuhao02
+lijiadong
+
+# Author 
+lijiadong
+
+# Reviewer
+liuhao02
+lijiadong

app/interface/openplatform/monitor-end/OWNERS

@@ -0,0 +1,14 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- lijiadong
+- liuhao02
+labels:
+- interface
+- interface/openplatform/monitor-end
+- openplatform
+options:
+  no_parent_owners: true
+reviewers:
+- lijiadong
+- liuhao02

app/interface/openplatform/monitor-end/README.md

@@ -0,0 +1,13 @@
+#### open-monitor
+
+##### 项目简介
+> 1.开放平台端监控服务  
+
+##### 编译环境
+> 请只用golang v1.9.x以上版本编译执行。  
+
+##### 依赖包
+> 1.公共包go-common  
+
+##### 特别说明
+> 1.model目录可能会被其他项目引用，请谨慎请改并通知各方。  

app/interface/openplatform/monitor-end/cmd/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["monitor.toml"],
+    importpath = "go-common/app/interface/openplatform/monitor-end/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/http:go_default_library",
+        "//app/interface/openplatform/monitor-end/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/cmd/main.go

@@ -0,0 +1,41 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/http"
+	"go-common/app/interface/openplatform/monitor-end/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	conf.Init()
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	trace.Init(nil)
+	defer trace.Close()
+	svr := service.New(conf.Conf)
+	defer svr.Close()
+	http.Init(conf.Conf, svr)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT:
+			time.Sleep(time.Second) // 休眠1s
+			return
+		case syscall.SIGHUP:
+		// TODO reload
+		default:
+			return
+		}
+	}
+}

app/interface/openplatform/monitor-end/cmd/monitor.toml

@@ -0,0 +1,24 @@
+# This is a TOML document. Boom.
+
+version = "1.0.0"
+user = "nobody"
+pid = "/tmp/monitor.pid"
+dir = "./"
+family = "monitor"
+trace = false
+debug = false
+apps = "mall", "ticket", "article", "mall", "ticket", "article"
+
+[xlog]
+    dir = "/data/log/monitor"
+
+[bm]
+    addr = "0.0.0.0:6888"
+    timeout = "1s"
+
+[monitor]
+    taskID  = "000069"
+    proto   = "tcp"
+    Addr    = "127.0.0.1:9988"
+    Chan    = 2048
+

app/interface/openplatform/monitor-end/conf/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/interface/openplatform/monitor-end/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/model/kafka:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/monitor:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/conf/conf.go

@@ -0,0 +1,156 @@
+package conf
+
+import (
+	"context"
+	"fmt"
+	"sync"
+
+	"go-common/app/interface/openplatform/monitor-end/model/kafka"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/cache/redis"
+	"go-common/library/conf/paladin"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+	bm "go-common/library/net/http/blademaster"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	// Conf is global config
+	Conf = &Config{Prom: &Prom{}}
+)
+
+// Config service config
+type Config struct {
+	// http
+	BM *bm.ServerConfig
+	// XLog
+	Log *log.Config
+	// monitor
+	Monitor *monitor.MonitorConfig
+	// kafka
+	Kafka       *kafka.Config
+	NeedConsume bool
+	// Prometheus
+	Prom  *Prom
+	mutex sync.Mutex
+	// mysql
+	MySQL *sql.Config
+	// redis
+	Redis *redis.Config
+	// infoc
+	CollectInfoc *infoc.Config
+	// collect fronend log
+	CollectFE bool
+	// products
+	Products string
+}
+
+// Prom .
+type Prom struct {
+	Promed   bool
+	IgnoreNA bool
+	Factor   int
+	Limit    uint64
+}
+
+// Set .
+func (e *Config) Set(text string) error {
+	var ec Config
+	if err := toml.Unmarshal([]byte(text), &ec); err != nil {
+		return err
+	}
+	if ec.Prom == nil {
+		e.Prom = &Prom{}
+		return nil
+	}
+	e.mutex.Lock()
+	e.Prom = &Prom{
+		Factor:   ec.Prom.Factor,
+		Promed:   ec.Prom.Promed,
+		Limit:    ec.Prom.Limit,
+		IgnoreNA: ec.Prom.IgnoreNA,
+	}
+	e.CollectFE = ec.CollectFE
+	e.Products = ec.Products
+	e.mutex.Unlock()
+	// *e = ec
+	return nil
+}
+
+// Init .
+func Init() {
+	if err := paladin.Init(); err != nil {
+		panic(err)
+	}
+	// var ec exampleConf
+	// var setter
+	if err := paladin.Watch("monitor-end.toml", Conf); err != nil {
+		panic(err)
+	}
+	if err := paladin.Get("monitor-end.toml").UnmarshalTOML(Conf); err != nil {
+		panic(err)
+	}
+
+	// use exampleConf
+	// watch key
+	go func() {
+		for event := range paladin.WatchEvent(context.TODO(), "monitor-end.toml") {
+			fmt.Println(event, Conf)
+		}
+	}()
+}
+
+// func init() {
+// 	flag.StringVar(&confPath, "conf", "", "config path")
+// }
+
+// func local() (err error) {
+// 	_, err = toml.DecodeFile(confPath, &Conf)
+// 	return
+// }
+
+// Init int config
+// func Init() error {
+// 	if confPath != "" {
+// 		return local()
+// 	}
+// 	return remote()
+// }
+
+// func remote() (err error) {
+// 	if client, err = conf.New(); err != nil {
+// 		return
+// 	}
+// 	if err = load(); err != nil {
+// 		return
+// 	}
+// 	go func() {
+// 		for range client.Event() {
+// 			log.Info("config reload")
+// 			if load() != nil {
+// 				log.Error("config reload error (%v)", err)
+// 			}
+// 		}
+// 	}()
+// 	return
+// }
+
+// func load() (err error) {
+// 	var (
+// 		s       string
+// 		ok      bool
+// 		tmpConf *Config
+// 	)
+// 	if s, ok = client.Toml2(); !ok {
+// 		return errors.New("load config center error")
+// 	}
+// 	if _, err = toml.Decode(s, &tmpConf); err != nil {
+// 		return errors.New("could not decode config")
+// 	}
+// 	*Conf = *tmpConf
+// 	return
+// }

app/interface/openplatform/monitor-end/dao/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "mysql.go",
+        "redis.go",
+    ],
+    importpath = "go-common/app/interface/openplatform/monitor-end/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/xstr:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/dao/dao.go

@@ -0,0 +1,44 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/library/cache/redis"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+// Dao .
+type Dao struct {
+	c     *conf.Config
+	db    *sql.DB
+	redis *redis.Pool
+}
+
+// New .
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:     c,
+		db:    sql.NewMySQL(c.MySQL),
+		redis: redis.NewPool(c.Redis),
+	}
+	return d
+}
+
+// Close .
+func (d *Dao) Close() {
+	d.db.Close()
+	d.redis.Close()
+}
+
+// Ping .
+func (d *Dao) Ping(c context.Context) (err error) {
+	if err = d.pingRedis(c); err != nil {
+		log.Error("d.pingRedis error(%+v)", err)
+		return
+	}
+	if err = d.db.Ping(c); err != nil {
+		log.Error("d.db.Ping error(%+v)", err)
+	}
+	return
+}

app/interface/openplatform/monitor-end/dao/mysql.go

@@ -0,0 +1,427 @@
+package dao
+
+import (
+	"context"
+	"database/sql"
+	"fmt"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+	"go-common/library/xstr"
+)
+
+var (
+	_allGroupsSQL     = "SELECT id, name, receivers, `interval`, ctime, mtime FROM `alert_group` WHERE is_deleted = 0 ORDER BY id DESC"
+	_groupSQL         = "SELECT id, name, receivers, `interval`, ctime, mtime FROM `alert_group` WHERE id = ? AND is_deleted = 0"
+	_groupsSQL        = "SELECT id, name, receivers, `interval`, ctime, mtime FROM `alert_group` WHERE id in (%s) AND is_deleted = 0"
+	_addGroupSQL      = "INSERT INTO `alert_group` (name,receivers,`interval`)VALUES(?,?,?)"
+	_groupByNameSQL   = "SELECT id, name, receivers, `interval`, ctime, mtime FROM `alert_group` WHERE name = ? AND is_deleted = 0"
+	_updateGroupSQL   = "UPDATE `alert_group` SET name = ?, receivers = ?, `interval` = ? WHERE id = ?"
+	_deleteGroupSQL   = "UPDATE `alert_group` SET is_deleted = 1 WHERE id = ?"
+	_targetSQL        = "SELECT id, sub_event, event, product, source, group_id, threshold, duration, state, ctime, mtime FROM alert_target WHERE id = ? AND deleted_time = 0"
+	_targetQuerySQL   = "SELECT id, sub_event, event, product, source, group_id, threshold, duration, state, ctime, mtime FROM alert_target"
+	_allTargetsSQL    = "SELECT id, sub_event, event, product, source, group_id, threshold, duration, state, ctime, mtime FROM alert_target WHERE deleted_time = 0 AND state = ?"
+	_countTargetSQL   = "SELECT count(id) as cnt FROM alert_target"
+	_addTargetSQL     = "INSERT INTO alert_target (sub_event, event, product, source, group_id, threshold, duration, state)VALUES(?,?,?,?,?,?,?,?)"
+	_updateTargetSQL  = "UPDATE alert_target SET sub_event = ?, event =? , product = ?, source = ?, group_id = ?, threshold = ?, duration = ?, state = ? WHERE id = ?"
+	_existTargetSQL   = "SELECT id FROM alert_target WHERE sub_event = ? AND event =? AND product = ? AND source = ? AND deleted_time = 0"
+	_targetSyncSQL    = "UPDATE alert_target set state = ? WHERE id = ?"
+	_deleteTargetSQL  = "UPDATE alert_target set deleted_time = ? WHERE id = ?"
+	_productSQL       = "SELECT id, name, group_id, ctime, mtime FROM alert_product WHERE id = ? AND is_deleted = 0"
+	_productByNameSQL = "SELECT id, name, group_id, ctime, mtime FROM alert_product WHERE name = ? AND is_deleted = 0"
+	_allProductsSQL   = "SELECT id, name, group_id, ctime, mtime FROM alert_product WHERE is_deleted = 0 AND state = 1 ORDER BY ctime desc"
+	_addProductSQL    = "INSERT INTO alert_product (name, group_id, state)VALUES(?,?,?)"
+	_updateProductSQL = "UPDATE alert_product SET name = ?, group_id = ?, state = ? WHERE id = ?"
+	_deleteProductSQL = "UPDATE alert_product SET is_deleted = 1 WHERE id = ?"
+)
+
+// Group query group by id.
+func (d *Dao) Group(c context.Context, id int64) (res *model.Group, err error) {
+	res = &model.Group{}
+	if err = d.db.QueryRow(c, _groupSQL, id).Scan(&res.ID, &res.Name, &res.Receivers, &res.Interval, &res.Ctime, &res.Mtime); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("d.Group.Scan error(%+v), id(%d)", err, id)
+	}
+	return
+}
+
+// Groups query groups by ids.
+func (d *Dao) Groups(c context.Context, ids []int64) (res []*model.Group, err error) {
+	if len(ids) == 0 {
+		return
+	}
+	var (
+		rows  *xsql.Rows
+		query = fmt.Sprintf(_groupsSQL, xstr.JoinInts(ids))
+	)
+	if rows, err = d.db.Query(c, query); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("d.Group.Scan error(%+v), id(%d)", err, ids)
+	}
+	for rows.Next() {
+		r := &model.Group{}
+		if err = rows.Scan(&r.ID, &r.Name, &r.Receivers, &r.Interval, &r.Ctime, &r.Mtime); err != nil {
+			log.Error("d.Group.Scan error(%+v), id(%d)", err, ids)
+			return
+		}
+		res = append(res, r)
+	}
+	err = rows.Err()
+	return
+}
+
+// GroupByName query group id by name.
+func (d *Dao) GroupByName(c context.Context, name string) (res *model.Group, err error) {
+	res = &model.Group{}
+	if err = d.db.QueryRow(c, _groupByNameSQL, name).Scan(&res.ID, &res.Name, &res.Receivers, &res.Interval, &res.Ctime, &res.Mtime); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		log.Error("d.GroupByName.Scan error(%+v), name(%s)", err, name)
+	}
+	return
+}
+
+// AddGroup add new group.
+func (d *Dao) AddGroup(c context.Context, g *model.Group) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _addGroupSQL, g.Name, g.Receivers, g.Interval); err != nil {
+		log.Error("d.AddGroup.Exec error(%+v), group(%+v)", err, g)
+		return
+	}
+	if res, err = r.LastInsertId(); err != nil {
+		log.Error("d.AddGroup.LastInsertId error(%+v), group(%+v)", err, g)
+	}
+	return
+}
+
+// UpdateGroup update group.
+func (d *Dao) UpdateGroup(c context.Context, g *model.Group) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _updateGroupSQL, g.Name, g.Receivers, g.Interval, g.ID); err != nil {
+		log.Error("d.UpdateGroup.Exec error(%+v), group(%+v)", err, g)
+		return
+	}
+	if res, err = r.RowsAffected(); err != nil {
+		log.Error("d.UpdateGroup.RowsAffected error(%+v), group(%+v)", err, g)
+	}
+	return
+}
+
+// AllGroups return all groups.
+func (d *Dao) AllGroups(c context.Context) (res []*model.Group, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _allGroupsSQL); err != nil {
+		log.Error("d.AllGroups.Query error(%+v)", err)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var g = &model.Group{}
+		if err = rows.Scan(&g.ID, &g.Name, &g.Receivers, &g.Interval, &g.Ctime, &g.Mtime); err != nil {
+			log.Error("d.AllGroups.Scan error(%+v)]", err)
+			return
+		}
+		res = append(res, g)
+	}
+	err = rows.Err()
+	return
+}
+
+// DeleteGroup delete group.
+func (d *Dao) DeleteGroup(c context.Context, id int64) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _deleteGroupSQL, id); err != nil {
+		log.Error("d.DeleteGroup.Exec error(%+v), id(%d)", err, id)
+		return
+	}
+	if res, err = r.RowsAffected(); err != nil {
+		log.Error("d.DeleteGroup.RowsAffected error(%+v), id(%d)", err, id)
+	}
+	return
+}
+
+// IsExisted return id if target existed by sub_event, event, product, source.
+func (d *Dao) IsExisted(c context.Context, t *model.Target) (res int64, err error) {
+	if err = d.db.QueryRow(c, _existTargetSQL, t.SubEvent, t.Event, t.Product, t.Source).Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		log.Error("d.Isexisted.Scan error(%+v), target(%+v)", err, t)
+	}
+	return
+}
+
+// Target get target by id.
+func (d *Dao) Target(c context.Context, id int64) (res *model.Target, err error) {
+	res = &model.Target{}
+	if err = d.db.QueryRow(c, _targetSQL, id).Scan(&res.ID, &res.SubEvent, &res.Event, &res.Product, &res.Source, &res.GroupIDs, &res.Threshold, &res.Duration, &res.State, &res.Ctime, &res.Mtime); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("d.Target.Scan error(%+v), id(%d)", err, id)
+	}
+	if res.GroupIDs != "" {
+		var gids []int64
+		if gids, err = xstr.SplitInts(res.GroupIDs); err != nil {
+			log.Error("d.Product.SplitInts error(%+v), group ids(%s)", err, res.GroupIDs)
+			return
+		}
+		if res.Groups, err = d.Groups(c, gids); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// AllTargets return all targets by state.
+func (d *Dao) AllTargets(c context.Context, state int) (res []*model.Target, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _allTargetsSQL, state); err != nil {
+		log.Error("d.AllTargets.Query error(%+v), sql(%s)", err, _allTargetsSQL)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var t = &model.Target{}
+		if err = rows.Scan(&t.ID, &t.SubEvent, &t.Event, &t.Product, &t.Source, &t.GroupIDs, &t.Threshold, &t.Duration, &t.State, &t.Ctime, &t.Mtime); err != nil {
+			log.Error("d.AllTargets.Scan error(%+v), sql(%s)", err, _allTargetsSQL)
+			return
+		}
+		if t.GroupIDs != "" {
+			var gids []int64
+			if gids, err = xstr.SplitInts(t.GroupIDs); err != nil {
+				log.Error("d.Product.SplitInts error(%+v), group ids(%s)", err, t.GroupIDs)
+				return
+			}
+			if t.Groups, err = d.Groups(c, gids); err != nil {
+				return
+			}
+		}
+		res = append(res, t)
+	}
+	err = rows.Err()
+	return
+}
+
+// AddTarget add a new target.
+func (d *Dao) AddTarget(c context.Context, t *model.Target) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _addTargetSQL, t.SubEvent, t.Event, t.Product, t.Source, t.GroupIDs, t.Threshold, t.Duration, t.State); err != nil {
+		log.Error("d.AddTarget.Exec error(%+v), target(%+v)", err, t)
+		return
+	}
+	if res, err = r.LastInsertId(); err != nil {
+		log.Error("d.AddTarget.LastInsertId error(%+v), target(%+v)", err, t)
+	}
+	return
+}
+
+// UpdateTarget uodate target.
+func (d *Dao) UpdateTarget(c context.Context, t *model.Target) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _updateTargetSQL, t.SubEvent, t.Event, t.Product, t.Source, t.GroupIDs, t.Threshold, t.Duration, t.State, t.ID); err != nil {
+		log.Error("d.UpdateGroup.Exec error(%+v), target(%+v)", err, t)
+		return
+	}
+	if res, err = r.RowsAffected(); err != nil {
+		log.Error("d.UpdateGroup.RowsAffected error(%+v), target(%+v)", err, t)
+	}
+	return
+}
+
+// DeleteTarget delete target by id.
+func (d *Dao) DeleteTarget(c context.Context, id int64) (res int64, err error) {
+	var (
+		r   sql.Result
+		now = time.Now()
+	)
+	if r, err = d.db.Exec(c, _deleteTargetSQL, now, id); err != nil {
+		log.Error("d.UpdateGroup.Exec error(%+v), target(%d)", err, id)
+		return
+	}
+	if res, err = r.RowsAffected(); err != nil {
+		log.Error("d.UpdateGroup.RowsAffected error(%+v), target(%d)", err, id)
+	}
+	return
+}
+
+// TargetsByQuery query targets by query.
+func (d *Dao) TargetsByQuery(c context.Context, where string) (res []*model.Target, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _targetQuerySQL+where); err != nil {
+		log.Error("d.TargetsByQuery.Query error(%+v), sql(%s)", err, _targetQuerySQL+where)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var t = &model.Target{}
+		if err = rows.Scan(&t.ID, &t.SubEvent, &t.Event, &t.Product, &t.Source, &t.GroupIDs, &t.Threshold, &t.Duration, &t.State, &t.Ctime, &t.Mtime); err != nil {
+			log.Error("d.TargetsByQuery.Scan error(%+v), sql(%s)", err, _targetQuerySQL+where)
+			return
+		}
+		if t.GroupIDs != "" {
+			var gids []int64
+			if gids, err = xstr.SplitInts(t.GroupIDs); err != nil {
+				log.Error("d.Product.SplitInts error(%+v), group ids(%s)", err, t.GroupIDs)
+				return
+			}
+			if t.Groups, err = d.Groups(c, gids); err != nil {
+				return
+			}
+		}
+		res = append(res, t)
+	}
+	return
+}
+
+// CountTargets .
+func (d *Dao) CountTargets(c context.Context, where string) (res int, err error) {
+	if err = d.db.QueryRow(c, _countTargetSQL+where).Scan(&res); err != nil {
+		if err == sql.ErrNoRows {
+			err = nil
+			return
+		}
+		log.Error("d.CountTargets.Scan error(%+v), sql(%s)", err, _countTargetSQL+where)
+	}
+	return
+}
+
+// TargetSync sync target state by id.
+func (d *Dao) TargetSync(c context.Context, id int64, state int) (err error) {
+	if _, err = d.db.Exec(c, _targetSyncSQL, state, id); err != nil {
+		log.Error("d.TargetSync.Exec error(%+v), id(%d), state(%d)", err, id, state)
+	}
+	return
+}
+
+// Product get product by id.
+func (d *Dao) Product(c context.Context, id int64) (res *model.Product, err error) {
+	res = &model.Product{}
+	if err = d.db.QueryRow(c, _productSQL, id).Scan(&res.ID, &res.Name, &res.GroupIDs, &res.Ctime, &res.Mtime); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("d.Product.Scan error(%+v), id(%d)", err, id)
+	}
+	if res.GroupIDs != "" {
+		var gids []int64
+		if gids, err = xstr.SplitInts(res.GroupIDs); err != nil {
+			log.Error("d.Product.SplitInts error(%+v), group ids(%s)", err, res.GroupIDs)
+			return
+		}
+		if res.Groups, err = d.Groups(c, gids); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// ProductByName get product bu name.
+func (d *Dao) ProductByName(c context.Context, name string) (res *model.Product, err error) {
+	res = &model.Product{}
+	if err = d.db.QueryRow(c, _productByNameSQL, name).Scan(&res.ID, &res.Name, &res.GroupIDs, &res.Ctime, &res.Mtime); err != nil {
+		if err == sql.ErrNoRows {
+			res = nil
+			err = nil
+			return
+		}
+		log.Error("d.ProductByName.Scan error(%+v), name(%s)", err, name)
+	}
+	if res.GroupIDs != "" {
+		var gids []int64
+		if gids, err = xstr.SplitInts(res.GroupIDs); err != nil {
+			log.Error("d.Product.SplitInts error(%+v), group ids(%s)", err, res.GroupIDs)
+			return
+		}
+		if res.Groups, err = d.Groups(c, gids); err != nil {
+			return
+		}
+	}
+	return
+}
+
+// AllProducts return all products.
+func (d *Dao) AllProducts(c context.Context) (res []*model.Product, err error) {
+	var rows *xsql.Rows
+	if rows, err = d.db.Query(c, _allProductsSQL); err != nil {
+		log.Error("d.AllProducts.Query error(%+v), sql(%s)", err, _allProductsSQL)
+		return
+	}
+	defer rows.Close()
+	for rows.Next() {
+		var p = &model.Product{}
+		if err = rows.Scan(&p.ID, &p.Name, &p.GroupIDs, &p.Ctime, &p.Mtime); err != nil {
+			log.Error("d.AllProducts.Scan error(%+v), sql(%s)", err, _allProductsSQL)
+			return
+		}
+		if p.GroupIDs != "" {
+			var gids []int64
+			if gids, err = xstr.SplitInts(p.GroupIDs); err != nil {
+				log.Error("d.Product.SplitInts error(%+v), group ids(%s)", err, p.GroupIDs)
+				return
+			}
+			if p.Groups, err = d.Groups(c, gids); err != nil {
+				return
+			}
+		}
+		res = append(res, p)
+	}
+	err = rows.Err()
+	return
+}
+
+// AddProduct add a new product.
+func (d *Dao) AddProduct(c context.Context, p *model.Product) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _addProductSQL, p.Name, p.GroupIDs, p.State); err != nil {
+		log.Error("d.AddProduct.Exec error(%+v), product(%+v)", err, p)
+		return
+	}
+	if res, err = r.LastInsertId(); err != nil {
+		log.Error("d.AddProduct.RowsAffected error(%+v), product(%+v)", err, p)
+	}
+	return
+}
+
+// UpdateProduct update product by id.
+func (d *Dao) UpdateProduct(c context.Context, p *model.Product) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _updateProductSQL, p.Name, p.GroupIDs, p.State, p.ID); err != nil {
+		log.Error("d.DeleteProduct.Exec error(%+v), product(%+v)", err, p)
+		return
+	}
+	if res, err = r.RowsAffected(); err != nil {
+		log.Error("d.DeleteProduct.RowsAffected error(%+v), product(%+v)", err, p)
+	}
+	return
+}
+
+// DeleteProduct delete a product by id.
+func (d *Dao) DeleteProduct(c context.Context, id int64) (res int64, err error) {
+	var r sql.Result
+	if r, err = d.db.Exec(c, _deleteProductSQL, id); err != nil {
+		log.Error("d.DeleteProduct.Exec error(%+v), id(%d)", err, id)
+		return
+	}
+	if res, err = r.RowsAffected(); err != nil {
+		log.Error("d.DeleteProduct.RowsAffected error(%+v), id(%d)", err, id)
+	}
+	return
+}

app/interface/openplatform/monitor-end/dao/redis.go

@@ -0,0 +1,67 @@
+package dao
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+func mailGroupKey(name string, t *model.Target, code string) string {
+	return fmt.Sprintf("%s:%s", name, targetKey(t, code))
+}
+
+func targetKey(t *model.Target, code string) string {
+	return fmt.Sprintf("%s_%s_%s_%s_%s", t.Source, t.Product, t.Event, t.SubEvent, code)
+}
+
+func (d *Dao) pingRedis(c context.Context) (err error) {
+	conn := d.redis.Get(c)
+	if _, err = conn.Do("SET", "PING", "PONG"); err != nil {
+		log.Error("remote redis: conn.Do(SET,PING,PONG) error(%+v)", err)
+	}
+	conn.Close()
+	return
+}
+
+// GetMailLock .
+func (d *Dao) GetMailLock(c context.Context, name string, interval int, t *model.Target, code string) (ok bool, err error) {
+	if name == "" || interval == 0 {
+		return
+	}
+	conn := d.redis.Get(c)
+	defer conn.Close()
+	key := mailGroupKey(name, t, code)
+	if ok, err = redis.Bool(conn.Do("SETNX", key, "1")); err != nil {
+		if err == redis.ErrNil {
+			err = nil
+		} else {
+			log.Error("d.redis.conn.Do(SETNX(%s)) error(%v)", key, err)
+			return
+		}
+	}
+	if ok {
+		conn.Do("EXPIRE", key, interval)
+	}
+	return
+}
+
+// TargetIncr get current target error amount.
+func (d *Dao) TargetIncr(c context.Context, t *model.Target, code string) (res int) {
+	var (
+		conn = d.redis.Get(c)
+		err  error
+		key  = targetKey(t, code)
+	)
+	defer conn.Close()
+	if res, err = redis.Int(conn.Do("INCR", key)); err != nil {
+		log.Error("d.redis.intr error(%+v)", err)
+		return
+	}
+	if res == 1 {
+		conn.Do("EXPIRE", key, t.Duration)
+	}
+	return
+}

app/interface/openplatform/monitor-end/http/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "alert.go",
+        "http.go",
+        "monitor.go",
+    ],
+    importpath = "go-common/app/interface/openplatform/monitor-end/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/model:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/monitor:go_default_library",
+        "//app/interface/openplatform/monitor-end/service:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/auth:go_default_library",
+        "//library/net/metadata:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/http/alert.go

@@ -0,0 +1,218 @@
+package http
+
+import (
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/metadata"
+	"strconv"
+)
+
+func groupList(c *bm.Context) {
+	var params = &model.GroupListParams{}
+	if err := c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(mfSvc.GroupList(c, params))
+}
+
+func groupAdd(c *bm.Context) {
+	var params = &model.Group{}
+
+	if err := c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(mfSvc.AddGroup(c, params))
+}
+
+func groupUpdate(c *bm.Context) {
+	var (
+		params = &model.Group{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	err = mfSvc.UpdateGroup(c, params)
+	c.JSON(nil, err)
+}
+
+func groupDelete(c *bm.Context) {
+	var (
+		params = &model.Target{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	err = mfSvc.DeleteGroup(c, params.ID)
+	c.JSON(nil, err)
+}
+
+func targetList(c *bm.Context) {
+	var (
+		params = &model.Target{}
+		pn, ps int
+		err    error
+		form   = c.Request.Form
+		sort   string
+		draw   int
+		res    *model.Targets
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	pnStr := form.Get("page")
+	psStr := form.Get("pagesize")
+	drawStr := form.Get("draw")
+	sort = form.Get("sort")
+	if sort == "" {
+		sort = "ctime,0"
+	}
+	if drawStr != "" {
+		if draw, err = strconv.Atoi(drawStr); err != nil {
+			err = ecode.RequestErr
+			c.JSON(nil, err)
+			return
+		}
+		pnStr = form.Get("start")
+		psStr = form.Get("length")
+	}
+	if pn, err = strconv.Atoi(pnStr); err != nil || pn < 0 {
+		err = ecode.RequestErr
+		c.JSON(nil, err)
+		return
+	}
+	if ps, err = strconv.Atoi(psStr); err != nil || ps < 0 {
+		err = ecode.RequestErr
+		c.JSON(nil, err)
+		return
+	}
+	if ps == 0 {
+		ps = 20
+	}
+	if draw > 0 {
+		pn = (pn + ps) / ps
+	}
+	if pn == 0 {
+		pn = 1
+	}
+	res, err = mfSvc.TargetList(c, params, pn, ps, sort)
+	if draw > 0 {
+		res.Draw = draw
+	}
+	c.JSON(res, err)
+}
+
+func targetAdd(c *bm.Context) {
+	var params = &model.Target{}
+	if err := c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(mfSvc.AddTarget(c, params))
+}
+
+func targetUpdate(c *bm.Context) {
+	var (
+		params = &model.Target{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	err = mfSvc.UpdateTarget(c, params)
+	c.JSON(nil, err)
+}
+
+func targetSync(c *bm.Context) {
+	var (
+		params = &model.Target{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	if params.ID == 0 {
+		err = ecode.RequestErr
+		c.JSON(nil, err)
+		return
+	}
+	err = mfSvc.TargetSync(c, params.ID, params.State)
+	c.JSON(nil, err)
+}
+
+func productAdd(c *bm.Context) {
+	var (
+		params = &model.Product{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(mfSvc.AddProduct(c, params))
+}
+
+func productUpdate(c *bm.Context) {
+	var (
+		params = &model.Product{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	err = mfSvc.UpdateProduct(c, params)
+	c.JSON(nil, err)
+}
+
+func productDelete(c *bm.Context) {
+	var (
+		params = &model.Product{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	if params.ID == 0 {
+		err = ecode.RequestErr
+		c.JSON(nil, err)
+		return
+	}
+	err = mfSvc.DeleteProduct(c, params.ID)
+	c.JSON(nil, err)
+}
+
+func productList(c *bm.Context) {
+	c.JSON(mfSvc.AllProducts(c))
+}
+
+func collect(c *bm.Context) {
+	var (
+		params = &monitor.Log{}
+		err    error
+	)
+	if err = c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	midInter, _ := c.Get("mid")
+	if midInter != nil {
+		params.Mid = strconv.FormatInt(midInter.(int64), 10)
+	}
+	params.IP = metadata.String(c, metadata.RemoteIP)
+	params.Buvid = c.Request.Header.Get("Buvid")
+	params.UserAgent = c.Request.Header.Get("User-Agent")
+	mfSvc.Collect(c, params)
+	c.JSON(nil, nil)
+}

app/interface/openplatform/monitor-end/http/http.go

@@ -0,0 +1,84 @@
+package http
+
+import (
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/auth"
+	"net/http"
+)
+
+var (
+	mfSvc   *service.Service
+	authSvr *auth.Auth
+)
+
+// Init init account service.
+func Init(c *conf.Config, s *service.Service) {
+	authSvr = auth.New(nil)
+	mfSvc = s
+	// engine
+	engine := bm.DefaultServer(c.BM)
+	innerRouter(engine)
+	// init inner server
+	if err := engine.Start(); err != nil {
+		log.Error("engine.Start() error(%v)", err)
+		panic(err)
+	}
+}
+
+// innerRouter init inner router.
+func innerRouter(e *bm.Engine) {
+	e.Ping(ping)
+	e.Register(register)
+	group := e.Group("/open/monitor", authSvr.Guest)
+	{
+		group.GET("/report", report)
+		group.GET("/err/collect", collect)
+	}
+	group = e.Group("open/internal/monitor", authSvr.Guest)
+	{
+		group.GET("/report", report)
+		group.GET("/consume/start", startConsume)
+		group.GET("/consume/stop", stopConsume)
+		group.GET("/consume/pause", pauseConsume)
+		r := group.Group("/alert")
+		{
+			cr := r.Group("/group")
+			{
+				cr.GET("/list", groupList)
+				cr.POST("/update", groupUpdate)
+				cr.POST("/add", groupAdd)
+				cr.POST("/delete", groupDelete)
+			}
+			cr = r.Group("/target")
+			{
+				cr.GET("/list", targetList)
+				cr.POST("/update", targetUpdate)
+				cr.POST("/add", targetAdd)
+				cr.POST("/sync", targetSync)
+			}
+			cr = r.Group("/product")
+			{
+				cr.POST("/add", productAdd)
+				cr.POST("/update", productUpdate)
+				cr.POST("/delete", productDelete)
+				cr.GET("/list", productList)
+			}
+		}
+	}
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	if err := mfSvc.Ping(c); err != nil {
+		log.Error("ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+// register support discovery.
+func register(c *bm.Context) {
+	c.JSON(map[string]struct{}{}, nil)
+}

app/interface/openplatform/monitor-end/http/monitor.go

@@ -0,0 +1,69 @@
+package http
+
+import (
+	"strconv"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/metadata"
+)
+
+func report(c *bm.Context) {
+	var (
+		params           = &model.LogParams{}
+		mid              int64
+		ip               = metadata.String(c, metadata.RemoteIP)
+		err              error
+		buvid, userAgent string
+	)
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	if err := c.Bind(params); err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	if cookie, _ := c.Request.Cookie("buvid3"); cookie != nil {
+		buvid = cookie.Value
+	}
+	if buvid == "" {
+		buvid = c.Request.Header.Get("Buvid")
+	}
+	userAgent = c.Request.Header.Get("User-Agent")
+	if err = mfSvc.Report(c, params, mid, ip, buvid, userAgent); err != nil {
+		err = ecode.RequestErr
+	}
+	c.JSON(nil, err)
+}
+
+func startConsume(c *bm.Context) {
+	var err error
+	if err = mfSvc.StartConsume(); err != nil {
+		c.JSON(err.Error(), nil)
+	}
+	c.JSON("success", nil)
+}
+
+func stopConsume(c *bm.Context) {
+	var err error
+	if err = mfSvc.StopConsume(); err != nil {
+		c.JSON(err.Error(), nil)
+	}
+	c.JSON("success", nil)
+}
+
+func pauseConsume(c *bm.Context) {
+	var (
+		t   int64
+		err error
+	)
+	duration := c.Request.Form.Get("duration")
+	if t, err = strconv.ParseInt(duration, 10, 64); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+	}
+	if err = mfSvc.PauseConsume(t); err != nil {
+		c.JSON(err.Error(), nil)
+	}
+	c.JSON("success", nil)
+}

app/interface/openplatform/monitor-end/model/BUILD

@@ -0,0 +1,34 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["params.go"],
+    importpath = "go-common/app/interface/openplatform/monitor-end/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/interface/openplatform/monitor-end/model/kafka:all-srcs",
+        "//app/interface/openplatform/monitor-end/model/monitor:all-srcs",
+        "//app/interface/openplatform/monitor-end/model/prom:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/model/kafka/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["kafka.go"],
+    importpath = "go-common/app/interface/openplatform/monitor-end/model/kafka",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/model/kafka/kafka.go

@@ -0,0 +1,8 @@
+package kafka
+
+// Config .
+type Config struct {
+	Addr       []string
+	Topic      string
+	Partitions []int32
+}

app/interface/openplatform/monitor-end/model/monitor/BUILD

@@ -0,0 +1,54 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "handler.go",
+        "monitor.go",
+    ],
+    importpath = "go-common/app/interface/openplatform/monitor-end/model/monitor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "handler_test.go",
+        "monitor_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//library/log:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)

app/interface/openplatform/monitor-end/model/monitor/handler.go

@@ -0,0 +1,246 @@
+package monitor
+
+import (
+	"bytes"
+	"context"
+	"fmt"
+	"net"
+	"sync"
+	"time"
+
+	"go-common/library/conf/env"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+	xtime "go-common/library/time"
+
+	"github.com/json-iterator/go"
+)
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+const (
+	// app_id
+	_appID = "app_id"
+	// time format
+	_timeFormat = "2006-01-02T15:04:05.999999"
+	//  log level name: INFO, WARN...
+	_level = "level"
+	// log time.
+	_time = "time"
+	// uniq ID from trace.
+	_tid = "traceid"
+	// default chan size
+	_defaultChan = 2048
+	// default agent timeout
+	_agentTimeout = xtime.Duration(20 * time.Millisecond)
+	// default merge wait time
+	_mergeWait = 1 * time.Second
+	// max buffer size
+	_maxBuffer = 10 * 1024 * 1024 // 10mb
+)
+
+var (
+	_defaultMonitorConfig = &MonitorConfig{
+		Proto: "unixgram",
+		Addr:  "/var/run/lancer/collector.sock",
+	}
+	_defaultTaskIDs = map[string]string{
+		env.DeployEnvFat1: "000069",
+		env.DeployEnvUat:  "000069",
+		env.DeployEnvPre:  "000161",
+		env.DeployEnvProd: "000161",
+	}
+	// log separator
+	_logSeparator = []byte("\u0001")
+)
+
+// MonitorConfig agent config.
+type MonitorConfig struct {
+	TaskID  string
+	Buffer  int
+	Proto   string         `dsn:"network"`
+	Addr    string         `dsn:"address"`
+	Chan    int            `dsn:"query.chan"`
+	Timeout xtime.Duration `dsn:"query.timeout"`
+}
+
+// MonitorHandler .
+type MonitorHandler struct {
+	c      *MonitorConfig
+	msgs   chan map[string]interface{}
+	waiter sync.WaitGroup
+	pool   sync.Pool
+}
+
+// NewMonitor a MonitorHandler.
+func NewMonitor(c *MonitorConfig) (a *MonitorHandler) {
+	if c == nil {
+		c = _defaultMonitorConfig
+	}
+	if c.Buffer == 0 {
+		c.Buffer = 1
+	}
+	if len(c.TaskID) == 0 {
+		c.TaskID = _defaultTaskIDs[env.DeployEnv]
+	}
+	c.Timeout = _agentTimeout
+	a = &MonitorHandler{c: c}
+	a.pool.New = func() interface{} {
+		return make(map[string]interface{}, 20)
+	}
+	a.msgs = make(chan map[string]interface{}, _defaultChan)
+	a.waiter.Add(1)
+	go a.writeproc()
+	return
+}
+
+// Log log to udp statsd daemon.
+func (h *MonitorHandler) Log(ctx context.Context, lv log.Level, appID string, args ...log.D) {
+	if args == nil {
+		return
+	}
+	d := h.data()
+	for _, arg := range args {
+		d[arg.Key] = arg.Value
+	}
+	if t, ok := trace.FromContext(ctx); ok {
+		d[_tid] = fmt.Sprintf("%s", t)
+	}
+	d[_appID] = env.AppID + "." + appID
+	d[_level] = lv.String()
+	d[_time] = time.Now().Format(_timeFormat)
+	select {
+	case h.msgs <- d:
+	default:
+	}
+}
+
+// writeproc write data into connection.
+func (h *MonitorHandler) writeproc() {
+	var (
+		buf   bytes.Buffer
+		conn  net.Conn
+		err   error
+		count int
+		quit  bool
+	)
+	defer h.waiter.Done()
+	taskID := []byte(h.c.TaskID)
+	tick := time.NewTicker(_mergeWait)
+	enc := json.NewEncoder(&buf)
+	for {
+		select {
+		case d := <-h.msgs:
+			if d == nil {
+				quit = true
+				goto DUMP
+			}
+			if buf.Len() >= _maxBuffer {
+				buf.Reset() // avoid oom
+			}
+			now := time.Now()
+			buf.Write(taskID)
+			buf.Write([]byte(fmt.Sprintf("%d", now.UnixNano()/1e6)))
+			enc.Encode(d)
+			h.free(d)
+			if count++; count < h.c.Buffer {
+				buf.Write(_logSeparator)
+				continue
+			}
+		case <-tick.C:
+		}
+		if conn == nil || err != nil {
+			if conn, err = net.DialTimeout(h.c.Proto, h.c.Addr, time.Duration(h.c.Timeout)); err != nil {
+				log.Error("net.DialTimeout(%s:%s) error(%v)\n", h.c.Proto, h.c.Addr, err)
+				continue
+			}
+		}
+	DUMP:
+		if conn != nil && buf.Len() > 0 {
+			count = 0
+			if _, err = conn.Write(buf.Bytes()); err != nil {
+				log.Error("conn.Write(%d bytes) error(%v)\n", buf.Len(), err)
+				conn.Close()
+			} else {
+				// only succeed reset buffer, let conn reconnect.
+				log.Info("conn Write(%d bytes) data(%v)\n", buf.Len(), string(buf.Bytes()))
+				buf.Reset()
+			}
+		}
+		if quit {
+			if conn != nil && err == nil {
+				conn.Close()
+			}
+			return
+		}
+	}
+}
+
+func (h *MonitorHandler) data() map[string]interface{} {
+	return h.pool.Get().(map[string]interface{})
+}
+
+func (h *MonitorHandler) free(d map[string]interface{}) {
+	for k := range d {
+		delete(d, k)
+	}
+	h.pool.Put(d)
+}
+
+// Close close the connection.
+func (h *MonitorHandler) Close() (err error) {
+	h.msgs <- nil
+	h.waiter.Wait()
+	return nil
+}
+
+// SetFormat .
+func (h *MonitorHandler) SetFormat(string) {
+	// discard setformat
+}
+
+// Info .
+func (h *MonitorHandler) Info(ctx context.Context, appID string, args ...log.D) {
+	h.Log(ctx, log.Level(1), appID, args...)
+}
+
+// Warn .
+func (h *MonitorHandler) Warn(ctx context.Context, appID string, args ...log.D) {
+	h.Log(ctx, log.Level(2), appID, args...)
+}
+
+// Error .
+func (h *MonitorHandler) Error(ctx context.Context, appID string, args ...log.D) {
+	h.Log(ctx, log.Level(3), appID, args...)
+}
+
+// CalCode handle codes.
+func (a *Log) CalCode() {
+	if a.HTTPCode == "" {
+		return
+	}
+	a.Codes = a.HTTPCode
+	var (
+		codes Codes
+		err   error
+	)
+	if err = json.Unmarshal([]byte(a.HTTPCode), &codes); err != nil {
+		log.Warn("s.CalCode error(%+v), codes(%s)", err, a.HTTPCode)
+		return
+	}
+	a.HTTPCode = fmt.Sprintf("%v", codes.HTTPCode)
+	a.BusinessCode = fmt.Sprintf("%v", codes.HTTPBusinessCode)
+	a.InnerCode = fmt.Sprintf("%v", codes.HTTPInnerCode)
+	// 电商inner_code 覆盖 business_code
+	if a.InnerCode != "-1" {
+		// 电商code 1 转成 0
+		if a.InnerCode == "1" {
+			a.BusinessCode = "0"
+		} else {
+			a.BusinessCode = a.InnerCode
+		}
+	}
+	if a.BusinessCode == "-1" {
+		a.BusinessCode = "0"
+	}
+}

app/interface/openplatform/monitor-end/model/monitor/handler_test.go

@@ -0,0 +1,40 @@
+package monitor
+
+import (
+	"context"
+	"testing"
+
+	"go-common/library/log"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	m *MonitorHandler
+	c = context.Background()
+)
+
+// TestNewMonitor .
+func TestNewMonitor(t *testing.T) {
+	Convey("new monitor", t, func() {
+		m = NewMonitor(nil)
+		So(m, ShouldNotBeNil)
+	})
+}
+
+// TestLog .
+func TestLog(t *testing.T) {
+	var err error
+	Convey("info", t, func() {
+		m.Info(c, "test", []log.D{}...)
+		So(err, ShouldBeNil)
+	})
+	Convey("warn", t, func() {
+		m.Warn(c, "test", []log.D{}...)
+		So(err, ShouldBeNil)
+	})
+	Convey("error", t, func() {
+		m.Error(c, "test", []log.D{}...)
+		So(err, ShouldBeNil)
+	})
+}

app/interface/openplatform/monitor-end/model/monitor/monitor.go

@@ -0,0 +1,139 @@
+package monitor
+
+import (
+	"bytes"
+	"errors"
+	"net/url"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"go-common/library/log"
+)
+
+const (
+	_regex     = `.*\d{2,}`
+	_fieldNums = 33
+)
+
+// ErrIncompleteLog .
+var ErrIncompleteLog = errors.New("Incomplete log")
+
+// Log .
+type Log struct {
+	// from app log
+	RequestURI   string           `form:"request_uri" json:"request_uri"`
+	TimeIso      string           `form:"time_iso" json:"time_iso"`
+	IP           string           `form:"ip" json:"ip"`
+	Version      string           `form:"version" json:"version"`
+	Buvid        string           `form:"buvid" json:"buvid"`
+	Fts          string           `form:"fts" json:"fts"`
+	Proid        string           `form:"proid" json:"proid"`
+	Chid         string           `form:"chid" json:"chid"`
+	Pid          string           `form:"pid" json:"pid"`
+	Brand        string           `form:"brand" json:"brand"`
+	Deviceid     string           `form:"deviceid" json:"deviceid"`
+	Model        string           `form:"model" json:"model"`
+	Osver        string           `form:"osver" json:"osver"`
+	Ctime        string           `form:"ctime" json:"ctime"`
+	Mid          string           `form:"mid" json:"mid"`
+	Ver          string           `form:"ver" json:"ver"`
+	Net          string           `form:"net" json:"net"`
+	Oid          string           `form:"oid" json:"oid"`
+	Product      string           `form:"product" json:"product"`
+	Createtime   string           `form:"createtime" json:"createtime"`
+	Event        string           `form:"event" json:"event"`
+	SubEvent     string           `form:"sub_event" json:"sub_event"`
+	LogType      string           `form:"log_type" json:"log_type"`
+	Duration     string           `form:"duration" json:"duration"`
+	Message      string           `form:"message" json:"message"`
+	Result       string           `form:"result" json:"result"`
+	ExtJSON      string           `form:"ext_json" json:"ext_json"`
+	Traceid      string           `form:"traceid" json:"traceid"`
+	Desc         string           `form:"desc" json:"desc"`
+	Network      string           `form:"network" json:"network"`
+	TraceidEnd   string           `form:"traceid_end" json:"traceid_end"`
+	HTTPCode     string           `form:"http_code" json:"http_code"`
+	SubProduct   string           `form:"sub_product" json:"sub_product"`
+	Codes        string           `json:"codes"`
+	BusinessCode string           `json:"business_code"`
+	InnerCode    string           `json:"inner_code"`
+	TraceidSvr   string           `json:"traceid_svr"`
+	Type         string           `json:"type"`
+	Query        string           `json:"query"`
+	UserAgent    string           `json:"user_agent"`
+	Details      map[string]int64 `json:"details"`
+}
+
+// Codes .
+type Codes struct {
+	HTTPCode         interface{} `json:"http_code"`
+	HTTPInnerCode    interface{} `json:"http_inner_code"`
+	HTTPBusinessCode interface{} `json:"http_business_code"`
+}
+
+// LogData .
+func (a *Log) LogData() (appID string, logType string, kv []log.D, err error) {
+	// if a.Product == "" {
+	// 	a.Product = "nameless"
+	// }
+	//TODO 去除非法appid
+	var (
+		ok    bool
+		regex = "^[A-Za-z_]{1,}$"
+	)
+	if ok, err = regexp.Match(regex, []byte(a.Product)); err != nil || !ok {
+		a.Product = "nameless"
+	}
+	appID = a.Product
+	logType = a.LogType
+	kv = a.kv()
+	return
+}
+
+func (a *Log) kv() (kv []log.D) {
+	t := reflect.TypeOf(*a)
+	v := reflect.ValueOf(*a)
+	for i := 0; i < t.NumField()-1; i++ {
+		if v.Field(i).String() == "" {
+			continue
+		}
+		tag := t.Field(i).Tag.Get("json")
+		// s, _ := url.QueryUnescape(v.Field(i).Interface().(string))
+		kv = append(kv, log.KV(tag, v.Field(i).Interface()))
+	}
+	if a.Details != nil {
+		kv = append(kv, log.KV("detail", a.Details))
+	}
+	return
+}
+
+// LogFromBytes .
+func LogFromBytes(msg []byte) *Log {
+	var (
+		e error
+		d string
+	)
+	a := &Log{}
+	data := bytes.Split(msg, []byte("|"))
+	v := reflect.ValueOf(a).Elem()
+	l := _fieldNums
+	if len(data) < l {
+		l = len(data)
+	}
+	for i := 0; i < l; i++ {
+		if d, e = url.QueryUnescape(string(data[i])); e != nil {
+			tmp := string(data[i])
+			tmp = tmp[:strings.LastIndex(tmp, "%")]
+			if d, e = url.QueryUnescape(tmp); e != nil {
+				d = string(data[i])
+				log.Warn("s.LogFromBytes url decode error(%+v), data(%s)", e, string(data[i]))
+			}
+		}
+		if len(d) > 512 {
+			d = d[:512]
+		}
+		v.Field(i).SetString(d)
+	}
+	return a
+}

app/interface/openplatform/monitor-end/model/monitor/monitor_test.go

@@ -0,0 +1,42 @@
+package monitor
+
+import (
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestCase .
+type TestCase struct {
+	tag      string
+	testData *Log
+	expected int
+}
+
+// TestLogData .
+func TestLogData(t *testing.T) {
+	var (
+		tcs = []TestCase{
+			TestCase{
+				tag:      "empty log",
+				testData: &Log{},
+				expected: 1,
+			},
+			TestCase{
+				tag:      "normal data",
+				testData: &Log{Product: "test", LogType: "1", Event: "test", SubEvent: "test"},
+				expected: 0,
+			},
+		}
+	)
+	for _, tc := range tcs {
+		Convey(tc.tag, t, func() {
+			_, _, _, err := tc.testData.LogData()
+			if tc.expected == 0 {
+				So(err, ShouldBeNil)
+			} else {
+				So(err, ShouldNotBeNil)
+			}
+		})
+	}
+}

app/interface/openplatform/monitor-end/model/params.go

@@ -0,0 +1,95 @@
+package model
+
+import "go-common/library/time"
+
+// LogParams .
+type LogParams struct {
+	Source string `form:"source"`
+	Log    string `form:"log"`
+	IsAPP  int    `form:"is_app"`
+}
+
+// CollectParams .
+type CollectParams struct {
+	SubEvent  string `form:"sub_event" json:"sub_event"`
+	Event     string `form:"event" json:"event"`
+	Product   string `form:"product" json:"product"`
+	Source    string `form:"source" json:"source"`
+	Code      int    `form:"code" json:"code"`
+	ExtJSON   string `form:"ext_json"`
+	Mid       int64
+	IP        string
+	Buvid     string
+	UserAgent string
+}
+
+// Group .
+type Group struct {
+	ID        int64     `form:"id" json:"id"`
+	Name      string    `form:"name" json:"name"`
+	Receivers string    `form:"receivers" json:"receivers"`
+	Interval  int       `form:"interval" json:"interval"`
+	Ctime     time.Time `json:"ctime"`
+	Mtime     time.Time `json:"mtime"`
+}
+
+// Target .
+type Target struct {
+	ID         int64     `form:"id" json:"id"`
+	SubEvent   string    `form:"sub_event" json:"sub_event"`
+	Event      string    `form:"event" json:"event"`
+	Product    string    `form:"product" json:"product"`
+	Source     string    `form:"source" json:"source"`
+	GroupIDs   string    `form:"gid" json:"-"`
+	Groups     []*Group  `json:"groups"`
+	States     string    `form:"states" json:"-"`
+	State      int       `form:"state" json:"state"`
+	Threshold  int       `form:"threshold" json:"threshold"`
+	Duration   int       `form:"duration" json:"duration"`
+	DeleteTime time.Time `json:"deleted_time"`
+	Ctime      time.Time `json:"ctime"`
+	Mtime      time.Time `json:"mtime"`
+}
+
+// Targets .
+type Targets struct {
+	Total    int       `json:"total"`
+	Page     int       `json:"page"`
+	PageSize int       `json:"pagesize"`
+	Draw     int       `form:"draw" json:"draw"`
+	Targets  []*Target `json:"targets"`
+}
+
+// GroupListParams .
+type GroupListParams struct {
+	Pn   int    `form:"pn" json:"pn"`
+	Ps   int    `form:"ps" json:"ps"`
+	Name string `form:"name" json:"name"`
+}
+
+// Groups .
+type Groups struct {
+	Total    int      `json:"total"`
+	Page     int      `json:"page"`
+	PageSize int      `json:"pagesize"`
+	Groups   []*Group `json:"groups"`
+}
+
+// Product .
+type Product struct {
+	ID       int64     `form:"id" json:"id"`
+	Name     string    `form:"name" json:"name"`
+	GroupIDs string    `form:"gid" json:"-"`
+	Groups   []*Group  `json:"groups"`
+	State    int       `form:"state" json:"state"`
+	Ctime    time.Time `json:"ctime"`
+	Mtime    time.Time `json:"mtime"`
+}
+
+// Products .
+type Products struct {
+	Total    int        `json:"total"`
+	Page     int        `json:"page"`
+	PageSize int        `json:"pagesize"`
+	Products []*Product `json:"products"`
+}

app/interface/openplatform/monitor-end/model/prom/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["prom.go"],
+    importpath = "go-common/app/interface/openplatform/monitor-end/model/prom",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//vendor/github.com/prometheus/client_golang/prometheus:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/model/prom/prom.go

@@ -0,0 +1,308 @@
+package prom
+
+import (
+	"math/rand"
+	"runtime"
+	"sync"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	_department   = "open"
+	_typeCommon   = "Common"
+	_typeDetailed = "Detailed"
+	_NaN          = "NaN"
+)
+
+var (
+	// HTTPClientSum HTTP Client request cost sum.
+	HTTPClientSum *Prom
+	// HTTPClientCount HTTP Client request count.
+	HTTPClientCount *Prom
+	// HTTPClientCode HTTP Client request server code count.
+	HTTPClientCode *Prom
+	// HTTPClientStatus HTTP Client request status.
+	HTTPClientStatus *Prom
+	// HTTPClientSummary HTTP Client request quantiles.
+	HTTPClientSummary *Prom
+)
+
+var c *conf.Config
+
+// Prom struct info
+type Prom struct {
+	timer   *prometheus.HistogramVec
+	counter *prometheus.CounterVec
+	state   *prometheus.GaugeVec
+	summary *prometheus.SummaryVec
+}
+
+// Init .
+func Init(ce *conf.Config) {
+	c = ce
+	NewProm(true)
+	go clearMemory()
+}
+
+func clearMemory() {
+	for {
+		// default 512MB accloc
+		var limit = uint64(512 * 1024 * 1024)
+		memStat := new(runtime.MemStats)
+		runtime.ReadMemStats(memStat)
+		used := memStat.Alloc
+		if c.Prom.Limit > 0 && c.Prom.Limit < 3072 {
+			limit = uint64(c.Prom.Limit * 1024 * 1024)
+		}
+		if used > limit {
+			NewProm(false)
+		}
+		time.Sleep(time.Minute)
+	}
+}
+
+// NewProm .
+func NewProm(isFirst bool) {
+	if isFirst {
+		HTTPClientSum = New().WithCounter("http_client_sum", []string{"target", "client_app", "department", "type", "method", "event", "version", "detail"})
+		HTTPClientCount = New().WithCounter("http_client_count", []string{"target", "client_app", "department", "type", "method", "event", "version", "detail"})
+		HTTPClientCode = New().WithCounter("http_client_code", []string{"target", "client_app", "department", "type", "method", "event", "version", "code"})
+		HTTPClientStatus = New().WithCounter("http_client_status", []string{"target", "client_app", "department", "type", "method", "event", "version", "status"})
+		HTTPClientSummary = New().WithQuantile("http_client_summary", []string{"target", "client_app", "department", "type", "method", "event", "version", "detail"})
+		return
+	}
+	var mutex sync.Mutex
+	mutex.Lock()
+	HTTPClientSum.Unregister()
+	HTTPClientSum = New().WithCounter("http_client_sum", []string{"target", "client_app", "department", "type", "method", "event", "version", "detail"})
+	HTTPClientCount.Unregister()
+	HTTPClientCount = New().WithCounter("http_client_count", []string{"target", "client_app", "department", "type", "method", "event", "version", "detail"})
+	HTTPClientCode.Unregister()
+	HTTPClientCode = New().WithCounter("http_client_code", []string{"target", "client_app", "department", "type", "method", "event", "version", "code"})
+	HTTPClientStatus.Unregister()
+	HTTPClientStatus = New().WithCounter("http_client_status", []string{"target", "client_app", "department", "type", "method", "event", "version", "status"})
+	HTTPClientSummary.Unregister()
+	HTTPClientSummary = New().WithQuantile("http_client_summary", []string{"target", "client_app", "department", "type", "method", "event", "version", "detail"})
+	mutex.Unlock()
+}
+
+// New creates a Prom instance.
+func New() *Prom {
+	return &Prom{}
+}
+
+// WithTimer with summary timer
+func (p *Prom) WithTimer(name string, labels []string) *Prom {
+	if p == nil || p.timer != nil {
+		return p
+	}
+	p.timer = prometheus.NewHistogramVec(
+		prometheus.HistogramOpts{
+			Name: name,
+			Help: name,
+		}, labels)
+	prometheus.MustRegister(p.timer)
+	return p
+}
+
+// WithCounter sets counter.
+func (p *Prom) WithCounter(name string, labels []string) *Prom {
+	if p == nil || p.counter != nil {
+		return p
+	}
+	p.counter = prometheus.NewCounterVec(
+		prometheus.CounterOpts{
+			Name: name,
+			Help: name,
+		}, labels)
+	prometheus.MustRegister(p.counter)
+	return p
+}
+
+// WithState sets state.
+func (p *Prom) WithState(name string, labels []string) *Prom {
+	if p == nil || p.state != nil {
+		return p
+	}
+	p.state = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name: name,
+			Help: name,
+		}, labels)
+	prometheus.MustRegister(p.state)
+	return p
+}
+
+// WithQuantile sets quantiles.
+func (p *Prom) WithQuantile(name string, labels []string) *Prom {
+	if p == nil || p.summary != nil {
+		return p
+	}
+	p.summary = prometheus.NewSummaryVec(
+		prometheus.SummaryOpts{
+			Name: name,
+			Help: name,
+		}, labels)
+	prometheus.MustRegister(p.summary)
+	return p
+}
+
+// Unregister .
+func (p *Prom) Unregister() {
+	if p.counter != nil {
+		prometheus.Unregister(p.counter)
+	}
+	if p.state != nil {
+		prometheus.Unregister(p.state)
+	}
+	if p.timer != nil {
+		prometheus.Unregister(p.timer)
+	}
+	if p.summary != nil {
+		prometheus.Unregister(p.summary)
+	}
+}
+
+// Timing log timing information (in milliseconds) without sampling
+func (p *Prom) Timing(name string, time int64, extra ...string) {
+	label := append([]string{name}, extra...)
+	if p.timer != nil {
+		p.timer.WithLabelValues(label...).Observe(float64(time))
+	}
+}
+
+// Incr increments one stat counter without sampling
+func (p *Prom) Incr(name string, extra ...string) {
+	label := append([]string{name}, extra...)
+	if p.counter != nil {
+		p.counter.WithLabelValues(label...).Inc()
+	}
+	if p.state != nil {
+		p.state.WithLabelValues(label...).Inc()
+	}
+}
+
+// Decr decrements one stat counter without sampling
+func (p *Prom) Decr(name string, extra ...string) {
+	if p.state != nil {
+		label := append([]string{name}, extra...)
+		p.state.WithLabelValues(label...).Dec()
+	}
+}
+
+// State set state
+func (p *Prom) State(name string, v int64, extra ...string) {
+	if p.state != nil {
+		label := append([]string{name}, extra...)
+		p.state.WithLabelValues(label...).Set(float64(v))
+	}
+}
+
+// Add add count    v must > 0
+func (p *Prom) Add(name string, v int64, extra ...string) {
+	label := append([]string{name}, extra...)
+	if p.counter != nil {
+		p.counter.WithLabelValues(label...).Add(float64(v))
+	}
+	if p.state != nil {
+		p.state.WithLabelValues(label...).Add(float64(v))
+	}
+}
+
+// AddCommonLog .
+func AddCommonLog(target string, app string, method string, event string, version string, v int64) {
+	if HTTPClientCount.counter == nil || HTTPClientSum.counter == nil || HTTPClientSummary.summary == nil {
+		return
+	}
+	if target == "" || app == "" || method == "" || event == "" {
+		return
+	}
+	var ok = true
+	if c.Prom.Factor > 0 && c.Prom.Factor < 100 {
+		if rand.Intn(100) >= c.Prom.Factor {
+			ok = false
+		}
+	}
+	if version == "" {
+		version = _NaN
+	}
+	// default 1ms per request
+	i := float64(v)
+	if i <= 0 {
+		i = 1
+	}
+	labels := []string{target, app, _department, _typeCommon, method, event, version, _NaN}
+	// labels := append(label, _NaN)
+	HTTPClientCount.counter.WithLabelValues(labels...).Inc()
+	HTTPClientSum.counter.WithLabelValues(labels...).Add(i)
+	if ok {
+		HTTPClientSummary.summary.WithLabelValues(labels...).Observe(i)
+	}
+}
+
+// AddDetailedLog .
+func AddDetailedLog(target string, app string, method string, event string, version string, details map[string]int64) {
+	if HTTPClientCount.counter == nil || HTTPClientSum.counter == nil || HTTPClientSummary.summary == nil {
+		return
+	}
+	if target == "" || app == "" || method == "" || event == "" || details == nil {
+		return
+	}
+	var ok = true
+	if c.Prom.Factor > 0 && c.Prom.Factor < 100 {
+		if rand.Intn(100) >= c.Prom.Factor {
+			ok = false
+		}
+	}
+	if version == "" {
+		version = _NaN
+	}
+	label := []string{target, app, _department, _typeDetailed, method, event, version}
+	for k, v := range details {
+		labels := append(label, k)
+		// default 1ms per request
+		i := float64(v)
+		if i <= 0 {
+			i = 1
+		}
+		HTTPClientCount.counter.WithLabelValues(labels...).Inc()
+		HTTPClientSum.counter.WithLabelValues(labels...).Add(i)
+		if ok {
+			HTTPClientSummary.summary.WithLabelValues(labels...).Observe(i)
+		}
+	}
+}
+
+// AddHTTPCode .
+func AddHTTPCode(target string, app string, method string, event string, version string, code string) {
+	if HTTPClientStatus.counter == nil {
+		return
+	}
+	if target == "" || app == "" || method == "" || event == "" || code == "" {
+		return
+	}
+	if version == "" {
+		version = _NaN
+	}
+	label := []string{target, app, _department, _typeCommon, method, event, version, code}
+	HTTPClientStatus.counter.WithLabelValues(label...).Inc()
+}
+
+// AddCode .
+func AddCode(target string, app string, method string, event string, version string, code string) {
+	if HTTPClientCode.counter == nil {
+		return
+	}
+	if target == "" || app == "" || method == "" || event == "" || code == "" {
+		return
+	}
+	if version == "" {
+		version = _NaN
+	}
+	label := []string{target, app, _department, _typeCommon, method, event, version, code}
+	HTTPClientCode.counter.WithLabelValues(label...).Inc()
+}

app/interface/openplatform/monitor-end/service/BUILD

@@ -0,0 +1,77 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "alert_test.go",
+        "consumer_test.go",
+        "monitor_test.go",
+        "service_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/model:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/kafka:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/monitor:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/container/pool:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "alert.go",
+        "consumer.go",
+        "infoc.go",
+        "mail.go",
+        "monitor.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/interface/openplatform/monitor-end/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/openplatform/monitor-end/conf:go_default_library",
+        "//app/interface/openplatform/monitor-end/dao:go_default_library",
+        "//app/interface/openplatform/monitor-end/model:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/kafka:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/monitor:go_default_library",
+        "//app/interface/openplatform/monitor-end/model/prom:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/log/infoc:go_default_library",
+        "//vendor/github.com/Shopify/sarama:go_default_library",
+        "//vendor/github.com/bsm/sarama-cluster:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+        "//vendor/github.com/scorredoira/email:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/openplatform/monitor-end/service/alert.go

@@ -0,0 +1,412 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"fmt"
+	"reflect"
+	"strings"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/ecode"
+)
+
+var (
+	_duplicateNameErr    = errors.New("组名已经存在！")
+	_duplicateTargetErr  = errors.New("已经存在该告警sub_event!")
+	_invalidTargetIDErr  = errors.New("无效的id！")
+	_duplicateProductErr = errors.New("重复的product name")
+)
+
+func targetKey(t *model.Target) string {
+	if t.Source == "" || t.Product == "" || t.Event == "" || t.SubEvent == "" {
+		return ""
+	}
+	return fmt.Sprintf("%s_%s_%s_%s", t.Source, t.Product, t.Event, t.SubEvent)
+
+}
+
+func productKey(p string) string {
+	if p == "" {
+		return ""
+	}
+	return fmt.Sprintf("%s", p)
+}
+
+func (s *Service) loadalertsettings() {
+	var (
+		c   = context.Background()
+		gm  = make(map[int64]*model.Group)
+		tm  = make(map[int64]*model.Target)
+		tmk = make(map[string]*model.Target)
+		tmn = make(map[string]*model.Target)
+		pm  = make(map[int64]*model.Product)
+		pmk = make(map[string]*model.Product)
+	)
+	if gs, err := s.dao.AllGroups(c); err == nil {
+		for _, g := range gs {
+			gm[g.ID] = g
+		}
+	}
+
+	if ts, err := s.dao.AllTargets(c, 1); err == nil {
+		for _, t := range ts {
+			tm[t.ID] = t
+			if key := targetKey(t); key != "" {
+				tmk[key] = t
+			}
+		}
+	}
+	if ts, err := s.dao.AllTargets(c, 0); err == nil {
+		for _, t := range ts {
+			if key := targetKey(t); key != "" {
+				tmn[key] = t
+			}
+		}
+	}
+	if ps, err := s.dao.AllProducts(c); err == nil {
+		for _, p := range ps {
+			pm[p.ID] = p
+			if key := productKey(p.Name); err == nil {
+				pmk[key] = p
+			}
+		}
+	}
+	s.mapMutex.Lock()
+	s.groups = gm
+	s.targets = tm
+	s.products = pm
+	s.targetKeys = tmk
+	s.productKeys = pmk
+	s.newTargets = tmn
+	s.mapMutex.Unlock()
+}
+
+// AddGroup add a new group.
+func (s *Service) AddGroup(c context.Context, group *model.Group) (id int64, err error) {
+	var g *model.Group
+	if group.Name == "" || group.Receivers == "" || group.Interval < 0 {
+		err = ecode.RequestErr
+		return
+	}
+	if g, err = s.GroupByName(c, group.Name); err != nil {
+		return
+	}
+	if g.ID > 0 {
+		err = _duplicateNameErr
+		return
+	}
+	if group.Interval == 0 {
+		group.Interval = 30
+	}
+	return s.dao.AddGroup(c, group)
+}
+
+// UpdateGroup update group.
+func (s *Service) UpdateGroup(c context.Context, group *model.Group) (err error) {
+	var g *model.Group
+	if group.ID == 0 || group.Name == "" || group.Receivers == "" || group.Interval < 0 {
+		err = ecode.RequestErr
+		return
+	}
+	if g, err = s.GroupByName(c, group.Name); err != nil {
+		return
+	}
+	if g.ID != 0 && g.ID != group.ID {
+		err = _duplicateNameErr
+		return
+	}
+	if group.Interval == 0 {
+		group.Interval = 30
+	}
+	_, err = s.dao.UpdateGroup(c, group)
+	return
+}
+
+// DeleteGroup delete group.
+func (s *Service) DeleteGroup(c context.Context, id int64) (err error) {
+	if id == 0 {
+		err = ecode.RequestErr
+		return
+	}
+	_, err = s.dao.DeleteGroup(c, id)
+	return
+}
+
+// GroupList return all groups.
+func (s *Service) GroupList(c context.Context, params *model.GroupListParams) (res *model.Groups, err error) {
+	res = &model.Groups{}
+	if res.Groups, err = s.dao.AllGroups(c); err != nil {
+		return
+	}
+	res.Total = len(res.Groups)
+	return
+}
+
+// GroupByName get group by name.
+func (s *Service) GroupByName(c context.Context, name string) (res *model.Group, err error) {
+	return s.dao.GroupByName(c, name)
+}
+
+// Target get target by id.
+func (s *Service) Target(c context.Context, id int64) (res *model.Target, err error) {
+	return s.dao.Target(c, id)
+}
+
+// AddTarget add a new target.
+func (s *Service) AddTarget(c context.Context, t *model.Target) (id int64, err error) {
+	if t.SubEvent == "" || t.Event == "" || t.Product == "" || t.Source == "" {
+		err = ecode.RequestErr
+		return
+	}
+	if err = s.checkTarget(c, t, true); err != nil {
+		return
+	}
+	return s.dao.AddTarget(c, t)
+}
+
+// UpdateTarget update target.
+func (s *Service) UpdateTarget(c context.Context, t *model.Target) (err error) {
+	var (
+		oldTarget *model.Target
+	)
+	if oldTarget, err = s.Target(c, t.ID); err != nil {
+		return
+	}
+	if oldTarget == nil {
+		err = _invalidTargetIDErr
+		return
+	}
+	mergeTarget(t, oldTarget)
+	if err = s.checkTarget(c, t, false); err != nil {
+		return
+	}
+	_, err = s.dao.UpdateTarget(c, t)
+	return
+}
+
+// TargetList .
+func (s *Service) TargetList(c context.Context, t *model.Target, pn int, ps int, sort string) (res *model.Targets, err error) {
+	// query := "SELECT id, sub_event, event, product, source, group_id, threshold, duration, state FROM target"
+	var (
+		where      = ""
+		order      = ""
+		empty      struct{}
+		sortFields = map[string]struct{}{
+			"sub_event": empty,
+			"mtime":     empty,
+			"ctime":     empty,
+			"state":     empty,
+		}
+		sortOrder = map[string]string{
+			"0": "DESC",
+			"1": "ASC",
+		}
+	)
+	if t.SubEvent != "" {
+		where += " sub_event LIKE '%" + t.SubEvent + "%'"
+	}
+	if t.Event != "" {
+		where += " event = '" + t.Event + "'"
+	}
+	if t.Product != "" {
+		where += " product = '" + t.Product + "'"
+	}
+	if t.Source != "" {
+		where += " source = '" + t.Source + "'"
+	}
+	if t.States != "" {
+		where += " state in (" + t.States + ")"
+	}
+	if where == "" {
+		where = " WHERE" + where + " deleted_time = 0"
+	} else {
+		where = " WHERE" + where + " AND deleted_time = 0"
+	}
+	countWhere := where
+	pn = (pn - 1) * ps
+	sorts := strings.Split(sort, ",")
+	if len(sorts) == 2 {
+		_, ok1 := sortFields[sorts[0]]
+		d, ok2 := sortOrder[sorts[1]]
+		if ok1 && ok2 {
+			order = " ORDER BY " + sorts[0] + " " + d
+		}
+	}
+	where += order
+	where += fmt.Sprintf(" LIMIT %d, %d", pn, ps)
+	res = &model.Targets{}
+	if res.Targets, err = s.dao.TargetsByQuery(c, where); err != nil {
+		return
+	}
+	if res.Total, err = s.dao.CountTargets(c, countWhere); err != nil {
+		return
+	}
+	res.Page = pn
+	res.PageSize = ps
+	return
+}
+
+// TargetSync sync target state.
+func (s *Service) TargetSync(c context.Context, id int64, state int) (err error) {
+	return s.dao.TargetSync(c, id, state)
+}
+
+// DeleteTarget delete target by id.
+func (s *Service) DeleteTarget(c context.Context, id int64) (err error) {
+	_, err = s.dao.DeleteTarget(c, id)
+	return
+}
+
+func (s *Service) checkTarget(c context.Context, t *model.Target, isNew bool) (err error) {
+	var id int64
+	t.SubEvent = induceSubEvent(t.SubEvent)
+	if id, err = s.dao.IsExisted(c, t); err != nil {
+		return
+	}
+	if isNew && id != 0 {
+		fmt.Println("id", id)
+		err = _duplicateTargetErr
+	}
+	if !isNew && id != 0 && t.ID != id {
+		err = _duplicateTargetErr
+	}
+	return
+}
+
+func mergeTarget(t *model.Target, o *model.Target) {
+	te := reflect.ValueOf(t).Elem()
+	oe := reflect.ValueOf(o).Elem()
+	for i := 0; i < te.NumField()-2; i++ {
+		switch v := te.Field(i).Interface().(type) {
+		case int, int64:
+			if v == 0 {
+				te.Field(i).Set(oe.Field(i))
+			}
+		case string:
+			if v == "" {
+				te.Field(i).Set(oe.Field(i))
+			}
+		}
+	}
+}
+
+// AddProduct add a new group.
+func (s *Service) AddProduct(c context.Context, p *model.Product) (id int64, err error) {
+	if p.Name == "" || p.GroupIDs == "" {
+		err = ecode.RequestErr
+		return
+	}
+	var a *model.Product
+	if a, err = s.dao.ProductByName(c, p.Name); err != nil {
+		return
+	}
+	if a != nil {
+		err = _duplicateProductErr
+		return
+	}
+	id, err = s.dao.AddProduct(c, p)
+	return
+}
+
+// UpdateProduct update product.
+func (s *Service) UpdateProduct(c context.Context, p *model.Product) (err error) {
+	if p.ID == 0 || p.Name == "" || p.GroupIDs == "" {
+		err = ecode.RequestErr
+		return
+	}
+	var a *model.Product
+	if a, err = s.dao.ProductByName(c, p.Name); err != nil {
+		return
+	}
+	if a != nil && a.ID != p.ID {
+		err = _duplicateProductErr
+		return
+	}
+	_, err = s.dao.UpdateProduct(c, p)
+	return
+}
+
+// DeleteProduct delete product.
+func (s *Service) DeleteProduct(c context.Context, id int64) (err error) {
+	_, err = s.dao.DeleteProduct(c, id)
+	return
+}
+
+// AllProducts return all products.
+func (s *Service) AllProducts(c context.Context) (res *model.Products, err error) {
+	res = &model.Products{}
+	if res.Products, err = s.dao.AllProducts(c); err != nil {
+		return
+	}
+	res.Total = len(res.Products)
+	return
+}
+
+// Collect collect.
+func (s *Service) Collect(c context.Context, p *monitor.Log) {
+	var (
+		curr int
+		key  string
+		t    *model.Target
+	)
+	target := &model.Target{
+		Source:   sourceFromLog(p),
+		Product:  p.Product,
+		Event:    p.Event,
+		SubEvent: induceSubEvent(p.SubEvent),
+		State:    0,
+	}
+	s.infoCh <- p
+	//TODO 获取buvid, ip等信息，过滤重复请求
+	if key = targetKey(target); key == "" {
+		return
+	}
+
+	if t = s.targetKeys[key]; t == nil {
+		if s.newTargets[key] == nil {
+			// 添加新的target
+			s.AddTarget(c, target)
+		}
+		s.mapMutex.Lock()
+		s.newTargets[key] = t
+		s.mapMutex.Unlock()
+		return
+	}
+	if t.Threshold == 0 {
+		return
+	}
+	p.CalCode()
+	code := codeFromLog(p)
+	curr = s.dao.TargetIncr(c, t, code)
+	if curr > t.Threshold {
+		go s.mail(c, p, t, curr, code)
+	}
+}
+
+func sourceFromLog(l *monitor.Log) string {
+	if l.Type == "web/h5" {
+		return l.Type
+	}
+	if l.RequestURI == "" {
+		return "app"
+	}
+	if res := strings.Split(l.RequestURI, "?"); len(res) > 1 {
+		return res[1]
+	}
+	return l.RequestURI
+}
+
+func codeFromLog(l *monitor.Log) string {
+	if l.Codes == "" {
+		return "999"
+	}
+	if l.HTTPCode != "" && l.HTTPCode != "200" {
+		return l.HTTPCode
+	}
+	if l.BusinessCode != "" && l.BusinessCode != "0" {
+		return l.BusinessCode
+	}
+	return "-999"
+}

app/interface/openplatform/monitor-end/service/alert_test.go

@@ -0,0 +1,237 @@
+package service
+
+import (
+	"context"
+	"testing"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+var (
+	testGroup = &model.Group{
+		Name:      "test",
+		Receivers: "lalalal",
+		Interval:  30,
+	}
+	tmpGroup, tmpGroup2 *model.Group
+)
+
+// TestAddGroup .
+func TestAddGroup(t *testing.T) {
+	c := context.Background()
+	Convey("add group ", t, func() {
+		id, err := svr.AddGroup(c, testGroup)
+		So(err, ShouldBeNil)
+		tmpGroup, err = svr.dao.Group(c, id)
+		So(err, ShouldBeNil)
+		testGroup.Name = "hahaha"
+		id, err = svr.AddGroup(c, testGroup)
+		So(err, ShouldBeNil)
+		tmpGroup2, err = svr.dao.Group(c, id)
+		So(err, ShouldBeNil)
+	})
+	Convey("add group again with duplicate name", t, func() {
+		_, err := svr.AddGroup(c, testGroup)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestUpdateGroup .
+func TestUpdateGroup(t *testing.T) {
+	c := context.Background()
+	Convey("update group ", t, func() {
+		tmpGroup.Name = "huhuhu"
+		err := svr.UpdateGroup(c, tmpGroup)
+		So(err, ShouldBeNil)
+	})
+	Convey("update group again with duplicate name", t, func() {
+		tmpGroup2.Name = "huhuhu"
+		err := svr.UpdateGroup(c, tmpGroup2)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestGroupList .
+func TestGroupList(t *testing.T) {
+	c := context.Background()
+	Convey("group list", t, func() {
+		_, err := svr.GroupList(c, nil)
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestDeleteGroup .
+func TestDeleteGroup(t *testing.T) {
+	c := context.Background()
+	Convey("delete group", t, func() {
+		err := svr.DeleteGroup(c, tmpGroup.ID)
+		So(err, ShouldBeNil)
+		err = svr.DeleteGroup(c, tmpGroup2.ID)
+		So(err, ShouldBeNil)
+	})
+}
+
+var (
+	testTarget = &model.Target{
+		SubEvent:  "mall.bilibili.com/home",
+		Event:     "test",
+		Product:   "test",
+		Source:    "test",
+		GroupIDs:  "1,2,3",
+		Threshold: 4,
+		Duration:  5,
+	}
+	tmpTarget, tmpTarget2 *model.Target
+)
+
+// TestAddTarget .
+func TestAddTarget(t *testing.T) {
+	c := context.Background()
+	Convey("add target ", t, func() {
+		id, err := svr.AddTarget(c, testTarget)
+		So(err, ShouldBeNil)
+		tmpTarget, err = svr.dao.Target(c, id)
+		So(err, ShouldBeNil)
+		testTarget.Product = "tttt"
+		id, err = svr.AddTarget(c, testTarget)
+		So(err, ShouldBeNil)
+		tmpTarget2, err = svr.dao.Target(c, id)
+		So(err, ShouldBeNil)
+	})
+	Convey("add Target again with duplicate name", t, func() {
+		_, err := svr.AddTarget(c, testTarget)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestUpdateTarget .
+func TestUpdateTarget(t *testing.T) {
+	c := context.Background()
+	Convey("update Target ", t, func() {
+		tmpTarget.Product = "xxxx"
+		err := svr.UpdateTarget(c, tmpTarget)
+		So(err, ShouldBeNil)
+	})
+	Convey("update Target again with duplicate name", t, func() {
+		tmpTarget2.Product = "xxxx"
+		err := svr.UpdateTarget(c, tmpTarget2)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestTargetList .
+func TestTargetList(t *testing.T) {
+	c := context.Background()
+	Convey("Target list", t, func() {
+		t := &model.Target{}
+		_, err := svr.TargetList(c, t, 1, 2, "mtime,0")
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestTargetSync .
+func TestTargetSync(t *testing.T) {
+	c := context.Background()
+	Convey("sync Target", t, func() {
+		err := svr.TargetSync(c, tmpTarget.ID, 1)
+		So(err, ShouldBeNil)
+		err = svr.TargetSync(c, tmpTarget2.ID, 1)
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestDeleteTarget .
+func TestDeleteTarget(t *testing.T) {
+	c := context.Background()
+	Convey("delete Target", t, func() {
+		err := svr.DeleteTarget(c, tmpTarget.ID)
+		So(err, ShouldBeNil)
+		err = svr.DeleteTarget(c, tmpTarget2.ID)
+		So(err, ShouldBeNil)
+	})
+}
+
+var (
+	testProduct = &model.Product{
+		Name:     "test",
+		GroupIDs: "1,2",
+		State:    1,
+	}
+	tmpProduct, tmpProduct2 *model.Product
+)
+
+// TestAddProduct .
+func TestAddProduct(t *testing.T) {
+	c := context.Background()
+	Convey("add product ", t, func() {
+		id, err := svr.AddProduct(c, testProduct)
+		So(err, ShouldBeNil)
+		tmpProduct, err = svr.dao.Product(c, id)
+		So(err, ShouldBeNil)
+		testProduct.Name = "hahaha"
+		id, err = svr.AddProduct(c, testProduct)
+		So(err, ShouldBeNil)
+		tmpProduct2, err = svr.dao.Product(c, id)
+		So(err, ShouldBeNil)
+	})
+	Convey("add product again with duplicate name", t, func() {
+		_, err := svr.AddProduct(c, testProduct)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestUpdateProduct .
+func TestUpdateProduct(t *testing.T) {
+	c := context.Background()
+	Convey("update product ", t, func() {
+		tmpProduct.Name = "huhuhu"
+		err := svr.UpdateProduct(c, tmpProduct)
+		So(err, ShouldBeNil)
+	})
+	Convey("update Product again with duplicate name", t, func() {
+		tmpProduct2.Name = "huhuhu"
+		err := svr.UpdateProduct(c, tmpProduct2)
+		So(err, ShouldNotBeNil)
+	})
+}
+
+// TestAllProducts .
+func TestAllProducts(t *testing.T) {
+	c := context.Background()
+	Convey("all products", t, func() {
+		_, err := svr.AllProducts(c)
+		So(err, ShouldBeNil)
+	})
+}
+
+// TestDeleteProduct .
+func TestDeleteProduct(t *testing.T) {
+	c := context.Background()
+	Convey("delete product", t, func() {
+		err := svr.DeleteProduct(c, tmpProduct.ID)
+		So(err, ShouldBeNil)
+		err = svr.DeleteProduct(c, tmpProduct2.ID)
+		So(err, ShouldBeNil)
+	})
+}
+
+var (
+	testCollect = &monitor.Log{
+		SubEvent: "aaa",
+		Event:    "bbb",
+		Product:  "bbb",
+		ExtJSON:  "xxx",
+		HTTPCode: "404",
+	}
+)
+
+// TestCollect .
+func TestCollect(t *testing.T) {
+	c := context.Background()
+	Convey("test collect", t, func() {
+		svr.Collect(c, testCollect)
+	})
+}

app/interface/openplatform/monitor-end/service/consumer.go

@@ -0,0 +1,140 @@
+package service
+
+import (
+	"errors"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/model/kafka"
+	"go-common/library/log"
+
+	"github.com/Shopify/sarama"
+	cluster "github.com/bsm/sarama-cluster"
+)
+
+const _group = "open-monitor-end"
+
+// Consumer .
+type Consumer struct {
+	// c      sarama.Consumer
+	c        *cluster.Consumer
+	Config   *kafka.Config
+	closed   bool
+	paused   bool
+	duration time.Duration
+	messages chan *sarama.ConsumerMessage
+}
+
+var (
+	errClosedMsgChannel    = errors.New("message channel is closed")
+	errClosedNotifyChannel = errors.New("notification channel is closed")
+	errConsumerOver        = errors.New("too many consumers")
+)
+
+// NewConsumer .
+func NewConsumer(c *conf.Config) (con *Consumer, err error) {
+	con = &Consumer{
+		Config:   c.Kafka,
+		messages: make(chan *sarama.ConsumerMessage, 1024),
+	}
+	cfg := cluster.NewConfig()
+	cfg.Version = sarama.V0_8_2_0
+	cfg.ClientID = fmt.Sprintf("%s-%s", _group, c.Kafka.Topic)
+	cfg.Net.KeepAlive = 30 * time.Second
+	// NOTE cluster auto commit offset interval
+	cfg.Consumer.Offsets.CommitInterval = time.Second * 1
+	// NOTE set fetch.wait.max.ms
+	cfg.Consumer.MaxWaitTime = time.Millisecond * 250
+	cfg.Consumer.MaxProcessingTime = 50 * time.Millisecond
+	// NOTE errors that occur during offset management,if enabled, c.Errors channel must be read
+	cfg.Consumer.Return.Errors = true
+	// NOTE notifications that occur during consumer, if enabled, c.Notifications channel must be read
+	cfg.Group.Return.Notifications = true
+	// con.c = sarama.NewConsumer(c.Kafka.Addr, nil)
+	// consumer.Partitions = consumer.c.Partitions(consumer.Config.Topic)
+	cfg.Consumer.Offsets.Initial = sarama.OffsetNewest
+	if con.c, err = cluster.NewConsumer(c.Kafka.Addr, _group, []string{c.Kafka.Topic}, cfg); err != nil {
+		log.Error("s.NewConsumer group(%s) topic(%s) addr(%s) cluster.NewConsumer() error(%v)", _group, c.Kafka.Topic, strings.Join(c.Kafka.Addr, ","), err)
+	} else {
+		log.Info("s.NewConsumer group(%s) topic(%s) addr(%s) cluster.NewConsumer() ok", _group, c.Kafka.Topic, strings.Join(c.Kafka.Addr, ","))
+	}
+	return
+}
+
+func (s *Service) consume() {
+	for {
+		if s.consumer.closed {
+			return
+		}
+		if err := s.consumer.message(); err != nil {
+			time.Sleep(time.Minute)
+		}
+	}
+}
+
+func (s *Service) handleMsg() {
+	for {
+		select {
+		case msg := <-s.consumer.messages:
+			s.HandleMsg(msg.Value)
+		}
+		if s.consumer.closed {
+			return
+		}
+	}
+}
+
+func (s *Consumer) message() (err error) {
+	var (
+		msg    *sarama.ConsumerMessage
+		notify *cluster.Notification
+		ok     bool
+	)
+	for {
+		if s.closed {
+			s.c.Close()
+			err = nil
+			return
+		}
+		if s.paused {
+			s.paused = false
+			time.Sleep(s.duration)
+		}
+		select {
+		case err = <-s.c.Errors():
+			log.Error("group(%s) topic(%s) addr(%s) catch error(%v)", _group, s.Config.Topic, s.Config.Addr, err)
+			return
+		case notify, ok = <-s.c.Notifications():
+			if !ok {
+				log.Info("notification notOk group(%s) topic(%s) addr(%v) catch error(%v)", _group, s.Config.Topic, s.Config.Addr, err)
+				err = errClosedNotifyChannel
+				return
+			}
+			switch notify.Type {
+			case cluster.UnknownNotification, cluster.RebalanceError:
+				log.Error("notification(%s) group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+				err = errClosedNotifyChannel
+				return
+			case cluster.RebalanceStart:
+				log.Info("notification(%s) group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+				continue
+			case cluster.RebalanceOK:
+				log.Info("notification(%s) group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+			}
+			if len(notify.Current[s.Config.Topic]) == 0 {
+				log.Warn("notification(%s) no topic group(%s) topic(%s) addr(%v) catch error(%v)", notify.Type, _group, s.Config.Topic, s.Config.Addr, err)
+				err = errConsumerOver
+				return
+			}
+		case msg, ok = <-s.c.Messages():
+			if !ok {
+				log.Error("group(%s) topic(%s) addr(%v) message channel closed", _group, s.Config.Topic, s.Config.Addr)
+				err = errClosedMsgChannel
+				return
+			}
+			s.messages <- msg
+		}
+	}
+}

app/interface/openplatform/monitor-end/service/consumer_test.go

@@ -0,0 +1,39 @@
+package service
+
+import (
+	"testing"
+	"time"
+
+	"github.com/Shopify/sarama"
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestStartConsume .
+func TestStartConsume(t *testing.T) {
+	Convey("start consume", t, func() {
+		err := svr.StartConsume()
+		So(err, ShouldNotBeNil)
+	})
+}
+
+func TestStartHandle(t *testing.T) {
+	go svr.handleMsg()
+}
+
+// TestHandle .
+func TestHandle(t *testing.T) {
+	Convey("handle msg", t, func() {
+		var l = `a|b|c|d|e|f|g|h|i|j|k|l|m|n|o|p|q|r|s|t|u|v|w|x|y|z|1|2|3|4|5|6|7|8`
+		msg := &sarama.ConsumerMessage{
+			Value: []byte(l),
+		}
+		svr.consumer.messages <- msg
+		time.Sleep(time.Second)
+		So(len(svr.consumer.messages), ShouldEqual, 0)
+	})
+}
+
+// TestClose .
+func TestClose(t *testing.T) {
+	svr.Close()
+}

app/interface/openplatform/monitor-end/service/infoc.go

@@ -0,0 +1,25 @@
+package service
+
+import (
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/library/log"
+	"go-common/library/log/infoc"
+)
+
+// writeInfoc
+func (s *Service) infocproc() {
+	var (
+		collectInfoc = infoc.New(s.c.CollectInfoc)
+	)
+	for {
+		i, ok := <-s.infoCh
+		if !ok {
+			log.Warn("infoc proc exit")
+			return
+		}
+		switch l := i.(type) {
+		case model.CollectParams:
+			collectInfoc.Info(l.Source, l.Product, l.Event, l.SubEvent, l.Code, l.ExtJSON, l.Mid, l.IP, l.Buvid, l.UserAgent)
+		}
+	}
+}

app/interface/openplatform/monitor-end/service/mail.go

@@ -0,0 +1,87 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"net/mail"
+	"net/smtp"
+	"strings"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/log"
+
+	"github.com/scorredoira/email"
+)
+
+var text = `消息时间:%s
+最近%d秒内，%s端%s服务%s的%s出现异常（%s)，异常数量超过告警阀值 %d
+`
+
+func (s *Service) mail(c context.Context, p *monitor.Log, t *model.Target, curr int, code string) {
+	if t == nil {
+		return
+	}
+	var groups = t.Groups
+	if groups == nil || len(t.Groups) == 0 {
+		product := s.productKeys[productKey(t.Product)]
+		if product == nil || len(product.Groups) == 0 {
+			return
+		}
+		groups = product.Groups
+	}
+	for _, g := range groups {
+		if g.Name == "" || g.Interval == 0 {
+			continue
+		}
+		if ok, err := s.dao.GetMailLock(c, g.Name, g.Interval, t, code); err != nil || !ok {
+			continue
+		}
+		go s.mailByGroup(c, g.Receivers, p, curr, t.Threshold, t.Duration, code)
+	}
+	return
+}
+
+func (s *Service) mailByGroup(c context.Context, receivers string, p *monitor.Log, curr int, threshold int, duration int, code string) {
+	tos := strings.Split(receivers, ",")
+	if len(tos) == 0 {
+		return
+	}
+	for i, t := range tos {
+		tos[i] = t + "@bilibili.com"
+	}
+	source := sourceFromLog(p)
+	now := time.Now().Format("2006-01-02 15:03:04")
+	title := fmt.Sprintf("【端监控告警】%s端%s出现异常", source, p.Product)
+	body := fmt.Sprintf(text, now, duration, source, p.Product, p.Event, p.SubEvent, code, threshold)
+	if err := send(tos, title, body, 0); err != nil {
+		log.Error("s.mailByGroup.send error(%+v), mail to(%s), title(%s), body(%s)", err, receivers, title, body)
+	} else {
+		log.Info("s.mailByGroup.send successed, mail to(%s), title(%s), body(%s)", receivers, title, body)
+	}
+}
+
+/*
+    * const HOST = 'smtp.exmail.qq.com';
+    * const USER = 'show@bilibili.com';
+    * const PASS = 'Kfpt2017';
+	* const NAME = 'bilibili演出票务';
+*/
+func send(to []string, title string, body string, mode int) error {
+	var (
+		m    *email.Message
+		host = "smtp.exmail.qq.com:25"
+	)
+	if mode == 0 {
+		m = email.NewMessage(title, body)
+	} else {
+		m = email.NewHTMLMessage(title, body)
+	}
+	m.From = mail.Address{
+		Name:    "kfc监控告警",
+		Address: "show@bilibili.com",
+	}
+	m.To = to
+	return email.Send(host, smtp.PlainAuth("", "show@bilibili.com", "Kfpt2017", "smtp.exmail.qq.com"), m)
+}

app/interface/openplatform/monitor-end/service/monitor.go

@@ -0,0 +1,431 @@
+package service
+
+import (
+	"context"
+	"regexp"
+	"strconv"
+	"strings"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/app/interface/openplatform/monitor-end/model/prom"
+	"go-common/library/log"
+
+	"github.com/json-iterator/go"
+)
+
+const _regex = `.*\d{2,}`
+
+var json = jsoniter.ConfigCompatibleWithStandardLibrary
+
+const (
+	_typeAPP       = "app"
+	_typeWeb       = "web/h5"
+	_typeAPPH      = "app_h"
+	_eventPage     = "page"
+	_eventAPI      = "api"
+	_eventResource = "resource"
+	_levelInfo     = "info"
+	_levelWarning  = "warning"
+	_levelError    = "error"
+	_logTypeOne    = "1"
+	_logTypeTwo    = "2"
+)
+
+// Report .
+func (s *Service) Report(c context.Context, params *model.LogParams, mid int64, ip string, buvid string, userAgent string) (err error) {
+	var l *monitor.Log
+	log.Info("get log report(%+v)", params)
+	if params.IsAPP == 1 {
+		_, err = s.nativeLog(c, params.Log)
+	} else {
+		l, err = s.frontendLog(c, params.Source, params.Log, mid, ip, buvid, userAgent)
+		go s.promsFE(context.TODO(), l)
+	}
+	return
+}
+func induceSubEvent(s string) string {
+	if strings.Contains(s, "?") {
+		s = s[:strings.Index(s, "?")]
+	}
+	if strings.Contains(s, "://") {
+		s = s[(strings.Index(s, "://") + 3):]
+	}
+	if len(s) > 512 {
+		s = s[:512]
+	}
+	var (
+		ok  bool
+		err error
+	)
+	for {
+		if ok, err = regexp.Match(_regex, []byte(s)); err != nil {
+			log.Error("s.Report.regexp error(%+v), data(%s)", err, s)
+			return s
+		}
+		if !ok {
+			break
+		}
+		if !strings.Contains(s, "/") {
+			return s
+		}
+		s = s[:strings.LastIndex(s, "/")]
+	}
+	return s
+}
+
+func (s *Service) promsFE(c context.Context, l *monitor.Log) {
+	if !s.c.Prom.Promed || l == nil {
+		return
+	}
+	var (
+		ok  bool
+		err error
+	)
+	l.SubEvent = induceSubEvent(l.SubEvent)
+	if l.Event == _eventResource {
+		regex := `.*i\d{1}.hdslb.com`
+		if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
+			log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
+			return
+		}
+		if ok {
+			l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
+		}
+		prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
+		return
+	}
+	prom.AddHTTPCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
+	prom.AddCode(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
+	if l.Details == nil {
+		var cost int64
+		if l.Duration == "" {
+			return
+		}
+		if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
+			log.Warn("s.Info.ParseInt can not convert duration(%s) to int64", l.Duration)
+			return
+		}
+		prom.AddCommonLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
+	} else {
+		prom.AddDetailedLog(l.Type, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.Details)
+	}
+}
+
+func (s *Service) nativeLog(c context.Context, data string) (l *monitor.Log, err error) {
+	l = &monitor.Log{}
+	if err = json.Unmarshal([]byte(data), l); err != nil {
+		log.Error("s.nativeLog.unmarshal error(%+v), data(%s)", err, data)
+		return
+	}
+	l.Type = _typeAPPH
+	s.handleLog(c, l)
+	return
+}
+
+// HandleMsg .
+func (s *Service) HandleMsg(msg []byte) {
+	l := monitor.LogFromBytes(msg)
+	l.Type = _typeAPP
+	s.handleLog(context.TODO(), l)
+}
+
+func (s *Service) handleLog(c context.Context, l *monitor.Log) {
+	var (
+		appID   string
+		kv      []log.D
+		err     error
+		logtype int
+		isInt   bool
+		cost    int64
+		ok      bool
+		app     = "android"
+	)
+	l.TraceidSvr = l.Traceid
+	l.Traceid = ""
+	l.CalCode()
+	if appID, _, kv, err = l.LogData(); err != nil {
+		log.Error("s.nativeLog.LogData error(%+v), data(%+v)", err, l)
+		return
+	}
+	if l.Result == "1" || l.Result == "" {
+		s.mh.Info(c, appID, kv...)
+	} else {
+		s.mh.Error(c, appID, kv...)
+	}
+	if l.SubProduct == "" {
+		l.SubProduct = l.Product
+	}
+	if strings.Contains(l.RequestURI, "ios") {
+		app = "ios"
+		isInt = true
+	}
+	if logtype, err = bStrToInt(l.LogType, isInt); err != nil {
+		return
+	}
+	if s.c.Prom.IgnoreNA && !s.checkProduct(l) {
+		return
+	}
+	//	丢弃老版本的network日志
+	if l.Event == "network" && l.Codes == "" {
+		return
+	}
+	l.SubEvent = induceSubEvent(l.SubEvent)
+	regex := `.*i\d{1}.hdslb.com`
+	if ok, err = regexp.Match(regex, []byte(l.SubEvent)); err != nil {
+		log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
+		return
+	}
+	if ok {
+		l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
+	}
+	if (logtype & 1) == 1 {
+		// 性能日志
+		if cost, err = strconv.ParseInt(l.Duration, 10, 64); err != nil {
+			log.Warn("s.handleLog.ParseInt can not convert duration(%s) to int64", l.Duration)
+			return
+		}
+		prom.AddCommonLog(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, cost)
+	}
+	if (logtype >> 2 & 1) == 1 {
+		// 成功/失败日志
+		if l.Event == "network" {
+			// 网络类型 上报业务码和http状态
+			prom.AddHTTPCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.HTTPCode)
+			prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, l.BusinessCode)
+		} else {
+			// 其他类型 只上报失败成功 result=1 表示成功
+			res := "999"
+			if l.Result == "1" {
+				res = "0"
+			}
+			prom.AddCode(app, l.SubProduct, l.SubEvent, l.Event, l.Ver, res)
+		}
+	}
+}
+
+func (s *Service) frontendLog(c context.Context, source string, data string, mid int64, ip string, buvid string, userAgent string) (l *monitor.Log, err error) {
+	var (
+		ms                []interface{}
+		kv                []log.D
+		logtype, loglevel string
+		url, query        string
+	)
+	if err = json.Unmarshal([]byte(data), &ms); err != nil {
+		log.Error("s.frontendLog.unmarshal error(%+v), data(%s)", err, data)
+		return
+	}
+	for _, m := range ms {
+		switch m := m.(type) {
+		case map[string]interface{}:
+			logtype = stringValueByKey(m, "1", "logtype")
+			loglevel = stringValueByKey(m, "info", "level")
+			tmp := stringValueByKey(m, "", "url")
+			if strings.Contains(tmp, "?") {
+				url = strings.Split(tmp, "?")[0]
+				query = strings.Split(tmp, "?")[1]
+			} else {
+				url = tmp
+			}
+			switch logtype {
+			case _logTypeOne:
+				l = &monitor.Log{
+					LogType:      _logTypeOne,
+					Type:         _typeWeb,
+					Product:      source,
+					IP:           ip,
+					Buvid:        buvid,
+					UserAgent:    userAgent,
+					Event:        _eventAPI,
+					Mid:          strconv.FormatInt(mid, 10),
+					SubEvent:     url,
+					Query:        query,
+					Duration:     stringValueByKey(m, "", "cost"),
+					TraceidSvr:   stringValueByKey(m, "", "traceid_svr"),
+					TraceidEnd:   stringValueByKey(m, "", "traceid_end"),
+					HTTPCode:     stringValueByKey(m, "200", "status"),
+					BusinessCode: stringValueByKey(m, "0", "errno", "code"),
+					SubProduct:   stringValueByKey(m, source, "sub_product"),
+					Result:       "0",
+				}
+				srcURL := stringValueByKey(m, "", "srcUrl")
+				if srcURL != "" {
+					l.Event = _eventResource
+					l.SubEvent = srcURL
+					l.BusinessCode = "-999"
+				}
+				if loglevel == _levelInfo {
+					l.Result = "1"
+				}
+				if _, _, kv, err = l.LogData(); err != nil {
+					return
+				}
+				kv = append(kv, extKV(m)...)
+			case _logTypeTwo:
+				var msg []byte
+				l = &monitor.Log{
+					LogType:      _logTypeTwo,
+					Type:         _typeWeb,
+					Product:      source,
+					Event:        _eventPage,
+					IP:           ip,
+					Buvid:        buvid,
+					UserAgent:    userAgent,
+					Mid:          strconv.FormatInt(mid, 10),
+					SubEvent:     url,
+					Query:        query,
+					HTTPCode:     "200",
+					Details:      details(m),
+					Result:       "1",
+					BusinessCode: "0",
+					SubProduct:   stringValueByKey(m, source, "sub_product"),
+				}
+				if msg, err = json.Marshal(m); err != nil {
+					log.Error("s.frontendLog.Marshal error(%v), data(%v)", err, m)
+					continue
+				}
+				l.Message = string(msg)
+				if _, _, kv, err = l.LogData(); err != nil {
+					return
+				}
+			default:
+				log.Error("s.frontendLog error logype(%v)", logtype)
+				return
+			}
+			switch loglevel {
+			case _levelInfo:
+				s.mh.Info(c, source, kv...)
+			case _levelWarning:
+				s.mh.Warn(c, source, kv...)
+			case _levelError:
+				s.collectFE(c, l)
+				s.mh.Error(c, source, kv...)
+			default:
+				s.mh.Info(c, source, kv...)
+			}
+		default:
+			log.Error("s.frontendLog log data is not json type: " + data)
+		}
+	}
+	return
+}
+
+func (s *Service) collectFE(c context.Context, l *monitor.Log) {
+	if s.c.CollectFE {
+		regex := `.*i\d{1}.hdslb.com`
+		if ok, err := regexp.Match(regex, []byte(l.SubEvent)); err != nil {
+			log.Error("s.Report.regexp error(%+v), data(%s)", err, l.SubEvent)
+			return
+		} else if ok {
+			l.SubEvent = l.SubEvent[:strings.Index(l.SubEvent, "hdslb.com")+9]
+		}
+		go s.Collect(context.TODO(), l)
+	}
+}
+
+func stringValueByKey(m map[string]interface{}, defValue string, keys ...string) (r string) {
+	r = defValue
+	if m == nil {
+		return
+	}
+	var (
+		res interface{}
+		ok  bool
+	)
+	for _, key := range keys {
+		if res, ok = m[key]; ok {
+			break
+		}
+		return
+	}
+	if res == nil {
+		return
+	}
+	switch d := res.(type) {
+	case int:
+		r = strconv.Itoa(d)
+	case float64:
+		r = strconv.FormatFloat(d, 'f', -1, 64)
+	case int64:
+		r = strconv.FormatInt(d, 10)
+	case string:
+		r = d
+	default:
+		log.Warn("s.stringValueByKey unexcept type of value(%v)", d)
+	}
+	return
+}
+
+func extKV(m map[string]interface{}) (kv []log.D) {
+	if m == nil {
+		return
+	}
+	var keys = []string{"level", "logtype", "url", "cost", "traceid_svr", "traceid_end", "status", "code", "errno", "sub_product"}
+	for _, key := range keys {
+		delete(m, key)
+	}
+	for k, v := range m {
+		kv = append(kv, log.KV(k, v))
+	}
+	return
+}
+
+func details(m map[string]interface{}) (res map[string]int64) {
+	res = make(map[string]int64)
+	if m == nil {
+		return
+	}
+	delete(m, "level")
+	delete(m, "logtype")
+	delete(m, "url")
+	for k, v := range m {
+		var d int64
+		if v == nil {
+			res[k] = 0
+			continue
+		}
+		switch v := v.(type) {
+		case int64:
+			d = v
+		case float64:
+			d = int64(v)
+		case int:
+			d = int64(v)
+		case int32:
+			d = int64(v)
+		case string:
+			d, _ = strconv.ParseInt(v, 10, 64)
+		default:
+			log.Warn("s.details unexcept type of value(%v)", v)
+		}
+		res[k] = d
+	}
+	return
+}
+
+func bStrToInt(str string, isInt bool) (r int, err error) {
+	if isInt {
+		if r, err = strconv.Atoi(str); err != nil {
+			log.Warn("s.bStrToInt error(%+v), string(%s)", err, str)
+		}
+		return
+	}
+	for _, i := range str {
+		if i == '1' {
+			r = r*2 + 1
+		} else {
+			r *= 2
+		}
+	}
+	return
+}
+
+func (s *Service) checkProduct(l *monitor.Log) bool {
+	if s.naProducts == nil {
+		return true
+	}
+	if s.naProducts[l.Product] {
+		return true
+	}
+	return false
+}

app/interface/openplatform/monitor-end/service/monitor_test.go

@@ -0,0 +1,100 @@
+package service
+
+import (
+	"testing"
+
+	"go-common/app/interface/openplatform/monitor-end/model"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+// TestFELog .
+func TestFELog(t *testing.T) {
+	var (
+		tcs = []TestCase{
+			TestCase{
+				tag:      "logtype1 normal",
+				testData: `[{"level":"info","logtype":1,"url":"https://show.bilibili.com/api/ticket/district/geocoder","status":200,"cost":323,"traceid_end":"2565683971","traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype1 empty data",
+				testData: "",
+				expected: 1,
+			},
+			TestCase{
+				tag:      "logtype1 valide data2",
+				testData: `[{"level":"info","logtype":1,"url":"https://show.bilibili.com/api/ticket/district/geocoder","status":"200","cost":"323","traceid_end":2565683971,"traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype1 valid data2",
+				testData: `[{"level":"info","logtype":1,"url":111,"status":"200","cost":"323","traceid_end":2565683971,"traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype1 valid data3",
+				testData: `[{"level":"error","logtype":1,"url":111,"status":"200","cost":"323","traceid_end":2565683971,"traceid_svr":"2565683971"}]`,
+				expected: 0,
+			},
+			TestCase{
+				tag:      "logtype2 data",
+				testData: `[{"level":"info","logtype":2,"url":"https://show.bilibili.com/platform/home.html","navigationStart":0,"redirectStart":0,"redirectEnd":0,"fetchStart":2,"domainLookupStart":2,"domainLookupEnd":2,"connectStart":2,"secureConnectionStart":0,"connectEnd":2,"requestStart":71,"responseStart":73,"responseEnd":89,"domLoading":107,"domInteractive":442,"domContentLoadedEventStart":442,"domContentLoadedEventEnd":471,"domComplete":873,"loadEventStart":873,"loadEventEnd":873,"firstPaint":301,"firstContentfulPaint":515}]`,
+				expected: 0,
+			},
+		}
+	)
+	for _, tc := range tcs {
+		Convey(tc.tag, t, func() {
+			err := svr.Report(ctx, &model.LogParams{Source: "aaa", Log: tc.testData}, 111, "1.1.1.1", "", "")
+			if tc.expected == 0 {
+				So(err, ShouldBeNil)
+			} else {
+				So(err, ShouldNotBeNil)
+			}
+		})
+	}
+}
+
+// TestAPPLog .
+func TestAPPLog(t *testing.T) {
+	var (
+		tcs = []TestCase{
+			TestCase{
+				tag:      "logtype1 normal",
+				testData: `{"request_uri":"\/log\/mobile?ios","time_iso":"1539331278223","ip":"163.142.141.67","version":"2","buvid":"6baad8d5278a2982d204cafd403f089e","fts":"1525072552","proid":"1","chid":"AppStore","pid":"11","brand":"Apple","deviceid":"6baad8d5278a2982d204cafd403f089e","model":"iPhone 7","osver":"11.4.1","ctime":"20181012160107","mid":"18260473","ver":"5.32(8170)","net":"1","oid":"","product":"payment","createtime":"20181012-16:01:07.029GMT+08:00","event":"payment_iap","sub_event":"fetch_products","log_type":"16","duration":"0","message":"\u4eceApple\u83b7\u53d6\u6240\u6709products","result":"0","ext_json":"{\"productIds\":\"tv.danmaku.bilianimex68Bcoin,tv.danmaku.bilianimexnewpanel4998Bcoin,tv.danmaku.bilianimex3BigBcoin,tv.danmaku.bilianimex3VIPbf1,tv.danmaku.bilianimexnewpanel158Bcoin,tv.danmaku.bilianimexnewpanel1598Bcoin,tv.danmaku.bilianimex998Bcoin,tv.danmaku.bilianimexnewpanel648Bcoin,tv.danmaku.bilianimexnewpanel68Bcoin,tv.danmaku.bilianimex12VIPbf1,tv.danmaku.bilianimex3VIP,tv.danmaku.bilianimex18Bcoin,tv.danmaku.bilianimex12VIP,tv.danmaku.bilianimexnewpanel388Bcoinbf","traceid":"","desc":"","network":"1"}`,
+				expected: 0,
+			},
+			TestCase{
+				tag: "logtype1 normal",
+				testData: `	{"request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"1","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 0,
+			},
+			TestCase{
+				tag: "logtype1 error duration",
+				testData: `	{"request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277a","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"1","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 0,
+			},
+			TestCase{
+				tag: "logtype1 no json",
+				testData: `	{request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"1","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 1,
+			},
+			TestCase{
+				tag: "logtype1 normal",
+				testData: `	{"request_uri":"\/log\/mobile?android","time_iso":"1539335958928","ip":"39.181.159.21","version":"2","buvid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGinfoc","fts":"1538901223","proid":"1","chid":"vivo","pid":"13","brand":"vivo","deviceid":"HCwYL0pzQSBDIhIrV2VXZVdmVGVcaggxBnoGdkEnSiZUIkUnUjFQZ1IjTA","model":"vivo Y75","osver":"7.1.1","ctime":"20181012171917","mid":"169534145","ver":"5.32.0","net":"1","oid":"46002","product":"music","createtime":"1539335957529","event":"network","sub_event":"https:\/\/api.bilibili.com\/audio\/music-service-c\/url","log_type":"10101","duration":"277","message":"{\"code\":0,\"msg\":\"success\",\"data\":{\"sid\":507989,\"type\":2,\"info\":\"\",\"timeout\":10800,\"size\":3463341,\"cdns\":[\"https:\/\/upos-hz-mirrorkodou.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757&platform=android&upsig=558186152304d9145b77172de6c8941d\",\"https:\/\/upos-hz-mirrorossu.acgvideo.com\/ugaxcode\/i180908tx2zju6l33laab3311t7j7gdi-320k.m4a?deadline=1539346757","result":"0","ext_json":"{\"code\":\"200\",\"detail\":\"{\\\"t_befSendReq\\\":\\\"153\\\",\\\"t_parse\\\":\\\"7\\\",\\\"t_ttfb\\\":\\\"33\\\"}\",\"respsize\":\"783\"}","traceid":"","desc":"access_key=f71acb8feb40b3b0effec5d84b9503fc&appkey=1d8b6e7d45233436&build=5320000&mid=169534145&mobi_app=android&platform=android&privilege=2&quality=2&songid=507989&ts=1539335957&sign=480ade7c9622d846e6d91499fdd8fc80","network":"1"}`,
+				expected: 0,
+			},
+		}
+	)
+	for _, tc := range tcs {
+		Convey(tc.tag, t, func() {
+			err := svr.Report(ctx, &model.LogParams{Source: "test", Log: tc.testData, IsAPP: 1}, 111, "1.1.1.1", "", "")
+			if tc.expected == 0 {
+				So(err, ShouldBeNil)
+			} else {
+				So(err, ShouldNotBeNil)
+			}
+		})
+	}
+}

app/interface/openplatform/monitor-end/service/service.go

@@ -0,0 +1,154 @@
+package service
+
+import (
+	"context"
+	"errors"
+	"strings"
+	"sync"
+	"time"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/dao"
+	"go-common/app/interface/openplatform/monitor-end/model"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/app/interface/openplatform/monitor-end/model/prom"
+	"go-common/library/log"
+)
+
+var (
+	_notConsumedErr = errors.New("未启动消费")
+	_pausedNowErr   = errors.New("暂停消费中")
+	_closedNowErr   = errors.New("已停止消费")
+	_consumedNowErr = errors.New("已启动消费")
+)
+
+// Service struct
+type Service struct {
+	c        *conf.Config
+	dao      *dao.Dao
+	mh       *monitor.MonitorHandler
+	consumer *Consumer
+	consumed bool
+	// settings
+	groups      map[int64]*model.Group
+	targets     map[int64]*model.Target
+	targetKeys  map[string]*model.Target
+	products    map[int64]*model.Product
+	productKeys map[string]*model.Product
+	newTargets  map[string]*model.Target
+	naProducts  map[string]bool
+	mapMutex    sync.Mutex
+	// infoc
+	infoCh chan interface{}
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:           c,
+		dao:         dao.New(c),
+		mh:          monitor.NewMonitor(c.Monitor),
+		groups:      make(map[int64]*model.Group),
+		targets:     make(map[int64]*model.Target),
+		targetKeys:  make(map[string]*model.Target),
+		newTargets:  make(map[string]*model.Target),
+		products:    make(map[int64]*model.Product),
+		productKeys: make(map[string]*model.Product),
+		naProducts:  make(map[string]bool),
+		infoCh:      make(chan interface{}, 1024),
+	}
+	for c.NeedConsume && !s.consumed {
+		var err error
+		if s.consumer, err = NewConsumer(c); err != nil {
+			log.Error("s.New.NewConsumer error(%+v)", err)
+			time.Sleep(10 * time.Second)
+			continue
+		}
+		go s.consume()
+		go s.handleMsg()
+		s.consumed = true
+		break
+	}
+	prom.Init(c)
+	go s.loadalertsettingproc()
+	go s.infocproc()
+	go s.loadNAProducts()
+	// go s.alertproc()
+	return
+}
+
+// Ping .
+func (s *Service) Ping(c context.Context) (err error) {
+	return s.dao.Ping(c)
+}
+
+// StartConsume .
+func (s *Service) StartConsume() (err error) {
+	if s.consumed {
+		return _consumedNowErr
+	}
+	if s.consumer, err = NewConsumer(s.c); err != nil {
+		log.Error("s.New.NewConsumer error(%+v)", err)
+		return
+	}
+	go s.consume()
+	go s.handleMsg()
+	s.consumed = true
+	return
+}
+
+// Close .
+func (s *Service) Close() {
+	s.dao.Close()
+	s.StopConsume()
+}
+
+// StopConsume .
+func (s *Service) StopConsume() error {
+	if s.consumed {
+		s.consumed = false
+	}
+	if s.consumer != nil {
+		s.consumer.closed = true
+	}
+	return nil
+}
+
+// PauseConsume .
+func (s *Service) PauseConsume(t int64) error {
+	if !s.consumed {
+		return _notConsumedErr
+	}
+	if s.consumer.closed {
+		return _closedNowErr
+	}
+	if s.consumer.paused {
+		return _pausedNowErr
+	}
+	s.consumer.paused = true
+	s.consumer.duration = time.Second * time.Duration(t)
+	return nil
+}
+
+func (s *Service) loadalertsettingproc() {
+	for {
+		s.loadalertsettings()
+		time.Sleep(time.Minute)
+	}
+}
+
+func (s *Service) loadNAProducts() {
+	for {
+		if s.c.Products != "" {
+			var m = make(map[string]bool)
+			ps := strings.Split(s.c.Products, ",")
+			for _, p := range ps {
+				m[p] = true
+			}
+			s.mapMutex.Lock()
+			s.naProducts = m
+			s.mapMutex.Unlock()
+		}
+		time.Sleep(time.Minute)
+	}
+}

app/interface/openplatform/monitor-end/service/service_test.go

@@ -0,0 +1,85 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/interface/openplatform/monitor-end/conf"
+	"go-common/app/interface/openplatform/monitor-end/model/kafka"
+	"go-common/app/interface/openplatform/monitor-end/model/monitor"
+	"go-common/library/cache/redis"
+	"go-common/library/container/pool"
+	"go-common/library/database/sql"
+	"go-common/library/log/infoc"
+	"go-common/library/time"
+)
+
+var (
+	ctx = context.Background()
+	svr *Service
+)
+
+type TestData map[string]string
+
+type TestCase struct {
+	tag      string
+	testData string
+	expected int
+}
+
+/*[mysql]
+	dsn = "root:123456@tcp(172.16.33.203:3306)/public_monitor?timeout=500s&readTimeout=500s&writeTimeout=500s&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+    active = 5
+    idle = 2
+    idleTimeout ="4h"
+    queryTimeout = "1000s"
+    execTimeout = "200s"#
+    tranTimeout = "2000s"
+
+[redis]
+    name = "article"
+    proto = "tcp"
+    addr = "172.16.33.203:6379"
+    idle = 10
+    active = 10
+    dialTimeout = "1s"
+    readTimeout = "1s"
+    writeTimeout = "1s"
+    idleTimeout = "10s"*/
+func init() {
+	c := conf.Conf
+	if c.Monitor == nil {
+		c = &conf.Config{
+			Monitor: &monitor.MonitorConfig{Proto: "tcp", Addr: "127.0.0.1:9988"},
+			Kafka: &kafka.Config{
+				Addr:  []string{"1.1.1.1"},
+				Topic: "test_topic",
+			},
+			NeedConsume: false,
+			Redis: &redis.Config{
+				Name:  "article",
+				Proto: "tcp",
+				Addr:  "172.16.33.203:6379",
+				Config: &pool.Config{
+					Idle:   2,
+					Active: 5,
+				},
+				DialTimeout:  time.Duration(int64(1000000000)),
+				ReadTimeout:  time.Duration(int64(1000000000)),
+				WriteTimeout: time.Duration(int64(1000000000)),
+			},
+			MySQL: &sql.Config{
+				DSN:          "root:123456@tcp(172.16.33.203:3306)/public_monitor?timeout=500s&readTimeout=500s&writeTimeout=500s&parseTime=true&loc=Local&charset=utf8,utf8mb4",
+				QueryTimeout: time.Duration(int64(10000000000)),
+				ExecTimeout:  time.Duration(int64(10000000000)),
+				TranTimeout:  time.Duration(int64(20000000000)),
+			},
+			Prom:         &conf.Prom{Limit: 520},
+			CollectInfoc: &infoc.Config{},
+		}
+	}
+	svr = New(c)
+
+	if err := svr.dao.Ping(context.Background()); err != nil {
+		panic(err)
+	}
+}