Initial commit

app/job/BUILD

@@ -0,0 +1,21 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq:all-srcs",
+        "//app/job/live:all-srcs",
+        "//app/job/live-userexp:all-srcs",
+        "//app/job/main:all-srcs",
+        "//app/job/openplatform:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

app/job/OWNERS

@@ -0,0 +1,5 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+labels:
+- job
+- new-project

app/job/README.md

@@ -0,0 +1,3 @@
+# go-common/app/job
+
+job服务，对应异步服务

app/job/bbq/BUILD

@@ -0,0 +1,19 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/cms:all-srcs",
+        "//app/job/bbq/comment:all-srcs",
+        "//app/job/bbq/recall:all-srcs",
+        "//app/job/bbq/video:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/CONTRIBUTORS.md

@@ -0,0 +1,2 @@
+# Owner
+daiwei

app/job/bbq/OWNERS

@@ -0,0 +1,8 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- daiwei
+labels:
+- bbq
+- job
+- new-project

app/job/bbq/cms/BUILD

@@ -0,0 +1,20 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/cms/cmd:all-srcs",
+        "//app/job/bbq/cms/internal/dao:all-srcs",
+        "//app/job/bbq/cms/internal/model:all-srcs",
+        "//app/job/bbq/cms/internal/server/http:all-srcs",
+        "//app/job/bbq/cms/internal/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/cms/CHANGELOG.md

@@ -0,0 +1,2 @@
+### v1.0.0
+1. 上线功能cms-job

app/job/bbq/cms/CONTRIBUTORS.md

@@ -0,0 +1,7 @@
+# Owner
+jiangdongqi
+
+# Author
+jiangdongqi
+# Reviewer
+jiangdongqi

app/job/bbq/cms/OWNERS

@@ -0,0 +1,12 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- jiangdongqi
+labels:
+- bbq
+- job
+- job/bbq/cms
+options:
+  no_parent_owners: true
+reviewers:
+- jiangdongqi

app/job/bbq/cms/README.md

@@ -0,0 +1,12 @@
+# cms-job
+
+## 项目简介
+1.
+
+## 编译环境
+
+
+## 依赖包
+
+
+## 编译执行

app/job/bbq/cms/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "go-common/app/job/bbq/cms/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/cms/internal/server/http:go_default_library",
+        "//app/job/bbq/cms/internal/service:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/cms/cmd/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/bbq/cms/internal/server/http"
+	"go-common/app/job/bbq/cms/internal/service"
+	"go-common/library/conf/paladin"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := paladin.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(nil) // debug flag: log.dir={path}
+	defer log.Close()
+	log.Info("cms-job start")
+	ecode.Init(nil)
+	svc := service.New()
+	httpSrv := http.New(svc)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			ctx, _ := context.WithTimeout(context.Background(), 35*time.Second)
+			httpSrv.Shutdown(ctx)
+			log.Info("cms-job exit")
+			svc.Close()
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}
+

app/job/bbq/cms/configs/application.toml

@@ -0,0 +1,2 @@
+
+# This is a TOML document. Boom

app/job/bbq/cms/configs/http.toml

@@ -0,0 +1,4 @@
+
+[server]
+    addr = "0.0.0.0:8000"
+    timeout = "1s"

app/job/bbq/cms/configs/memcache.toml

@@ -0,0 +1,13 @@
+
+demoExpire = "24h"
+
+[demo]
+	name = "cms"
+	proto = "tcp"
+	addr = "127.0.0.1:11211"
+	active = 50
+	idle = 10
+	dialTimeout = "100ms"
+	readTimeout = "200ms"
+	writeTimeout = "300ms"
+	idleTimeout = "80s"

app/job/bbq/cms/configs/mysql.toml

@@ -0,0 +1,11 @@
+
+[demo]
+	addr = "127.0.0.1:3306"
+	dsn = "{user}:{password}@tcp(127.0.0.1:3306)/{database}?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8"
+	readDSN = ["{user}:{password}@tcp(127.0.0.2:3306)/{database}?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8","{user}:{password}@tcp(127.0.0.3:3306)/{database}?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 20
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "200ms"
+	execTimeout = "300ms"
+	tranTimeout = "400ms"

app/job/bbq/cms/configs/redis.toml

@@ -0,0 +1,13 @@
+
+demoExpire = "24h"
+
+[demo]
+	name = "cms"
+	proto = "tcp"
+	addr = "127.0.0.1:6389"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"

app/job/bbq/cms/internal/dao/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dao.go"],
+    importpath = "go-common/app/job/bbq/cms/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/cms/internal/dao/dao.go

@@ -0,0 +1,96 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf/paladin"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+// Dao dao.
+type Dao struct {
+	db          *sql.DB
+	redis       *redis.Pool
+	redisExpire int32
+	mc          *memcache.Pool
+	mcExpire    int32
+}
+
+func checkErr(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// New new a dao and return.
+func New() (dao *Dao) {
+	var (
+		dc struct {
+			Demo *sql.Config
+		}
+		rc struct {
+			Demo		*redis.Config
+			DemoExpire xtime.Duration
+		}
+		mc struct {
+			Demo		*memcache.Config
+			DemoExpire xtime.Duration
+		}
+	)
+	checkErr(paladin.Get("mysql.toml").UnmarshalTOML(&dc))
+	checkErr(paladin.Get("redis.toml").UnmarshalTOML(&rc))
+	checkErr(paladin.Get("memcache.toml").UnmarshalTOML(&mc))
+	dao = &Dao{
+		// mysql
+		db: sql.NewMySQL(dc.Demo),
+		// redis
+		redis:       redis.NewPool(rc.Demo),
+		redisExpire: int32(time.Duration(rc.DemoExpire) / time.Second),
+		// memcache
+		mc:       memcache.NewPool(mc.Demo),
+		mcExpire: int32(time.Duration(mc.DemoExpire) / time.Second),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.mc.Close()
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping ping the resource.
+func (d *Dao) Ping(ctx context.Context) (err error) {
+	if err = d.pingMC(ctx); err != nil {
+		return
+	}
+	if err = d.pingRedis(ctx); err != nil {
+		return
+	}
+	return d.db.Ping(ctx)
+}
+
+
+func (d *Dao) pingMC(ctx context.Context) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
+		log.Error("conn.Set(PING) error(%v)", err)
+	}
+	return
+}
+
+func (d *Dao) pingRedis(ctx context.Context) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	if _, err = conn.Do("SET", "ping", "pong"); err != nil {
+		log.Error("conn.Set(PING) error(%v)", err)
+	}
+	return
+}

app/job/bbq/cms/internal/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/bbq/cms/internal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/cms/internal/model/model.go

@@ -0,0 +1 @@
+package model

app/job/bbq/cms/internal/server/http/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/bbq/cms/internal/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/cms/internal/service:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/cms/internal/server/http/http.go

@@ -0,0 +1,61 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/bbq/cms/internal/service"
+	"go-common/library/conf/paladin"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	svc *service.Service
+)
+
+// New new a bm server.
+func New(s *service.Service) (engine *bm.Engine) {
+	var(
+		hc struct {
+			Server *bm.ServerConfig
+		}
+	)
+	if err := paladin.Get("http.toml").UnmarshalTOML(&hc); err != nil {
+		if err != paladin.ErrNotExist {
+			panic(err)
+		}
+	}
+	svc = s
+	engine = bm.DefaultServer(hc.Server)
+	initRouter(engine, verify.New(nil))
+	if err := engine.Start(); err != nil {
+		panic(err)
+	}
+	return
+}
+
+func initRouter(e *bm.Engine, v *verify.Verify) {
+	e.Ping(ping)
+	e.Register(register)
+	g := e.Group("/x/cms")
+	{
+		g.GET("/start", v.Verify, howToStart)
+	}
+}
+
+func ping(ctx *bm.Context) {
+	if err := svc.Ping(ctx); err != nil {
+		log.Error("ping error(%v)", err)
+		ctx.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+// example for http request handler.
+func howToStart(c *bm.Context) {
+	c.String(0, "Golang 大法好 !!!")
+}

app/job/bbq/cms/internal/service/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["service.go"],
+    importpath = "go-common/app/job/bbq/cms/internal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/cms/internal/dao:go_default_library",
+        "//library/conf/paladin:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/cms/internal/service/service.go

@@ -0,0 +1,37 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/bbq/cms/internal/dao"
+	"go-common/library/conf/paladin"
+)
+
+// Service service.
+type Service struct {
+	ac  *paladin.Map
+	dao *dao.Dao
+}
+
+// New new a service and return.
+func New() (s *Service) {
+	var ac = new(paladin.TOML)
+	if err := paladin.Watch("application.toml", ac); err != nil {
+		panic(err)
+	}
+	s = &Service{
+		ac:  ac,
+		dao: dao.New(),
+	}
+	return s
+}
+
+// Ping ping the resource.
+func (s *Service) Ping(ctx context.Context) (err error) {
+	return s.dao.Ping(ctx)
+}
+
+// Close close the resource.
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/job/bbq/comment/BUILD

@@ -0,0 +1,20 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/comment/cmd:all-srcs",
+        "//app/job/bbq/comment/internal/dao:all-srcs",
+        "//app/job/bbq/comment/internal/model:all-srcs",
+        "//app/job/bbq/comment/internal/server/http:all-srcs",
+        "//app/job/bbq/comment/internal/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/comment/CHANGELOG.md

@@ -0,0 +1,2 @@
+### v1.0.0
+1. 上线功能xxx

app/job/bbq/comment/CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+# Owner
+daiwei
+
+# Author
+
+# Reviewer

app/job/bbq/comment/OWNERS

@@ -0,0 +1,10 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- daiwei
+labels:
+- bbq
+- job
+- job/bbq/comment
+options:
+  no_parent_owners: true

app/job/bbq/comment/README.md

@@ -0,0 +1,12 @@
+# comment-job
+
+## 项目简介
+1.
+
+## 编译环境
+
+
+## 依赖包
+
+
+## 编译执行

app/job/bbq/comment/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "go-common/app/job/bbq/comment/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/comment/internal/server/http:go_default_library",
+        "//app/job/bbq/comment/internal/service:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/comment/cmd/main.go

@@ -0,0 +1,48 @@
+package main
+
+import (
+	"context"
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/bbq/comment/internal/server/http"
+	"go-common/app/job/bbq/comment/internal/service"
+	"go-common/library/conf/paladin"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+)
+
+func main() {
+	flag.Parse()
+	if err := paladin.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(nil) // debug flag: log.dir={path}
+	defer log.Close()
+	log.Info("comment-job start")
+	ecode.Init(nil)
+	svc := service.New()
+	httpSrv := http.New(svc)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			ctx, cancel := context.WithTimeout(context.Background(), 35*time.Second)
+			defer cancel()
+			httpSrv.Shutdown(ctx)
+			log.Info("comment-job exit")
+			svc.Close()
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/bbq/comment/configs/application.toml

@@ -0,0 +1,2 @@
+
+# This is a TOML document. Boom

app/job/bbq/comment/configs/http.toml

@@ -0,0 +1,4 @@
+
+[server]
+    addr = "0.0.0.0:8000"
+    timeout = "1s"

app/job/bbq/comment/configs/memcache.toml

@@ -0,0 +1,13 @@
+
+demoExpire = "24h"
+
+[demo]
+	name = "comment"
+	proto = "tcp"
+	addr = "127.0.0.1:11211"
+	active = 50
+	idle = 10
+	dialTimeout = "100ms"
+	readTimeout = "200ms"
+	writeTimeout = "300ms"
+	idleTimeout = "80s"

app/job/bbq/comment/configs/mysql.toml

@@ -0,0 +1,11 @@
+
+[demo]
+	addr = "127.0.0.1:3306"
+	dsn = "{user}:{password}@tcp(127.0.0.1:3306)/{database}?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8"
+	readDSN = ["{user}:{password}@tcp(127.0.0.2:3306)/{database}?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8mb4,utf8","{user}:{password}@tcp(127.0.0.3:3306)/{database}?timeout=1s&readTimeout=1s&writeTimeout=1s&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 20
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "200ms"
+	execTimeout = "300ms"
+	tranTimeout = "400ms"

app/job/bbq/comment/configs/redis.toml

@@ -0,0 +1,13 @@
+
+demoExpire = "24h"
+
+[demo]
+	name = "comment"
+	proto = "tcp"
+	addr = "127.0.0.1:6389"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"

app/job/bbq/comment/internal/dao/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["dao.go"],
+    importpath = "go-common/app/job/bbq/comment/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/comment/internal/dao/dao.go

@@ -0,0 +1,95 @@
+package dao
+
+import (
+	"context"
+	"time"
+
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf/paladin"
+	"go-common/library/database/sql"
+	"go-common/library/log"
+	xtime "go-common/library/time"
+)
+
+// Dao dao.
+type Dao struct {
+	db          *sql.DB
+	redis       *redis.Pool
+	redisExpire int32
+	mc          *memcache.Pool
+	mcExpire    int32
+}
+
+func checkErr(err error) {
+	if err != nil {
+		panic(err)
+	}
+}
+
+// New new a dao and return.
+func New() (dao *Dao) {
+	var (
+		dc struct {
+			Demo *sql.Config
+		}
+		rc struct {
+			Demo       *redis.Config
+			DemoExpire xtime.Duration
+		}
+		mc struct {
+			Demo       *memcache.Config
+			DemoExpire xtime.Duration
+		}
+	)
+	checkErr(paladin.Get("mysql.toml").UnmarshalTOML(&dc))
+	checkErr(paladin.Get("redis.toml").UnmarshalTOML(&rc))
+	checkErr(paladin.Get("memcache.toml").UnmarshalTOML(&mc))
+	dao = &Dao{
+		// mysql
+		db: sql.NewMySQL(dc.Demo),
+		// redis
+		redis:       redis.NewPool(rc.Demo),
+		redisExpire: int32(time.Duration(rc.DemoExpire) / time.Second),
+		// memcache
+		mc:       memcache.NewPool(mc.Demo),
+		mcExpire: int32(time.Duration(mc.DemoExpire) / time.Second),
+	}
+	return
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.mc.Close()
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping ping the resource.
+func (d *Dao) Ping(ctx context.Context) (err error) {
+	if err = d.pingMC(ctx); err != nil {
+		return
+	}
+	if err = d.pingRedis(ctx); err != nil {
+		return
+	}
+	return d.db.Ping(ctx)
+}
+
+func (d *Dao) pingMC(ctx context.Context) (err error) {
+	conn := d.mc.Get(ctx)
+	defer conn.Close()
+	if err = conn.Set(&memcache.Item{Key: "ping", Value: []byte("pong"), Expiration: 0}); err != nil {
+		log.Error("conn.Set(PING) error(%v)", err)
+	}
+	return
+}
+
+func (d *Dao) pingRedis(ctx context.Context) (err error) {
+	conn := d.redis.Get(ctx)
+	defer conn.Close()
+	if _, err = conn.Do("SET", "ping", "pong"); err != nil {
+		log.Error("conn.Set(PING) error(%v)", err)
+	}
+	return
+}

app/job/bbq/comment/internal/model/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/bbq/comment/internal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/comment/internal/model/model.go

@@ -0,0 +1 @@
+package model

app/job/bbq/comment/internal/server/http/BUILD

@@ -0,0 +1,35 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["http.go"],
+    importpath = "go-common/app/job/bbq/comment/internal/server/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/comment/internal/service:go_default_library",
+        "//library/conf/paladin:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/comment/internal/server/http/http.go

@@ -0,0 +1,61 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/job/bbq/comment/internal/service"
+	"go-common/library/conf/paladin"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	svc *service.Service
+)
+
+// New new a bm server.
+func New(s *service.Service) (engine *bm.Engine) {
+	var (
+		hc struct {
+			Server *bm.ServerConfig
+		}
+	)
+	if err := paladin.Get("http.toml").UnmarshalTOML(&hc); err != nil {
+		if err != paladin.ErrNotExist {
+			panic(err)
+		}
+	}
+	svc = s
+	engine = bm.DefaultServer(hc.Server)
+	initRouter(engine, verify.New(nil))
+	if err := engine.Start(); err != nil {
+		panic(err)
+	}
+	return
+}
+
+func initRouter(e *bm.Engine, v *verify.Verify) {
+	e.Ping(ping)
+	e.Register(register)
+	g := e.Group("/x/comment")
+	{
+		g.GET("/start", v.Verify, howToStart)
+	}
+}
+
+func ping(ctx *bm.Context) {
+	if err := svc.Ping(ctx); err != nil {
+		log.Error("ping error(%v)", err)
+		ctx.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}
+
+func register(c *bm.Context) {
+	c.JSON(map[string]interface{}{}, nil)
+}
+
+// example for http request handler.
+func howToStart(c *bm.Context) {
+	c.String(0, "Golang 大法好 !!!")
+}

app/job/bbq/comment/internal/service/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["service.go"],
+    importpath = "go-common/app/job/bbq/comment/internal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/comment/internal/dao:go_default_library",
+        "//library/conf/paladin:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/comment/internal/service/service.go

@@ -0,0 +1,37 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/bbq/comment/internal/dao"
+	"go-common/library/conf/paladin"
+)
+
+// Service service.
+type Service struct {
+	ac  *paladin.Map
+	dao *dao.Dao
+}
+
+// New new a service and return.
+func New() (s *Service) {
+	var ac = new(paladin.TOML)
+	if err := paladin.Watch("application.toml", ac); err != nil {
+		panic(err)
+	}
+	s = &Service{
+		ac:  ac,
+		dao: dao.New(),
+	}
+	return s
+}
+
+// Ping ping the resource.
+func (s *Service) Ping(ctx context.Context) (err error) {
+	return s.dao.Ping(ctx)
+}
+
+// Close close the resource.
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/job/bbq/recall/BUILD

@@ -0,0 +1,22 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/recall/api/v1:all-srcs",
+        "//app/job/bbq/recall/cmd:all-srcs",
+        "//app/job/bbq/recall/internal/conf:all-srcs",
+        "//app/job/bbq/recall/internal/dao:all-srcs",
+        "//app/job/bbq/recall/internal/model:all-srcs",
+        "//app/job/bbq/recall/internal/service:all-srcs",
+        "//app/job/bbq/recall/proto:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/CHANGELOG.md

@@ -0,0 +1,24 @@
+### v1.0.7
+1. 新发视频标签时效改为24小时内视频
+
+### v1.0.6
+1. 实时倒排取ctime为当天的视频
+
+### v1.0.5
+1. 增加新发视频实时倒排索引
+2. 正排质量信息增加曝光数
+
+### v1.0.4
+1. 正排标签逻辑优化
+
+### v1.0.3
+1. 正排离线三级标签
+
+### v1.0.2
+1. 正排部署脚本修改
+
+### v1.0.1
+1. 离线数据字段修改
+
+### v1.0.0
+1. BBQ召回正排数据离线计算

app/job/bbq/recall/CONTRIBUTORS.md

@@ -0,0 +1,6 @@
+# Owner
+daiwei
+
+# Author
+
+# Reviewer

app/job/bbq/recall/OWNERS

@@ -0,0 +1,10 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- daiwei
+labels:
+- bbq
+- job
+- job/bbq/recall
+options:
+  no_parent_owners: true

app/job/bbq/recall/README.md

@@ -0,0 +1,12 @@
+# recall-job
+
+## 项目简介
+1.
+
+## 编译环境
+
+
+## 依赖包
+
+
+## 编译执行

app/job/bbq/recall/api/v1/BUILD

@@ -0,0 +1,48 @@
+load(
+    "@io_bazel_rules_go//proto:def.bzl",
+    "go_proto_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+proto_library(
+    name = "v1_proto",
+    srcs = ["api.proto"],
+    tags = ["automanaged"],
+)
+
+go_proto_library(
+    name = "v1_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:go_proto"],
+    importpath = "go-common/app/job/bbq/recall/api/v1",
+    proto = ":v1_proto",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["generate.go"],
+    embed = [":v1_go_proto"],
+    importpath = "go-common/app/job/bbq/recall/api/v1",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/api/v1/api.proto

@@ -0,0 +1,16 @@
+// 定义项目 API 的 proto 文件 可以同时描述 gRPC 和 HTTP API
+// protobuf 文件参考:
+//  - https://developers.google.com/protocol-buffers/
+//  - http://info.bilibili.co/display/documentation/gRPC+Proto
+// protobuf 生成 HTTP 工具:
+//  - http://git.bilibili.co/platform/go-common/tree/master/app/tool/protoc-gen-bm
+syntax = "proto3";
+
+// package 命名使用 {discovery_id}.{version} 的方式, version 形如 v1, v2, v1beta ..
+// NOTE: 不知道的 discovery_id 请询问大佬, 新项目找大佬申请 discovery_id，先到先得抢注
+// e.g. account.service.v1
+// package {discovery_id}.{version}
+
+// NOTE: 最后请删除这些无用的注释 (゜-゜)つロ 
+
+option go_package = "v1";

app/job/bbq/recall/api/v1/generate.go

@@ -0,0 +1,4 @@
+package v1
+
+// 生成 gRPC 代码
+//go:generate $GOPATH/src/go-common/app/tool/warden/protoc.sh

app/job/bbq/recall/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["test.toml"],
+    importpath = "go-common/app/job/bbq/recall/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/recall/internal/conf:go_default_library",
+        "//app/job/bbq/recall/internal/service:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/cmd/main.go

@@ -0,0 +1,61 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	"go-common/app/job/bbq/recall/internal/service"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+var (
+	_serviceName string
+)
+
+func init() {
+	flag.StringVar(&_serviceName, "service", "", "run service name")
+}
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("recall-job start")
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	svc := service.New(conf.Conf)
+	defer svc.Close()
+
+	if _serviceName != "" {
+		svc.RunSrv(_serviceName)
+	} else {
+		svc.InitCron()
+		deamon()
+	}
+}
+
+func deamon() {
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("recall-job exit")
+			time.Sleep(time.Second)
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/bbq/recall/cmd/test.toml

@@ -0,0 +1,83 @@
+[log]
+	stdout = true
+
+[mysql]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq?allowNativePasswords=true&timeout=800ms&readTimeout=1200ms&writeTimeout=800ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq?allowNativePasswords=true&timeout=800ms&readTimeout=1200ms&writeTimeout=800ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 20
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "800ms"
+	execTimeout = "800ms"
+	tranTimeout = "1000ms"
+
+[offlineMysql]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq?allowNativePasswords=true&timeout=800ms&readTimeout=1200ms&writeTimeout=800ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq?allowNativePasswords=true&timeout=800ms&readTimeout=1200ms&writeTimeout=800ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 20
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "800ms"
+	execTimeout = "800ms"
+	tranTimeout = "1000ms"
+
+[cmsMysql]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq_cms?allowNativePasswords=true&timeout=800ms&readTimeout=1200ms&writeTimeout=800ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq_cms?allowNativePasswords=true&timeout=800ms&readTimeout=1200ms&writeTimeout=800ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 20
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "800ms"
+	execTimeout = "800ms"
+	tranTimeout = "1000ms"
+
+[redis]
+	name = "bbq-web"
+	proto = "tcp"
+	addr = "172.16.38.91:6379"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[bfredis]
+	name = "bbq-bf"
+	proto = "tcp"
+	addr = "172.16.38.91:6379"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[berserker]
+	[[berserker.keys]]
+		owner = "daiwei"
+		appkey = "66ed588a742c72408bc2d876afa8f7ca"
+		secret = "7d7a63e37fe6af52a58fb4755b62360e"
+	[[berserker.api]]
+		name = "video_quality"
+		url = "http://berserker.bilibili.co/avenger/api/175/query"
+	[[berserker.api]]
+		name = "video_view"
+		url = "http://berserker.bilibili.co/avenger/api/154/query"
+
+[job]
+	[job.forwardIndex]
+		jobName = "genForwardIndex"
+		schedule = "@every 30m"
+		input = "/Users/daiwei/Desktop/"
+		output = "/Users/daiwei/forward_index.txt"
+	[job.bloomfilter]
+		jobName = "genBloomFilter"
+		schedule = "@every 30m"
+		input = ""
+		output = ""

app/job/bbq/recall/internal/conf/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "conf.go",
+    ],
+    importpath = "go-common/app/job/bbq/recall/internal/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/internal/conf/berserker.go

@@ -0,0 +1,20 @@
+package conf
+
+// BerserkerConfig .
+type BerserkerConfig struct {
+	Keys []*BerserkerKey
+	API  []*BerserkerAPI
+}
+
+// BerserkerAPI .
+type BerserkerAPI struct {
+	Name string
+	URL  string
+}
+
+// BerserkerKey .
+type BerserkerKey struct {
+	Owner  string
+	AppKey string
+	Secret string
+}

app/job/bbq/recall/internal/conf/conf.go

@@ -0,0 +1,113 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/rpc/warden"
+	"go-common/library/net/trace"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log          *log.Config
+	BM           *bm.ServerConfig
+	Verify       *verify.Config
+	Tracer       *trace.Config
+	Redis        *redis.Config
+	BfRedis      *redis.Config
+	MySQL        *sql.Config
+	OfflineMySQL *sql.Config
+	CmsMySQL     *sql.Config
+	Ecode        *ecode.Config
+	Berserker    *BerserkerConfig
+	Job          *Jobs
+	GRPCClient   map[string]*GRPCConfig
+}
+
+// Jobs .
+type Jobs struct {
+	ForwardIndex *JobConfig
+	BloomFilter  *JobConfig
+}
+
+// JobConfig .
+type JobConfig struct {
+	JobName  string
+	Schedule string
+	Input    string
+	Output   string
+}
+
+// GRPCConfig .
+type GRPCConfig struct {
+	WardenConf *warden.ClientConfig
+	Addr       string
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/bbq/recall/internal/dao/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "bloomfilter.go",
+        "dao.go",
+        "hdfs.go",
+        "inverted_index.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/recall/internal/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/recall/internal/conf:go_default_library",
+        "//app/job/bbq/recall/internal/model:go_default_library",
+        "//app/job/bbq/recall/proto:go_default_library",
+        "//app/job/bbq/recall/proto/quality:go_default_library",
+        "//app/service/bbq/recsys-recall/api/grpc/v1:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//vendor/github.com/Dai0522/go-hash/bloomfilter:go_default_library",
+        "//vendor/github.com/golang/snappy:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/internal/dao/berserker.go

@@ -0,0 +1,14 @@
+package dao
+
+import (
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+)
+
+func (d *Dao) berserkerSign(ak, sk, dt, ver string) string {
+	str := fmt.Sprintf("%sappKey%stimestamp%sversion%s%s", sk, ak, dt, ver, sk)
+	b := md5.Sum([]byte(str))
+	sign := hex.EncodeToString(b[:])
+	return sign
+}

app/job/bbq/recall/internal/dao/bloomfilter.go

@@ -0,0 +1,68 @@
+package dao
+
+import (
+	"context"
+	"strings"
+	"time"
+
+	"github.com/Dai0522/go-hash/bloomfilter"
+)
+
+// FetchMidView .
+func (d *Dao) FetchMidView(c context.Context) (result []string, err error) {
+	dt := time.Now().AddDate(0, 0, -1).Format("20060102")
+	hdfs, err := d.scanHDFSPath(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/mid/")
+	if err != nil {
+		return
+	}
+	for _, v := range hdfs.Result {
+		raw, err := d.loadHDFSFile(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/mid/"+v)
+		if err != nil {
+			break
+		}
+		lines := strings.Split(string(*raw), "\n")
+		result = append(result, lines...)
+	}
+	return
+}
+
+// FetchBuvidView .
+func (d *Dao) FetchBuvidView(c context.Context) (result []string, err error) {
+	dt := time.Now().AddDate(0, 0, -1).Format("20060102")
+	hdfs, err := d.scanHDFSPath(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/buvid/")
+	if err != nil {
+		return
+	}
+	for _, v := range hdfs.Result {
+		raw, err := d.loadHDFSFile(c, d.c.Berserker.API[1].URL, d.c.Berserker.Keys[0], "/"+dt+"/buvid/"+v)
+		if err != nil {
+			break
+		}
+		lines := strings.Split(string(*raw), "\n")
+		result = append(result, lines...)
+	}
+	return
+}
+
+// InsertBloomFilter 构建BF，插入redis
+func (d *Dao) InsertBloomFilter(c context.Context, key string, svidList []uint64) error {
+	bfK := "BBQ:BF:V1:" + key
+	bf, err := bloomfilter.New(uint64(len(svidList)), 0.0001)
+	if err != nil {
+		return err
+	}
+	for _, v := range svidList {
+		bf.PutUint64(v)
+	}
+
+	b := bf.Serialized()
+	return d.SetBloomFilter(c, bfK, b)
+}
+
+// SetBloomFilter .
+func (d *Dao) SetBloomFilter(c context.Context, key string, b *[]byte) error {
+	conn := d.bfredis.Get(c)
+	defer conn.Close()
+	_, err := conn.Do("SETEX", []byte(key), 86400, *b)
+	return err
+}

app/job/bbq/recall/internal/dao/dao.go

@@ -0,0 +1,56 @@
+package dao
+
+import (
+	"context"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	recall "go-common/app/service/bbq/recsys-recall/api/grpc/v1"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/net/rpc/warden"
+)
+
+// Dao dao
+type Dao struct {
+	c            *conf.Config
+	redis        *redis.Pool
+	bfredis      *redis.Pool
+	db           *xsql.DB
+	dbOffline    *xsql.DB
+	dbCms        *xsql.DB
+	recallClient recall.RecsysRecallClient
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:            c,
+		redis:        redis.NewPool(c.Redis),
+		bfredis:      redis.NewPool(c.BfRedis),
+		db:           xsql.NewMySQL(c.MySQL),
+		dbOffline:    xsql.NewMySQL(c.OfflineMySQL),
+		dbCms:        xsql.NewMySQL(c.CmsMySQL),
+		recallClient: newRecallClient(c.GRPCClient["recall"]),
+	}
+	return
+}
+
+func newRecallClient(cfg *conf.GRPCConfig) recall.RecsysRecallClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return recall.NewRecsysRecallClient(cc)
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(ctx context.Context) error {
+	// TODO: add mc,redis... if you use
+	return d.db.Ping(ctx)
+}

app/job/bbq/recall/internal/dao/hdfs.go

@@ -0,0 +1,66 @@
+package dao
+
+import (
+	"context"
+	"io/ioutil"
+	"net/http"
+	"net/url"
+	"time"
+
+	"github.com/json-iterator/go"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	"go-common/app/job/bbq/recall/internal/model"
+)
+
+func (d *Dao) queryHDFS(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) {
+	dt := time.Now().Format("2006-01-02 15:04:05")
+	sign := d.berserkerSign(key.AppKey, key.Secret, dt, "1.0")
+
+	params := &url.Values{}
+	params.Set("appKey", key.AppKey)
+	params.Set("timestamp", dt)
+	params.Set("version", "1.0")
+	params.Set("signMethod", "md5")
+	params.Set("sign", sign)
+
+	fileSuffix := struct {
+		FileSuffix string `json:"fileSuffix"`
+	}{
+		FileSuffix: suffix,
+	}
+	j, err := jsoniter.Marshal(fileSuffix)
+	params.Set("query", string(j))
+
+	for retry := 0; retry < 3; retry++ {
+		resp, err := http.DefaultClient.Get(api + "?" + params.Encode())
+		if err != nil {
+			continue
+		}
+
+		b, err := ioutil.ReadAll(resp.Body)
+		resp.Body.Close()
+		if err == nil && len(b) > 0 {
+			result = &b
+			break
+		}
+		// sleep 5s berserker limit
+		time.Sleep(5 * time.Second)
+	}
+
+	return
+}
+
+func (d *Dao) scanHDFSPath(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *model.HDFSResult, err error) {
+	b, err := d.queryHDFS(c, api, key, suffix)
+	if err != nil {
+		return
+	}
+	result = &model.HDFSResult{}
+	err = jsoniter.Unmarshal(*b, result)
+	return
+}
+
+func (d *Dao) loadHDFSFile(c context.Context, api string, key *conf.BerserkerKey, suffix string) (result *[]byte, err error) {
+	return d.queryHDFS(c, api, key, suffix)
+}

app/job/bbq/recall/internal/dao/inverted_index.go

@@ -0,0 +1,16 @@
+package dao
+
+import (
+	"context"
+
+	recall "go-common/app/service/bbq/recsys-recall/api/grpc/v1"
+)
+
+// SetInvertedIndex 倒排写入redis
+func (d *Dao) SetInvertedIndex(c context.Context, key string, svids []int64) error {
+	_, err := d.recallClient.NewIncomeVideo(c, &recall.NewIncomeVideoRequest{
+		Key:   key,
+		SVIDs: svids,
+	})
+	return err
+}

app/job/bbq/recall/internal/dao/video.go

@@ -0,0 +1,143 @@
+package dao
+
+import (
+	"context"
+	"encoding/hex"
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/job/bbq/recall/internal/model"
+	"go-common/app/job/bbq/recall/proto"
+	"go-common/app/job/bbq/recall/proto/quality"
+	"go-common/library/log"
+
+	"github.com/golang/snappy"
+)
+
+const (
+	// _fetchVideo    = "select `id`, `title`, `content`, `mid`, `avid`, `cid`, `pubtime`, `ctime`, `mtime`, `duration`, `state`, `tid`, `sub_tid` from video where pubtime > ? limit ?, ?;"
+	_fetchVideo          = "select `svid`, `title`, `content`, `mid`, `avid`, `cid`, `pubtime`, `ctime`, `mtime`, `duration`, `state`, `tid`, `sub_tid` from video limit ?, ?;"
+	_fetchVideoTag       = "select `id`, `name`, `type` from `tag` where `id` = ? and `status` = 1;"
+	_fetchVideoTagAll    = "select `id`, `name`, `type` from `tag` where `status` = 1;"
+	_fetchVideoTextTag   = "select `tag` from `video_repository` where `svid` = ? limit 1;"
+	_queryVideoQuality   = "select `stat_info` from `video_forward_index_stat_info` where `svid` = ?;"
+	_fetchNewIncomeVideo = "select `svid` from `video` where ctime > ? and state in (%s);"
+)
+
+// FetchVideoInfo .
+func (d *Dao) FetchVideoInfo(c context.Context, offset, size int) (result []*model.Video, err error) {
+	// rows, err := d.db.Query(c, _fetchVideo, ptime.Format("2006-01-02 15:04:05"), offset, size)
+	rows, err := d.db.Query(c, _fetchVideo, offset, size)
+	if err != nil {
+		return nil, err
+	}
+
+	for rows.Next() {
+		tmp := &model.Video{}
+		if err = rows.Scan(&tmp.SVID, &tmp.Title, &tmp.Content, &tmp.MID, &tmp.AVID, &tmp.CID, &tmp.PubTime, &tmp.CTime, &tmp.MTime, &tmp.Duration, &tmp.State, &tmp.TID, &tmp.SubTID); err != nil {
+			log.Error("FetchVideoInfo: %v", err)
+			return nil, err
+		}
+		result = append(result, tmp)
+	}
+
+	return result, nil
+}
+
+// FetchVideoTagAll .
+func (d *Dao) FetchVideoTagAll(c context.Context) (result []*proto.Tag, err error) {
+	result = make([]*proto.Tag, 0)
+	rows, err := d.db.Query(c, _fetchVideoTagAll)
+	if err != nil {
+		return
+	}
+
+	for rows.Next() {
+		tmp := new(proto.Tag)
+		if err = rows.Scan(&tmp.TagID, &tmp.TagName, &tmp.TagType); err != nil {
+			log.Error("FetchVideoTag: %v", err)
+			continue
+		}
+		result = append(result, tmp)
+	}
+
+	return
+}
+
+// FetchVideoTag .
+func (d *Dao) FetchVideoTag(c context.Context, tid int32) (result *proto.Tag, err error) {
+	row := d.db.QueryRow(c, _fetchVideoTag, tid)
+
+	result = new(proto.Tag)
+	if err = row.Scan(&result.TagID, &result.TagName, &result.TagType); err != nil {
+		log.Error("FetchVideoTag: %v", err)
+		return
+	}
+
+	return
+}
+
+// FetchVideoTextTag .
+func (d *Dao) FetchVideoTextTag(c context.Context, svid int64) (result []string, err error) {
+	row := d.dbCms.QueryRow(c, _fetchVideoTextTag, svid)
+	var tags string
+	if err = row.Scan(&tags); err != nil {
+		log.Errorv(c, log.KV("log", "_fetchVideoTextTag failed"), log.KV("error", err), log.KV("svid", svid))
+		return
+	}
+
+	result = strings.Split(tags, ",")
+	return
+}
+
+// FetchVideoQuality .
+func (d *Dao) FetchVideoQuality(c context.Context, svid uint64) (result *quality.VideoQuality, err error) {
+	var raw string
+	row := d.dbOffline.QueryRow(c, _queryVideoQuality, svid)
+	row.Scan(&raw)
+	if raw == "" {
+		return
+	}
+
+	trimed := strings.Trim(raw, "\n")
+	hexDst, err := hex.DecodeString(trimed)
+	if err != nil {
+		log.Error("FetchVideoQuality: %v src[%s] raw[%s]", err, trimed, trimed)
+		return
+	}
+
+	snappyDst, err := snappy.Decode(nil, hexDst)
+	if err != nil {
+		log.Error("FetchVideoQuality: %v src[%s] raw[%s]", err, string(hexDst), trimed)
+		return
+	}
+
+	result = &quality.VideoQuality{}
+	result.Unmarshal(snappyDst)
+	if err != nil {
+		log.Error("FetchVideoQuality: %v src[%s] raw[%s]", err, snappyDst, trimed)
+	}
+	return
+}
+
+// FetchNewincomeVideo .
+func (d *Dao) FetchNewincomeVideo() (res []int64, err error) {
+	duration, _ := time.ParseDuration("-24h")
+	today := time.Now().Add(duration).Format("2006-01-02")
+	_query := fmt.Sprintf(_fetchNewIncomeVideo, strings.Join(model.RecommendVideoState, ","))
+	row, err := d.db.Query(context.Background(), _query, today)
+	if err != nil {
+		return
+	}
+	res = make([]int64, 0)
+	for row.Next() {
+		var tmp int64
+		if err = row.Scan(&tmp); err != nil {
+			return
+		}
+		res = append(res, tmp)
+	}
+
+	return
+}

app/job/bbq/recall/internal/model/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["model.go"],
+    importpath = "go-common/app/job/bbq/recall/internal/model",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/time:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/internal/model/model.go

@@ -0,0 +1,32 @@
+package model
+
+import (
+	xtime "go-common/library/time"
+)
+
+// RecommendVideoState 所有进推荐池的新发视频
+var RecommendVideoState = []string{"5", "4", "3", "1", "0"}
+
+// HDFSResult .
+type HDFSResult struct {
+	Code   int16    `json:"code"`
+	Msg    string   `json:"msg"`
+	Result []string `json:"result"`
+}
+
+// Video .
+type Video struct {
+	SVID     int64
+	Title    string
+	Content  string
+	MID      int64
+	AVID     int64
+	CID      int64
+	PubTime  xtime.Time
+	CTime    xtime.Time
+	MTime    xtime.Time
+	Duration int32
+	State    int16
+	TID      int32
+	SubTID   int32
+}

app/job/bbq/recall/internal/service/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "bloomfilter.go",
+        "forward_index.go",
+        "inverted_index.go",
+        "service.go",
+    ],
+    importpath = "go-common/app/job/bbq/recall/internal/service",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/recall/internal/conf:go_default_library",
+        "//app/job/bbq/recall/internal/dao:go_default_library",
+        "//app/job/bbq/recall/proto:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/golang/snappy:go_default_library",
+        "//vendor/github.com/robfig/cron:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/internal/service/bloomfilter.go

@@ -0,0 +1,52 @@
+package service
+
+import (
+	"context"
+	"strconv"
+	"strings"
+
+	"go-common/library/log"
+)
+
+// GenBloomFilter .
+func (s *Service) GenBloomFilter() {
+	log.Info("run [%s]", "GenBloomFilter")
+	result, err := s.dao.FetchMidView(context.Background())
+	if err != nil {
+		log.Error("FetchMidView: %v", err)
+		return
+	}
+	s.bloomFilter(result)
+
+	result, err = s.dao.FetchBuvidView(context.Background())
+	if err != nil {
+		log.Error("FetchBuvidView: %v", err)
+		return
+	}
+	s.bloomFilter(result)
+	log.Info("finish [%s]", "GenBloomFilter")
+}
+
+func (s *Service) bloomFilter(result []string) {
+	m := make(map[string][]uint64)
+	for _, v := range result {
+		items := strings.Split(v, "\u0001")
+		if len(items) != 2 {
+			continue
+		}
+		if _, ok := m[items[0]]; !ok {
+			m[items[0]] = []uint64{}
+		}
+		svid, _ := strconv.Atoi(items[1])
+		m[items[0]] = append(m[items[0]], uint64(svid))
+	}
+	for k, v := range m {
+		if k == "" {
+			continue
+		}
+		if err := s.dao.InsertBloomFilter(context.Background(), k, v); err != nil {
+			log.Error("InsertBloomFilter: %v", err)
+			continue
+		}
+	}
+}

app/job/bbq/recall/internal/service/forward_index.go

@@ -0,0 +1,135 @@
+package service
+
+import (
+	"context"
+	"encoding/hex"
+	"os"
+	"os/exec"
+
+	"go-common/app/job/bbq/recall/proto"
+	"go-common/library/log"
+
+	"github.com/golang/snappy"
+)
+
+// GenForwardIndex 生产正排索引
+func (s *Service) GenForwardIndex() {
+	log.Info("run [%s]", "GenForwardIndex")
+	c := context.Background()
+	vInfo, err := s.videoBasicInfo(c)
+	if err != nil {
+		log.Error("video info: %v", err)
+		return
+	}
+
+	outputFile, err := os.OpenFile(s.c.Job.ForwardIndex.Output, os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Error("open file: %v", err)
+		return
+	}
+	shadowFile, _ := os.OpenFile(s.c.Job.ForwardIndex.Output+".bak", os.O_RDWR|os.O_CREATE|os.O_TRUNC, 0666)
+	defer outputFile.Close()
+	defer shadowFile.Close()
+
+	for _, v := range vInfo {
+		qu, _ := s.dao.FetchVideoQuality(c, v.SVID)
+		tmp := &proto.ForwardIndex{
+			SVID:         v.SVID,
+			BasicInfo:    v,
+			VideoQuality: qu,
+		}
+		raw, err := tmp.Marshal()
+		if err != nil {
+			log.Error("json marshal: %v", err)
+			continue
+		}
+		_, err = outputFile.WriteString(hex.EncodeToString(snappy.Encode(nil, raw)))
+		if err != nil {
+			log.Error("output: %v", err)
+		}
+		outputFile.Write([]byte("\n"))
+		if err != nil {
+			log.Error("output endline: %v", err)
+		}
+		shadowFile.WriteString(tmp.String())
+		if err != nil {
+			log.Error("shadow: %v", err)
+		}
+		shadowFile.Write([]byte("\n"))
+		if err != nil {
+			log.Error("shadow endline: %v", err)
+		}
+	}
+	exec.Command(s.c.Job.ForwardIndex.Output + ".sh").Run()
+
+	log.Info("finish [GenForwardIndex]")
+
+	s.GenRealTimeInvertedIndex()
+}
+
+func (s *Service) videoBasicInfo(c context.Context) (result []*proto.VideoInfo, err error) {
+	// fetch tag info from db
+	tags, err := s.dao.FetchVideoTagAll(c)
+	if err != nil {
+		return
+	}
+	tagIDMap := make(map[int32]*proto.Tag)
+	tagNameMap := make(map[string]*proto.Tag)
+	for _, v := range tags {
+		tagIDMap[v.TagID] = v
+		tagNameMap[v.TagName] = v
+	}
+
+	// fetch video info from db
+	offset := 0
+	size := 1000
+	basic, err := s.dao.FetchVideoInfo(c, offset, size)
+	if err != nil {
+		log.Error("FetchVideoInfo: %v", err)
+		return
+	}
+	for len(basic) > 0 && err == nil {
+		log.Info("FetchVideoInfo: %v", len(result))
+		for _, v := range basic {
+			vInfo := &proto.VideoInfo{
+				SVID:     uint64(v.SVID),
+				Title:    v.Title,
+				Content:  v.Content,
+				MID:      uint64(v.MID),
+				AVID:     uint64(v.AVID),
+				CID:      uint64(v.CID),
+				PubTime:  v.PubTime.Time().Unix(),
+				CTime:    v.CTime.Time().Unix(),
+				MTime:    v.MTime.Time().Unix(),
+				Duration: uint32(v.Duration),
+				State:    int32(v.State),
+			}
+			vTags := make([]*proto.Tag, 0)
+			// 一级标签
+			if tag, ok := tagIDMap[v.TID]; ok {
+				vTags = append(vTags, tag)
+			}
+			// 二级标签
+			if subTag, ok := tagIDMap[v.SubTID]; ok {
+				vTags = append(vTags, subTag)
+			}
+			// 三级标签
+			if textTags, e := s.dao.FetchVideoTextTag(c, v.SVID); e == nil {
+				for _, v := range textTags {
+					if tmp, ok := tagNameMap[v]; ok {
+						vTags = append(vTags, tmp)
+					}
+				}
+			}
+
+			vInfo.Tags = vTags
+			result = append(result, vInfo)
+		}
+		offset += size
+		basic, err = s.dao.FetchVideoInfo(c, offset, size)
+		if err != nil {
+			log.Error("FetchVideoInfo: %v", err)
+		}
+	}
+	return
+}

app/job/bbq/recall/internal/service/inverted_index.go

@@ -0,0 +1,57 @@
+package service
+
+import (
+	"context"
+	"fmt"
+	"go-common/library/log"
+	"math/rand"
+	"sort"
+	"time"
+)
+
+const (
+	_nblocks     = 5
+	_redisPrefix = "RECALL:NEWPUB:%d"
+)
+
+// GenRealTimeInvertedIndex 实时倒排标签
+func (s *Service) GenRealTimeInvertedIndex() {
+	svids, err := s.dao.FetchNewincomeVideo()
+	if err != nil || svids == nil || len(svids) == 0 {
+		log.Error("GenRealTimeInvertedIndex FetchNewincomeVideo err[%v] svids[%v]", err, svids)
+		return
+	}
+	// svid 乱序
+	rand.Seed(time.Now().Unix())
+	sort.Slice(svids, func(i int, j int) bool {
+		return rand.Float32() > 0.5
+	})
+
+	// 平均分为5份
+	offset := 0
+	blocks := len(svids) / _nblocks
+	invertedIndex := make([][]int64, _nblocks)
+	for i := 0; i < _nblocks; i++ {
+		invertedIndex[i] = make([]int64, 0)
+		for j := 0; j < blocks; j++ {
+			invertedIndex[i] = append(invertedIndex[i], svids[offset+j])
+		}
+		offset = offset + blocks
+	}
+	if blocks*_nblocks < len(svids) {
+		invertedIndex[_nblocks-1] = append(invertedIndex[_nblocks-1], svids[len(svids)-1])
+	}
+
+	log.Info("GenRealTimeInvertedIndex invertedIndex[%v]", invertedIndex)
+
+	// 序列化后写入redis
+	for i, v := range invertedIndex {
+		key := fmt.Sprintf(_redisPrefix, i)
+		err = s.dao.SetInvertedIndex(context.Background(), key, v)
+		if err != nil {
+			log.Error("GenRealTimeInvertedIndex SetInvertedIndex err[%v]", err)
+		}
+	}
+
+	log.Info("finish [GenRealTimeInvertedIndex]")
+}

app/job/bbq/recall/internal/service/service.go

@@ -0,0 +1,64 @@
+package service
+
+import (
+	"context"
+
+	"go-common/app/job/bbq/recall/internal/conf"
+	"go-common/app/job/bbq/recall/internal/dao"
+	"go-common/library/log"
+
+	"github.com/robfig/cron"
+)
+
+// Service struct
+type Service struct {
+	c    *conf.Config
+	dao  *dao.Dao
+	sche *cron.Cron
+}
+
+// New init
+func New(c *conf.Config) (s *Service) {
+	s = &Service{
+		c:    c,
+		dao:  dao.New(c),
+		sche: cron.New(),
+	}
+	return s
+}
+
+// InitCron .
+func (s *Service) InitCron() {
+	s.sche.AddFunc("@every 3s", s.HeartBeat)
+	s.sche.AddFunc(s.c.Job.ForwardIndex.Schedule, s.GenForwardIndex)
+	s.sche.AddFunc(s.c.Job.BloomFilter.Schedule, s.GenBloomFilter)
+	s.sche.Start()
+}
+
+// RunSrv .
+func (s *Service) RunSrv(name string) {
+	log.Info("run job{%s}", name)
+	switch name {
+	case s.c.Job.ForwardIndex.JobName:
+		s.GenForwardIndex()
+	case s.c.Job.BloomFilter.JobName:
+		s.GenBloomFilter()
+	default:
+		s.HeartBeat()
+	}
+}
+
+// HeartBeat .
+func (s *Service) HeartBeat() {
+	log.Info("alive...")
+}
+
+// Ping Service
+func (s *Service) Ping(ctx context.Context) (err error) {
+	return s.dao.Ping(ctx)
+}
+
+// Close Service
+func (s *Service) Close() {
+	s.dao.Close()
+}

app/job/bbq/recall/proto/BUILD

@@ -0,0 +1,64 @@
+load(
+    "@io_bazel_rules_go//proto:def.bzl",
+    "go_proto_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+proto_library(
+    name = "proto_proto",
+    srcs = ["forward_index.proto"],
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/bbq/recall/proto/quality:quality_proto",
+        "@gogo_special_proto//github.com/gogo/protobuf/gogoproto",
+    ],
+)
+
+go_proto_library(
+    name = "proto_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:gogofast_proto"],
+    importpath = "go-common/app/job/bbq/recall/proto",
+    proto = ":proto_proto",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/bbq/recall/proto/quality:quality_go_proto",
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [],
+    embed = [":proto_go_proto"],
+    importpath = "go-common/app/job/bbq/recall/proto",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/recall/proto/quality:go_default_library",
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/recall/proto/quality:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/proto/forward_index.proto

@@ -0,0 +1,38 @@
+syntax = "proto3";
+
+package bbq.job.recall.forwardindex;
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+import "app/job/bbq/recall/proto/quality/quality.proto";
+
+option go_package = "proto";
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+
+message ForwardIndex {
+    uint64 svid = 1 [(gogoproto.customname) = "SVID"];        //视频id
+    VideoInfo basic_info = 2 [(gogoproto.customname) = "BasicInfo"];
+    VideoQuality video_quality = 3 [(gogoproto.customname) = "VideoQuality"]; //月 度视频质量指标
+}
+
+message VideoInfo {
+    uint64 svid = 1 [(gogoproto.customname) = "SVID"];
+    string title = 2 [(gogoproto.customname) = "Title"];
+    string content = 3 [(gogoproto.customname) = "Content"];
+    uint64 mid = 4 [(gogoproto.customname) = "MID"];
+    uint64 avid = 5 [(gogoproto.customname) = "AVID"];
+    uint64 cid = 6 [(gogoproto.customname) = "CID"];
+    int64 pubtime = 7 [(gogoproto.customname) = "PubTime"];
+    int64 ctime = 8 [(gogoproto.customname) = "CTime"];
+    int64 mtime = 9 [(gogoproto.customname) = "MTime"];
+    uint32 duration = 10 [(gogoproto.customname) = "Duration"];
+    int32 state = 11 [(gogoproto.customname) = "State"];
+    repeated Tag tags = 12 [(gogoproto.customname) = "Tags"];
+}
+
+message Tag {
+    string name = 1 [(gogoproto.customname) = "TagName"];
+    int32 type = 2 [(gogoproto.customname) = "TagType"];
+    int32 ID = 3 [(gogoproto.customname) = "TagID"];
+}

app/job/bbq/recall/proto/quality/BUILD

@@ -0,0 +1,54 @@
+load(
+    "@io_bazel_rules_go//proto:def.bzl",
+    "go_proto_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+proto_library(
+    name = "quality_proto",
+    srcs = ["quality.proto"],
+    tags = ["automanaged"],
+    deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
+)
+
+go_proto_library(
+    name = "quality_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:gogofast_proto"],
+    importpath = "go-common/app/job/bbq/recall/proto/quality",
+    proto = ":quality_proto",
+    tags = ["automanaged"],
+    deps = ["@com_github_gogo_protobuf//gogoproto:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [],
+    embed = [":quality_go_proto"],
+    importpath = "go-common/app/job/bbq/recall/proto/quality",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/recall/proto/quality/quality.proto

@@ -0,0 +1,44 @@
+syntax = "proto3";
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option go_package = "quality";
+option (gogoproto.goproto_getters_all) = false;
+option (gogoproto.goproto_stringer_all) = false;
+option (gogoproto.stringer_all) =  true;
+
+message VideoQuality {
+    string svid = 1;        //视频id
+ 
+    Quality quality_info_m = 2; //月 度视频质量指标
+    Quality quality_info_w = 3; //周 视频质量指标
+    Quality quality_info_h = 4; //天 视频质量指标
+ 
+    Quality quality_info_ms_m = 5; //月度 主站视频质量指标
+    Quality quality_info_ms_w = 6; //周 主站视频质量指标
+    Quality quality_info_ms_h = 7; //天 主站视频质量指标
+}
+
+//质量
+message Quality {
+    uint64 pub_time = 1;       //发布时间
+ 
+    uint32 play_cnt = 2;          //播放数
+    uint32 absolute_play_cnt = 3; //播完数
+    uint32 fav_cnt = 4;            //收藏数
+    uint32 like_cnt = 5;           //点赞数
+    uint32 coin_cnt = 6;           //投币数
+    uint32 share_cnt = 7;          //分享数
+    uint32 danmu_cnt = 8;          //弹幕数
+    uint32 neg_eval_cnt = 9;       //负面评价数
+    uint32 comment_add_cnt = 10;   //评论数
+    uint32 comment_like_cnt = 11;  //评论点赞
+    uint32 comment_report_cnt = 12; //评论回复
+ 
+    double absolute_play_rate = 13; //播完率:平滑 播完 / 播放数
+    double like_rate = 14;          //点赞率:平滑 点赞 / 播放数
+    double share_rate = 15;         //分享率:平滑 分享 / 播放数
+    double reply_rate = 16;         //回复率:平滑 (评论 + 评论回复 + 评论点赞) / 播放数
+    
+    uint32 imp_cnt = 17;  //曝光数
+}

app/job/bbq/video/BUILD

@@ -0,0 +1,21 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/job/bbq/video/cmd:all-srcs",
+        "//app/job/bbq/video/conf:all-srcs",
+        "//app/job/bbq/video/dao:all-srcs",
+        "//app/job/bbq/video/model:all-srcs",
+        "//app/job/bbq/video/server/http:all-srcs",
+        "//app/job/bbq/video/service:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/CHANGELOG.md

@@ -0,0 +1,39 @@
+# v1.0.10
+修复user_type的设置
+
+# v1.0.9
+1. fix dead  loop
+2. pr
+# v1.0.8
+修复video消费
+
+# v1.0.7
+1. 增加video表的state消费，同步修改到topic
+
+# v1.0.6
+1. 删除无效方法
+
+# v1.0.5
+1. 增加观看历史bloomfilter小时级重建
+2. 导入视频删除同步b站同步信息
+
+# v1.0.4
+1. 增加binlog补全用户信息
+2. merge master
+
+# v1.0.3
+1. 添加25w临时导入脚本
+2. merge master
+
+# v1.0.2
+1. 修改视频导入逻辑
+2. merge master
+3. 添加 std log
+
+# v1.0.1
+1. 添加批量注册评论服务
+2. 上线功能xxx
+
+# v1.0.6
+1. 删除无效方法
+2. 修复

app/job/bbq/video/CONTRIBUTORS.md

@@ -0,0 +1,9 @@
+# Owner
+yangyucheng
+luxiaowei
+daiwei
+jiangdongqi
+
+# Author
+
+# Reviewer

app/job/bbq/video/OWNERS

@@ -0,0 +1,13 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- daiwei
+- jiangdongqi
+- luxiaowei
+- yangyucheng
+labels:
+- bbq
+- job
+- job/bbq/video
+options:
+  no_parent_owners: true

app/job/bbq/video/README.md

@@ -0,0 +1,15 @@
+# video-job
+
+# 项目简介
+1.BBQ项目脚本
+
+# 编译环境
+
+
+# 依赖包
+
+
+# 编译执行
+```bash
+# go run main.go -conf test.toml -deploy.env uat
+```

app/job/bbq/video/api/http/api.md

@@ -0,0 +1 @@
+# HTTP API文档

app/job/bbq/video/cmd/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    data = ["test.toml"],
+    importpath = "go-common/app/job/bbq/video/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/server/http:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/trace:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/cmd/cids

@@ -0,0 +1 @@
+10150892

app/job/bbq/video/cmd/main.go

@@ -0,0 +1,40 @@
+package main
+
+import (
+	"flag"
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/server/http"
+	"go-common/library/log"
+	"go-common/library/net/trace"
+)
+
+func main() {
+	flag.Parse()
+	if err := conf.Init(); err != nil {
+		panic(err)
+	}
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("start")
+	trace.Init(conf.Conf.Tracer)
+	defer trace.Close()
+	http.Init(conf.Conf)
+	c := make(chan os.Signal, 1)
+	signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
+	for {
+		s := <-c
+		log.Info("get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
+			log.Info("exit")
+			return
+		case syscall.SIGHUP:
+		default:
+			return
+		}
+	}
+}

app/job/bbq/video/cmd/sug

@@ -0,0 +1,2 @@
+111001917,抹茶,bbq_user,0
+4,舞蹈,bbq_video,0

app/job/bbq/video/cmd/test.toml

@@ -0,0 +1,366 @@
+[log]
+    # dir = "/tmp/log/bbq"
+	stdout = true
+
+[download]
+	file = "/data/"
+
+[bm]
+    [bm.server]
+        addr = "0.0.0.0:9999"
+        timeout = "1s"
+    [bm.client]
+        key = "c8c48e784e05acfb"
+        secret = "aa63ee0a10afa358d02a07e7abcec546"
+        dial = "1000ms"
+        timeout = "30s"
+        keepAlive = "60s"
+        timer = 10
+    [bm.client.breaker]
+        window  = "3s"
+        sleep   = "1000ms"
+        bucket  = 10
+        ratio   = 0.5
+        request = 100
+
+[mysql]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 10
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "1000ms"
+	execTimeout = "2000ms"
+	tranTimeout = "2000ms"
+[mysqlcms]
+	addr = "172.16.38.91:3306"
+	dsn = "root:123456@tcp(172.16.38.91:3306)/bbq_cms?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"
+	readDSN = ["root:123456@tcp(172.16.38.91:3306)/bbq_cms?timeout=2000ms&readTimeout=2000ms&writeTimeout=2000ms&parseTime=true&loc=Local&charset=utf8,utf8mb4"]
+	active = 10
+	idle = 10
+	idleTimeout ="4h"
+	queryTimeout = "1000ms"
+	execTimeout = "2000ms"
+	tranTimeout = "2000ms"
+
+[redis]
+	name = "video-job"
+	proto = "tcp"
+	addr = "172.16.38.91:6379"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[bfredis]
+	name = "video-job-bloomfilter"
+	proto = "tcp"
+	addr = "172.16.38.91:6379"
+	idle = 10
+	active = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "1m"
+
+[memcache]
+	name = "video-job"
+	proto = "tcp"
+	addr = ""
+	active = 50
+	idle = 10
+	dialTimeout = "1s"
+	readTimeout = "1s"
+	writeTimeout = "1s"
+	idleTimeout = "10s"
+	expire = "24h"
+
+[grpcClient]
+	[grpcClient.video]
+        addr = "discovery://default/bbq.service.video"
+        [grpcClient.video.wardenconf]
+            dial = "300ms"
+            timeout = "2s"
+    [grpcClient.search]
+		addr = "discovery://default/bbq.service.search"
+		[grpcClient.search.wardenconf]
+        	dial = "100ms"
+        	timeout = "500ms"
+	[grpcClient.account]
+		addr = "discovery://default/account.service"
+        [grpcClient.account.wardenconf]
+            dial = "1s"
+            timeout = "2s"
+    [grpcClient.notice]
+        addr = "discovery://default/bbq.service.notice"
+        [grpcClient.notice.wardenconf]
+            dial = "3000ms"
+            timeout = "5000ms"
+
+[scheduler]
+    # #每天4点，同步全量视频数据到es
+    # checkVideo2ES = "0 0 4 * * *"
+    # #每天0点10分，同步用户画像
+    # syncUserDmg = "0 30 0 * * *"
+    # #每天3点00分，同步主站视频库
+    # SyncVideo = "0 0 3 * * *"
+    # #每天1点00分，同步主站up主信息
+    # SyncUpUserDmg = "0 0 1 * * *"
+    # #心跳测试
+    # Test = "*/1 * * * * *"
+    #每1分，视频增量脚本
+    checkVideo = "0 */1 * * * *"
+    checkVideoSt = "0 */1 * * * *"
+    checkVideoStHv = "0 */1 * * * *"
+    checkVideoTag = "0 */1 * * * *"
+    checkTag = "0 */1 * * * *"
+    #运营筛选视频导入，1分钟check一次信号
+    SyncVideoOper = "0 0 2 * * *"
+    #同步cms任务
+    TransToCheckBack = "1 * * * * *"
+	#每天晚上9点更新userbase
+	SyncUsrSta = "0 30 21 * * *"
+    # checkVideo = "0 */1 * * * *"
+    # checkVideoSt = "0 */1 * * * *"
+    # checkVideoStHv = "0 */1 * * * *"
+    # checkVideoTag = "0 */1 * * * *"
+    # checkTag = "0 */1 * * * *"
+    # #运营筛选视频导入，1分钟check一次信号
+    # SyncVideoOper = "0 0 2 * * *"
+    # #同步cms任务
+    # DeliveryNewVideoToCms = "0 0 3 * * *"
+	## 去重BloomFilter重建
+	videoViewHistory = "0 */1 * * * *"
+	# 系统消息任务
+	SysMsgTask = "0 */1 * * * *"
+	## UsrProfileBbq
+	UserProfileBbq = "0 0 9 * * *"
+
+[berserker]
+	[berserker.key]
+		[berserker.key.yyc]
+		    appkey = "dcf555f4f314ccdfad7cb8a3288f1643"
+		    secret = "4efd3443561e66832b3ba9128a1ad720"
+		[berserker.key.hsc]
+		    appkey = "0bcf67ecf921576350a4f916c1edecbf"
+		    secret = "1003e5a9babe66ef6834fe839ce8223f"
+		[berserker.key.lzq]
+			appkey = "e51d04f43e9c9bca8678dcb4586c76db"
+			secret = "7c53bf6d17c2d51253b927bc057abeae"
+		[berserker.key.lj]
+			appkey = "4ddb82b57e863ebd888713061a08ba10"
+			secret = "49c4e3eb4fd8aff9d3862447113e032f"
+		[berserker.key.dw]
+			appkey = "66ed588a742c72408bc2d876afa8f7ca"
+			secret = "7d7a63e37fe6af52a58fb4755b62360e"
+		[berserker.key.hm]
+			appkey = "0b685519ff66471faf3f000cc67661eb"
+			secret = "e345a3281f0009b51016fab9a9076877"
+	[berserker.api]
+        rankmonthly = "http://berserker.bilibili.co/avenger/api/92/query"
+		rankdaily = "http://berserker.bilibili.co/avenger/api/102/query"
+		userdmg = "http://berserker.bilibili.co/avenger/api/137/query"
+		upuserdmg = "http://berserker.bilibili.co/avenger/api/125/query"
+		operaonce = "http://berserker.bilibili.co/avenger/api/121/query"
+		userbasic = "http://berserker.bilibili.co/avenger/api/147/query"
+		upmid = "http://berserker.bilibili.co/avenger/api/149/query"
+		videoView = "http://berserker.bilibili.co/avenger/api/154/query"
+		UserProfile = "http://berserker.bilibili.co/avenger/api/178/query"
+		UserProfileBuvid = "http://berserker.bilibili.co/avenger/api/186/query"
+
+[tagmap]
+	[tagmap.tagtidmap]
+		1   = "动画"
+		3   = "音乐"
+		4   = "游戏"
+		5   = "娱乐"
+		11  = "电视剧"
+		13  = "番剧"
+		23  = "电影"
+		36  = "科技"
+		119 = "鬼畜"
+		129 = "舞蹈"
+		155 = "时尚"
+		160 = "生活"
+		165 = "广告"
+		167 = "国创"
+		177 = "纪录片"
+		181 = "影视"
+	[tagmap.tagsubtidmap]
+		24=  "MAD·AMV"
+		25=  "MMD·3D"
+		27=  "综合"
+		47=  "短片·手书·配音"
+		28=  "原创音乐"
+		29=  "三次元音乐"
+		30=  "VOCALOID·UTAU"
+		31=  "翻唱"
+		54=  "OP/ED/OST"
+		59=  "演奏"
+		130= "音乐选集"
+		17=  "单机联机"
+		19=  "Mugen"
+		65=  "网络游戏"
+		121= "GMV"
+		136= "音游"
+		171= "电子竞技"
+		172= "手机游戏"
+		173= "桌游棋牌"
+		71=  "综艺"
+		131= "Korea相关"
+		137= "明星"
+		185= "国产剧"
+		187= "海外剧"
+		32=  "完结动画"
+		33=  "连载动画"
+		51=  "资讯"
+		152= "官方延伸"
+		83=  "其他国家"
+		145= "欧美电影"
+		146= "日本电影"
+		147= "国产电影"
+		39=  "演讲• 公开课"
+		95=  "数码"
+		96=  "星海"
+		98=  "机械"
+		122= "野生技术协会"
+		124= "趣味科普人文"
+		176= "汽车"
+		22=  "鬼畜调教"
+		26=  "音MAD"
+		126= "人力VOCALOID"
+		127= "教程演示"
+		20=  "宅舞"
+		154= "三次元舞蹈"
+		156= "舞蹈教程"
+		157= "美妆"
+		158= "服饰"
+		159= "资讯"
+		164= "健身"
+		21=  "日常"
+		75=  "动物圈"
+		76=  "美食圈"
+		138= "搞笑"
+		161= "手工"
+		162= "绘画"
+		163= "运动"
+		174= "其他"
+		175= "ASMR"
+		166= "广告"
+		153= "国产动画"
+		168= "国产原创相关"
+		169= "布袋戏"
+		170= "资讯"
+		37=  "人文历史"
+		178= "科学探索"
+		179= "热血军事"
+		180= "舌尖上的旅行"
+		85=  "短片"
+		86=  "特摄"
+		182= "影视杂谈"
+		183= "影视剪辑"
+		184= "预告·资讯"
+
+[databus]
+    [databus.videosub]
+        key = "36ff3e402f7c310a"
+        secret = "dbd11b140486dc0bc263cf7ec540186c"
+        group = "BBQVideo-BbqBbq-S"
+        topic = "BBQVideo-T"
+        action = "sub"
+        buffer = 1024
+        name = "app-job/video"
+        proto = "tcp"
+        addr = "172.22.33.174:6205 "
+        idle = 1
+        active = 1
+        dialTimeout = "1s"
+        readTimeout = "60s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+	[databus.videosub.discovery]
+			domain = "uat-api.bilibili.co"
+			key = "c8c48e784e05acfb"
+			secret = "aa63ee0a10afa358d02a07e7abcec546"
+			region = "sh"
+			zone = "sh001"
+			env = "uat"
+    [databus.videorep]
+        key = "36ff3e402f7c310a"
+        secret = "dbd11b140486dc0bc263cf7ec540186c"
+        group = "BBQVideoRep-BbqBbq-S"
+        topic = "BBQVideoRep-T"
+        action = "sub"
+        buffer = 1024
+        name = "history"
+        proto = "tcp"
+        addr = "172.22.33.174:6205 "
+        idle = 1
+        active = 1
+        dialTimeout = "1s"
+        readTimeout = "60s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+		[databus.videorep.discovery]
+			domain = "uat-api.bilibili.co"
+			key = "c8c48e784e05acfb"
+			secret = "aa63ee0a10afa358d02a07e7abcec546"
+			region = "sh"
+			zone = "sh001"
+			env = "uat"
+	[databus.bvcsub]
+        key = "36ff3e402f7c310a"
+        secret = "dbd11b140486dc0bc263cf7ec540186c"
+        group = "BBQCms-BbqBbq-S"
+        topic = "BBQCms-T"
+        action = "sub"
+        buffer = 1024
+        name = "history"
+        proto = "tcp"
+        addr = "172.22.33.174:6205"
+        idle = 1
+        active = 1
+        dialTimeout = "1s"
+        readTimeout = "60s"
+        writeTimeout = "1s"
+        idleTimeout = "10s"
+		[databus.bvcsub.discovery]
+			domain = "uat-api.bilibili.co"
+			key = "c8c48e784e05acfb"
+			secret = "aa63ee0a10afa358d02a07e7abcec546"
+			region = "sh"
+			zone = "sh001"
+			env = "uat"
+		
+[mail]
+host = "smtp.exmail.qq.com"
+port = 465
+from = "jiangdongqi@bilibili.com"
+password = "l"
+to = ["jiangdongqi@bilibili.com"]
+
+[urls]
+	reply_reg = "http://uat-api.bilibili.co/x/internal/v2/reply/subject/regist"
+	account = "http://uat-api.bilibili.co/x/internal/v3/account/info"
+	bvc_push = "http://127.0.0.1:8802/bbq/internal/sv/trans/commit"
+
+[ftp]
+ addr = "172.16.33.203:21"
+ user = "work"
+ password = "MhxzKhl"
+ 	[ftp.remotepath]
+		search = "/home/work/open/%s"
+ timeout = "1000ms"
+	[ftp.localpath]
+		search = "/tmp/"
+[vst]
+	tmpstatus = -3
+[path]
+	cids="/Users/Cheney/go/src/go-common/app/job/bbq/video/cmd/cids"
+[subbvccontrol]
+	control = 1

app/job/bbq/video/conf/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/job/bbq/video/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/cache/memcache:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode/tip:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/net/trace:go_default_library",
+        "//library/queue/databus:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/conf/conf.go

@@ -0,0 +1,223 @@
+package conf
+
+import (
+	"errors"
+	"flag"
+	"go-common/library/cache/memcache"
+	"go-common/library/cache/redis"
+	"go-common/library/conf"
+	"go-common/library/database/sql"
+	ecode "go-common/library/ecode/tip"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/verify"
+	"go-common/library/net/trace"
+	"go-common/library/queue/databus"
+	xtime "go-common/library/time"
+
+	"go-common/library/net/rpc/warden"
+
+	"github.com/BurntSushi/toml"
+)
+
+var (
+	confPath string
+	client   *conf.Client
+	// Conf config
+	Conf = &Config{}
+)
+
+// Config .
+type Config struct {
+	Log          *log.Config
+	BM           *HTTPGeneral
+	Verify       *verify.Config
+	Tracer       *trace.Config
+	Redis        *redis.Config
+	BfRedis      *redis.Config
+	Memcache     *memcache.Config
+	MySQL        *sql.Config
+	MySQLCms     *sql.Config
+	MySQLOffline *sql.Config
+	Ecode        *ecode.Config
+	Scheduler    *Scheduler
+	GRPCClient   map[string]*GRPCConf
+	//berserker
+	Berserker     *Berserker
+	TagMap        *TagMap
+	Databus       map[string]*databus.Config
+	Mail          *Mail
+	URLs          map[string]string
+	Download      *Download
+	FTP           *FTP
+	VST           *Vst
+	Path          map[string]string
+	SubBvcControl *Sub
+}
+
+//Sub ...
+type Sub struct {
+	Control int8
+}
+
+//Vst ...
+type Vst struct {
+	TmpStatus int64
+}
+
+// FTP FTP.
+type FTP struct {
+	Addr       string
+	User       string
+	Password   string
+	RemotePath map[string]string
+	Timeout    xtime.Duration
+	LocalPath  map[string]string
+}
+
+//GRPCConf .
+type GRPCConf struct {
+	WardenConf *warden.ClientConfig
+	Addr       string
+}
+
+// Download .
+type Download struct {
+	File string
+}
+
+//Mail ...
+type Mail struct {
+	Host     string
+	Port     int
+	From     string
+	Password string
+	To       []string
+}
+
+//TagMap ...
+type TagMap struct {
+	TagTidMap    map[string]string
+	TagSubTidMap map[string]string
+}
+
+// HTTPGeneral conf
+type HTTPGeneral struct {
+	Server *bm.ServerConfig
+	Client *bm.ClientConfig
+}
+
+// Berserker conf
+type Berserker struct {
+	Key *BerSerkerKeyList
+	API *BerserkerAPI
+}
+
+// BerserkerAPI conf
+type BerserkerAPI struct {
+	Rankdaily        string
+	Userdmg          string
+	Upuserdmg        string
+	Operaonce        string
+	Userbasic        string
+	Upmid            string
+	VideoView        string
+	UserProfile      string
+	UserProfileBuvid string
+}
+
+// BerSerkerKeyList conf
+type BerSerkerKeyList struct {
+	YYC *BerSerkerKey
+	HSC *BerSerkerKey
+	LZQ *BerSerkerKey
+	LJ  *BerSerkerKey
+	DW  *BerSerkerKey
+	HM  *BerSerkerKey
+}
+
+// BerSerkerKey conf
+type BerSerkerKey struct {
+	Appkey string
+	Secret string
+}
+
+//Scheduler .
+type Scheduler struct {
+	CheckVideo2ES         string
+	SyncUserDmg           string
+	Test                  string
+	SyncUpUserDmg         string
+	CheckVideo            string
+	CheckVideoSt          string
+	CheckVideoStHv        string
+	CheckVideoTag         string
+	CheckTag              string
+	SyncVideoOper         string
+	DeliveryNewVideoToCms string
+	SyncUsrSta            string
+	SyncSearch            string
+	VideoViewHistory      string
+	SysMsgTask            string
+	UserProfileBbq        string
+	TransToReview         string
+	TransToCheckBack      string
+}
+
+// Databus .
+type Databus struct {
+	Video    *databus.Config
+	VideoRep *databus.Config
+	BvcSub   *databus.Config
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "default config path")
+}
+
+// Init init conf
+func Init() error {
+	if confPath != "" {
+		return local()
+	}
+	return remote()
+}
+
+func local() (err error) {
+	_, err = toml.DecodeFile(confPath, &Conf)
+	return
+}
+
+func remote() (err error) {
+	if client, err = conf.New(); err != nil {
+		return
+	}
+	if err = load(); err != nil {
+		return
+	}
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			}
+		}
+	}()
+	return
+}
+
+func load() (err error) {
+	var (
+		s       string
+		ok      bool
+		tmpConf *Config
+	)
+	if s, ok = client.Toml2(); !ok {
+		return errors.New("load config center error")
+	}
+	if _, err = toml.Decode(s, &tmpConf); err != nil {
+		return errors.New("could not decode config")
+	}
+	*Conf = *tmpConf
+	return
+}

app/job/bbq/video/dao/BUILD

@@ -0,0 +1,58 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "berserker.go",
+        "cmsvideo.go",
+        "comment.go",
+        "dao.go",
+        "email.go",
+        "notice.go",
+        "user.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/job/bbq/video/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/bbq/video/conf:go_default_library",
+        "//app/job/bbq/video/model:go_default_library",
+        "//app/service/bbq/notice-service/api/v1:go_default_library",
+        "//app/service/bbq/search/api/grpc/v1:go_default_library",
+        "//app/service/bbq/sys-msg/api/v1:go_default_library",
+        "//app/service/bbq/video/api/grpc/v1:go_default_library",
+        "//app/service/main/account/api:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/metadata:go_default_library",
+        "//library/net/rpc/warden:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+        "//vendor/gopkg.in/gomail.v2:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/bbq/video/dao/berserker.go

@@ -0,0 +1,454 @@
+package dao
+
+import (
+	"bytes"
+	"context"
+	"crypto/md5"
+	"encoding/hex"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	"go-common/app/job/bbq/video/model"
+	"go-common/library/conf/env"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	xhttp "net/http"
+	"net/url"
+	"os"
+	"sort"
+	"strings"
+	"time"
+
+	"io/ioutil"
+
+	pkgerr "github.com/pkg/errors"
+)
+
+const (
+	_jobStatusSuccess = 1
+	_jobStatusFailed  = 2
+	_jobStatusDoing   = 3
+	_jobStatusWaiting = 4
+	//_httpHeaderUser     = "x1-bilispy-user"
+	//_httpHeaderColor    = "x1-bilispy-color"
+	//_httpHeaderTimeout  = "x1-bilispy-timeout"
+	_httpHeaderRemoteIP = "x-backend-bili-real-ip"
+	_userAgent          = "User-Agent"
+	_noKickUserAgent    = "yangyucheng@bilibili.com"
+	_queryJSON          = `{"select":[],"where":{"log_date":{"in":["%s"]}},"page":{"limit":1000},"sort":{"play":-1}}`
+	_queryJSONOper      = `{"select":[],"where":{"log_date":{"in":["%s"]},"cid":{"gt":%d}},"page":{"limit":5000},"sort":{"cid":1}}`
+	_hscUserAgent       = "huangshancheng@bilibili.com"
+	_lzqUserAgent       = "liuzhiquan@bilibili.com"
+	_chmUserAgent       = "caiheming@bilibili.com"
+	_ljUserAgent        = "liujin@bilibili.com"
+	//_userDmgQueryJSON   = `{"select":[],"where":{"log_date":{"in":["%s"]},"mid":{"gt":"%s"}},"sort":{"mid":1},"page":{"limit":200}}`
+	_upUserDmgQueryJSON = `{"select":[],"where":{"mid":{"gt":%d}},"sort":{"mid":1},"page":{"limit":200}}`
+	_userDmgQueryHive   = `select mid, gender, age, geo, content_tag, viewed_video, content_zone, content_count, follow_ups from sycpb.hbase_dmp_tag where last_active_date >= %s and length(viewed_video) > 0`
+	_upMidQueryHive     = `select mid from ods.ods_member_relation_stat where log_date = %s  and follower>= 10000 limit 100`
+	//_upMidQueryHive = `{"select":["name":"mid"],"where":{"log_date":{"in":["%s"]},"follower":{"gte":10000}, "pages":{"limit":10}}`
+	_basePathUserProfile      = "/tmp/"
+	_basePathUserProfileBuvid = "/data/"
+)
+
+var (
+	signParams = []string{"appKey", "timestamp", "version"}
+)
+
+// QueryPlayDaily get video play rank list from berserker
+func (d *Dao) QueryPlayDaily(c context.Context, date string) (vlist []*model.VideoHiveInfo, err error) {
+	v := make(url.Values, 8)
+	query := fmt.Sprintf(_queryJSON, date)
+	v.Set("query", query)
+	var res struct {
+		Code   int                   `json:"code"`
+		Result []model.VideoHiveInfo `json:"result"`
+	}
+
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Rankdaily, "", v, d.c.Berserker.Key.YYC, _noKickUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+	if res.Code != 200 || len(res.Result) == 0 {
+		err = ecode.NothingFound
+		log.Warn("Berserker return err, url:%s;res:%d", d.c.Berserker.API.Rankdaily+"?"+v.Encode(), res.Code)
+		return
+	}
+	for _, info := range res.Result {
+		i := info
+		vlist = append(vlist, &i)
+	}
+	return
+}
+
+//QueryOperaVideo query operation video once
+func (d *Dao) QueryOperaVideo(c context.Context, date string, ch chan<- *model.VideoHiveInfo) (err error) {
+	i := int64(0)
+	var mid int64
+	for {
+		v := make(url.Values, 8)
+		var res struct {
+			Code   int                   `json:"code"`
+			Result []model.VideoHiveInfo `json:"result"`
+		}
+		query := fmt.Sprintf(_queryJSONOper, date, i)
+		v.Set("query", query)
+		if err = d.doHTTPGet(c, d.c.Berserker.API.Operaonce, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res); err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+			return
+		}
+		if res.Code == 200 && len(res.Result) == 0 {
+			return
+		}
+		if res.Code != 200 {
+			err = ecode.NothingFound
+			log.Warn("Berserker return err, url:%s;res:%d", d.c.Berserker.API.Operaonce+"?"+v.Encode(), res.Code)
+			return
+		}
+
+		for _, info := range res.Result {
+			ch <- &info
+			mid = info.CID
+		}
+		i = mid
+	}
+}
+
+//QueryUserBasic ...
+func (d *Dao) QueryUserBasic(c context.Context) (jobURL string, err error) {
+
+	v := make(url.Values, 8)
+	var res struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+	query := "{}"
+	v.Set("query", query)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Userbasic, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+	for i, file := range res.Result {
+		query = fmt.Sprintf("{\"fileSuffix\": \"%s\"}", file)
+		v.Set("query", query)
+		bs, err := d.doHTTPGetRaw(c, d.c.Berserker.API.Userbasic, "", v, d.c.Berserker.Key.LZQ, _lzqUserAgent, &res)
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf("/data/basic_profile/part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+		}
+
+	}
+
+	return
+}
+
+//UserProfileGet ...
+func (d *Dao) UserProfileGet(c context.Context) (jobURL []string, err error) {
+	//
+	v := make(url.Values, 8)
+	var res struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+	query := "{}"
+	v.Set("query", query)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.UserProfile, "", v, d.c.Berserker.Key.HM, _chmUserAgent, &res); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+
+	for i, file := range res.Result {
+		query = fmt.Sprintf("{\"fileSuffix\": \"/%s\"}", file)
+		//fmt.Printf("query: %v\n", query)
+		v.Set("query", query)
+
+		time.Sleep(3 * time.Second)
+		var bs []byte
+		bs, err = d.doHTTPGetRaw(c, d.c.Berserker.API.UserProfile, "", v, d.c.Berserker.Key.HM, _chmUserAgent, &res)
+
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf(_basePathUserProfile+"part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+			d.ReadLine(fmt.Sprintf(_basePathUserProfile+"part_%d", i), d.HandlerUserBbqDmg)
+			os.RemoveAll(fmt.Sprintf(_basePathUserProfile+"part_%d", i))
+		}
+
+	}
+
+	time.Sleep(3 * time.Second)
+	v2 := make(url.Values, 8)
+	var res2 struct {
+		Code   int      `json:"code"`
+		Msg    string   `json:"msg"`
+		Result []string `json:"result"`
+	}
+
+	query2 := "{}"
+	v2.Set("query2", query2)
+	if err = d.doHTTPGet(c, d.c.Berserker.API.UserProfileBuvid, "", v2, d.c.Berserker.Key.HM, _chmUserAgent, &res2); err != nil {
+		log.Error("d.doHTTPGet err[%v]", err)
+		return
+	}
+
+	for i, file := range res2.Result {
+		query2 = fmt.Sprintf("{\"fileSuffix\": \"/%s\"}", file)
+		//fmt.Printf("query: %v\n", query)
+		v2.Set("query", query2)
+
+		time.Sleep(3 * time.Second)
+		bs, err := d.doHTTPGetRaw(c, d.c.Berserker.API.UserProfileBuvid, "", v2, d.c.Berserker.Key.HM, _chmUserAgent, &res2)
+
+		if err != nil {
+			log.Error("d.doHTTPGet err[%v]", err)
+		} else {
+			fileName := fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i)
+			if ioutil.WriteFile(fileName, bs, 0644) == nil {
+				log.Info("write file success")
+			} else {
+				log.Error("write file error(%v)", err)
+			}
+			d.ReadLine(fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i), d.HandlerUserBbqDmgBuvid)
+			os.RemoveAll(fmt.Sprintf(_basePathUserProfileBuvid+"part_%d", i))
+		}
+
+	}
+	return
+}
+
+// doHttpRequest make a http request for data platform api
+func (d *Dao) doHTTPGet(c context.Context, uri, realIP string, params url.Values, key *conf.BerSerkerKey, userAgent string, res interface{}) (err error) {
+
+	enc, err := d.berserkeSign(params, key)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+
+	fmt.Printf("Req: %s ", req.URL)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+
+	req.Header.Set(_userAgent, userAgent+" "+env.AppID)
+	if err != nil {
+		return
+	}
+
+	if realIP != "" {
+		req.Header.Set(_httpHeaderRemoteIP, realIP)
+	}
+
+	return d.HTTPClient.Do(c, req, res)
+}
+
+// doHTTPGetRaw make a http request for data platform api
+func (d *Dao) doHTTPGetRaw(c context.Context, uri, realIP string, params url.Values, key *conf.BerSerkerKey, userAgent string, res interface{}) (bs []byte, err error) {
+	enc, err := d.berserkeSign(params, key)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "uri:%s,params:%v", uri, params)
+		return
+	}
+	if enc != "" {
+		uri = uri + "?" + enc
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, uri, nil)
+	if err != nil {
+		err = pkgerr.Wrapf(err, "method:%s,uri:%s", xhttp.MethodGet, uri)
+		return
+	}
+
+	req.Header.Set(_userAgent, userAgent+" "+env.AppID)
+	if err != nil {
+		return
+	}
+
+	if realIP != "" {
+		req.Header.Set(_httpHeaderRemoteIP, realIP)
+	}
+
+	return d.HTTPClient.Raw(c, req)
+}
+
+// Sign calc appkey and appsecret sign.
+func (d *Dao) berserkeSign(params url.Values, key *conf.BerSerkerKey) (query string, err error) {
+	params.Set("appKey", key.Appkey)
+	params.Set("signMethod", "md5")
+	params.Set("timestamp", time.Now().Format("2006-01-02 15:04:05"))
+	params.Set("version", "1.0")
+	tmp := params.Encode()
+	signTmp := d.encode(params)
+	if strings.IndexByte(tmp, '+') > -1 {
+		tmp = strings.Replace(tmp, "+", "%20", -1)
+	}
+	var b bytes.Buffer
+	b.WriteString(key.Secret)
+	b.WriteString(signTmp)
+	b.WriteString(key.Secret)
+	mh := md5.Sum(b.Bytes())
+	// query
+	var qb bytes.Buffer
+	qb.WriteString(tmp)
+	qb.WriteString("&sign=")
+	qb.WriteString(strings.ToUpper(hex.EncodeToString(mh[:])))
+	query = qb.String()
+	return
+}
+
+// Encode encodes the values into ``URL encoded'' form
+// ("bar=baz&foo=quux") sorted by key.
+func (d *Dao) encode(v url.Values) string {
+	if v == nil {
+		return ""
+	}
+	var buf bytes.Buffer
+	keys := make([]string, 0, len(v))
+	for k := range v {
+		keys = append(keys, k)
+	}
+	sort.Strings(keys)
+	for _, k := range keys {
+		found := false
+		for _, p := range signParams {
+			if p == k {
+				found = true
+				break
+			}
+		}
+		if !found {
+			continue
+		}
+		vs := v[k]
+		prefix := k
+		for _, v := range vs {
+			buf.WriteString(prefix)
+			buf.WriteString(v)
+		}
+	}
+	return buf.String()
+}
+
+// QueryUserDmg .
+func (d *Dao) QueryUserDmg(c context.Context) (jobURL string, err error) {
+	logDay := time.Now().AddDate(0, 0, -1).Format("20060102")
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_userDmgQueryHive, logDay))
+	var res struct {
+		Code         int    `json:"code"`
+		Msg          string `json:"msg"`
+		JobStatusURL string `json:"jobStatusUrl"`
+	}
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Userdmg, "", params, d.c.Berserker.Key.HSC, _hscUserAgent, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		log.Error("Berserker user_dmg err(%v)", err)
+		return
+	}
+	jobURL = res.JobStatusURL
+	return
+}
+
+// QueryJobStatus 查询hive脚本执行结果
+func (d *Dao) QueryJobStatus(c context.Context, jobURL string) (urls []string, err error) {
+	var res struct {
+		Code      int      `json:"code"`
+		Msg       string   `json:"msg"`
+		StatusID  int      `json:"statusId"`
+		StatusMsg string   `json:"statusMsg"`
+		HdfsPath  []string `json:"hdfsPath"`
+	}
+	req, err := xhttp.NewRequest(xhttp.MethodGet, jobURL, nil)
+	if err != nil {
+		log.Error("QueryJobStatus NewRequest, err(%v)", err)
+		return
+	}
+	for {
+		if err = d.HTTPClient.Do(c, req, &res); err != nil {
+			log.Error("QueryJobStatus do get failed, joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.Code != 200 {
+			log.Error("QueryJobStatus http code error, joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.StatusID == _jobStatusDoing || res.StatusID == _jobStatusWaiting {
+			//等待1min
+			log.Info("QueryJobStatus got job status %v, joburl(%v)", res.StatusID, jobURL)
+			time.Sleep(60 * time.Second)
+			continue
+		}
+		if res.StatusID == _jobStatusFailed {
+			log.Error("QueryJobStatus got job status failed joburl(%v), err(%v)", jobURL, err)
+			return
+		}
+		if res.StatusID == _jobStatusSuccess {
+			log.Info("QueryJobStatus got job status success joburl(%v), err(%v)", jobURL, err)
+			urls = res.HdfsPath
+			return
+		}
+		if res.StatusID != _jobStatusSuccess && res.StatusID != _jobStatusFailed && res.StatusID != _jobStatusDoing && res.StatusID != _jobStatusWaiting {
+			log.Error("QueryJobStatus got wrong job status status(%v), joburl(%v)", res.StatusID, jobURL)
+			return
+		}
+	}
+}
+
+//QueryUpUserDmg .
+func (d *Dao) QueryUpUserDmg(c context.Context, mid int64) (upUserDmg []*model.UpUserDmg, err error) {
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_upUserDmgQueryJSON, mid))
+	var res struct {
+		Code   int                `json:"code"`
+		Result []*model.UpUserDmg `json:"result"`
+	}
+
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Upuserdmg, "", params, d.c.Berserker.Key.HSC, _hscUserAgent, &res); err != nil {
+		return
+	}
+	if res.Code != 200 {
+		log.Error("Berserker up_user_dmg err(%v)", err)
+		return
+	}
+	upUserDmg = res.Result
+	return
+}
+
+//QueryUpMid .发起hive查询，取粉丝数大于1万的up mid
+func (d *Dao) QueryUpMid(c context.Context, date string) (jobURL string, err error) {
+	params := url.Values{}
+	params.Set("query", fmt.Sprintf(_upMidQueryHive, date))
+	var res struct {
+		Code         int    `json:"code"`
+		Msg          string `json:"msg"`
+		JobStatusURL string `json:"jobStatusUrl"`
+	}
+	if err = d.doHTTPGet(c, d.c.Berserker.API.Upmid, "", params, d.c.Berserker.Key.LJ, _ljUserAgent, &res); err != nil {
+		log.Error("hive QueryUpMid failed, err(%v)", err)
+		return
+	}
+	if res.Code != 200 {
+		fmt.Println(res.Code)
+		log.Error("hive QueryUpMid failed, err(%v), httpcode(%v)", err, res.Code)
+		return
+	}
+	jobURL = res.JobStatusURL
+	fmt.Println(jobURL)
+	return
+}

app/job/bbq/video/dao/cmsvideo.go

@@ -0,0 +1,175 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/job/bbq/video/model"
+	xsql "go-common/library/database/sql"
+	"go-common/library/log"
+)
+
+const (
+	_updatevideostatus    = "update video set state = ? where svid = ?"
+	_updateCmsvideoStatus = "insert into cms_video (svid,cms_status,sv_status,`from`,title,pubtime,mid) values (?,?,?,?,?,?,?) on duplicate key update cms_status = values(cms_status),sv_status=values(sv_status),cms_uname = '',`from` = values(`from`),title=values(title),pubtime=values(pubtime),mid = values(mid)"
+	_selectVideoRows      = "select id,svid,title,mid,`from`,pubtime from video where state = ? and id > ? limit 1000"
+	_updateVR             = "update video_repository set state = ? where svid=?"
+)
+
+//UpdateCms ..
+func (d *Dao) UpdateCms(c context.Context, v *model.VideoRaw) (err error) {
+	if _, err = d.dbCms.Exec(c,
+		_updateCmsvideoStatus,
+		v.SVID,
+		v.State,
+		v.State,
+		v.From,
+		v.Title,
+		v.Pubtime,
+		v.MID,
+	); err != nil {
+		log.Error("DeliveryNewVdieoToCms insert cms_video err,svid : %v,err :%v", v.SVID, err)
+		return
+	}
+	return
+}
+
+//TransToCheckBack ..
+func (d *Dao) TransToCheckBack() (err error) {
+	var (
+		rows  *xsql.Rows
+		count int64
+		id    int64
+		c     = context.Background()
+	)
+	for {
+		if rows, err = d.db.Query(c, _selectVideoRows, model.VideoStPassReview, count); err != nil {
+			log.Error("DeliveryNewVdieoToCms select video failed ,err:%v", err)
+			return
+		}
+		flag := false
+		for rows.Next() {
+			videoinfo := model.VideoInfo{}
+			if err = rows.Scan(
+				&id,
+				&videoinfo.SVID,
+				&videoinfo.Title,
+				&videoinfo.MID,
+				&videoinfo.From,
+				&videoinfo.Pubtime,
+			); err != nil {
+				if err == xsql.ErrNoRows {
+					return
+				}
+				continue
+			}
+			count = id
+			//满足运营导入规则
+			if d.CmsRule(videoinfo.SVID) {
+				if _, err = d.dbCms.Exec(c,
+					_updateCmsvideoStatus,
+					videoinfo.SVID,
+					model.VideoStCheckBack,
+					model.VideoStCheckBack,
+					videoinfo.From,
+					videoinfo.Title,
+					videoinfo.Pubtime,
+					videoinfo.MID,
+				); err != nil {
+					log.Error("DeliveryNewVdieoToCms insert cms_video err,svid : %v,err :%v", videoinfo.SVID, err)
+					continue
+				}
+				if _, err = d.db.Exec(c,
+					_updatevideostatus,
+					model.VideoStCheckBack,
+					videoinfo.SVID,
+				); err != nil {
+					log.Error("DeliveryNewVdieoToCms update video status err : %v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+				if _, err = d.dbCms.Exec(c, _updateVR, model.VideoStCheckBack, videoinfo.SVID); err != nil {
+					log.Error("DeliveryNewVdieoToCms update vr err :%v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+			}
+			flag = true
+		}
+		rows.Close()
+		if !flag {
+			return
+		}
+	}
+}
+
+// CmsRule ...
+func (d *Dao) CmsRule(svid int64) (flag bool) {
+	return true
+}
+
+//TransToReview ...
+func (d *Dao) TransToReview() (err error) {
+	var (
+		rows  *xsql.Rows
+		count int64
+		id    int64
+		c     = context.Background()
+	)
+	for {
+		if rows, err = d.db.Query(c, _selectVideoRows, model.VideoStPendingPassReview, count); err != nil {
+			log.Error("TransToReview select video failed ,err:%v", err)
+			continue
+		}
+		flag := false
+		for rows.Next() {
+			videoinfo := model.VideoInfo{}
+			if err = rows.Scan(
+				&id,
+				&videoinfo.SVID,
+				&videoinfo.Title,
+				&videoinfo.MID,
+				&videoinfo.From,
+				&videoinfo.Pubtime,
+			); err != nil {
+				if err == xsql.ErrNoRows {
+					return
+				}
+				continue
+			}
+			count = id
+			//满足运营导入规则
+			if d.CmsRule(videoinfo.SVID) {
+				var st int
+				if videoinfo.From == model.VideoFromBILI || videoinfo.From == model.VideoFromCMS {
+					st = model.VideoStPassReview
+				} else {
+					st = model.VideoStPassReviewReject
+				}
+
+				if _, err = d.dbCms.Exec(c,
+					_updateCmsvideoStatus,
+					videoinfo.SVID,
+					st,
+					st,
+					videoinfo.From,
+					videoinfo.Title,
+					videoinfo.Pubtime,
+					videoinfo.MID,
+				); err != nil {
+					log.Error("TransToReview insert cms_video err,svid : %v,err :%v", videoinfo.SVID, err)
+					continue
+				}
+				if _, err = d.db.Exec(c,
+					_updatevideostatus,
+					st,
+					videoinfo.SVID,
+				); err != nil {
+					log.Error("TransToReview update video status err : %v,svid : %v", err, videoinfo.SVID)
+					continue
+				}
+			}
+			flag = true
+		}
+		rows.Close()
+		if !flag {
+			return
+		}
+	}
+}

app/job/bbq/video/dao/comment.go

@@ -0,0 +1,13 @@
+package dao
+
+import (
+	"context"
+	"go-common/library/net/metadata"
+)
+
+//ReplyReg 评论注册/冻结
+func (d *Dao) ReplyReg(c context.Context, req map[string]interface{}) (err error) {
+	ip := metadata.String(c, metadata.RemoteIP)
+	_, err = replyHTTPCommon(c, d.HTTPClient, d.c.URLs["reply_reg"], "POST", req, ip)
+	return
+}

app/job/bbq/video/dao/dao.go

@@ -0,0 +1,157 @@
+package dao
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"go-common/app/job/bbq/video/conf"
+	notice "go-common/app/service/bbq/notice-service/api/v1"
+	searchv1 "go-common/app/service/bbq/search/api/grpc/v1"
+	videov1 "go-common/app/service/bbq/video/api/grpc/v1"
+	account "go-common/app/service/main/account/api"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/rpc/warden"
+	"net/url"
+	"reflect"
+	"strconv"
+
+	jsoniter "github.com/json-iterator/go"
+	gomail "gopkg.in/gomail.v2"
+)
+
+// Dao dao
+type Dao struct {
+	c             *conf.Config
+	redis         *redis.Pool
+	bfredis       *redis.Pool
+	db            *xsql.DB
+	dbCms         *xsql.DB
+	HTTPClient    *bm.Client
+	SearchClient  searchv1.SearchClient
+	VideoClient   videov1.VideoClient
+	AccountClient account.AccountClient
+	email         *gomail.Dialer
+	noticeClient  notice.NoticeClient
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:             c,
+		redis:         redis.NewPool(c.Redis),
+		bfredis:       redis.NewPool(c.BfRedis),
+		db:            xsql.NewMySQL(c.MySQL),
+		dbCms:         xsql.NewMySQL(c.MySQLCms),
+		HTTPClient:    bm.NewClient(c.BM.Client),
+		SearchClient:  newSearchClient(c.GRPCClient["search"]),
+		VideoClient:   newVideoClient(c.GRPCClient["video"]),
+		AccountClient: newAccountClient(c.GRPCClient["account"]),
+		email:         gomail.NewDialer(c.Mail.Host, c.Mail.Port, c.Mail.From, c.Mail.Password),
+		noticeClient:  newNoticeClient(c.GRPCClient["notice"]),
+	}
+	return
+}
+
+// newNoticeClient .
+func newNoticeClient(cfg *conf.GRPCConf) notice.NoticeClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return notice.NewNoticeClient(cc)
+}
+
+//newSearchClient .
+func newSearchClient(cfg *conf.GRPCConf) searchv1.SearchClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return searchv1.NewSearchClient(cc)
+}
+
+//newAccountClient .
+func newAccountClient(cfg *conf.GRPCConf) account.AccountClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return account.NewAccountClient(cc)
+}
+
+//newVideoClient
+func newVideoClient(cfg *conf.GRPCConf) videov1.VideoClient {
+	cc, err := warden.NewClient(cfg.WardenConf).Dial(context.Background(), cfg.Addr)
+	if err != nil {
+		panic(err)
+	}
+	return videov1.NewVideoClient(cc)
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return d.db.Ping(c)
+}
+
+// BeginTran begin mysql transaction
+func (d *Dao) BeginTran(c context.Context) (*xsql.Tx, error) {
+	return d.db.Begin(c)
+}
+
+// ReplyHTTPCommon 评论公用请求
+func replyHTTPCommon(c context.Context, httpClient *bm.Client, path string, method string, data map[string]interface{}, ip string) (r []byte, err error) {
+
+	params := url.Values{}
+	t := reflect.TypeOf(data).Kind()
+	if t == reflect.Map {
+		for k, v := range data {
+			// params.Set(k, v.(string))
+			switch reflect.TypeOf(v).Kind() {
+			case reflect.Int64:
+				params.Set(k, strconv.FormatInt(v.(int64), 10))
+			case reflect.Int16:
+				params.Set(k, strconv.FormatInt(int64(v.(int16)), 10))
+			case reflect.String:
+				params.Set(k, v.(string))
+			case reflect.Int:
+				params.Set(k, strconv.FormatInt(int64(v.(int)), 10))
+			}
+		}
+	}
+	log.V(5).Infov(c, log.KV("log", fmt.Sprintf("reply req url(%s)", path+"?"+params.Encode())))
+	req, err := httpClient.NewRequest(method, path, ip, params)
+	if err != nil {
+		log.Errorv(c, log.KV("log", fmt.Sprintf("reply url(%s) error(%v)", path+"?"+params.Encode(), err)))
+		return
+	}
+	var res struct {
+		Code int             `json:"code"`
+		Msg  string          `json:"message"`
+		Data json.RawMessage `json:"data"`
+	}
+	var json = jsoniter.ConfigCompatibleWithStandardLibrary
+	if err = httpClient.Do(c, req, &res); err != nil {
+		str, _ := json.Marshal(res)
+		log.Errorv(c, log.KV("log", fmt.Sprintf("reply ret data(%s) err[%v]", str, err)))
+		return
+	}
+	str, _ := json.Marshal(res)
+	log.V(5).Infov(c, log.KV("log", fmt.Sprintf("reply ret data(%s)", str)))
+	if res.Code != 0 {
+		err = ecode.Int(res.Code)
+		log.Warnv(c, log.KV("log", fmt.Sprintf("reply url(%s) error(%v)", path+"?"+params.Encode(), err)))
+	}
+	r = res.Data
+	return
+}