Initial commit

app/job/main/videoup-report/dao/redis/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = [
+        "dao_test.go",
+        "redis_test.go",
+    ],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/job/main/videoup-report/conf:go_default_library",
+        "//app/job/main/videoup-report/model/task:go_default_library",
+        "//vendor/github.com/smartystreets/goconvey/convey:go_default_library",
+    ],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "monitor.go",
+        "redis.go",
+        "task.go",
+    ],
+    importpath = "go-common/app/job/main/videoup-report/dao/redis",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/job/main/videoup-report/conf:go_default_library",
+        "//app/job/main/videoup-report/model/task:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/job/main/videoup-report/dao/redis/dao.go

@@ -0,0 +1,31 @@
+package redis
+
+import (
+	"go-common/app/job/main/videoup-report/conf"
+	"go-common/library/cache/redis"
+)
+
+// Dao is redis dao.
+type Dao struct {
+	c         *conf.Config
+	redis     *redis.Pool
+	secondary *redis.Pool
+}
+
+// New new a archive dao.
+func New(c *conf.Config) (d *Dao) {
+	d = &Dao{
+		c:         c,
+		redis:     redis.NewPool(c.Redis.Track.Config),
+		secondary: redis.NewPool(c.Redis.Secondary.Config),
+	}
+	return d
+}
+
+// Close close the redis connection
+func (d *Dao) Close() (err error) {
+	if err = d.secondary.Close(); err != nil {
+		return
+	}
+	return d.redis.Close()
+}

app/job/main/videoup-report/dao/redis/dao_test.go

@@ -0,0 +1,19 @@
+package redis
+
+import (
+	"flag"
+	"path/filepath"
+
+	"go-common/app/job/main/videoup-report/conf"
+)
+
+var (
+	d *Dao
+)
+
+func init() {
+	dir, _ := filepath.Abs("../../cmd/videoup-report-job.toml")
+	flag.Set("conf", dir)
+	conf.Init()
+	d = New(conf.Conf)
+}

app/job/main/videoup-report/dao/redis/monitor.go

@@ -0,0 +1,52 @@
+package redis
+
+import (
+	"context"
+	"go-common/library/log"
+	"time"
+)
+
+// AddMonitorStats add stay stats
+func (d *Dao) AddMonitorStats(c context.Context, key string, oid int64) (err error) {
+	var (
+		conn = d.secondary.Get(c)
+		now  = time.Now().Unix()
+		age  = 7 * 24 * 60 * 60
+	)
+	defer conn.Close()
+	if _, err = conn.Do("ZADD", key, now, oid); err != nil {
+		log.Error("conn.Do(ZADD, %s, %d, %d) error(%v)", key, now, oid, err)
+		return
+	}
+	if _, err = conn.Do("EXPIRE", key, age); err != nil {
+		log.Error("conn.Do(EXPIRE, %s, %d) error(%v)", key, age, err)
+	}
+	return
+}
+
+// RemMonitorStats remove stay stats
+func (d *Dao) RemMonitorStats(c context.Context, key string, oid int64) (err error) {
+	var (
+		conn = d.secondary.Get(c)
+	)
+	defer conn.Close()
+	if _, err = conn.Do("ZREM", key, oid); err != nil {
+		log.Error("conn.Do(ZREM, %s, %d) error(%v)", key, oid, err)
+	}
+	return
+}
+
+// ClearMonitorStats clear expire stats
+func (d *Dao) ClearMonitorStats(c context.Context, key string) (err error) {
+	var (
+		conn = d.secondary.Get(c)
+		now  = time.Now().Unix()
+		min  int64
+		max  = now - 7*24*60*60
+	)
+	defer conn.Close()
+	if _, err = conn.Do("ZREMRANGEBYSCORE", key, min, max); err != nil {
+		log.Error("conn.Do(ZREMRANGEBYSCORE, %s, %d, %d) error(%v)", key, min, max, err)
+	}
+	return
+}

app/job/main/videoup-report/dao/redis/redis.go

@@ -0,0 +1,91 @@
+package redis
+
+import (
+	"context"
+	"fmt"
+	"go-common/library/log"
+	"time"
+)
+
+const (
+	_videoJamTime = "va_v_jam_time"
+)
+
+// SetVideoJam set video traffic jam time
+func (d *Dao) SetVideoJam(c context.Context, jamTime int) (err error) {
+
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("SET", _videoJamTime, jamTime); err != nil {
+		log.Error("conn.Do(SET, %s, %d) error(%v)", _videoJamTime, jamTime, err)
+	}
+	return
+}
+
+// TrackAddRedis add track redis
+func (d *Dao) TrackAddRedis(c context.Context, key, value string) (err error) {
+	if key == "" {
+		log.Warn("TrackAddRedis add empty key(%s) value(%v)", key, value)
+		return fmt.Errorf("empty key")
+	}
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("ZADD", key, time.Now().Unix(), value); err != nil {
+		log.Error("conn.Do(ZADD, %s, %s) error(%v)", key, value, err)
+	}
+	return
+}
+
+// TrackAddVideosRedis add track video redis
+func (d *Dao) TrackAddVideosRedis(c context.Context, keys []string, value string) (err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	for _, key := range keys {
+		if err = conn.Send("ZADD", key, time.Now().Unix(), value); err != nil {
+			log.Error("conn.Send(ZADD, %s, %s) error(%v)", key, value, err)
+		}
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("add conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < len(keys); i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("add conn.Receive(%d) error(%v)", i+1, err)
+			return
+		}
+	}
+	return
+}
+
+// TrackRemRedis remove track redis
+func (d *Dao) TrackRemRedis(c context.Context, key, value string) (err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	if _, err = conn.Do("ZREM", key, value); err != nil {
+		log.Error("conn.Do(ZREM, %s, %s) error(%v)", key, value, err)
+	}
+	return
+}
+
+// TrackRemVideosRedis remove video track redis
+func (d *Dao) TrackRemVideosRedis(c context.Context, keys []string, value string) (err error) {
+	var conn = d.redis.Get(c)
+	defer conn.Close()
+	for _, key := range keys {
+		if err = conn.Send("ZREM", key, value); err != nil {
+			log.Error("conn.Send(ZREM, %s, %s) error(%v)", key, value, err)
+		}
+	}
+	if err = conn.Flush(); err != nil {
+		log.Error("add conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < len(keys); i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("add conn.Receive(%d) error(%v)", i+1, err)
+			return
+		}
+	}
+	return
+}

app/job/main/videoup-report/dao/redis/redis_test.go

@@ -0,0 +1,47 @@
+package redis
+
+import (
+	"context"
+	"testing"
+
+	tmod "go-common/app/job/main/videoup-report/model/task"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func Test_SetVideoJam(t *testing.T) {
+	Convey("SetVideoJam", t, func() {
+		_ = d.SetVideoJam(context.TODO(), 1)
+	})
+}
+
+func Test_TrackAddRedis(t *testing.T) {
+	Convey("TrackAddRedis", t, func() {
+		err := d.TrackAddRedis(context.TODO(), "", "")
+		So(err, ShouldNotBeNil)
+	})
+}
+
+func Test_SetWeight(t *testing.T) {
+	Convey("SetWeight", t, func() {
+		err := d.SetWeight(context.TODO(), map[int64]*tmod.WeightParams{
+			1: &tmod.WeightParams{
+				TaskID: 1,
+				Weight: 1,
+			},
+			2: &tmod.WeightParams{
+				TaskID: 2,
+				Weight: 2,
+			},
+		})
+		So(err, ShouldBeNil)
+	})
+}
+
+func Test_GetWeight(t *testing.T) {
+	Convey("GetWeight", t, func() {
+		mcases, err := d.GetWeight(context.TODO(), []int64{1, 2})
+		So(err, ShouldBeNil)
+		So(mcases, ShouldNotBeNil)
+	})
+}

app/job/main/videoup-report/dao/redis/task.go

@@ -0,0 +1,85 @@
+package redis
+
+import (
+	"context"
+	"encoding/json"
+	"fmt"
+	"time"
+
+	tmod "go-common/app/job/main/videoup-report/model/task"
+	"go-common/library/cache/redis"
+	"go-common/library/log"
+)
+
+const (
+	_twexpire = 24 * 60 * 60 // 1 day
+)
+
+func key(id int64) string {
+	return fmt.Sprintf("tw_%d", id)
+}
+
+//SetWeight 设置权重配置
+func (d *Dao) SetWeight(c context.Context, mcases map[int64]*tmod.WeightParams) (err error) {
+	conn := d.secondary.Get(c)
+	defer conn.Close()
+
+	log.Info("SetWeight before len(%d) time(%v)", len(mcases), time.Now())
+	for tid, mcase := range mcases {
+		var bs []byte
+		key := key(tid)
+		if bs, err = json.Marshal(mcase); err != nil {
+			log.Error("json.Marshal(%+v) error(%v)", mcase, err)
+			continue
+		}
+
+		if err = conn.Send("SET", key, bs); err != nil {
+			log.Error("SET error(%v)", err)
+			continue
+		}
+		if err = conn.Send("EXPIRE", key, _twexpire); err != nil {
+			log.Error("EXPIRE error(%v)", err)
+			continue
+		}
+	}
+
+	if err = conn.Flush(); err != nil {
+		log.Error("conn.Flush error(%v)", err)
+		return
+	}
+	for i := 0; i < 2*len(mcases); i++ {
+		if _, err = conn.Receive(); err != nil {
+			log.Error("conn.Receive() error(%v)", err)
+			return
+		}
+	}
+	log.Info("SetWeight end len(%d) time(%v)", len(mcases), time.Now())
+	return
+}
+
+//GetWeight 获取实时任务的权重配置
+func (d *Dao) GetWeight(c context.Context, ids []int64) (mcases map[int64]*tmod.WeightParams, err error) {
+	conn := d.secondary.Get(c)
+	defer conn.Close()
+	mcases = make(map[int64]*tmod.WeightParams)
+	for _, id := range ids {
+		var bs []byte
+		key := key(int64(id))
+		if bs, err = redis.Bytes(conn.Do("GET", key)); err != nil {
+			if err == redis.ErrNil {
+				err = nil
+			} else {
+				log.Error("conn.Do(GET, %v) error(%v)", key, err)
+			}
+			continue
+		}
+		p := &tmod.WeightParams{}
+		if err = json.Unmarshal(bs, p); err != nil {
+			log.Error("json.Unmarshal(%s) error(%v)", string(bs), err)
+			err = nil
+			continue
+		}
+		mcases[int64(id)] = p
+	}
+	return
+}