Initial commit

app/service/bbq/search/dao/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "dao.go",
+        "video.go",
+    ],
+    importpath = "go-common/app/service/bbq/search/dao",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/bbq/search/api/grpc/v1:go_default_library",
+        "//app/service/bbq/search/conf:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/database/sql:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/json-iterator/go:go_default_library",
+        "//vendor/gopkg.in/olivere/elastic.v5:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/bbq/search/dao/dao.go

@@ -0,0 +1,97 @@
+package dao
+
+import (
+	"context"
+	"net"
+	"net/http"
+	"time"
+
+	"fmt"
+	"go-common/app/service/bbq/search/conf"
+	"go-common/library/cache/redis"
+	xsql "go-common/library/database/sql"
+	"gopkg.in/olivere/elastic.v5"
+)
+
+// Dao dao
+type Dao struct {
+	c          *conf.Config
+	redis      *redis.Pool
+	db         *xsql.DB
+	esPool     map[string]*elastic.Client
+	httpClient []*http.Client
+}
+
+// New init mysql db
+func New(c *conf.Config) (dao *Dao) {
+	dao = &Dao{
+		c:          c,
+		redis:      redis.NewPool(c.Redis),
+		db:         xsql.NewMySQL(c.MySQL),
+		esPool:     newEsPool(c.Es),
+		httpClient: dao.createHTTPClient(),
+	}
+	dao.createESIndex(_bbqEsName, _videoIndex, _videoMapping)
+	return
+}
+
+// newEsPool new es cluster action
+func newEsPool(esInfo map[string]*conf.Es) (esCluster map[string]*elastic.Client) {
+	esCluster = make(map[string]*elastic.Client)
+	for esName, e := range esInfo {
+		client, err := elastic.NewClient(elastic.SetURL(e.Addr...), elastic.SetSniff(false))
+		if err != nil {
+			panic(fmt.Sprintf("es:集群连接失败, cluster: %s, %v", esName, err))
+		}
+		esCluster[esName] = client
+	}
+	return
+}
+
+// createESIndex 初始化es索引
+func (d *Dao) createESIndex(esName, index, mapping string) {
+	exist, err := d.esPool[esName].IndexExists(index).Do(context.Background())
+	if err != nil {
+		panic(fmt.Sprintf("check if index exists, name(%s) error (%v)", esName, err))
+	}
+	if exist {
+		return
+	}
+	if _, err = d.esPool[esName].CreateIndex(index).Body(mapping).Do(context.Background()); err != nil {
+		panic(fmt.Sprintf("create index, name(%s) error (%v)", esName, err))
+	}
+}
+
+// createHTTPClient .
+func (d *Dao) createHTTPClient() []*http.Client {
+	clients := make([]*http.Client, 0)
+	for i := 0; i < 10; i++ {
+		client := &http.Client{
+			Transport: &http.Transport{
+				Proxy: http.ProxyFromEnvironment,
+				DialContext: (&net.Dialer{
+					Timeout:   30 * time.Second,
+					KeepAlive: 30 * time.Second,
+				}).DialContext,
+				MaxIdleConns:        100,
+				MaxIdleConnsPerHost: 100,
+				IdleConnTimeout:     30 * time.Second,
+			},
+			Timeout: 2 * time.Second,
+		}
+		clients = append(clients, client)
+	}
+	return clients
+}
+
+// Close close the resource.
+func (d *Dao) Close() {
+	d.redis.Close()
+	d.db.Close()
+}
+
+// Ping dao ping
+func (d *Dao) Ping(c context.Context) error {
+	// TODO: if you need use mc,redis, please add
+	return d.db.Ping(c)
+}

app/service/bbq/search/dao/video.go

@@ -0,0 +1,158 @@
+package dao
+
+import (
+	"context"
+	"go-common/app/service/bbq/search/api/grpc/v1"
+	"go-common/app/service/bbq/search/conf"
+	"go-common/library/log"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"strconv"
+	"strings"
+	"time"
+
+	"github.com/json-iterator/go"
+
+	"gopkg.in/olivere/elastic.v5"
+)
+
+const (
+	_bbqEsName    = "bbq"
+	_videoIndex   = "video"
+	_videoType    = "video_info"
+	_videoMapping = `
+{
+	"settings":{
+		"number_of_shards":5,
+		"number_of_replicas":2,
+		"index":{
+			"analysis.analyzer.default.type":"ik_smart"
+		}
+	}
+}
+`
+)
+
+//SaveVideo 保存视频信息
+func (d *Dao) SaveVideo(c context.Context, videos *v1.SaveVideoRequest) (err error) {
+	d.createESIndex(_bbqEsName, _videoIndex, _videoMapping)
+	bulkRequest := d.esPool[_bbqEsName].Bulk()
+	for _, v := range videos.List {
+		request := elastic.NewBulkUpdateRequest().Index(_videoIndex).Type(_videoType).Id(strconv.Itoa(int(v.SVID))).Doc(v).DocAsUpsert(true)
+		bulkRequest.Add(request)
+	}
+	if _, err = bulkRequest.Do(c); err != nil {
+		log.Error("save es [%d] err(%v)", 1, err)
+	}
+	return
+}
+
+//RecVideoDataElastic 获取视频信息
+func (d *Dao) RecVideoDataElastic(c context.Context, query elastic.Query, script *elastic.ScriptSort, from, size int) (total int64, list []*v1.RecVideoInfo, err error) {
+	search := d.esPool[_bbqEsName].Search().Index(_videoIndex).Type(_videoType)
+	if query != nil {
+		search.Query(query)
+	}
+	if script != nil {
+		search.SortBy(script)
+	}
+	log.Error("start time(%d)", time.Now().UnixNano())
+	res, err := search.From(from).Size(size).Timeout(d.c.Es[_bbqEsName].Timeout).Do(c)
+	if err != nil {
+		log.Error("video search es (%s) err(%v)", _bbqEsName, err)
+		return
+	}
+	log.Error("do time(%d)", time.Now().UnixNano())
+	total = res.TotalHits()
+	list = []*v1.RecVideoInfo{}
+	for _, value := range res.Hits.Hits {
+		tmp := new(v1.RecVideoInfo)
+		byte, _ := jsoniter.Marshal(value.Source)
+		jsoniter.Unmarshal(byte, tmp)
+		if value.Score != nil {
+			tmp.ESScore = float64(*value.Score)
+		}
+		if value.Sort != nil {
+			for _, v := range value.Sort {
+				tmp.CustomScore = append(tmp.CustomScore, v.(float64))
+			}
+		}
+		list = append(list, tmp)
+	}
+	return
+}
+
+//VideoData 获取视频信息
+func (d *Dao) VideoData(c context.Context, query elastic.Query, from, size int) (total int64, list []*v1.VideoESInfo, err error) {
+	res, err := d.esPool[_bbqEsName].Search().Index(_videoIndex).Type(_videoType).Query(query).From(from).Size(size).Timeout(d.c.Es[_bbqEsName].Timeout).Do(c)
+	if err != nil {
+		log.Error("video search es (%s) err(%v)", _bbqEsName, err)
+		return
+	}
+	total = res.TotalHits()
+	list = []*v1.VideoESInfo{}
+	for _, value := range res.Hits.Hits {
+		tmp := new(v1.VideoESInfo)
+		byte, _ := jsoniter.Marshal(value.Source)
+		jsoniter.Unmarshal(byte, tmp)
+		list = append(list, tmp)
+	}
+	return
+}
+
+//ESVideoData 获取视频信息
+func (d *Dao) ESVideoData(c context.Context, query string) (total int64, list []*v1.RecVideoInfo, err error) {
+	i := rand.Intn(len(conf.Conf.Es["bbq"].Addr))
+	req, err := http.NewRequest("POST", conf.Conf.Es["bbq"].Addr[i]+"/video/_search", strings.NewReader(query))
+	if err != nil {
+		log.Error("conn es err(%v)", err)
+		return
+	}
+	req.Header.Set("Content-Type", "application/json; charset=UTF-8")
+	j := rand.Intn(len(d.httpClient))
+	res, err := d.httpClient[j].Do(req)
+	if err != nil || res.StatusCode != 200 {
+		log.Error("conn es http err(%v)", err)
+		return
+	}
+	body, err := ioutil.ReadAll(res.Body)
+	if err != nil {
+		log.Error("es read body err(%v)", err)
+		return
+	}
+	log.Infov(c, log.KV("query", query), log.KV("response", string(body)))
+
+	videos := new(elastic.SearchResult)
+	jsoniter.Unmarshal(body, &videos)
+	if videos == nil {
+		return
+	}
+
+	list = make([]*v1.RecVideoInfo, 0)
+	total = videos.TotalHits()
+	for _, value := range videos.Hits.Hits {
+		tmp := new(v1.RecVideoInfo)
+		byte, _ := jsoniter.Marshal(value.Source)
+		jsoniter.Unmarshal(byte, tmp)
+		list = append(list, tmp)
+	}
+	return
+}
+
+// DelVideoDataBySVID 根据svid删除视频
+func (d *Dao) DelVideoDataBySVID(c context.Context, svid int64) (err error) {
+	i := rand.Intn(len(conf.Conf.Es["bbq"].Addr))
+	url := conf.Conf.Es["bbq"].Addr[i] + "/video/video_info/" + strconv.Itoa(int(svid))
+	req, err := http.NewRequest("DELETE", url, nil)
+	if err != nil {
+		log.Error("conn es err(%v)", err)
+		return
+	}
+	j := rand.Intn(len(d.httpClient))
+	res, err := d.httpClient[j].Do(req)
+	if err != nil || res.StatusCode != 200 {
+		log.Error("conn read body err(%v)", err)
+	}
+	return
+}