Initial commit

This commit is contained in:
Donny
2019-04-22 20:46:32 +08:00
commit 49ab8aadd1
25441 changed files with 4055000 additions and 0 deletions

View File

@@ -0,0 +1,21 @@
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [
":package-srcs",
"//app/interface/openplatform/seo/cmd:all-srcs",
"//app/interface/openplatform/seo/conf:all-srcs",
"//app/interface/openplatform/seo/dao:all-srcs",
"//app/interface/openplatform/seo/model:all-srcs",
"//app/interface/openplatform/seo/server/http:all-srcs",
"//app/interface/openplatform/seo/service:all-srcs",
],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,12 @@
# v1.0.3
1. 增加电商SEO
2. 配置优化
# v1.0.2
1. 返回站点地图接口
# v1.0.1
1. 增加访问来源和URL的日志
# v1.0.0
1. 票务项目详情页SEO

View File

@@ -0,0 +1,11 @@
# Owner
liuguodong
liuzhan
penggenping
# Author
penggenping
# Reviewer
liuguodong
liuzhan

View File

@@ -0,0 +1,16 @@
# See the OWNERS docs at https://go.k8s.io/owners
approvers:
- liuguodong
- liuzhan
- penggenping
labels:
- interface
- interface/openplatform/seo
- openplatform
options:
no_parent_owners: true
reviewers:
- liuguodong
- liuzhan
- penggenping

View File

@@ -0,0 +1,12 @@
# seo-interface
# 项目简介
1.
# 编译环境
# 依赖包
# 编译执行

View File

@@ -0,0 +1 @@
# HTTP API文档

View File

@@ -0,0 +1,43 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_binary",
"go_library",
)
go_binary(
name = "cmd",
embed = [":go_default_library"],
tags = ["automanaged"],
)
go_library(
name = "go_default_library",
srcs = ["main.go"],
data = ["conf.toml"],
importpath = "go-common/app/interface/openplatform/seo/cmd",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/interface/openplatform/seo/conf:go_default_library",
"//app/interface/openplatform/seo/server/http:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/trace:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,37 @@
[memcache]
name = "seo-interface"
proto = "tcp"
addr = "172.22.33.137:11221"
active = 50
idle = 10
dialTimeout = "1s"
readTimeout = "1s"
writeTimeout = "1s"
idleTimeout = "10s"
expire = "24h"
#[log]
# dir = "/tmp/log/seo"
[seo]
expire = 3600
maxAge = 3600
botList = ["bot", "spider"]
[[sitemaps]]
host = "show.bilibili.com"
url = "http://uat-bfs.bilibili.co/openplatform/sitemaps/project.xml"
[[sitemaps]]
host = "mall.bilibili.com"
url = "http://uat-bfs.bilibili.co/openplatform/sitemaps/item.xml"
[[pages]]
name = "pro"
url = "http://uat-show.bilibili.com/platform/detail.html?id=%d"
bfs = "http://uat-bfs.bilibili.co/openplatform/project/%d.html"
path = "/tmp/seo/html/project/%d.html"
[[pages]]
name = "item"
url = "http://uat-mall.bilibili.com/detail.html?itemsId=%d"
bfs = "http://uat-bfs.bilibili.co/openplatform/item/%d.html"
path = "/tmp/seo/html/item/%d.html"

View File

@@ -0,0 +1,40 @@
package main
import (
"flag"
"os"
"os/signal"
"syscall"
"go-common/app/interface/openplatform/seo/conf"
"go-common/app/interface/openplatform/seo/server/http"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
"go-common/library/net/trace"
)
func main() {
flag.Parse()
if err := conf.Init(); err != nil {
panic(err)
}
log.Init(conf.Conf.Log)
defer log.Close()
trace.Init(conf.Conf.Tracer)
defer trace.Close()
ecode.Init(conf.Conf.Ecode)
http.Init(conf.Conf)
c := make(chan os.Signal, 1)
signal.Notify(c, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT)
for {
s := <-c
log.Info("get a signal %s", s.String())
switch s {
case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT:
return
case syscall.SIGHUP:
default:
return
}
}
}

View File

@@ -0,0 +1,38 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["conf.go"],
importpath = "go-common/app/interface/openplatform/seo/conf",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//library/cache/memcache:go_default_library",
"//library/conf:go_default_library",
"//library/ecode/tip:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
"//library/net/trace:go_default_library",
"//vendor/github.com/BurntSushi/toml:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,128 @@
package conf
import (
"errors"
"flag"
"go-common/library/cache/memcache"
"go-common/library/conf"
ecode "go-common/library/ecode/tip"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
"go-common/library/net/trace"
"github.com/BurntSushi/toml"
)
// Conf global var
var (
Conf = &Config{}
confPath string
client *conf.Client
)
// Config .
type Config struct {
Log *log.Config
BM *bm.ServerConfig
Verify *verify.Config
Tracer *trace.Config
Memcache *memcache.Config
Ecode *ecode.Config
Seo *Seo
Pages []*Page
Sitemaps []*Sitemap
}
// Seo config
type Seo struct {
Expire int32
MaxAge int32
BotList []string
}
// Page pro, item ...
type Page struct {
Name string
Url string
Bfs string
Path string
}
// Sitemap app sitemap
type Sitemap struct {
Host string
Url string
}
func init() {
flag.StringVar(&confPath, "conf", "", "default config path")
}
// Init .
func Init() error {
if confPath != "" {
return local()
}
return remote()
}
func local() (err error) {
_, err = toml.DecodeFile(confPath, &Conf)
return
}
func remote() (err error) {
if client, err = conf.New(); err != nil {
return
}
if err = load(); err != nil {
return
}
go func() {
for range client.Event() {
log.Info("config reload")
if load() != nil {
log.Error("config reload error (%v)", err)
}
}
}()
return
}
func load() (err error) {
var (
s string
ok bool
tmpConf *Config
)
if s, ok = client.Toml2(); !ok {
return errors.New("load config center error")
}
if _, err = toml.Decode(s, &tmpConf); err != nil {
return errors.New("could not decode config")
}
*Conf = *tmpConf
return
}
// GetPage get page config by name
func GetPage(name string) *Page {
for _, p := range Conf.Pages {
if p.Name == name {
return p
}
}
return nil
}
// GetSitemap get sitemap url
func GetSitemap(host string) *Sitemap {
for _, s := range Conf.Sitemaps {
if s.Host == host {
return s
}
}
return nil
}

View File

@@ -0,0 +1,38 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"dao.go",
"item.go",
"pro.go",
"sitemap.go",
],
importpath = "go-common/app/interface/openplatform/seo/dao",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/interface/openplatform/seo/conf:go_default_library",
"//library/cache/memcache:go_default_library",
"//library/log:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,145 @@
package dao
import (
"context"
"fmt"
"io/ioutil"
"net/http"
"go-common/app/interface/openplatform/seo/conf"
"go-common/library/cache/memcache"
"go-common/library/log"
)
const (
_pro = "pro"
_item = "item"
)
// Dao dao
type Dao struct {
c *conf.Config
mc *memcache.Pool
}
// New init memcache
func New(c *conf.Config) (dao *Dao) {
dao = &Dao{
c: c,
mc: memcache.NewPool(c.Memcache),
}
return
}
// Close close the resource.
func (d *Dao) Close() {
d.mc.Close()
}
// Ping dao ping
func (d *Dao) Ping(c context.Context) (err error) {
if err = d.pingMC(c); err != nil {
log.Error("pingMC error(%+v)", err)
return
}
return
}
// pingMc ping memcache
func (d *Dao) pingMC(c context.Context) (err error) {
con := d.mc.Get(c)
defer con.Close()
item := memcache.Item{Key: "ping", Value: []byte{1}, Expiration: 1}
err = con.Set(&item)
return
}
// getUrl get page url
// page: pro, item
func getUrl(id int, name string, bot bool) string {
p := conf.GetPage(name)
if p == nil {
return ""
}
url := p.Bfs
if !bot {
url = p.Url
}
return fmt.Sprintf(url, id)
}
// getKey get page cache key
// name: pro, item
// return key: pro:bot:1001, pro:app:1001
// return key: item:bot:1001, item:app:1001
func getKey(id int, name string, bot bool) string {
ua := "bot"
if !bot {
ua = "app"
}
return fmt.Sprintf("%s:%s:%d", name, ua, id)
}
// GetFile get page from file
func (d *Dao) GetFile(c context.Context, path string) (res []byte, err error) {
log.Info(path)
res, err = ioutil.ReadFile(path)
return
}
// GetUrl get page from url
func (d *Dao) GetUrl(c context.Context, url string) (res []byte, err error) {
log.Info(url)
r, err := http.Get(url)
if err != nil {
return
}
defer r.Body.Close()
if err == nil {
res, err = ioutil.ReadAll(r.Body)
}
return
}
// GetCache get page from cache
func (d *Dao) GetCache(c context.Context, key string) (res []byte, err error) {
log.Info(key)
con := d.mc.Get(c)
defer con.Close()
item, err := con.Get(key)
if err != nil {
return
}
err = con.Scan(item, &res)
return
}
// AddCache add page to cache
func (d *Dao) AddCache(c context.Context, key string, val []byte) (err error) {
log.Info(key)
item := &memcache.Item{
Key: key,
Value: val,
Flags: memcache.FlagRAW,
Expiration: conf.Conf.Seo.Expire,
}
con := d.mc.Get(c)
defer con.Close()
if err = con.Set(item); err != nil {
log.Error("key(%s) error(%v)", key, err)
}
return
}
// DelCache delete page cache
func (d *Dao) DelCache(c context.Context, key string) (err error) {
con := d.mc.Get(c)
defer con.Close()
return con.Delete(key)
}
// ClearCache clear all page cache
func (d *Dao) ClearCache(c context.Context) (err error) {
return
}

View File

@@ -0,0 +1,21 @@
package dao
import (
"context"
)
// GetItem get item page
func (d *Dao) GetItem(c context.Context, id int, bot bool) (res []byte, err error) {
key := getKey(id, _item, bot)
res, err = d.GetCache(c, key)
if err == nil && res != nil {
return
}
url := getUrl(id, _item, bot)
res, err = d.GetUrl(c, url)
if err == nil {
d.AddCache(c, key, res)
}
return
}

View File

@@ -0,0 +1,21 @@
package dao
import (
"context"
)
// GetPro get project page
func (d *Dao) GetPro(c context.Context, id int, bot bool) (res []byte, err error) {
key := getKey(id, _pro, bot)
res, err = d.GetCache(c, key)
if err == nil && res != nil {
return
}
url := getUrl(id, _pro, bot)
res, err = d.GetUrl(c, url)
if err == nil {
d.AddCache(c, key, res)
}
return
}

View File

@@ -0,0 +1,20 @@
package dao
import (
"context"
"errors"
"go-common/app/interface/openplatform/seo/conf"
)
// Sitemap 生成站点地图
func (d *Dao) Sitemap(c context.Context, host string) (res []byte, err error) {
s := conf.GetSitemap(host)
if s == nil || s.Url == "" {
err = errors.New(host + " sitemap config not exist")
return
}
res, err = d.GetUrl(c, s.Url)
return
}

View File

@@ -0,0 +1,28 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = ["model.go"],
importpath = "go-common/app/interface/openplatform/seo/model",
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,11 @@
package model
// ArgProID .
type ArgProID struct {
ID int `form:"id" validate:"required,min=1,gte=1"`
}
// ArgItemID .
type ArgItemID struct {
ID int `form:"itemsId" validate:"required,min=1,gte=1"`
}

View File

@@ -0,0 +1,41 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"http.go",
"item.go",
"pro.go",
"sitemap.go",
],
importpath = "go-common/app/interface/openplatform/seo/server/http",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/interface/openplatform/seo/conf:go_default_library",
"//app/interface/openplatform/seo/model:go_default_library",
"//app/interface/openplatform/seo/service:go_default_library",
"//library/log:go_default_library",
"//library/net/http/blademaster:go_default_library",
"//library/net/http/blademaster/middleware/verify:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,99 @@
package http
import (
"crypto/sha1"
"fmt"
"net/http"
"strings"
"go-common/app/interface/openplatform/seo/conf"
"go-common/app/interface/openplatform/seo/service"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
"go-common/library/net/http/blademaster/middleware/verify"
)
var (
srv *service.Service
vfy *verify.Verify
)
// Init init seo service
func Init(c *conf.Config) {
srv = service.New(c)
vfy = verify.New(c.Verify)
engine := bm.DefaultServer(c.BM)
router(engine)
if err := engine.Start(); err != nil {
log.Error("innerEngine.Start() error(%v)", err)
panic(err)
}
}
func router(e *bm.Engine) {
e.Ping(ping)
e.Register(register)
g := e.Group("/platform")
{
g.GET("/home.html", proList)
g.GET("/detail.html", proInfo)
}
e.GET("/detail.html", itemInfo)
e.GET("/sitemap.xml", sitemap)
}
func ping(c *bm.Context) {
if err := srv.Ping(c); err != nil {
log.Error("ping error(%v)", err)
c.AbortWithStatus(http.StatusServiceUnavailable)
}
}
func register(c *bm.Context) {
c.JSON(map[string]interface{}{}, nil)
}
// isBot check if user-agent is bot
func isBot(c *bm.Context) bool {
ua := c.Request.Header.Get("User-Agent")
ua = strings.ToLower(ua)
for _, bot := range conf.Conf.Seo.BotList {
if strings.Contains(ua, strings.ToLower(bot)) {
return true
}
}
return false
}
// FullUrl get request full url
func FullUrl(c *bm.Context) string {
return c.Request.Host + c.Request.RequestURI
}
func setCache(c *bm.Context, res []byte) bool {
h := c.Writer.Header()
h.Set("Content-Type", "text/html; charset=utf8")
h.Set("Cache-Control", fmt.Sprintf("max-age=%d", conf.Conf.Seo.MaxAge))
etag := ETag(res)
h.Set("ETag", etag)
if etag == c.Request.Header.Get("If-None-Match") {
c.Writer.WriteHeader(http.StatusNotModified)
return true
}
return false
}
// ETag get etag for cache
func ETag(res []byte) string {
return fmt.Sprintf(`W/"%x-%x"`, len(res), sha1.Sum(res))
}
// logUA log User-Agent and Url
func logUA(c *bm.Context) {
log.Infov(c,
log.KV("ua", c.Request.Header.Get("User-Agent")),
log.KV("url", fmt.Sprintf("%s%s", c.Request.Host, c.Request.URL)),
)
}

View File

@@ -0,0 +1,27 @@
package http
import (
"go-common/app/interface/openplatform/seo/model"
bm "go-common/library/net/http/blademaster"
)
func itemInfo(c *bm.Context) {
logUA(c)
arg := new(model.ArgItemID)
if err := c.Bind(arg); err != nil {
return
}
bot := isBot(c)
res, err := srv.GetItem(c, arg.ID, bot)
if err != nil {
c.String(0, "%v", err)
return
}
if setCache(c, res) {
return
}
c.String(0, string(res))
}

View File

@@ -0,0 +1,31 @@
package http
import (
"go-common/app/interface/openplatform/seo/model"
bm "go-common/library/net/http/blademaster"
)
func proList(c *bm.Context) {
c.String(0, "project list")
}
func proInfo(c *bm.Context) {
logUA(c)
arg := new(model.ArgProID)
if err := c.Bind(arg); err != nil {
return
}
bot := isBot(c)
res, err := srv.GetPro(c, arg.ID, bot)
if err != nil {
c.String(0, "%v", err)
return
}
if setCache(c, res) {
return
}
c.String(0, string(res))
}

View File

@@ -0,0 +1,20 @@
package http
import (
"net/http"
"go-common/library/log"
bm "go-common/library/net/http/blademaster"
)
func sitemap(c *bm.Context) {
logUA(c)
res, err := srv.Sitemap(c, c.Request.Host)
if err != nil {
log.Error(err.Error())
return
}
c.Writer.Header().Set("Content-Type", "text/xml;charset=utf-8")
c.String(http.StatusOK, string(res))
}

View File

@@ -0,0 +1,36 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_library",
)
go_library(
name = "go_default_library",
srcs = [
"item.go",
"pro.go",
"service.go",
],
importpath = "go-common/app/interface/openplatform/seo/service",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = [
"//app/interface/openplatform/seo/conf:go_default_library",
"//app/interface/openplatform/seo/dao:go_default_library",
],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)

View File

@@ -0,0 +1,11 @@
package service
import (
"context"
)
// GetItem get item detail
func (s *Service) GetItem(c context.Context, id int, bot bool) (res []byte, err error) {
res, err = s.dao.GetItem(c, id, bot)
return
}

View File

@@ -0,0 +1,11 @@
package service
import (
"context"
)
// GetPro get project detail
func (s *Service) GetPro(c context.Context, id int, bot bool) (res []byte, err error) {
res, err = s.dao.GetPro(c, id, bot)
return
}

View File

@@ -0,0 +1,38 @@
package service
import (
"context"
"go-common/app/interface/openplatform/seo/conf"
"go-common/app/interface/openplatform/seo/dao"
)
// Service struct
type Service struct {
c *conf.Config
dao *dao.Dao
}
// New init
func New(c *conf.Config) (s *Service) {
s = &Service{
c: c,
dao: dao.New(c),
}
return s
}
// Ping .
func (s *Service) Ping(c context.Context) (err error) {
return s.dao.Ping(c)
}
// Close .
func (s *Service) Close() {
s.dao.Close()
}
// Sitemap 生成站点地图
func (s *Service) Sitemap(c context.Context, host string) ([]byte, error) {
return s.dao.Sitemap(c, host)
}