Initial commit

app/interface/main/upload/http/BUILD

@@ -0,0 +1,47 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "http.go",
+        "record.go",
+    ],
+    importpath = "go-common/app/interface/main/upload/http",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/upload/conf:go_default_library",
+        "//app/interface/main/upload/http/antispam:go_default_library",
+        "//app/interface/main/upload/model:go_default_library",
+        "//app/interface/main/upload/service:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/log:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/binding:go_default_library",
+        "//library/net/http/blademaster/middleware/auth:go_default_library",
+        "//library/net/http/blademaster/middleware/verify:go_default_library",
+        "//library/net/http/blademaster/render:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/interface/main/upload/http/antispam:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/interface/main/upload/http/antispam/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+    "go_test",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["antispam.go"],
+    importpath = "go-common/app/interface/main/upload/http/antispam",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/interface/main/upload/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/ecode:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/net/http/blademaster/binding:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["antispam_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = [
+        "//app/interface/main/upload/model:go_default_library",
+        "//library/cache/redis:go_default_library",
+        "//library/container/pool:go_default_library",
+        "//library/net/http/blademaster:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)

app/interface/main/upload/http/antispam/antispam.go

@@ -0,0 +1,159 @@
+package antispam
+
+import (
+	"fmt"
+	"strings"
+	"time"
+
+	"go-common/app/interface/main/upload/model"
+	"go-common/library/cache/redis"
+	"go-common/library/ecode"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+)
+
+const (
+	_prefixRate  = "r_%d_%s_%d"
+	_prefixTotal = "t_%d_%s_%d"
+)
+
+// Antispam is a antispam instance.
+type Antispam struct {
+	redis     *redis.Pool
+	limitFunc func(bucket, dir string) (model.DirRateConfig, bool)
+	conf      *Config
+}
+
+// Config antispam config.
+// On     bool // switch on/off
+// Second int  // every N second allow N requests.
+// N      int  // one unit allow N requests.
+// Hour   int  // every N hour allow M requests.
+// M      int  // one winodw allow M requests.
+type Config struct {
+	On     bool // switch on/off
+	Second int  // every N second allow N requests.
+	N      int  // one unit allow N requests.
+	Hour   int  // every N hour allow M requests.
+	M      int  // one winodw allow M requests.
+
+	Redis *redis.Config
+}
+
+// New new a antispam service.
+func New(c *Config, l func(bucket, dir string) (model.DirRateConfig, bool)) (s *Antispam) {
+	if c == nil {
+		panic("antispam config nil")
+	}
+	s = &Antispam{
+		limitFunc: l,
+		redis:     redis.NewPool(c.Redis),
+	}
+	s.conf = c
+	return s
+}
+
+// NativeRate limit user + path second level
+func (s *Antispam) NativeRate(c *bm.Context, path string, mid interface{}) (err error) {
+	curSecond := int(time.Now().Unix())
+	burst := curSecond - curSecond%s.conf.Second
+	key := rateKey(mid.(int64), path, burst)
+	return s.antispam(c, key, s.conf.Second, s.conf.N)
+}
+
+// Rate antispam by user + bucket + dir.
+func (s *Antispam) Rate(c *bm.Context) (err error) {
+	mid, ok := c.Get("mid")
+	if !ok {
+		return
+	}
+	ap := new(struct {
+		Bucket string `form:"bucket" json:"bucket"`
+		Dir    string `form:"dir" json:"dir"`
+	})
+	if err = c.BindWith(ap, binding.FormMultipart); err != nil {
+		return s.NativeRate(c, c.Request.URL.Path, mid)
+	}
+	if ap.Bucket == "" || ap.Dir == "" { //not need dir limit
+		return s.NativeRate(c, c.Request.URL.Path, mid)
+	}
+	limit, ok := s.limitFunc(ap.Bucket, ap.Dir)
+	if !ok {
+		return s.NativeRate(c, c.Request.URL.Path, mid)
+	}
+	if limit.SecondQPS == 0 || limit.CountQPS == 0 {
+		return s.NativeRate(c, c.Request.URL.Path, mid)
+	}
+	path := strings.Join([]string{ap.Bucket, ap.Dir}, "_")
+	curSecond := int(time.Now().Unix())
+	burst := curSecond - curSecond%limit.SecondQPS
+	key := rateKey(mid.(int64), path, burst)
+	return s.antispam(c, key, limit.SecondQPS, limit.CountQPS)
+}
+
+func totalKey(mid int64, path string, burst int) string {
+	return fmt.Sprintf(_prefixTotal, mid, path, burst)
+}
+
+// Total antispam by user + path hour level
+func (s *Antispam) Total(c *bm.Context, hour, count int) (err error) {
+	second := hour * 3600
+	mid, ok := c.Get("mid")
+	if !ok {
+		return
+	}
+	curHour := int(time.Now().Unix() / 3600)
+	burst := curHour - curHour%hour
+	key := totalKey(mid.(int64), c.Request.URL.Path, burst)
+	return s.antispam(c, key, second, count)
+}
+
+func (s *Antispam) antispam(c *bm.Context, key string, interval, count int) (err error) {
+	conn := s.redis.Get(c)
+	defer conn.Close()
+	cur, err := redis.Int(conn.Do("GET", key))
+	if err != nil && err != redis.ErrNil {
+		err = nil
+		return
+	}
+	if cur >= count {
+		err = ecode.LimitExceed
+		return
+	}
+	err = nil
+	conn.Send("INCR", key)
+	conn.Send("EXPIRE", key, interval)
+	if err1 := conn.Flush(); err1 != nil {
+		return
+	}
+	for i := 0; i < 2; i++ {
+		if _, err1 := conn.Receive(); err1 != nil {
+			return
+		}
+	}
+	return
+}
+
+func rateKey(mid int64, path string, burst int) string {
+	return fmt.Sprintf(_prefixRate, mid, path, burst)
+}
+
+func (s *Antispam) ServeHTTP(ctx *bm.Context) {
+	// user + bucket + dir.
+	if err := s.Rate(ctx); err != nil {
+		ctx.JSON(nil, ecode.ServiceUnavailable)
+		ctx.Abort()
+		return
+	}
+	// user + path
+	if err := s.Total(ctx, s.conf.Hour, s.conf.M); err != nil {
+		ctx.JSON(nil, ecode.ServiceUnavailable)
+		ctx.Abort()
+		return
+	}
+}
+
+// Handler is antispam handle.
+func (s *Antispam) Handler() bm.HandlerFunc {
+	return s.ServeHTTP
+}

app/interface/main/upload/http/antispam/antispam_test.go

@@ -0,0 +1,98 @@
+package antispam
+
+import (
+	"context"
+	"io/ioutil"
+	"math/rand"
+	"net/http"
+	"strconv"
+	"testing"
+	"time"
+
+	"go-common/app/interface/main/upload/model"
+	"go-common/library/cache/redis"
+	"go-common/library/container/pool"
+	bm "go-common/library/net/http/blademaster"
+	xtime "go-common/library/time"
+)
+
+func TestAntiSpamHandler(t *testing.T) {
+	anti := New(
+		&Config{
+			On:     true,
+			Second: 1,
+			N:      1,
+			Hour:   1,
+			M:      1,
+			Redis: &redis.Config{
+				Config: &pool.Config{
+					Active:      10,
+					Idle:        10,
+					IdleTimeout: xtime.Duration(time.Second * 60),
+				},
+				Name:         "test",
+				Proto:        "tcp",
+				Addr:         "172.16.33.54:6380",
+				DialTimeout:  xtime.Duration(time.Second),
+				ReadTimeout:  xtime.Duration(time.Second),
+				WriteTimeout: xtime.Duration(time.Second),
+			}}, GetGetRateLimit)
+
+	engine := bm.New()
+	engine.UseFunc(func(c *bm.Context) {
+		mid, _ := strconv.ParseInt(c.Request.Form.Get("mid"), 10, 64)
+		c.Set("mid", mid)
+		c.Next()
+	})
+	engine.Use(anti.Handler())
+	engine.GET("/antispam", func(c *bm.Context) {
+		c.String(200, "pass")
+	})
+	go engine.Run(":18080")
+
+	time.Sleep(time.Millisecond * 50)
+	mid := rand.Int()
+	_, content, err := httpGet("http://127.0.0.1:18080/antispam?mid=" + strconv.Itoa(mid) + "&bucket=a&dir=b")
+	if err != nil {
+		t.Logf("http get failed, err:=%v", err)
+		t.FailNow()
+	}
+	if string(content) != "pass" {
+		t.Logf("request should block by limiter, but passed")
+		t.FailNow()
+	}
+
+	_, content, err = httpGet("http://127.0.0.1:18080/antispam?mid=" + strconv.Itoa(mid) + "&bucket=a&dir=b")
+	if err != nil {
+		t.Logf("http get failed, err:=%v", err)
+		t.FailNow()
+	}
+	if string(content) == "pass" {
+		t.Logf("request should block by limiter, but passed")
+		t.FailNow()
+	}
+
+	engine.Server().Shutdown(context.TODO())
+}
+
+func httpGet(url string) (code int, content []byte, err error) {
+	resp, err := http.Get(url)
+	if err != nil {
+		return
+	}
+	defer resp.Body.Close()
+
+	content, err = ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return
+	}
+	code = resp.StatusCode
+	return
+}
+
+func GetGetRateLimit(bucket, dir string) (model.DirRateConfig, bool) {
+	return model.DirRateConfig{
+		SecondQPS: 1,
+		CountQPS:  1,
+	}, true
+}

app/interface/main/upload/http/http.go

@@ -0,0 +1,68 @@
+package http
+
+import (
+	"net/http"
+
+	"go-common/app/interface/main/upload/conf"
+	xanti "go-common/app/interface/main/upload/http/antispam"
+	"go-common/app/interface/main/upload/service"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/middleware/auth"
+	"go-common/library/net/http/blademaster/middleware/verify"
+)
+
+var (
+	uploadSvr    *service.Service
+	authInterSvr *auth.Auth
+	authSvr      *auth.Auth
+	verifySvr    *verify.Verify
+	anti         *xanti.Antispam
+)
+
+// Init init http
+func Init(c *conf.Config, s *service.Service) {
+	initService(c, s)
+	engine := bm.DefaultServer(c.BM)
+	initRouter(engine)
+	// init Outer serve
+	if err := engine.Start(); err != nil {
+		log.Error("xhttp.Serve error(%v)", err)
+		panic(err)
+	}
+}
+
+func initService(c *conf.Config, s *service.Service) {
+	uploadSvr = s
+	authInterSvr = auth.New(c.AuthInter)
+	authSvr = auth.New(c.AuthOut)
+	verifySvr = verify.New(nil)
+	anti = xanti.New(c.Antispam, s.GetRateLimit) //mid+dir 限流
+}
+
+func initRouter(e *bm.Engine) {
+	e.Ping(ping)
+	uploadInternal := e.Group("/x/internal")
+	{
+		uploadInternal.POST("/upload", verifySvr.Verify, internalUpload)
+		uploadInternal.POST("/upload/image", authInterSvr.User, anti.Handler(), internalUploadImage)
+		uploadInternal.POST("/upload/admin/image", verifySvr.Verify, anti.Handler(), internalUploadAdminImage)
+		uploadInternal.POST("/image/gen", verifySvr.Verify, anti.Handler(), genImageUpload)
+	}
+
+	upload := e.Group("/x/upload")
+	{
+		upload.POST("/image", uploadImagePublic)
+		upload.POST("/app/image", authSvr.UserMobile, anti.Handler(), uploadMobileImage)
+		upload.POST("/web/image", authSvr.UserWeb, anti.Handler(), uploadWebImage)
+	}
+}
+
+// ping check server ok.
+func ping(c *bm.Context) {
+	var err error
+	if err = uploadSvr.Ping(c); err != nil {
+		log.Error("upload service ping error(%v)", err)
+		c.AbortWithStatus(http.StatusServiceUnavailable)
+	}
+}

app/interface/main/upload/http/record.go

@@ -0,0 +1,250 @@
+package http
+
+import (
+	"bytes"
+	"io"
+	"net/http"
+	"strconv"
+
+	"go-common/app/interface/main/upload/model"
+	"go-common/library/ecode"
+	"go-common/library/log"
+	bm "go-common/library/net/http/blademaster"
+	"go-common/library/net/http/blademaster/binding"
+	"go-common/library/net/http/blademaster/render"
+)
+
+const (
+	_defaultDistance = 1
+)
+
+// genImageUpload .
+func genImageUpload(c *bm.Context) {
+	params := c.Request.Form
+	uploadKey := params.Get("upload_key")
+	wmKey := params.Get("wm_key")
+	wmText := params.Get("wm_text")
+	if len(wmText) > 20 {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	distance, err := strconv.Atoi(params.Get("distance"))
+	if err != nil {
+		distance = _defaultDistance
+	}
+	vertical, err := strconv.ParseBool(params.Get("wm_vertical"))
+	if err != nil {
+		vertical = true
+	}
+	c.JSON(uploadSvr.GenImageUpload(c, uploadKey, wmKey, wmText, distance, vertical))
+}
+
+// uploadImagePublic .
+func uploadImagePublic(c *bm.Context) {
+	if err := c.Request.ParseMultipartForm(model.MaxUploadSize); err != nil {
+		c.JSON(nil, ecode.BfsUploadFileTooLarge)
+		return
+	}
+	params := c.Request.Form
+	uploadKey := params.Get("upload_key")
+	uploadToken := params.Get("upload_token")
+	file, header, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadImage.file.illegal,err:(%v)", err.Error())
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	result, err := uploadSvr.Upload(c, uploadKey, uploadToken, header.Header.Get("Content-Type"), buf.Bytes())
+	if err != nil {
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(result, nil)
+}
+
+// internalUpload upload by key and sign.
+func internalUpload(c *bm.Context) {
+	var (
+		err error
+		mid int64
+	)
+	up := new(model.UploadParam)
+	if err = c.BindWith(up, binding.FormMultipart); err != nil {
+		return
+	}
+	up.WMInit()
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	file, _, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadImage.file.illegal,err::%v", err.Error())
+		c.JSON(nil, ecode.FileNotExists)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	c.JSON(uploadSvr.UploadRecord(c, model.UploadInternal, mid, up, buf.Bytes()))
+}
+
+// internalUploadImage .
+func internalUploadImage(c *bm.Context) {
+	var (
+		err error
+		mid int64
+	)
+	up := new(model.UploadParam)
+	if err = c.BindWith(up, binding.FormMultipart); err != nil {
+		return
+	}
+	up.WMInit()
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	file, _, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadImage.file.illegal,err::%v", err.Error())
+		c.JSON(nil, ecode.FileNotExists)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	result, err := uploadSvr.UploadRecord(c, model.UploadInternal, mid, up, buf.Bytes())
+	if err != nil {
+		log.Error("uploadSvr.UploadRecord(%d,%+v) error(%v)", mid, up, err)
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(result, nil)
+}
+
+// internalUploadAdminImage .
+func internalUploadAdminImage(c *bm.Context) {
+	var err error
+	up := new(model.UploadParam)
+	if err = c.BindWith(up, binding.FormMultipart); err != nil {
+		return
+	}
+	up.WMInit()
+	file, _, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadImage.file.illegal,err::%v", err.Error())
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	result, err := uploadSvr.UploadAdminRecord(c, model.UploadInternalAdmin, up, buf.Bytes())
+	if err != nil {
+		log.Error("uploadSrv.Upload(%v,%v,%v) error(%v)", up.Bucket, up.FileName, err)
+		c.JSON(nil, err)
+		return
+	}
+	c.JSON(result, nil)
+}
+
+// uploadMobileImage .
+func uploadMobileImage(c *bm.Context) {
+	var (
+		err error
+		mid int64
+	)
+	up := new(model.UploadParam)
+	if err = c.BindWith(up, binding.FormMultipart); err != nil {
+		return
+	}
+	up.WMInit()
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	if mid <= 0 {
+		c.Render(http.StatusOK, render.JSON{
+			Code:    ecode.UserNotExist.Code(),
+			Message: "invalid or not exist mid",
+			Data:    nil,
+		})
+		return
+	}
+	file, _, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadMobileImage.file.illegal,err::%v", err.Error())
+		c.JSON(nil, ecode.FileNotExists)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	result, err := uploadSvr.UploadRecord(c, model.UploadApp, mid, up, buf.Bytes())
+	if err != nil {
+		log.Error("uploadSrv.UploadRecord(%v,%v,%v) error(%v)", mid, up.Bucket, up.FileName, err)
+		c.JSON(nil, err)
+		return
+	}
+	log.Info("app/upload param (%+v) result (%+v)", up, result)
+	c.JSON(result, nil)
+}
+
+// uploadWebImage .
+func uploadWebImage(c *bm.Context) {
+	var (
+		err error
+		mid int64
+	)
+	up := new(model.UploadParam)
+	if err = c.BindWith(up, binding.FormMultipart); err != nil {
+		return
+	}
+	up.WMInit()
+	if midInter, ok := c.Get("mid"); ok {
+		mid = midInter.(int64)
+	}
+	if mid <= 0 {
+		c.Render(http.StatusOK, render.JSON{
+			Code:    ecode.UserNotExist.Code(),
+			Message: "invalid or not exist mid",
+			Data:    nil,
+		})
+		return
+	}
+	file, _, err := c.Request.FormFile("file")
+	if err != nil {
+		log.Error("upload.UploadWebImage.file.illegal,err::%v", err.Error())
+		c.JSON(nil, ecode.FileNotExists)
+		return
+	}
+	defer file.Close()
+	buf := new(bytes.Buffer)
+	if _, err = io.Copy(buf, file); err != nil {
+		c.JSON(nil, ecode.RequestErr)
+		return
+	}
+	result, err := uploadSvr.UploadRecord(c, model.UploadWeb, mid, up, buf.Bytes())
+	if err != nil {
+		log.Error("uploadSrv.UploadRecord(%v,%v,%v) error(%v)", mid, up.Bucket, up.FileName, err)
+		c.JSON(nil, err)
+		return
+	}
+	log.Info("web/upload param (%+v) result (%+v)", up, result)
+	c.JSON(result, nil)
+}