Initial commit

This commit is contained in:
Donny
2019-04-22 20:46:32 +08:00
commit 49ab8aadd1
25441 changed files with 4055000 additions and 0 deletions

56
library/conf/dsn/BUILD Normal file
View File

@@ -0,0 +1,56 @@
package(default_visibility = ["//visibility:public"])
load(
"@io_bazel_rules_go//go:def.bzl",
"go_test",
"go_library",
)
go_test(
name = "go_default_test",
srcs = [
"dsn_test.go",
"query_test.go",
],
embed = [":go_default_library"],
rundir = ".",
tags = ["automanaged"],
deps = ["//library/time:go_default_library"],
)
go_library(
name = "go_default_library",
srcs = [
"doc.go",
"dsn.go",
"query.go",
],
importpath = "go-common/library/conf/dsn",
tags = ["automanaged"],
visibility = ["//visibility:public"],
deps = ["//vendor/gopkg.in/go-playground/validator.v9:go_default_library"],
)
filegroup(
name = "package-srcs",
srcs = glob(["**"]),
tags = ["automanaged"],
visibility = ["//visibility:private"],
)
filegroup(
name = "all-srcs",
srcs = [":package-srcs"],
tags = ["automanaged"],
visibility = ["//visibility:public"],
)
go_test(
name = "go_default_xtest",
srcs = ["example_test.go"],
tags = ["automanaged"],
deps = [
"//library/conf/dsn:go_default_library",
"//library/time:go_default_library",
],
)

63
library/conf/dsn/doc.go Normal file
View File

@@ -0,0 +1,63 @@
// Package dsn implements dsn parse with struct bind
/*
DSN 格式类似 URI, DSN 结构如下图
network:[//[username[:password]@]address[:port][,address[:port]]][/path][?query][#fragment]
与 URI 的主要区别在于 scheme 被替换为 network, host 被替换为 address 并且支持多个 address.
network 与 net 包中 network 意义相同, tcp、udp、unix 等, address 支持多个使用 ',' 分割, 如果
network 为 unix 等本地 sock 协议则使用 Path, 有且只有一个
dsn 包主要提供了 Parse, Bind 和 validate 功能
Parse 解析 dsn 字符串成 DSN struct, DSN struct 与 url.URL 几乎完全一样
Bind 提供将 DSN 数据绑定到一个 struct 的功能, 通过 tag dsn:"key,[default]" 指定绑定的字段, 目前支持两种类型的数据绑定
内置变量 key:
network string tcp, udp, unix 等, 参考 net 包中的 network
username string
password string
address string or []string address 可以绑定到 string 或者 []string, 如果为 string 则取 address 第一个
Query: 通过 query.name 可以取到 query 上的数据
数组可以通过传递多个获得
array=1&array=2&array3 -> []int `tag:"query.array"`
struct 支持嵌套
foo.sub.name=hello&foo.tm=hello
struct Foo {
Tm string `dsn:"query.tm"`
Sub struct {
Name string `dsn:"query.name"`
} `dsn:"query.sub"`
}
默认值: 通过 dsn:"key,[default]" 默认值暂时不支持数组
忽略 Bind: 通过 dsn:"-" 忽略 Bind
自定义 Bind: 可以同时实现 encoding.TextUnmarshaler 自定义 Bind 实现
Validate: 参考 https://github.com/go-playground/validator
使用参考: example_test.go
DSN 命名规范:
没有历史遗留的情况下,尽量使用 Address, Network, Username, Password 等命名,代替之前的 Proto 和 Addr 等命名
Query 命名参考, 使用驼峰小写开头:
timeout 通用超时
dialTimeout 连接建立超时
readTimeout 读操作超时
writeTimeout 写操作超时
readsTimeout 批量读超时
writesTimeout 批量写超时
*/
package dsn

108
library/conf/dsn/dsn.go Normal file
View File

@@ -0,0 +1,108 @@
// Package dsn provide parse dsn and bind to struct
// see http://git.bilibili.co/platform/go-common/issues/279
package dsn
import (
"net/url"
"reflect"
"strings"
"gopkg.in/go-playground/validator.v9"
)
var _validator *validator.Validate
func init() {
_validator = validator.New()
}
// DSN a DSN represents a parsed DSN as same as url.URL.
type DSN struct {
*url.URL
}
// Bind dsn to specify struct and validate use use go-playground/validator format
//
// The bind of each struct field can be customized by the format string
// stored under the 'dsn' key in the struct field's tag. The format string
// gives the name of the field, possibly followed by a comma-separated
// list of options. The name may be empty in order to specify options
// without overriding the default field name.
//
// A two type data you can bind to struct
// built-in values, use below keys to bind built-in value
// username
// password
// address
// network
// the value in query string, use query.{name} to bind value in query string
//
// As a special case, if the field tag is "-", the field is always omitted.
// NOTE: that a field with name "-" can still be generated using the tag "-,".
//
// Examples of struct field tags and their meanings:
// // Field bind username
// Field string `dsn:"username"`
// // Field is ignored by this package.
// Field string `dsn:"-"`
// // Field bind value from query
// Field string `dsn:"query.name"`
//
func (d *DSN) Bind(v interface{}) (url.Values, error) {
assignFuncs := make(map[string]assignFunc)
if d.User != nil {
username := d.User.Username()
password, ok := d.User.Password()
if ok {
assignFuncs["password"] = stringsAssignFunc(password)
}
assignFuncs["username"] = stringsAssignFunc(username)
}
assignFuncs["address"] = addressesAssignFunc(d.Addresses())
assignFuncs["network"] = stringsAssignFunc(d.Scheme)
query, err := bindQuery(d.Query(), v, assignFuncs)
if err != nil {
return nil, err
}
return query, _validator.Struct(v)
}
func addressesAssignFunc(addresses []string) assignFunc {
return func(v reflect.Value, to tagOpt) error {
if v.Kind() == reflect.String {
if addresses[0] == "" && to.Default != "" {
v.SetString(to.Default)
} else {
v.SetString(addresses[0])
}
return nil
}
if !(v.Kind() == reflect.Slice && v.Type().Elem().Kind() == reflect.String) {
return &BindTypeError{Value: strings.Join(addresses, ","), Type: v.Type()}
}
vals := reflect.MakeSlice(v.Type(), len(addresses), len(addresses))
for i, address := range addresses {
vals.Index(i).SetString(address)
}
if v.CanSet() {
v.Set(vals)
}
return nil
}
}
// Addresses parse host split by ','
// For Unix networks, return ['path']
func (d *DSN) Addresses() []string {
switch d.Scheme {
case "unix", "unixgram", "unixpacket":
return []string{d.Path}
}
return strings.Split(d.Host, ",")
}
// Parse parses rawdsn into a URL structure.
func Parse(rawdsn string) (*DSN, error) {
u, err := url.Parse(rawdsn)
return &DSN{URL: u}, err
}

View File

@@ -0,0 +1,79 @@
package dsn
import (
"net/url"
"reflect"
"testing"
"time"
xtime "go-common/library/time"
)
type config struct {
Network string `dsn:"network"`
Addresses []string `dsn:"address"`
Username string `dsn:"username"`
Password string `dsn:"password"`
Timeout xtime.Duration `dsn:"query.timeout"`
Sub Sub `dsn:"query.sub"`
Def string `dsn:"query.def,hello"`
}
type Sub struct {
Foo int `dsn:"query.foo"`
}
func TestBind(t *testing.T) {
var cfg config
rawdsn := "tcp://root:toor@172.12.23.34,178.23.34.45?timeout=1s&sub.foo=1&hello=world"
dsn, err := Parse(rawdsn)
if err != nil {
t.Fatal(err)
}
values, err := dsn.Bind(&cfg)
if err != nil {
t.Error(err)
}
if !reflect.DeepEqual(values, url.Values{"hello": {"world"}}) {
t.Errorf("unexpect values get %v", values)
}
cfg2 := config{
Network: "tcp",
Addresses: []string{"172.12.23.34", "178.23.34.45"},
Password: "toor",
Username: "root",
Sub: Sub{Foo: 1},
Timeout: xtime.Duration(time.Second),
Def: "hello",
}
if !reflect.DeepEqual(cfg, cfg2) {
t.Errorf("unexpect config get %v, expect %v", cfg, cfg2)
}
}
type config2 struct {
Network string `dsn:"network"`
Address string `dsn:"address"`
Timeout xtime.Duration `dsn:"query.timeout"`
}
func TestUnix(t *testing.T) {
var cfg config2
rawdsn := "unix:///run/xxx.sock?timeout=1s&sub.foo=1&hello=world"
dsn, err := Parse(rawdsn)
if err != nil {
t.Fatal(err)
}
_, err = dsn.Bind(&cfg)
if err != nil {
t.Error(err)
}
cfg2 := config2{
Network: "unix",
Address: "/run/xxx.sock",
Timeout: xtime.Duration(time.Second),
}
if !reflect.DeepEqual(cfg, cfg2) {
t.Errorf("unexpect config2 get %v, expect %v", cfg, cfg2)
}
}

View File

@@ -0,0 +1,31 @@
package dsn_test
import (
"log"
"go-common/library/conf/dsn"
xtime "go-common/library/time"
)
// Config struct
type Config struct {
Network string `dsn:"network" validate:"required"`
Host string `dsn:"host" validate:"required"`
Username string `dsn:"username" validate:"required"`
Password string `dsn:"password" validate:"required"`
Timeout xtime.Duration `dsn:"query.timeout,1s"`
Offset int `dsn:"query.offset" validate:"gte=0"`
}
func ExampleParse() {
cfg := &Config{}
d, err := dsn.Parse("tcp://root:toor@172.12.12.23:2233?timeout=10s")
if err != nil {
log.Fatal(err)
}
_, err = d.Bind(cfg)
if err != nil {
log.Fatal(err)
}
log.Printf("%v", cfg)
}

422
library/conf/dsn/query.go Normal file
View File

@@ -0,0 +1,422 @@
package dsn
import (
"encoding"
"net/url"
"reflect"
"runtime"
"strconv"
"strings"
)
const (
_tagID = "dsn"
_queryPrefix = "query."
)
// InvalidBindError describes an invalid argument passed to DecodeQuery.
// (The argument to DecodeQuery must be a non-nil pointer.)
type InvalidBindError struct {
Type reflect.Type
}
func (e *InvalidBindError) Error() string {
if e.Type == nil {
return "Bind(nil)"
}
if e.Type.Kind() != reflect.Ptr {
return "Bind(non-pointer " + e.Type.String() + ")"
}
return "Bind(nil " + e.Type.String() + ")"
}
// BindTypeError describes a query value that was
// not appropriate for a value of a specific Go type.
type BindTypeError struct {
Value string
Type reflect.Type
}
func (e *BindTypeError) Error() string {
return "cannot decode " + e.Value + " into Go value of type " + e.Type.String()
}
type assignFunc func(v reflect.Value, to tagOpt) error
func stringsAssignFunc(val string) assignFunc {
return func(v reflect.Value, to tagOpt) error {
if v.Kind() != reflect.String || !v.CanSet() {
return &BindTypeError{Value: "string", Type: v.Type()}
}
if val == "" {
v.SetString(to.Default)
} else {
v.SetString(val)
}
return nil
}
}
// bindQuery parses url.Values and stores the result in the value pointed to by v.
// if v is nil or not a pointer, bindQuery returns an InvalidDecodeError
func bindQuery(query url.Values, v interface{}, assignFuncs map[string]assignFunc) (url.Values, error) {
if assignFuncs == nil {
assignFuncs = make(map[string]assignFunc)
}
d := decodeState{
data: query,
used: make(map[string]bool),
assignFuncs: assignFuncs,
}
err := d.decode(v)
ret := d.unused()
return ret, err
}
type tagOpt struct {
Name string
Default string
}
func parseTag(tag string) tagOpt {
vs := strings.SplitN(tag, ",", 2)
if len(vs) == 2 {
return tagOpt{Name: vs[0], Default: vs[1]}
}
return tagOpt{Name: vs[0]}
}
type decodeState struct {
data url.Values
used map[string]bool
assignFuncs map[string]assignFunc
}
func (d *decodeState) unused() url.Values {
ret := make(url.Values)
for k, v := range d.data {
if !d.used[k] {
ret[k] = v
}
}
return ret
}
func (d *decodeState) decode(v interface{}) (err error) {
defer func() {
if r := recover(); r != nil {
if _, ok := r.(runtime.Error); ok {
panic(r)
}
err = r.(error)
}
}()
rv := reflect.ValueOf(v)
if rv.Kind() != reflect.Ptr || rv.IsNil() {
return &InvalidBindError{reflect.TypeOf(v)}
}
return d.root(rv)
}
func (d *decodeState) root(v reflect.Value) error {
var tu encoding.TextUnmarshaler
tu, v = d.indirect(v)
if tu != nil {
return tu.UnmarshalText([]byte(d.data.Encode()))
}
// TODO support map, slice as root
if v.Kind() != reflect.Struct {
return &BindTypeError{Value: d.data.Encode(), Type: v.Type()}
}
tv := v.Type()
for i := 0; i < tv.NumField(); i++ {
fv := v.Field(i)
field := tv.Field(i)
to := parseTag(field.Tag.Get(_tagID))
if to.Name == "-" {
continue
}
if af, ok := d.assignFuncs[to.Name]; ok {
if err := af(fv, tagOpt{}); err != nil {
return err
}
continue
}
if !strings.HasPrefix(to.Name, _queryPrefix) {
continue
}
to.Name = to.Name[len(_queryPrefix):]
if err := d.value(fv, "", to); err != nil {
return err
}
}
return nil
}
func combinekey(prefix string, to tagOpt) string {
key := to.Name
if prefix != "" {
key = prefix + "." + key
}
return key
}
func (d *decodeState) value(v reflect.Value, prefix string, to tagOpt) (err error) {
key := combinekey(prefix, to)
d.used[key] = true
var tu encoding.TextUnmarshaler
tu, v = d.indirect(v)
if tu != nil {
if val, ok := d.data[key]; ok {
return tu.UnmarshalText([]byte(val[0]))
}
if to.Default != "" {
return tu.UnmarshalText([]byte(to.Default))
}
return
}
switch v.Kind() {
case reflect.Bool:
err = d.valueBool(v, prefix, to)
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
err = d.valueInt64(v, prefix, to)
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
err = d.valueUint64(v, prefix, to)
case reflect.Float32, reflect.Float64:
err = d.valueFloat64(v, prefix, to)
case reflect.String:
err = d.valueString(v, prefix, to)
case reflect.Slice:
err = d.valueSlice(v, prefix, to)
case reflect.Struct:
err = d.valueStruct(v, prefix, to)
case reflect.Ptr:
if !d.hasKey(combinekey(prefix, to)) {
break
}
if !v.CanSet() {
break
}
nv := reflect.New(v.Type().Elem())
v.Set(nv)
err = d.value(nv, prefix, to)
}
return
}
func (d *decodeState) hasKey(key string) bool {
for k := range d.data {
if strings.HasPrefix(k, key+".") || k == key {
return true
}
}
return false
}
func (d *decodeState) valueBool(v reflect.Value, prefix string, to tagOpt) error {
key := combinekey(prefix, to)
val := d.data.Get(key)
if val == "" {
if to.Default == "" {
return nil
}
val = to.Default
}
return d.setBool(v, val)
}
func (d *decodeState) setBool(v reflect.Value, val string) error {
bval, err := strconv.ParseBool(val)
if err != nil {
return &BindTypeError{Value: val, Type: v.Type()}
}
v.SetBool(bval)
return nil
}
func (d *decodeState) valueInt64(v reflect.Value, prefix string, to tagOpt) error {
key := combinekey(prefix, to)
val := d.data.Get(key)
if val == "" {
if to.Default == "" {
return nil
}
val = to.Default
}
return d.setInt64(v, val)
}
func (d *decodeState) setInt64(v reflect.Value, val string) error {
ival, err := strconv.ParseInt(val, 10, 64)
if err != nil {
return &BindTypeError{Value: val, Type: v.Type()}
}
v.SetInt(ival)
return nil
}
func (d *decodeState) valueUint64(v reflect.Value, prefix string, to tagOpt) error {
key := combinekey(prefix, to)
val := d.data.Get(key)
if val == "" {
if to.Default == "" {
return nil
}
val = to.Default
}
return d.setUint64(v, val)
}
func (d *decodeState) setUint64(v reflect.Value, val string) error {
uival, err := strconv.ParseUint(val, 10, 64)
if err != nil {
return &BindTypeError{Value: val, Type: v.Type()}
}
v.SetUint(uival)
return nil
}
func (d *decodeState) valueFloat64(v reflect.Value, prefix string, to tagOpt) error {
key := combinekey(prefix, to)
val := d.data.Get(key)
if val == "" {
if to.Default == "" {
return nil
}
val = to.Default
}
return d.setFloat64(v, val)
}
func (d *decodeState) setFloat64(v reflect.Value, val string) error {
fval, err := strconv.ParseFloat(val, 64)
if err != nil {
return &BindTypeError{Value: val, Type: v.Type()}
}
v.SetFloat(fval)
return nil
}
func (d *decodeState) valueString(v reflect.Value, prefix string, to tagOpt) error {
key := combinekey(prefix, to)
val := d.data.Get(key)
if val == "" {
if to.Default == "" {
return nil
}
val = to.Default
}
return d.setString(v, val)
}
func (d *decodeState) setString(v reflect.Value, val string) error {
v.SetString(val)
return nil
}
func (d *decodeState) valueSlice(v reflect.Value, prefix string, to tagOpt) error {
key := combinekey(prefix, to)
strs, ok := d.data[key]
if !ok {
strs = strings.Split(to.Default, ",")
}
if len(strs) == 0 {
return nil
}
et := v.Type().Elem()
var setFunc func(reflect.Value, string) error
switch et.Kind() {
case reflect.Bool:
setFunc = d.setBool
case reflect.Int8, reflect.Int16, reflect.Int32, reflect.Int64, reflect.Int:
setFunc = d.setInt64
case reflect.Uint8, reflect.Uint16, reflect.Uint32, reflect.Uint64, reflect.Uint:
setFunc = d.setUint64
case reflect.Float32, reflect.Float64:
setFunc = d.setFloat64
case reflect.String:
setFunc = d.setString
default:
return &BindTypeError{Type: et, Value: strs[0]}
}
vals := reflect.MakeSlice(v.Type(), len(strs), len(strs))
for i, str := range strs {
if err := setFunc(vals.Index(i), str); err != nil {
return err
}
}
if v.CanSet() {
v.Set(vals)
}
return nil
}
func (d *decodeState) valueStruct(v reflect.Value, prefix string, to tagOpt) error {
tv := v.Type()
for i := 0; i < tv.NumField(); i++ {
fv := v.Field(i)
field := tv.Field(i)
fto := parseTag(field.Tag.Get(_tagID))
if fto.Name == "-" {
continue
}
if af, ok := d.assignFuncs[fto.Name]; ok {
if err := af(fv, tagOpt{}); err != nil {
return err
}
continue
}
if !strings.HasPrefix(fto.Name, _queryPrefix) {
continue
}
fto.Name = fto.Name[len(_queryPrefix):]
if err := d.value(fv, to.Name, fto); err != nil {
return err
}
}
return nil
}
func (d *decodeState) indirect(v reflect.Value) (encoding.TextUnmarshaler, reflect.Value) {
v0 := v
haveAddr := false
if v.Kind() != reflect.Ptr && v.Type().Name() != "" && v.CanAddr() {
haveAddr = true
v = v.Addr()
}
for {
if v.Kind() == reflect.Interface && !v.IsNil() {
e := v.Elem()
if e.Kind() == reflect.Ptr && !e.IsNil() && e.Elem().Kind() == reflect.Ptr {
haveAddr = false
v = e
continue
}
}
if v.Kind() != reflect.Ptr {
break
}
if v.Elem().Kind() != reflect.Ptr && v.CanSet() {
break
}
if v.IsNil() {
v.Set(reflect.New(v.Type().Elem()))
}
if v.Type().NumMethod() > 0 {
if u, ok := v.Interface().(encoding.TextUnmarshaler); ok {
return u, reflect.Value{}
}
}
if haveAddr {
v = v0
haveAddr = false
} else {
v = v.Elem()
}
}
return nil, v
}

View File

@@ -0,0 +1,128 @@
package dsn
import (
"net/url"
"reflect"
"testing"
"time"
xtime "go-common/library/time"
)
type cfg1 struct {
Name string `dsn:"query.name"`
Def string `dsn:"query.def,hello"`
DefSlice []int `dsn:"query.defslice,1,2,3,4"`
Ignore string `dsn:"-"`
FloatNum float64 `dsn:"query.floatNum"`
}
type cfg2 struct {
Timeout xtime.Duration `dsn:"query.timeout"`
}
type cfg3 struct {
Username string `dsn:"username"`
Timeout xtime.Duration `dsn:"query.timeout"`
}
type cfg4 struct {
Timeout xtime.Duration `dsn:"query.timeout,1s"`
}
func TestDecodeQuery(t *testing.T) {
type args struct {
query url.Values
v interface{}
assignFuncs map[string]assignFunc
}
tests := []struct {
name string
args args
want url.Values
cfg interface{}
wantErr bool
}{
{
name: "test generic",
args: args{
query: url.Values{
"name": {"hello"},
"Ignore": {"test"},
"floatNum": {"22.33"},
"adb": {"123"},
},
v: &cfg1{},
},
want: url.Values{
"Ignore": {"test"},
"adb": {"123"},
},
cfg: &cfg1{
Name: "hello",
Def: "hello",
DefSlice: []int{1, 2, 3, 4},
FloatNum: 22.33,
},
},
{
name: "test go-common/library/time",
args: args{
query: url.Values{
"timeout": {"1s"},
},
v: &cfg2{},
},
want: url.Values{},
cfg: &cfg2{xtime.Duration(time.Second)},
},
{
name: "test empty go-common/library/time",
args: args{
query: url.Values{},
v: &cfg2{},
},
want: url.Values{},
cfg: &cfg2{},
},
{
name: "test go-common/library/time",
args: args{
query: url.Values{},
v: &cfg4{},
},
want: url.Values{},
cfg: &cfg4{xtime.Duration(time.Second)},
},
{
name: "test build-in value",
args: args{
query: url.Values{
"timeout": {"1s"},
},
v: &cfg3{},
assignFuncs: map[string]assignFunc{"username": stringsAssignFunc("hello")},
},
want: url.Values{},
cfg: &cfg3{
Timeout: xtime.Duration(time.Second),
Username: "hello",
},
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := bindQuery(tt.args.query, tt.args.v, tt.args.assignFuncs)
if (err != nil) != tt.wantErr {
t.Errorf("DecodeQuery() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("DecodeQuery() = %v, want %v", got, tt.want)
}
if !reflect.DeepEqual(tt.args.v, tt.cfg) {
t.Errorf("DecodeQuery() = %v, want %v", tt.args.v, tt.cfg)
}
})
}
}