Initial commit

app/service/ep/saga-agent/BUILD

@@ -0,0 +1,20 @@
+package(default_visibility = ["//visibility:public"])
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ep/saga-agent/cmd:all-srcs",
+        "//app/service/ep/saga-agent/conf:all-srcs",
+        "//app/service/ep/saga-agent/path:all-srcs",
+        "//app/service/ep/saga-agent/service/agent:all-srcs",
+    ],
+    tags = ["automanaged"],
+)

app/service/ep/saga-agent/CHANGELOG.md

@@ -0,0 +1,5 @@
+# gitlab-CI相关的工具
+
+# v1.0.0
+1.创建工具agent用于Caster里自动拉取配置动态注册gitlab-runner
+2.调整changelog和path两个小工具到该目录下

app/service/ep/saga-agent/CONTRIBUTORS.md

@@ -0,0 +1,12 @@
+# Owner
+muyang
+fangrongchang
+
+# Author 
+muyang
+yubaihai
+wangweizhen
+fangrongchang
+
+# Reviewer
+muyang

app/service/ep/saga-agent/OWNERS

@@ -0,0 +1,18 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- fangrongchang
+- muyang
+- wangweizhen
+- yubaihai
+labels:
+- ep
+- service
+- service/ep/saga-agent
+options:
+  no_parent_owners: true
+reviewers:
+- fangrongchang
+- muyang
+- wangweizhen
+- yubaihai

app/service/ep/saga-agent/cmd/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "go-common/app/service/ep/saga-agent/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ep/saga-agent/conf:go_default_library",
+        "//app/service/ep/saga-agent/service/agent:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ep/saga-agent/cmd/main.go

@@ -0,0 +1,37 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+
+	"go-common/app/service/ep/saga-agent/conf"
+	"go-common/app/service/ep/saga-agent/service/agent"
+	"go-common/library/log"
+)
+
+func listenSignal() {
+	sigs := make(chan os.Signal, 1)
+	signal.Notify(sigs, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT, syscall.SIGUSR1,
+		syscall.SIGUSR2, syscall.SIGTSTP)
+	select {
+	case <-sigs:
+		log.Info("get sig=%v\n, RunnerUnRegisterAll", sigs)
+		agent.RunnerUnRegisterAll()
+	default:
+		log.Info("get sig=%v\n", sigs)
+	}
+}
+
+func main() {
+	log.Info("agent start......")
+	err := conf.Init()
+	if err != nil {
+		panic(err)
+	}
+	go listenSignal()
+	go agent.UpdateRegister()
+	agent.ExecRegister()
+	agent.RunnerStart()
+	agent.RunnerUnRegisterAll()
+}

app/service/ep/saga-agent/conf/BUILD

@@ -0,0 +1,44 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["conf_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+    deps = ["//vendor/github.com/smartystreets/goconvey/convey:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/service/ep/saga-agent/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ep/saga-agent/conf/conf.go

@@ -0,0 +1,117 @@
+package conf
+
+import (
+	"flag"
+	"os"
+
+	"go-common/library/conf"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+	"github.com/pkg/errors"
+)
+
+// Runner ...
+type Runner struct {
+	URL   string
+	Token string
+	Name  string
+}
+
+type config struct {
+	Offline bool
+	Runner  []Runner
+}
+
+// Conf ...
+var (
+	confPath  string
+	client    *conf.Client
+	filename  string
+	reload    chan bool
+	Conf      config
+	RunnerMap map[string]Runner
+	HostName  string
+)
+
+func load() (err error) {
+	var (
+		s  string
+		ok bool
+	)
+	if s, ok = client.Value(filename); !ok {
+		err = errors.Errorf("load config center error [%s]", filename)
+		return
+	}
+
+	log.Info("config data: %s", s)
+	Conf = config{}
+	if _, err = toml.Decode(s, &Conf); err != nil {
+		err = errors.Wrapf(err, "could not decode config err(%+v)", err)
+		return
+	}
+	return
+}
+
+func configCenter() (err error) {
+	if filename == "" {
+		return errors.New("filename is null,please set the environment(RUNNER_REPONAME)")
+	}
+	if client, err = conf.New(); err != nil {
+		panic(err)
+	}
+	if err = load(); err != nil {
+		return
+	}
+	client.Watch(filename)
+	go func() {
+		for range client.Event() {
+			log.Info("config reload")
+
+			if load() != nil {
+				log.Error("config reload error (%v)", err)
+			} else {
+				reload <- true
+			}
+		}
+	}()
+	return
+}
+
+func osHostName() string {
+	var (
+		host string
+		err  error
+	)
+	if host, err = os.Hostname(); err != nil {
+		host = "unKown-Hostname"
+		log.Error("%+v", errors.WithStack(err))
+	}
+	return host
+}
+
+func init() {
+	flag.StringVar(&confPath, "conf", "", "config path")
+}
+
+// Init ...
+func Init() (err error) {
+	filename = os.Getenv("CONF_FILENAME")
+	RunnerMap = make(map[string]Runner)
+	HostName = osHostName()
+	reload = make(chan bool, 1)
+	if confPath == "" {
+		err = configCenter()
+		return
+	}
+	if _, err = toml.DecodeFile(confPath, &Conf); err != nil {
+		log.Error("toml.DecodeFile(%s) err(%+v)", confPath, err)
+		return
+	}
+	return
+}
+
+// ReloadEvent ...
+func ReloadEvent() <-chan bool {
+	return reload
+}

app/service/ep/saga-agent/conf/conf_test.go

@@ -0,0 +1,30 @@
+package conf
+
+import (
+	"flag"
+	"fmt"
+	"testing"
+
+	. "github.com/smartystreets/goconvey/convey"
+)
+
+func showConf() {
+	for _, v := range Conf.Runner {
+		fmt.Println(v.URL, v.Token)
+	}
+}
+
+func init() {
+	var err error
+	flag.Set("conf", "../service/agent/runners.toml")
+	if err = Init(); err != nil {
+		panic(err)
+	}
+	showConf()
+}
+
+func TestConf(t *testing.T) {
+	Convey("test conf", t, func() {
+		So(Conf.Runner, ShouldNotBeEmpty)
+	})
+}

app/service/ep/saga-agent/path/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "path",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "go-common/app/service/ep/saga-agent/path",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//vendor/github.com/pkg/errors:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ep/saga-agent/path/build.sh

@@ -0,0 +1 @@
+env GOOS=linux GOARCH=amd64 go build -o pathchecker

app/service/ep/saga-agent/path/main.go

@@ -0,0 +1,99 @@
+package main
+
+import (
+	"flag"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	flagDep     = flag.String("dep", "main,live,openplatform,ep", "department list , split by comma")
+	flagPrefix  = flag.String("prefix", `business`, "prefix path")
+	flagService = flag.String("serivce", "interface,job,admin,service", "service type")
+	// flagWhite prefix下允许的dir名称
+	flagWhite = flag.String("white", "", "white subpath from prefix , split by comma")
+)
+
+const (
+	codeSuccess = 0
+	codeFail    = 1
+)
+
+func main() {
+	flag.Parse()
+	var (
+		depList      []string
+		serviceList  []string
+		filePathList []string
+		whiteDirList []string
+	)
+	filePathList = flag.Args()
+	if len(filePathList) <= 0 {
+		fmt.Println("No file to check")
+		os.Exit(codeSuccess)
+	}
+
+	depList = strings.Split(*flagDep, ",")
+	serviceList = strings.Split(*flagService, ",")
+	for _, wd := range strings.Split(*flagWhite, ",") {
+		if wd != "" {
+			whiteDirList = append(whiteDirList, strings.Join([]string{*flagPrefix, wd}, "/"))
+		}
+	}
+	code := check(filePathList, serviceList, depList, whiteDirList)
+	os.Exit(code)
+}
+
+func check(filePathList []string, serviceTypeList []string, depList []string, whiteDirList []string) (code int) {
+	var (
+		regDep      = strings.Join(depList, "|")
+		serviceType = strings.Join(serviceTypeList, "|")
+		regStr      = fmt.Sprintf(`%s/(%s)/(%s)`, *flagPrefix, serviceType, regDep)
+		reg         *regexp.Regexp
+		flag        = true
+		failedFiles []string
+		err         error
+	)
+	regStr = strings.Replace(regStr, "/", `\/`, -1)
+	if reg, err = regexp.Compile(regStr); err != nil {
+		err = errors.Wrapf(err, "regexp : %s", regStr)
+		fmt.Printf("%+v\n", err)
+		code = codeFail
+		return
+	}
+	for _, p := range filePathList {
+		if strings.HasPrefix(p, *flagPrefix) {
+			if whiteCheck(whiteDirList, p) {
+				continue
+			}
+			if !reg.MatchString(p) {
+				failedFiles = append(failedFiles, p)
+				flag = false
+				break
+			}
+		}
+	}
+	if !flag {
+		fmt.Println("invalid files : ")
+		for _, f := range failedFiles {
+			fmt.Printf("\t%s\n", f)
+		}
+		code = codeFail
+	} else {
+		code = codeSuccess
+	}
+	return
+}
+
+func whiteCheck(whiteDirList []string, path string) bool {
+	for _, wd := range whiteDirList {
+		if strings.HasPrefix(path, wd) {
+			return true
+		}
+	}
+	return false
+}

app/service/ep/saga-agent/service/agent/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_test",
+    "go_library",
+)
+
+go_test(
+    name = "go_default_test",
+    srcs = ["agent_test.go"],
+    embed = [":go_default_library"],
+    rundir = ".",
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["agent.go"],
+    data = ["runners.toml"],
+    importpath = "go-common/app/service/ep/saga-agent/service/agent",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ep/saga-agent/conf:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/pkg/errors:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ep/saga-agent/service/agent/agent.go

@@ -0,0 +1,146 @@
+package agent
+
+import (
+	"bytes"
+	"fmt"
+	"os/exec"
+	"runtime/debug"
+	"strings"
+
+	"go-common/app/service/ep/saga-agent/conf"
+	"go-common/library/log"
+
+	"github.com/pkg/errors"
+)
+
+var (
+	hostName string
+)
+
+const _runnerWorkDir = "/data/gitlab-runner"
+
+func shellExec(dir string, name string, args ...string) (stdout string, stderr string, err error) {
+	defer func() {
+		log.Info("Shell Exec dir[%s] name[%s] args[%+v] stdout[%s] stderr[%s] err[%+v]", dir, name, args, stdout, stderr, err)
+	}()
+	if _, err = exec.LookPath(name); err != nil {
+		err = errors.Wrapf(err, "LookPath(%s) failed", name)
+		return
+	}
+	var (
+		stdo = new(bytes.Buffer)
+		stde = new(bytes.Buffer)
+		cmd  *exec.Cmd
+	)
+	cmd = exec.Command(name, args...)
+	cmd.Stdout = stdo
+	cmd.Stderr = stde
+	cmd.Dir = dir
+	if err = cmd.Run(); err != nil {
+		err = errors.Wrapf(err, "cmd.Run(%s,%s,%v) failed", dir, name, args)
+	}
+	stdout = stdo.String()
+	stderr = stde.String()
+	return
+}
+
+// RunnerRegister ...
+func RunnerRegister(url, token, name string) (stdout string, stderr string, err error) {
+	var args string
+
+	if (url == "") || (token == "") {
+		err = fmt.Errorf("RunnerRegister invalid parameters(url:%s token:%s)", url, token)
+		log.Error(err.Error())
+		return
+	}
+
+	log.Info("gitlab-runner register url:%s token:%s", url, token)
+	args = fmt.Sprintf("gitlab-runner register -u %s -r %s --name %s --executor shell -n true -c /data/gitlab-runner/config.toml",
+		url, token, name)
+	if stdout, stderr, err = shellExec(_runnerWorkDir, "/bin/sh", "-c", args); err != nil {
+		log.Error(strings.TrimSpace(stdout + "\n" + stderr))
+	}
+	return
+}
+
+// RunnerUnRegister ...
+func RunnerUnRegister(name string) (stdout string, stderr string, err error) {
+	var args string
+	log.Info("gitlab-runner unregister name:%s", name)
+	args = fmt.Sprintf("gitlab-runner unregister -n %s", name)
+	if stdout, stderr, err = shellExec(_runnerWorkDir, "/bin/sh", "-c", args); err != nil {
+		log.Error(strings.TrimSpace(stdout + "\n" + stderr))
+	}
+	return
+}
+
+// RunnerUnRegisterAll ...
+func RunnerUnRegisterAll() (stdout string, stderr string, err error) {
+	var args string
+	log.Info("gitlab-runner RunnerUnregisterAll")
+	args = "gitlab-runner unregister --all-runners"
+	if stdout, stderr, err = shellExec(_runnerWorkDir, "/bin/sh", "-c", args); err != nil {
+		log.Error(strings.TrimSpace(stdout + "\n" + stderr))
+	}
+	return
+}
+
+// RunnerStart ...
+func RunnerStart() (stdout string, stderr string, err error) {
+	var args string
+	log.Info("gitlab-runner start...")
+	args = fmt.Sprintf("gitlab-runner run -d %s", _runnerWorkDir)
+	if stdout, stderr, err = shellExec(_runnerWorkDir, "/bin/sh", "-c", args); err != nil {
+		log.Error(strings.TrimSpace(stdout + "\n" + stderr))
+	}
+	return
+
+}
+
+// ExecRegister ...
+func ExecRegister() {
+	var tmpMap map[string]conf.Runner
+
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("execRegister: %+v %s", x, debug.Stack())
+		}
+	}()
+
+	log.Info("execRegister... runner.offline %t,runner.len:%d", conf.Conf.Offline, len(conf.Conf.Runner))
+	if conf.Conf.Offline {
+		conf.RunnerMap = make(map[string]conf.Runner)
+		RunnerUnRegisterAll()
+		return
+	}
+	tmpMap = make(map[string]conf.Runner)
+	for _, v := range conf.Conf.Runner {
+		tmpMap[v.Token] = v
+		_, ok := conf.RunnerMap[v.Token]
+		if !ok {
+			RunnerRegister(v.URL, v.Token, hostName+"-"+v.Name)
+			conf.RunnerMap[v.Token] = v
+		}
+	}
+	for _, v := range conf.RunnerMap {
+		_, ok := tmpMap[v.Token]
+		if !ok {
+			RunnerUnRegister(hostName + "-" + v.Name)
+			delete(conf.RunnerMap, v.Token)
+		}
+	}
+}
+
+// UpdateRegister ...
+func UpdateRegister() {
+	defer func() {
+		if x := recover(); x != nil {
+			log.Error("updateRegister: %+v %s", x, debug.Stack())
+		}
+	}()
+
+	for range conf.ReloadEvent() {
+		log.Info("updateRegister")
+		ExecRegister()
+	}
+}

app/service/ep/saga-agent/service/agent/agent_test.go

@@ -0,0 +1,33 @@
+package agent
+
+import (
+	"testing"
+)
+
+func TestRunnerStart(t *testing.T) {
+	stdout, stderr, err := RunnerStart()
+	if err != nil {
+		t.Error(stdout, stderr)
+	}
+}
+
+func TestRunnerRegister(t *testing.T) {
+	stdout, stderr, err := RunnerRegister("http://gitlab.bilibili.co/", "pxZPKWk1JQHLHNzYyj5p", "mac-test-1")
+	if err != nil {
+		t.Error(stdout, stderr)
+	}
+}
+
+func TestRunnerUnRegister(t *testing.T) {
+	stdout, stderr, err := RunnerUnRegister("mac-test-1")
+	if err != nil {
+		t.Error(stdout, stderr)
+	}
+}
+
+func TestRunnerUnRegisterAll(t *testing.T) {
+	stdout, stderr, err := RunnerUnRegisterAll()
+	if err != nil {
+		t.Error(stdout, stderr)
+	}
+}

app/service/ep/saga-agent/service/agent/runners.toml

@@ -0,0 +1,10 @@
+ID=123
+[[runner]]
+url="http://gitlab.bilibili.co/"
+token="pxZPKWk1JQHLHNzYyj5p"
+name="mac-test-1"
+
+[[runner]]
+url="http://gitlab.bilibili.co/"
+token="crkm4z2ZQjfgQRi_puup"
+name="mac-test-2"