Initial commit

app/service/ops/OWNERS

@@ -0,0 +1,6 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+labels:
+- new-project
+- ops
+- service

app/service/ops/log-agent/BUILD

@@ -0,0 +1,29 @@
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/cmd:all-srcs",
+        "//app/service/ops/log-agent/conf:all-srcs",
+        "//app/service/ops/log-agent/event:all-srcs",
+        "//app/service/ops/log-agent/input:all-srcs",
+        "//app/service/ops/log-agent/output:all-srcs",
+        "//app/service/ops/log-agent/pipeline:all-srcs",
+        "//app/service/ops/log-agent/pkg/bufio:all-srcs",
+        "//app/service/ops/log-agent/pkg/common:all-srcs",
+        "//app/service/ops/log-agent/pkg/flowmonitor:all-srcs",
+        "//app/service/ops/log-agent/pkg/httpstream:all-srcs",
+        "//app/service/ops/log-agent/pkg/lancermonitor:all-srcs",
+        "//app/service/ops/log-agent/pkg/lancerroute:all-srcs",
+        "//app/service/ops/log-agent/pkg/limit:all-srcs",
+        "//app/service/ops/log-agent/processor:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/CHANGELOG.md

@@ -0,0 +1,65 @@
+### log-agent
+##### 2.1.0(20181228)
+> 1. 支持文件采集
+
+##### 2.0.6(20181224)
+> 1. priority==high的时候不采样
+
+##### 2.0.5(20181224)
+> 1. 支持grpc与lancer交互
+> 2. 支持priority=high标识高优先级日志
+
+##### 2.0.4(20181212)
+> 1. fix fmt import
+
+##### 2.0.3(20181205)
+> 1. 调整日志默认最长为32K
+
+##### 2.0.2(20181128)
+> 1. 支持lancer route table
+
+##### 2.0.1(20181120)
+> 1. fix: revocer from bufio.Write error
+
+##### 2.0.0(20181110)
+> 1. 插件化架构
+> 2. 支持非日志类数据上报（非000161）
+> 3. flowmonitor 增加kind字段，标识是否为错误
+
+##### 1.1.8.2(20180830)
+> 1. httpstream查看日志
+
+##### 1.1.8.1(20180821)
+> 1. flush buf first when recycle conn caused by expired
+
+##### 1.1.8.0(20180820)
+> 1. lancer新上报协议
+
+##### 1.1.7.2(20180810)
+> 1. 修复sendChan 中bytes.buffer 的data race问题
+
+##### 1.1.7.1(20180726)
+> 1. 支持日志聚合发送
+> 2. 重构连接池：并发发送 + buf和conn隔离
+
+##### 1.1.7(20180726)
+> 1. 修复buf size
+> 2. 修复getConn时错误的处理，nil不会被putConn
+
+##### 1.1.6(20180621)
+> 1. 校验日志长度是否大于_logLancerHeaderLen
+
+##### 1.1.4(20180619)
+> 1. 增加telnet功能，本地流式查看日志
+> 2. 重构日志流监控
+
+##### 1.1.3(20180604)
+> 1. 修改日志级别策略，先检查路由表再检查app_id合规性
+
+##### 1.1.2(20180529)
+> 1. 增加路由表功能：根据日志分级策略路由到不同的logid
+> 2. 批量接收日志（collector_tcp.sock）
+
+##### 1.1.1
+> 1. 通过conn pool连接到lancer
+

app/service/ops/log-agent/CONTRIBUTORS.md

@@ -0,0 +1,11 @@
+# Owner
+maojian
+wangxiangyu
+
+# Author 
+wangxiangyu
+
+# Reviewer
+maojian
+weicheng
+chenzhihui

app/service/ops/log-agent/OWNERS

@@ -0,0 +1,16 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- maojian
+- wangxiangyu
+labels:
+- ops
+- service
+- service/ops/log-agent
+options:
+  no_parent_owners: true
+reviewers:
+- chenzhihui
+- maojian
+- wangxiangyu
+- weicheng

app/service/ops/log-agent/README.md

@@ -0,0 +1,22 @@
+# log-agent
+
+##### 项目简介
+> 1. 公司级日志收集组件，收集应用日志，然后发送给lancer
+> 2. 支持本地缓存，确保数据可靠传输
+> 3. 暴露http api，支持日志流式查看。
+> 4. 支持日志采样
+> 5. app级别日志流实时监控
+
+##### 编译环境
+> 1. 请使用golang v1.7.x以上版本编译执行。
+
+##### 依赖包
+> 1. 依赖github.com/prometheus/client_golang/prometheus
+
+##### 编译执行
+> 1. 编译后在物理机上启动log-agent即可。
+> 2. 基于rms进程agent的管理与升级
+
+##### 使用方式
+> 1. [日志规范](http://info.bilibili.co/pages/viewpage.action?pageId=3674729)
+> 2. [日志接入流程](http://info.bilibili.co/pages/viewpage.action?pageId=3678680)

app/service/ops/log-agent/cmd/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_binary",
+    "go_library",
+)
+
+go_binary(
+    name = "cmd",
+    embed = [":go_default_library"],
+    tags = ["automanaged"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["main.go"],
+    importpath = "go-common/app/service/ops/log-agent/cmd",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf:go_default_library",
+        "//app/service/ops/log-agent/pipeline:go_default_library",
+        "//app/service/ops/log-agent/pipeline/dockerlogcollector:go_default_library",
+        "//app/service/ops/log-agent/pipeline/hostlogcollector:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/httpstream:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancermonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancerroute:go_default_library",
+        "//app/service/ops/log-agent/pkg/limit:go_default_library",
+        "//library/conf/env:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//library/net/ip:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/cmd/main.go

@@ -0,0 +1,158 @@
+package main
+
+import (
+	"os"
+	"os/signal"
+	"syscall"
+	"time"
+	"context"
+	"flag"
+	"fmt"
+	"net/http"
+
+	"go-common/app/service/ops/log-agent/conf"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pipeline"
+	"go-common/app/service/ops/log-agent/pipeline/hostlogcollector"
+	"go-common/app/service/ops/log-agent/pipeline/dockerlogcollector"
+	"go-common/app/service/ops/log-agent/pkg/limit"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+	"go-common/app/service/ops/log-agent/pkg/httpstream"
+	"go-common/app/service/ops/log-agent/pkg/lancermonitor"
+	"go-common/app/service/ops/log-agent/pkg/lancerroute"
+	"go-common/library/conf/env"
+	xip "go-common/library/net/ip"
+	"go-common/library/naming/discovery"
+	"go-common/library/naming"
+)
+
+const AppVersion = "2.1.0"
+
+type Agent struct {
+	limit      *limit.Limit
+	httpstream *httpstream.HttpStream
+}
+
+func main() {
+	var (
+		err         error
+		ctx, cancel = context.WithCancel(context.Background())
+		//cancel context.CancelFunc
+	)
+	version := flag.Bool("v", false, "show version and exit")
+	flag.Parse()
+	if *version {
+		fmt.Println(AppVersion)
+		os.Exit(0)
+	}
+
+	if err = conf.Init(); err != nil {
+		panic(err)
+	}
+
+	agent := new(Agent)
+
+	// set context
+	ctx = context.WithValue(ctx, "GlobalConfig", conf.Conf)
+	ctx = context.WithValue(ctx, "MetaPath", conf.Conf.HostLogCollector.MetaPath)
+
+	// init xlog
+	conf.Conf.Log.Stdout = true // ooooo just for test
+	log.Init(conf.Conf.Log)
+	defer log.Close()
+	log.Info("log agent [version: %s] start", AppVersion)
+
+	// resource limit by cgroup
+	if conf.Conf.Limit != nil {
+		conf.Conf.Limit.AppName = "log-agent"
+		if agent.limit, err = limit.LimitRes(conf.Conf.Limit); err != nil {
+			log.Warn("resource limit disabled: %s", err)
+		}
+	}
+
+	// /debug/vars
+	if conf.Conf.DebugAddr != "" {
+		go http.ListenAndServe(conf.Conf.DebugAddr, nil)
+	}
+
+	// init lancer route
+	err = lancerroute.InitLancerRoute()
+	if err != nil {
+		panic(err)
+	}
+
+	// httpstream
+	if agent.httpstream, err = httpstream.NewHttpStream(conf.Conf.HttpStream); err != nil {
+		log.Warn("httpstream disabled: %s", err)
+	}
+
+	// start pipeline management
+	err = pipeline.InitPipelineMng(ctx)
+	if err != nil {
+		panic(err)
+	}
+
+	// start host log collector
+	err = hostlogcollector.InitHostLogCollector(ctx, conf.Conf.HostLogCollector)
+	if err != nil {
+		panic(err)
+	}
+
+	// start docker log collector
+	err = dockerlogcollector.InitDockerLogCollector(ctx, conf.Conf.DockerLogCollector)
+	if err != nil {
+		log.Error("failed to start docker log collector: %s", err)
+	}
+
+	// flow monitor
+	if conf.Conf.Flowmonitor != nil {
+		if err = flowmonitor.InitFlowMonitor(conf.Conf.Flowmonitor); err != nil {
+			panic(err)
+		}
+	}
+
+	// lancer monitor
+	if conf.Conf.LancerMonitor != nil {
+		if _, err = lancermonitor.InitLancerMonitor(conf.Conf.LancerMonitor); err != nil {
+			panic(err)
+		}
+	}
+
+	// start discovery register
+	if env.IP == "" {
+		ip := xip.InternalIP()
+		dis := discovery.New(conf.Conf.Discovery)
+		ins := &naming.Instance{
+			Zone:     env.Zone,
+			Env:      env.DeployEnv,
+			AppID:    env.AppID,
+			Hostname: env.Hostname,
+			Version:  AppVersion,
+			Addrs: []string{
+				ip,
+			},
+		}
+		_, err = dis.Register(ctx, ins)
+		if err != nil {
+			panic(err)
+		}
+	}
+
+	// signal
+	ch := make(chan os.Signal, 1)
+	signal.Notify(ch, syscall.SIGHUP, syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGINT, syscall.SIGSTOP)
+	for {
+		s := <-ch
+		log.Info("agent get a signal %s", s.String())
+		switch s {
+		case syscall.SIGQUIT, syscall.SIGTERM, syscall.SIGSTOP, syscall.SIGINT, syscall.SIGHUP:
+			if cancel != nil {
+				cancel()
+			}
+			time.Sleep(time.Second)
+			return
+		default:
+			return
+		}
+	}
+}

app/service/ops/log-agent/conf/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["conf.go"],
+    importpath = "go-common/app/service/ops/log-agent/conf",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//app/service/ops/log-agent/pipeline/dockerlogcollector:go_default_library",
+        "//app/service/ops/log-agent/pipeline/hostlogcollector:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/httpstream:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancermonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/limit:go_default_library",
+        "//library/log:go_default_library",
+        "//library/naming/discovery:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/conf/configcenter:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/conf/conf.go

@@ -0,0 +1,105 @@
+package conf
+
+import (
+	"errors"
+	"time"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/limit"
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+	"go-common/app/service/ops/log-agent/pkg/httpstream"
+	"go-common/app/service/ops/log-agent/pkg/lancermonitor"
+	"go-common/app/service/ops/log-agent/pipeline/hostlogcollector"
+	"go-common/app/service/ops/log-agent/pipeline/dockerlogcollector"
+	"go-common/library/naming/discovery"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	config = "agent.toml"
+)
+
+var (
+	// Conf conf
+	Conf = &Config{}
+)
+
+type Config struct {
+	// discovery
+	Discovery *discovery.Config `toml:"discovery"`
+	// log
+	Log *log.Config `toml:"log"`
+	// flow monitor
+	Flowmonitor *flowmonitor.Config `toml:"flowmonitor"`
+	// limit
+	Limit *limit.LimitConf `toml:"limit"`
+	// debug
+	DebugAddr string `toml:"debugAddr"`
+	// httpstream
+	HttpStream *httpstream.Config `toml:"httpstream"`
+	// lancermonitor
+	LancerMonitor *lancermonitor.Config `toml:"lancermonitor"`
+	// hostlogcollector
+	HostLogCollector *hostlogcollector.Config `toml:"hostlogcollector"`
+	// docker log collector
+	DockerLogCollector *dockerlogcollector.Config `toml:"dockerLogCollector"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of log agent can't be nil")
+	}
+
+	if c.DockerLogCollector == nil {
+		c.DockerLogCollector = new(dockerlogcollector.Config)
+	}
+
+	if c.HostLogCollector == nil {
+		c.HostLogCollector = new(hostlogcollector.Config)
+	}
+
+	return nil
+}
+
+// initConfig init config
+func Init() (err error) {
+	configcenter.InitConfigCenter()
+
+	if err = readConfig(); err != nil {
+		return
+	}
+
+	go func() {
+		currentVersion := configcenter.Version
+		for {
+			if currentVersion != configcenter.Version {
+				log.Info("lancer route config reload")
+				if err := readConfig(); err != nil {
+					log.Error("lancer route config reload error (%v", err)
+				}
+				currentVersion = configcenter.Version
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+	return Conf.ConfigValidate()
+}
+
+//// readConfig read config from config center
+func readConfig() (err error) {
+	var (
+		ok        bool
+		value     string
+		tmpConfig *Config
+	)
+	//config
+	if value, ok = configcenter.Client.Value(config); !ok {
+		return errors.New("failed to get agent.toml")
+	}
+	if _, err = toml.Decode(value, &tmpConfig); err != nil {
+		return err
+	}
+	Conf = tmpConfig
+	return
+}

app/service/ops/log-agent/conf/configcenter/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["sven.go"],
+    importpath = "go-common/app/service/ops/log-agent/conf/configcenter",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/conf:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/conf/configcenter/sven.go

@@ -0,0 +1,28 @@
+package configcenter
+
+import (
+	"go-common/library/log"
+	"go-common/library/conf"
+)
+
+var (
+	// Conf conf
+	Client  *conf.Client
+	Version int
+)
+
+func InitConfigCenter() {
+	var err error
+	if Client, err = conf.New(); err != nil {
+		panic(err)
+	}
+
+	// watch update and update Version
+	Client.WatchAll()
+	go func() {
+		for range Client.Event() {
+			log.Info("config reload")
+			Version += 1
+		}
+	}()
+}

app/service/ops/log-agent/event/BUILD

@@ -0,0 +1,29 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["event.go"],
+    importpath = "go-common/app/service/ops/log-agent/event",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/log:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/event/event.go

@@ -0,0 +1,71 @@
+package event
+
+import (
+	"sync"
+	"time"
+	"go-common/library/log"
+)
+
+var pool sync.Pool
+
+// event between input and processor
+type ProcessorEvent struct {
+	Source       string
+	Destination  string
+	LogId        string
+	AppId        []byte
+	Level        []byte
+	Time         time.Time
+	Body         []byte
+	Priority     string
+	Length       int
+	TimeRangeKey string
+	Fields       map[string]interface{}
+	ParsedFields map[string]string
+	Tags         []string
+}
+
+//  GetEvent get event from pool.
+func GetEvent() (e *ProcessorEvent) {
+	var (
+		ok  bool
+		tmp = pool.Get()
+	)
+	if e, ok = tmp.(*ProcessorEvent); !ok {
+		e = &ProcessorEvent{Body: make([]byte, 1024*64), Tags: make([]string, 0, 1)} // max 64K, should be longer than max log lentth
+	}
+	e.LogId = ""
+	e.Length = 0
+	e.AppId = nil
+	e.Level = nil
+	e.Time = time.Time{}
+	e.TimeRangeKey = ""
+	e.Source = ""
+	e.Priority = ""
+	e.Destination = ""
+	e.Tags = e.Tags[:0]
+	e.Fields = make(map[string]interface{})
+	e.ParsedFields = make(map[string]string)
+	return e
+}
+
+// PutEvent put event back to pool
+func PutEvent(e *ProcessorEvent) {
+	pool.Put(e)
+}
+
+func (e *ProcessorEvent) Bytes() []byte {
+	return e.Body[:e.Length]
+}
+
+func (e *ProcessorEvent) String() string {
+	return string(e.Body[:e.Length])
+}
+
+func (e *ProcessorEvent) Write(b []byte) {
+	if len(b) > cap(e.Body) {
+		log.Error("bytes write beyond e.Body capacity")
+	}
+
+	e.Length = copy(e.Body, b)
+}

app/service/ops/log-agent/input/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["input.go"],
+    importpath = "go-common/app/service/ops/log-agent/input",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/input/file:all-srcs",
+        "//app/service/ops/log-agent/input/sock:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/input/file/BUILD

@@ -0,0 +1,46 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "file.go",
+        "glob.go",
+        "harvester.go",
+        "multiline.go",
+        "reader.go",
+        "registrar.go",
+        "state.go",
+        "states.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/input/file",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/input:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancerroute:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/input/file/config.go

@@ -0,0 +1,103 @@
+package file
+
+import (
+	"errors"
+	"time"
+
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	Paths           []string               `toml:"paths"`
+	Symlinks        bool                   `toml:"symlinks"`
+	AppId           string                 `toml:"appId"`
+	LogId           string                 `toml:"logId"`
+	ConfigPath      string                 `toml:"-"`
+	MetaPath        string                 `toml:"-"`
+	ID              string                 `toml:"-"`
+	ReadFrom        string                 `toml:"readFrom"`
+	MaxLength       int                    `toml:"maxLength"`
+	IgnoreOlder     xtime.Duration         `toml:"ignoreOlder"`
+	CleanFilesOlder xtime.Duration         `toml:"cleanFilesOlder"`
+	ScanFrequency   xtime.Duration         `toml:"scanFrequency"`
+	CleanInactive   xtime.Duration         `toml:"cleanInactive"`
+	HarvesterTTL    xtime.Duration         `toml:"harvesterTTL"` // harvester will stop itself if inactive longer than HarvesterTTL
+	Multiline       *MultilineConf         `toml:"multiline"`
+	Timeout         xtime.Duration         `toml:"timeout"`
+	Fields          map[string]interface{} `toml:"fields"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of file Input is nil")
+	}
+
+	if len(c.Paths) == 0 {
+		return errors.New("paths of file Input can't be nil")
+	}
+
+	if c.LogId == "" {
+		c.LogId = "000161"
+	}
+
+	if c.AppId == "" {
+		return errors.New("appId of file Input can't be nil")
+	}
+
+	if c.IgnoreOlder == 0 {
+		c.IgnoreOlder = xtime.Duration(time.Hour * 24)
+	}
+
+	if c.ScanFrequency == 0 {
+		c.ScanFrequency = xtime.Duration(time.Second * 10)
+	}
+
+	// Note: CleanInactive should be greater chan ignore_older + scan_frequency
+	if c.CleanInactive == 0 {
+		c.CleanInactive = xtime.Duration(time.Hour * 24 * 7)
+	}
+
+	if c.CleanInactive < c.IgnoreOlder+c.ScanFrequency {
+		return errors.New("CleanInactive must be greater than ScanFrequency + IgnoreOlder")
+	}
+
+	if c.HarvesterTTL == 0 {
+		c.HarvesterTTL = xtime.Duration(time.Hour * 1)
+	}
+
+	if c.Timeout == 0 {
+		c.Timeout = xtime.Duration(time.Second * 5)
+	}
+
+	if c.ReadFrom != "" && c.ReadFrom != "newest" && c.ReadFrom != "oldest" {
+		return errors.New("ReadFrom of file input can only be newest or oldest")
+	}
+
+	if c.ReadFrom == "" {
+		c.ReadFrom = "newest"
+	}
+
+	if c.MaxLength == 0 || c.MaxLength > 1024*10*64 {
+		c.MaxLength = 1024 * 10 * 64
+	}
+	// Symlinks is always disabled
+	c.Symlinks = false
+
+	if c.Multiline != nil {
+		if err := c.Multiline.ConfigValidate(); err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/input/file/file.go

@@ -0,0 +1,213 @@
+package file
+
+import (
+	"context"
+	"fmt"
+	"time"
+	"path"
+	"crypto/sha1"
+	"encoding/base64"
+	"errors"
+	"os"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/input"
+	"go-common/library/log"
+)
+
+type File struct {
+	c        *Config
+	output   chan<- *event.ProcessorEvent
+	ctx      context.Context
+	cancel   context.CancelFunc
+	register *Registrar
+}
+
+func init() {
+	err := input.Register("file", NewFile)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func NewFile(ctx context.Context, config interface{}, output chan<- *event.ProcessorEvent) (input.Input, error) {
+	f := new(File)
+	if c, ok := config.(*Config); !ok {
+		return nil, fmt.Errorf("Error config for File Input")
+	} else {
+		if err := c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		f.c = c
+	}
+	f.output = output
+	f.ctx, f.cancel = context.WithCancel(ctx)
+
+	// set config by ctx
+	if f.c.ConfigPath == "" {
+		configPath := ctx.Value("configPath")
+		if configPath == nil {
+			return nil, errors.New("can't get configPath from context")
+		}
+		f.c.ConfigPath = configPath.(string)
+	}
+
+	if f.c.ID == "" {
+		hasher := sha1.New()
+		hasher.Write([]byte(f.c.ConfigPath))
+		f.c.ID = base64.URLEncoding.EncodeToString(hasher.Sum(nil))
+	}
+
+	if f.c.MetaPath == "" {
+		f.c.MetaPath = ctx.Value("MetaPath").(string)
+	}
+
+	// init register
+	if err := f.initRegister(); err != nil {
+		return nil, err
+	}
+
+	return f, nil
+}
+
+func (f *File) Run() (err error) {
+	log.Info("start collect log configured in %s", f.c.ConfigPath)
+
+	f.scan()
+
+	ticker := time.Tick(time.Duration(f.c.ScanFrequency))
+	go func() {
+		for {
+			select {
+			case <-f.ctx.Done():
+				return
+			case <-ticker:
+				f.scan()
+			}
+		}
+	}()
+	return nil
+}
+
+func (f *File) Stop() {
+	f.cancel()
+}
+
+func (f *File) Ctx() (context.Context) {
+	return f.ctx
+}
+
+func (f *File) initRegister() error {
+	path := path.Join(f.c.MetaPath, f.c.ID)
+
+	register, err := NewRegistry(f.ctx, path)
+	if err != nil {
+		return err
+	}
+	f.register = register
+	return nil
+}
+
+// Scan starts a scanGlob for each provided path/glob
+func (f *File) scan() {
+	paths := f.getFiles()
+
+	// clean files older than
+	if time.Duration(f.c.CleanFilesOlder) != 0 {
+		f.cleanOldFiles(paths)
+	}
+
+	for path, info := range paths {
+		select {
+		case <-f.ctx.Done():
+			return
+		default:
+		}
+
+		newState, err := getFileState(path, info)
+		if err != nil {
+			log.Error("Skipping file %s due to error %s", path, err)
+			continue
+		}
+
+		// Load last state
+		lastState := f.register.FindPrevious(newState)
+
+		// Ignores all files which fall under ignore_older
+		if f.isIgnoreOlder(newState) {
+			continue
+		}
+
+		// Decides if previous state exists
+		if lastState.IsEmpty() {
+			log.Info("Start harvester for new file: %s, inode: %d", newState.Source, newState.Inode)
+			ctx := context.WithValue(f.ctx, "firstRun", true)
+			err := f.startHarvester(ctx, f.c, f.register, newState, 0)
+			if err != nil {
+				log.Error("Harvester could not be started on new file: %s, Err: %s", newState.Source, err)
+			}
+		} else {
+			ctx := context.WithValue(f.ctx, "firstRun", false)
+			f.harvestExistingFile(ctx, f.c, f.register, newState, lastState)
+		}
+	}
+}
+
+func (f *File) cleanOldFiles(paths map[string]os.FileInfo) {
+	if time.Duration(f.c.CleanFilesOlder) == 0 {
+		return
+	}
+
+	var latestFile *State
+	for path, info := range paths {
+		newState, err := getFileState(path, info)
+
+		if err != nil {
+			log.Error("Skipping file %s due to error %s", path, err)
+			continue
+		}
+
+		if latestFile == nil {
+			latestFile = &newState
+			continue
+		}
+
+		if newState.Fileinfo.ModTime().After(latestFile.Fileinfo.ModTime()) {
+			// delete latestFile if newer file existing and modtime of latestFile is older than f.c.CleanFilesOlder
+			if time.Since(latestFile.Fileinfo.ModTime()) > time.Duration(f.c.CleanFilesOlder) {
+				if err := os.Remove(latestFile.Source); err != nil {
+					log.Error("Failed to delete file %s", latestFile.Source)
+				} else {
+					log.Info("Delete file %s older than %s", latestFile.Source, time.Duration(f.c.CleanFilesOlder).String())
+				}
+			}
+			latestFile = &newState
+			continue
+		}
+
+		if newState.Fileinfo.ModTime().Before(latestFile.Fileinfo.ModTime()) {
+			if time.Since(newState.Fileinfo.ModTime()) > time.Duration(f.c.CleanFilesOlder) {
+				if err := os.Remove(newState.Source); err != nil {
+					log.Error("Failed to delete file %s", newState.Source)
+				} else {
+					log.Info("Delete file %s older than %s", newState.Source, time.Duration(f.c.CleanFilesOlder))
+				}
+			}
+		}
+	}
+}
+
+// isIgnoreOlder checks if the given state reached ignore_older
+func (f *File) isIgnoreOlder(state State) bool {
+	// ignore_older is disable
+	if f.c.IgnoreOlder == 0 {
+		return false
+	}
+
+	modTime := state.Fileinfo.ModTime()
+	if time.Since(modTime) > time.Duration(f.c.IgnoreOlder) {
+		return true
+	}
+
+	return false
+}

app/service/ops/log-agent/input/file/glob.go

@@ -0,0 +1,74 @@
+package file
+
+import (
+	"os"
+	"path/filepath"
+	"path"
+
+	"go-common/library/log"
+)
+
+func (f *File) getFiles() map[string]os.FileInfo {
+	paths := map[string]os.FileInfo{}
+
+	for _, p := range f.c.Paths {
+		// logs in docker overlay2
+		if MergedDir := f.ctx.Value("MergedDir"); MergedDir != nil {
+			p = path.Join(MergedDir.(string), p)
+		}
+
+		matches, err := filepath.Glob(p)
+		if err != nil {
+			log.Error("glob(%s) failed: %v", p, err)
+			continue
+		}
+
+		// Check any matched files to see if we need to start a harvester
+		for _, file := range matches {
+			// check if the file is in the exclude_files list
+			//if f.isFileExcluded(file) {
+			//	log.Info("input", "Exclude file: %s", file)
+			//	continue
+			//}
+
+			// Fetch Lstat File info to detected also symlinks
+			fileInfo, err := os.Lstat(file)
+			if err != nil {
+				log.Warn("lstat(%s) failed: %s", file, err)
+				continue
+			}
+
+			if fileInfo.IsDir() {
+				log.Warn("Skipping directory: %s", file)
+				continue
+			}
+
+			isSymlink := fileInfo.Mode()&os.ModeSymlink > 0
+			if isSymlink && !f.c.Symlinks {
+				log.Warn("File %s skipped as it is a symlink.", file)
+				continue
+			}
+
+			// Fetch Stat file info which fetches the inode. In case of a symlink, the original inode is fetched
+			fileInfo, err = os.Stat(file)
+			if err != nil {
+				log.Warn("stat(%s) failed: %s", file, err)
+				continue
+			}
+
+			// If symlink is enabled, it is checked that original is not part of same input
+			// It original is harvested by other input, states will potentially overwrite each other
+			//if p.config.Symlinks {
+			//	for _, finfo := range paths {
+			//		if os.SameFile(finfo, fileInfo) {
+			//			log.Info("Same file found as symlink and originap. Skipping file: %s", file)
+			//			continue OUTER
+			//		}
+			//	}
+			//}
+			paths[file] = fileInfo
+		}
+	}
+
+	return paths
+}

app/service/ops/log-agent/input/file/harvester.go

@@ -0,0 +1,503 @@
+package file
+
+import (
+	"fmt"
+	"os"
+	"errors"
+	"io"
+	"time"
+	"context"
+	"bytes"
+	"strconv"
+	"regexp"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/lancerroute"
+)
+
+type Source interface {
+	io.ReadCloser
+	Name() string
+	Stat() (os.FileInfo, error)
+	Continuable() bool // can we continue processing after EOF?
+	HasState() bool    // does this source have a state?
+}
+
+type Hfile struct {
+	*os.File
+}
+
+var lineReadTimeout = errors.New("lineReadTimeout")
+
+func (Hfile) Continuable() bool { return true }
+func (Hfile) HasState() bool    { return true }
+
+// Harvester contains all harvester related data
+type Harvester struct {
+	config          *Config
+	source          *os.File
+	ctx             context.Context
+	cancel          context.CancelFunc
+	state           State
+	register        *Registrar
+	reader          Reader
+	output          chan<- *event.ProcessorEvent
+	active          time.Time
+	lineBuffer      *bytes.Buffer
+	multilineBuffer *bytes.Buffer
+	firstLine       []byte
+	readFunc        func() ([]byte, error)
+}
+
+// startHarvester starts a new harvester with the given offset
+func (f *File) startHarvester(ctx context.Context, c *Config, register *Registrar, state State, offset int64) (err error) {
+	// Set state to "not" finished to indicate that a harvester is running
+	state.Finished = false
+	state.Offset = offset
+
+	// Create harvester with state
+	h, err := NewHarvester(c, register, state, f.output)
+
+	if err != nil {
+		return err
+	}
+	h.ctx, h.cancel = context.WithCancel(ctx)
+
+	err = h.Setup()
+	if err != nil {
+		return fmt.Errorf("error setting up harvester: %s", err)
+	}
+
+	// Update state before staring harvester
+	// This makes sure the states is set to Finished: false
+	// This is synchronous state update as part of the scan
+	h.register.SendStateUpdate(h.state)
+	h.active = time.Now()
+	h.lineBuffer = bytes.NewBuffer(make([]byte, 0, h.config.MaxLength))
+	h.multilineBuffer = bytes.NewBuffer(make([]byte, 0, h.config.MaxLength))
+
+	if h.config.Multiline != nil {
+		h.readFunc = h.readMultiLine
+	} else {
+		h.readFunc = h.readOneLine
+	}
+
+	go h.stateUpdatePeriodically()
+	go h.Run()
+	go h.activeCheck()
+
+	return err
+}
+
+// harvestExistingFile continues harvesting a file with a known state if needed
+func (f *File) harvestExistingFile(ctx context.Context, c *Config, register *Registrar, newState State, oldState State) {
+	//log.Info("Update existing file for harvesting: %s, offset: %v", newState.Source, oldState.Offset)
+
+	// No harvester is running for the file, start a new harvester
+	// It is important here that only the size is checked and not modification time, as modification time could be incorrect on windows
+	// https://blogs.technet.microsoft.com/asiasupp/2010/12/14/file-date-modified-property-are-not-updating-while-modifying-a-file-without-closing-it/
+
+	if oldState.Finished && newState.Fileinfo.Size() > oldState.Offset {
+		// Resume harvesting of an old file we've stopped harvesting from
+		// This could also be an issue with force_close_older that a new harvester is started after each scan but not needed?
+		// One problem with comparing modTime is that it is in seconds, and scans can happen more then once a second
+		log.Info("Resuming harvesting of file: %s, offset: %d, new size: %d", newState.Source, oldState.Offset, newState.Fileinfo.Size())
+		err := f.startHarvester(ctx, c, register, newState, oldState.Offset)
+		if err != nil {
+			log.Error("Harvester could not be started on existing file: %s, Err: %s", newState.Source, err)
+		}
+		return
+	}
+
+	// File size was reduced -> truncated file
+	if newState.Fileinfo.Size() < oldState.Offset {
+		log.Info("Old file was truncated. Starting from the beginning: %s, old size: %d  new size: %d", newState.Source, oldState.Offset, newState.Fileinfo.Size())
+		if oldState.Finished {
+			err := f.startHarvester(ctx, c, register, newState, 0)
+			if err != nil {
+				log.Error("Harvester could not be started on truncated file: %s, Err: %s", newState.Source, err)
+			}
+			return
+		}
+		// just stop old harvester
+		h := f.register.GetHarvester(oldState.Inode)
+		if h != nil {
+			h.Stop()
+		}
+		return
+	}
+
+	// Check if file was renamed
+	if oldState.Source != "" && oldState.Source != newState.Source {
+		// This does not start a new harvester as it is assume that the older harvester is still running
+		// or no new lines were detected. It sends only an event status update to make sure the new name is persisted.
+		log.Info("File rename was detected: %s -> %s, Current offset: %v", oldState.Source, newState.Source, oldState.Offset)
+
+		oldState.Source = newState.Source
+		f.register.SendStateUpdate(oldState)
+	}
+
+	if !oldState.Finished {
+		// Nothing to do. Harvester is still running and file was not renamed
+		log.V(1).Info("Harvester for file is still running: %s, inode %d", newState.Source, newState.Inode)
+	} else {
+		log.V(1).Info("File didn't change: %s, inode %d", newState.Source, newState.Inode)
+	}
+}
+
+// NewHarvester creates a new harvester
+func NewHarvester(c *Config, register *Registrar, state State, output chan<- *event.ProcessorEvent) (*Harvester, error) {
+	h := &Harvester{
+		config:   c,
+		state:    state,
+		register: register,
+		output:   output,
+	}
+
+	// Add ttl if cleanInactive is set
+	if h.config.CleanInactive > 0 {
+		h.state.TTL = time.Duration(h.config.CleanInactive)
+	}
+
+	// Add outlet signal so harvester can also stop itself
+	return h, nil
+}
+
+// Setup opens the file handler and creates the reader for the harvester
+func (h *Harvester) Setup() error {
+	err := h.openFile()
+	if err != nil {
+		return fmt.Errorf("Harvester setup failed. Unexpected file opening error: %s", err)
+	}
+
+	h.reader, err = h.newLogFileReader()
+	if err != nil {
+		if h.source != nil {
+			h.source.Close()
+		}
+		return fmt.Errorf("Harvester setup failed. Unexpected encoding line reader error: %s", err)
+	}
+
+	return nil
+}
+
+// openFile opens a file and checks for the encoding. In case the encoding cannot be detected
+// or the file cannot be opened because for example of failing read permissions, an error
+// is returned and the harvester is closed. The file will be picked up again the next time
+// the file system is scanned
+func (h *Harvester) openFile() error {
+	f, err := os.OpenFile(h.state.Source, os.O_RDONLY, os.FileMode(0))
+	if err != nil {
+		return fmt.Errorf("Failed opening %s: %s", h.state.Source, err)
+	}
+
+	// Makes sure file handler is also closed on errors
+	err = h.validateFile(f)
+	if err != nil {
+		f.Close()
+		return err
+	}
+
+	h.source = f
+	return nil
+}
+
+func (h *Harvester) validateFile(f *os.File) error {
+	info, err := f.Stat()
+	if err != nil {
+		return fmt.Errorf("Failed getting stats for file %s: %s", h.state.Source, err)
+	}
+
+	if !info.Mode().IsRegular() {
+		return fmt.Errorf("Tried to open non regular file: %q %s", info.Mode(), info.Name())
+	}
+
+	// Compares the stat of the opened file to the state given by the input. Abort if not match.
+	if !os.SameFile(h.state.Fileinfo, info) {
+		return errors.New("file info is not identical with opened file. Aborting harvesting and retrying file later again")
+	}
+
+	// get file offset. Only update offset if no error
+	offset, err := h.initFileOffset(f)
+	if err != nil {
+		return err
+	}
+
+	log.V(1).Info("harvester Setting offset for file: %s inode %d. Offset: %d ", h.state.Source, h.state.Inode, offset)
+	h.state.Offset = offset
+
+	return nil
+}
+
+// initFileOffset set offset for file handler
+func (h *Harvester) initFileOffset(file *os.File) (int64, error) {
+	// continue from last known offset
+	if h.state.Offset > 0 {
+		return file.Seek(h.state.Offset, os.SEEK_SET)
+	}
+
+	var firstRun = false
+	if v := h.ctx.Value("firstRun"); v != nil {
+		firstRun = v.(bool)
+	}
+	if h.config.ReadFrom == "newest" && firstRun {
+		return file.Seek(0, os.SEEK_END)
+	}
+	return file.Seek(0, os.SEEK_CUR)
+}
+
+func (h *Harvester) newLogFileReader() (Reader, error) {
+	return NewLineReader(h.source, h.config.MaxLength)
+}
+
+func (h *Harvester) WriteToProcessor(message []byte) {
+	e := event.GetEvent()
+	e.Write(message)
+	e.AppId = []byte(h.config.AppId)
+	e.LogId = h.config.LogId
+	e.Source = "file"
+	// update fields
+	for k, v := range h.config.Fields {
+		e.Fields[k] = v
+	}
+	e.Fields["file"] = h.state.Source
+	e.Destination = lancerroute.GetLancerByLogid(e.LogId)
+	// time maybe overwrite by processor
+	e.Time = time.Now()
+	e.TimeRangeKey = strconv.FormatInt(e.Time.Unix()/100*100, 10)
+
+	h.output <- e
+}
+
+func (h *Harvester) stateUpdatePeriodically() {
+	interval := time.Tick(time.Second * 5)
+	var offset int64
+	for {
+		select {
+		case <-interval:
+			if h.state.Offset > offset {
+				offset = h.state.Offset
+				h.register.SendStateUpdate(h.state)
+				h.active = time.Now()
+			}
+		case <-h.ctx.Done():
+			h.register.SendStateUpdate(h.state)
+			return
+		}
+	}
+}
+
+func (h *Harvester) Stop() {
+	h.state.Finished = true
+	h.register.SendStateUpdate(h.state)
+	h.cancel()
+	log.Info("Harvester for File: %s Inode %d Existed", h.state.Source, h.state.Inode)
+}
+
+func (h *Harvester) activeCheck() {
+	interval := time.Tick(time.Minute * 1)
+	for {
+		select {
+		case <-interval:
+			if time.Now().Sub(h.active) > time.Duration(h.config.HarvesterTTL) {
+				log.Info("Harvester for file: %s, inode: %d is inactive longer than HarvesterTTL, Ended", h.state.Source, h.state.Inode)
+				h.Stop()
+				return
+			}
+		case <-h.ctx.Done():
+			return
+		}
+	}
+}
+
+func (h *Harvester) readMultiLine() (b []byte, err error) {
+	h.multilineBuffer.Reset()
+	counter := 0
+	if h.firstLine != nil {
+		h.multilineBuffer.Write(h.firstLine)
+	}
+
+	ctx, _ := context.WithTimeout(h.ctx, time.Duration(h.config.Timeout))
+	for {
+		select {
+		case <-ctx.Done():
+			h.firstLine = nil
+			return h.multilineBuffer.Bytes(), lineReadTimeout
+		default:
+		}
+
+		message, err := h.readOneLine()
+
+		if err != nil && err != io.EOF && err != lineReadTimeout {
+			h.firstLine = nil
+			if h.multilineBuffer.Len() == 0 {
+				return message, nil
+			}
+			if len(message) > 0 {
+				h.multilineBuffer.Write([]byte{'\n'})
+				h.multilineBuffer.Write(message)
+			}
+			return h.multilineBuffer.Bytes(), err
+		}
+
+		if len(message) == 0 {
+			continue
+		}
+
+		matched, err := regexp.Match(h.config.Multiline.Pattern, message)
+
+		if matched {
+			// old multiline ended
+			if h.firstLine != nil {
+				h.firstLine = message
+				return h.multilineBuffer.Bytes(), nil
+			}
+			// pure new multiline
+			if h.firstLine == nil {
+				h.firstLine = message
+				h.multilineBuffer.Write(message)
+				continue
+			}
+		}
+
+		if !matched {
+			// multiline not begin
+			if h.firstLine == nil {
+				return message, nil
+			}
+
+			if h.firstLine != nil {
+				h.multilineBuffer.Write([]byte{'\n'})
+				h.multilineBuffer.Write(message)
+				counter += 1
+			}
+		}
+
+		if counter > h.config.Multiline.MaxLines || h.multilineBuffer.Len() > h.config.MaxLength {
+			h.firstLine = nil
+			return h.multilineBuffer.Bytes(), nil
+		}
+	}
+}
+
+func (h *Harvester) readOneLine() (b []byte, err error) {
+	h.lineBuffer.Reset()
+	ctx, _ := context.WithTimeout(h.ctx, time.Duration(h.config.Timeout))
+
+	for {
+		select {
+		case <-ctx.Done():
+			return h.lineBuffer.Bytes(), lineReadTimeout
+		default:
+		}
+
+		message, advance, err := h.reader.Next()
+		// update offset
+		h.state.Offset += int64(advance)
+
+		if err == nil && h.lineBuffer.Len() == 0 {
+			return message, nil
+		}
+
+		h.lineBuffer.Write(message)
+
+		if err == nil {
+			return h.lineBuffer.Bytes(), nil
+		}
+
+		if err == io.EOF && advance == 0 {
+			time.Sleep(time.Millisecond * 100)
+			continue
+		}
+
+		if err == io.EOF && advance > 0 && h.lineBuffer.Len() >= h.config.MaxLength {
+			return h.lineBuffer.Bytes(), nil
+		}
+
+		if err != nil {
+			return h.lineBuffer.Bytes(), err
+		}
+	}
+}
+
+//// Run start the harvester and reads files line by line and sends events to the defined output
+//func (h *Harvester) Run() {
+//	log.V(1).Info("Harvester started for file: %s, inode %d", h.state.Source, h.state.Inode)
+//	h.register.RegisterHarvester(h)
+//	defer h.register.UnRegisterHarvester(h)
+//
+//	var line = make([]byte, 0, h.config.MaxLength)
+//	for {
+//		select {
+//		case <-h.ctx.Done():
+//			return
+//		default:
+//		}
+//		//TODO MaxLength check
+//		message, advance, err := h.reader.Next()
+//		// update offset
+//		h.state.Offset += int64(advance)
+//
+//		if err == nil {
+//			if len(line) == 0 {
+//				h.WriteToProcessor(message)
+//				continue
+//			}
+//			line = append(line, message...)
+//			h.WriteToProcessor(line)
+//			line = line[:0]
+//			continue
+//		}
+//
+//		if err == io.EOF && advance == 0 {
+//			time.Sleep(time.Millisecond * 100)
+//			continue
+//		}
+//
+//		if err == io.EOF && advance > 0 {
+//			line = append(line, message...)
+//			if len(line) >= h.config.MaxLength {
+//				h.WriteToProcessor(line)
+//				line = line[:0]
+//			}
+//			continue
+//		}
+//
+//		if err != nil {
+//			log.Error("Harvester Read line error: %v; File: %v", err, h.state.Source)
+//			h.Stop()
+//			return
+//		}
+//	}
+//}
+
+// Run start the harvester and reads files line by line and sends events to the defined output
+func (h *Harvester) Run() {
+	log.V(1).Info("Harvester started for file: %s, inode %d", h.state.Source, h.state.Inode)
+	h.register.RegisterHarvester(h)
+	defer h.register.UnRegisterHarvester(h)
+
+	for {
+		select {
+		case <-h.ctx.Done():
+			return
+		default:
+		}
+
+		message, err := h.readFunc()
+
+		if err == lineReadTimeout {
+			log.V(1).Info("lineReadTimeout when harvesting %s", h.state.Source)
+		}
+
+		if len(message) > 0 {
+			h.WriteToProcessor(message)
+		}
+
+		if err != nil && err != lineReadTimeout && err != io.EOF {
+			log.Error("Harvester Read line error: %v; File: %v", err, h.state.Source)
+			h.Stop()
+			return
+		}
+	}
+}

app/service/ops/log-agent/input/file/multiline.go

@@ -0,0 +1,31 @@
+package file
+
+import (
+	"errors"
+	"regexp"
+	"fmt"
+)
+
+type MultilineConf struct {
+	Pattern  string `toml:"pattern"`
+	MaxLines int    `toml:"maxLines"`
+}
+
+func (c *MultilineConf) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of Multiline  is nil")
+	}
+
+	if c.Pattern == "" {
+		return errors.New("Pattern in Multiline can't be nil")
+	}
+
+	if _, err := regexp.Compile(c.Pattern); err != nil {
+		return fmt.Errorf("Multiline pattern compile error: %s", err)
+	}
+
+	if c.MaxLines == 0 {
+		c.MaxLines = 200
+	}
+	return nil
+}

app/service/ops/log-agent/input/file/reader.go

@@ -0,0 +1,80 @@
+package file
+
+import (
+	"io"
+	"bufio"
+	"bytes"
+)
+
+// Message represents a reader event with timestamp, content and actual number
+// of bytes read from input before decoding.
+//type Message struct {
+//	Ts      time.Time // timestamp the content was read
+//	Content []byte    // actual content read
+//	Bytes   int       // total number of bytes read to generate the message
+//	//Fields  common.MapStr // optional fields that can be added by reader
+//}
+
+type Reader interface {
+	Next() ([]byte, int, error)
+}
+
+type LineReader struct {
+	reader     io.Reader
+	rb         *bufio.Reader
+	bufferSize int
+	nl         []byte
+	nlSize     int
+	scan       *bufio.Scanner
+}
+
+// New creates a new reader object
+func NewLineReader(input io.Reader, bufferSize int) (*LineReader, error) {
+	nl := []byte{'\n'}
+
+	r := &LineReader{
+		reader:     input,
+		bufferSize: bufferSize,
+		nl:         nl,
+		nlSize:     len(nl),
+	}
+	r.rb = bufio.NewReaderSize(input, r.bufferSize)
+	r.scan = bufio.NewScanner(r.rb)
+	r.scan.Split(ScanLines)
+
+	return r, nil
+}
+
+func ScanLines(data []byte, atEOF bool) (advance int, token []byte, err error) {
+	if atEOF && len(data) == 0 {
+		return 0, nil, nil
+	}
+
+	if i := bytes.IndexByte(data, '\n'); i >= 0 {
+		// We have a full newline-terminated line.
+		return i + 1, data[0:i], nil
+	}
+
+	// Request more data.
+	return 0, nil, nil
+}
+
+// Next reads the next line until the new line character
+func (r *LineReader) Next() ([]byte, int, error) {
+	body, err := r.rb.ReadBytes('\n')
+	advance := len(body)
+	//if err == io.EOF && advance > 0 {
+	//	return body, advance, err
+	//}
+	// remove '\n'
+	if len(body) > 0 && body[len(body)-1] == '\n' {
+		body = body[0:len(body)-1]
+	}
+
+	// remove '\r'
+	if len(body) > 0 && body[len(body)-1] == '\r' {
+		body = body[0: len(body)-1]
+	}
+
+	return body, advance, err
+}

app/service/ops/log-agent/input/file/registrar.go

@@ -0,0 +1,288 @@
+package file
+
+import (
+	"sync"
+	"os"
+	"fmt"
+	"time"
+	"io"
+	"path/filepath"
+	"encoding/json"
+	"context"
+
+	"go-common/library/log"
+)
+
+type Registrar struct {
+	Channel              chan State
+	registryFile         string // Path to the Registry File
+	wg                   sync.WaitGroup
+	states               *States // Map with all file paths inside and the corresponding state
+	bufferedStateUpdates int
+	flushInterval        time.Duration
+	harvesters           map[uint64]*Harvester
+	hLock                sync.RWMutex
+	ctx                  context.Context
+	cancel               context.CancelFunc
+}
+
+// New creates a new Registrar instance, updating the registry file on
+// `file.State` updates. New fails if the file can not be opened or created.
+func NewRegistry(ctx context.Context, registryFile string) (*Registrar, error) {
+	r := &Registrar{
+		registryFile:  registryFile,
+		states:        NewStates(),
+		Channel:       make(chan State, 100),
+		wg:            sync.WaitGroup{},
+		flushInterval: time.Second * 5,
+		harvesters:    make(map[uint64]*Harvester),
+	}
+	r.ctx, r.cancel = context.WithCancel(ctx)
+	err := r.Init()
+	if err != nil {
+		return nil, err
+	}
+	go r.Run()
+
+	return r, err
+}
+
+func (r *Registrar) RegisterHarvester(h *Harvester) error {
+	r.hLock.Lock()
+	defer r.hLock.Unlock()
+
+	if _, ok := r.harvesters[h.state.Inode]; !ok {
+		r.harvesters[h.state.Inode] = h
+		return nil
+	}
+	return fmt.Errorf("harvestor of inode %s Re registered", h.state.Inode)
+}
+
+func (r *Registrar) UnRegisterHarvester(h *Harvester) error {
+	r.hLock.Lock()
+	defer r.hLock.Unlock()
+
+	if _, ok := r.harvesters[h.state.Inode]; ok {
+		delete(r.harvesters, h.state.Inode)
+		return nil
+	}
+
+	return fmt.Errorf("harvestor of inode %d not found", h.state.Inode)
+}
+
+func (r *Registrar) GetHarvester(i uint64) *Harvester {
+	r.hLock.RLock()
+	defer r.hLock.RUnlock()
+	if h, ok := r.harvesters[i]; ok {
+		return h
+	}
+	return nil
+}
+
+// Init sets up the Registrar and make sure the registry file is setup correctly
+func (r *Registrar) Init() (err error) {
+	// Create directory if it does not already exist.
+	registryPath := filepath.Dir(r.registryFile)
+	err = os.MkdirAll(registryPath, 0750)
+	if err != nil {
+		return fmt.Errorf("Failed to created registry file dir %s: %v", registryPath, err)
+	}
+
+	// Check if files exists
+	fileInfo, err := os.Lstat(r.registryFile)
+	if os.IsNotExist(err) {
+		log.Info("No registry file found under: %s. Creating a new registry file.", r.registryFile)
+		// No registry exists yet, write empty state to check if registry can be written
+		return r.writeRegistry()
+	}
+	if err != nil {
+		return err
+	}
+
+	// Check if regular file, no dir, no symlink
+	if !fileInfo.Mode().IsRegular() {
+		// Special error message for directory
+		if fileInfo.IsDir() {
+			return fmt.Errorf("Registry file path must be a file. %s is a directory.", r.registryFile)
+		}
+		return fmt.Errorf("Registry file path is not a regular file: %s", r.registryFile)
+	}
+
+	log.Info("Registry file set to: %s", r.registryFile)
+
+	// load states
+	if err = r.loadStates(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+// writeRegistry writes the new json registry file to disk.
+func (r *Registrar) writeRegistry() error {
+	// First clean up states
+	r.gcStates()
+
+	// TODO lock for reading r.states.states
+	tempfile, err := writeTmpFile(r.registryFile, r.states.states)
+	if err != nil {
+		return err
+	}
+
+	err = SafeFileRotate(r.registryFile, tempfile)
+	if err != nil {
+		return err
+	}
+
+	log.V(1).Info("Registry file %s updated. %d states written.", r.registryFile, len(r.states.states))
+	return nil
+}
+
+// SafeFileRotate safely rotates an existing file under path and replaces it with the tempfile
+func SafeFileRotate(path, tempfile string) error {
+	parent := filepath.Dir(path)
+
+	if e := os.Rename(tempfile, path); e != nil {
+		return e
+	}
+
+	// best-effort fsync on parent directory. The fsync is required by some
+	// filesystems, so to update the parents directory metadata to actually
+	// contain the new file being rotated in.
+	f, err := os.Open(parent)
+	if err != nil {
+		return nil // ignore error
+	}
+	defer f.Close()
+	f.Sync()
+
+	return nil
+}
+
+// loadStates fetches the previous reading state from the configure RegistryFile file
+// The default file is `registry` in the data path.
+func (r *Registrar) loadStates() error {
+	f, err := os.Open(r.registryFile)
+	if err != nil {
+		return err
+	}
+
+	defer f.Close()
+
+	log.Info("Loading registrar data from %s", r.registryFile)
+
+	states, err := readStatesFrom(f)
+	if err != nil {
+		return err
+	}
+
+	states = r.preProcessStates(states)
+
+	r.states.SetStates(states)
+	log.V(1).Info("States Loaded from registrar%s : %+v", r.registryFile, len(states))
+
+	return nil
+}
+
+func (r *Registrar) preProcessStates(states map[uint64]State) map[uint64]State {
+	for key, state := range states {
+		// set all states to finished
+		state.Finished = true
+		states[key] = state
+	}
+	return states
+}
+
+func readStatesFrom(in io.Reader) (map[uint64]State, error) {
+	states := make(map[uint64]State)
+	decoder := json.NewDecoder(in)
+	if err := decoder.Decode(&states); err != nil {
+		return nil, fmt.Errorf("Error decoding states: %s", err)
+	}
+	return states, nil
+}
+
+func writeTmpFile(baseName string, states map[uint64]State) (string, error) {
+	tempfile := baseName + ".new"
+	f, err := os.OpenFile(tempfile, os.O_RDWR|os.O_CREATE|os.O_TRUNC|os.O_SYNC, 0640)
+	if err != nil {
+		log.Error("Failed to create tempfile (%s) for writing: %s", tempfile, err)
+		return "", err
+	}
+
+	defer f.Close()
+
+	encoder := json.NewEncoder(f)
+
+	if err := encoder.Encode(states); err != nil {
+		log.Error("Error when encoding the states: %s", err)
+		return "", err
+	}
+
+	// Commit the changes to storage to avoid corrupt registry files
+	if err = f.Sync(); err != nil {
+		log.Error("Error when syncing new registry file contents: %s", err)
+		return "", err
+	}
+
+	return tempfile, nil
+}
+
+// gcStates runs a registry Cleanup. The method check if more event in the
+// registry can be gc'ed in the future. If no potential removable state is found,
+// the gcEnabled flag is set to false, indicating the current registrar state being
+// stable. New registry update events can re-enable state gc'ing.
+func (r *Registrar) gcStates() {
+	//if !r.gcRequired {
+	//	return
+	//}
+
+	beforeCount := len(r.states.states)
+	cleanedStates := r.states.Cleanup()
+
+	log.V(1).Info(
+		"Registrar states %s cleaned up. Before: %d, After: %d", r.registryFile,
+		beforeCount, beforeCount-cleanedStates)
+}
+
+// FindPrevious lookups a registered state, that matching the new state.
+// Returns a zero-state if no match is found.
+func (r *Registrar) FindPrevious(newState State) State {
+	return r.states.FindPrevious(newState)
+}
+
+func (r *Registrar) Run() {
+	log.Info("Starting Registrar for: %s", r.registryFile)
+	flushC := time.Tick(r.flushInterval)
+	for {
+		select {
+		case <-flushC:
+			r.flushRegistry()
+		case state := <-r.Channel:
+			r.processEventStates(state)
+		case <-r.ctx.Done():
+			r.flushRegistry()
+			return
+		}
+	}
+}
+
+func (r *Registrar) flushRegistry() {
+	if err := r.writeRegistry(); err != nil {
+		log.Error("Writing of registry returned error: %v. Continuing...", err)
+	}
+}
+
+// processEventStates gets the states from the events and writes them to the registrar state
+func (r *Registrar) processEventStates(state State) {
+	r.states.UpdateWithTs(state, time.Now())
+}
+
+func (r *Registrar) SendStateUpdate(state State) {
+	r.Channel <- state
+	//select {
+	//case r.Channel <- state:
+	//default:
+	//	log.Warn("state update receiving chan full")
+	//}
+}

app/service/ops/log-agent/input/file/state.go

@@ -0,0 +1,63 @@
+package file
+
+import (
+	"time"
+	"os"
+	"fmt"
+	"path/filepath"
+	"syscall"
+)
+
+type State struct {
+	Source    string            `json:"source"`
+	Offset    int64             `json:"offset"`
+	Inode     uint64            `json:"inode"`
+	Fileinfo  os.FileInfo       `json:"-"` // the file info
+	Timestamp time.Time         `json:"timestamp"`
+	Finished  bool              `json:"finished"`
+	Meta      map[string]string `json:"meta"`
+	TTL       time.Duration     `json:"ttl"`
+}
+
+// NewState creates a new file state
+func NewState(fileInfo os.FileInfo, path string) State {
+	stat := fileInfo.Sys().(*syscall.Stat_t)
+	return State{
+		Fileinfo:  fileInfo,
+		Inode:     stat.Ino,
+		Source:    path,
+		Finished:  false,
+		Timestamp: time.Now(),
+		TTL:       -1, // By default, state does have an infinite ttl
+		Meta:      nil,
+	}
+}
+
+func (s *State) ID() uint64 {
+	return s.Inode
+}
+
+// IsEqual compares the state to an other state supporting stringer based on the unique string
+func (s *State) IsEqual(c *State) bool {
+	return s.ID() == c.ID()
+}
+
+// IsEmpty returns true if the state is empty
+func (s *State) IsEmpty() bool {
+	return s.Inode == 0 &&
+		s.Source == "" &&
+		len(s.Meta) == 0 &&
+		s.Timestamp.IsZero()
+}
+
+func getFileState(path string, info os.FileInfo) (State, error) {
+	var err error
+	var absolutePath string
+	absolutePath, err = filepath.Abs(path)
+	if err != nil {
+		return State{}, fmt.Errorf("could not fetch abs path for file %s: %s", absolutePath, err)
+	}
+	// Create new state for comparison
+	newState := NewState(info, absolutePath)
+	return newState, nil
+}

app/service/ops/log-agent/input/file/states.go

@@ -0,0 +1,101 @@
+package file
+
+import (
+	"sync"
+	"time"
+	"go-common/library/log"
+)
+
+// States handles list of FileState. One must use NewStates to instantiate a
+// file states registry. Using the zero-value is not safe.
+type States struct {
+	sync.RWMutex
+
+	// states store
+	states map[uint64]State
+}
+
+// NewStates generates a new states registry.
+func NewStates() *States {
+	return &States{
+		states: map[uint64]State{},
+	}
+}
+
+// Update updates a state. If previous state didn't exist, new one is created
+func (s *States) Update(newState State) {
+	s.Lock()
+	defer s.Unlock()
+
+	id := newState.ID()
+
+	if _, ok := s.states[id]; ok {
+		s.states[id] = newState
+		return
+	}
+	log.V(1).Info("New state added for %s", id)
+	s.states[id] = newState
+}
+
+// Cleanup cleans up the state array. All states which are older then `older` are removed
+// The number of states that were cleaned up is returned.
+func (s *States) Cleanup() (int) {
+	s.Lock()
+	defer s.Unlock()
+
+	currentTime := time.Now()
+	statesBefore := len(s.states)
+
+	for inode, state := range s.states {
+		if state.Finished && state.TTL > 0 && currentTime.Sub(state.Timestamp) > state.TTL {
+			delete(s.states, inode)
+		}
+	}
+
+	return statesBefore - len(s.states)
+}
+
+// GetStates creates copy of the file states.
+func (s *States) GetState(id uint64) State {
+	s.RLock()
+	defer s.RUnlock()
+
+	if _, ok := s.states[id]; ok {
+		return s.states[id]
+	}
+	return State{}
+}
+
+// FindPrevious lookups a registered state, that matching the new state.
+// Returns a zero-state if no match is found.
+func (s *States) FindPrevious(newState State) State {
+	s.RLock()
+	defer s.RUnlock()
+
+	if s, ok := s.states[newState.ID()]; ok {
+		return s
+	}
+
+	return State{}
+}
+
+// SetStates overwrites all internal states with the given states array
+func (s *States) SetStates(states map[uint64]State) {
+	s.Lock()
+	defer s.Unlock()
+	s.states = states
+}
+
+// UpdateWithTs updates a state, assigning the given timestamp.
+// If previous state didn't exist, new one is created
+func (s *States) UpdateWithTs(newState State, ts time.Time) {
+	id := newState.ID()
+	oldState := s.FindPrevious(newState)
+	newState.Timestamp = ts
+	s.Lock()
+	defer s.Unlock()
+	s.states[id] = newState
+	if oldState.IsEmpty() {
+		log.V(1).Info("New state added for %s", newState.Source)
+	}
+}

app/service/ops/log-agent/input/input.go

@@ -0,0 +1,44 @@
+package input
+
+import (
+	"fmt"
+	"context"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/event"
+)
+
+type Input interface {
+	Run() (err error)
+	Stop()
+	Ctx() (ctx context.Context)
+}
+
+// Factory is used to register functions creating new Input instances.
+type Factory = func(ctx context.Context, config interface{}, connector chan<- *event.ProcessorEvent) (Input, error)
+
+var registry = make(map[string]Factory)
+
+func Register(name string, factory Factory) error {
+	log.Info("Registering input factory")
+	if name == "" {
+		return fmt.Errorf("Error registering input: name cannot be empty")
+	}
+	if factory == nil {
+		return fmt.Errorf("Error registering input '%v': factory cannot be empty", name)
+	}
+	if _, exists := registry[name]; exists {
+		return fmt.Errorf("Error registering input '%v': already registered", name)
+	}
+
+	registry[name] = factory
+	log.Info("Successfully registered input")
+
+	return nil
+}
+
+func GetFactory(name string) (Factory, error) {
+	if _, exists := registry[name]; !exists {
+		return nil, fmt.Errorf("Error creating input. No such input type exist: '%v'", name)
+	}
+	return registry[name], nil
+}

app/service/ops/log-agent/input/sock/BUILD

@@ -0,0 +1,41 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "sock.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/input/sock",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/input:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancermonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancerroute:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/input/sock/conf.go

@@ -0,0 +1,53 @@
+package sock
+
+import (
+	"errors"
+	"time"
+
+	xtime "go-common/library/time"
+)
+
+type Config struct {
+	TcpAddr          string         `toml:"tcpAddr"`
+	UdpAddr          string         `toml:"udpAddr"`
+	ReadChanSize     int            `toml:"readChanSize"`
+	TcpBatchMaxBytes int            `toml:"tcpBatchMaxBytes"`
+	UdpPacketMaxSize int            `toml:"udpPacketMaxSize"`
+	LogMaxBytes      int            `toml:"logMaxBytes"`
+	UdpReadTimeout   xtime.Duration `toml:"udpReadTimeout"`
+	TcpReadTimeout   xtime.Duration `toml:"tcpReadTimeout"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of Sock Input is nil")
+	}
+
+	if c.TcpAddr == "" {
+		c.TcpAddr = "/var/run/lancer/collector_tcp.sock"
+	}
+	if c.UdpAddr == "" {
+		c.UdpAddr = "/var/run/lancer/collector.sock"
+	}
+
+	if c.ReadChanSize == 0 {
+		c.ReadChanSize = 5000
+	}
+
+	if c.TcpBatchMaxBytes == 0 {
+		c.TcpBatchMaxBytes = 10240000 // 10MB
+	}
+
+	if c.UdpPacketMaxSize == 0 {
+		c.UdpPacketMaxSize = 1024 * 64 //64KB
+	}
+
+	if c.UdpReadTimeout == 0 {
+		c.UdpReadTimeout = xtime.Duration(time.Second * 10)
+	}
+
+	if c.TcpReadTimeout == 0 {
+		c.TcpReadTimeout = xtime.Duration(time.Minute * 5)
+	}
+	return nil
+}

app/service/ops/log-agent/input/sock/sock.go

@@ -0,0 +1,254 @@
+package sock
+
+import (
+	"bufio"
+	"runtime"
+	"errors"
+	"net"
+	"os"
+	"time"
+	"io"
+	"path/filepath"
+	"fmt"
+	"context"
+	"strings"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/input"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+	"go-common/app/service/ops/log-agent/pkg/lancermonitor"
+	"go-common/app/service/ops/log-agent/pkg/lancerroute"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	_logIdLen           = 6
+	_logLancerHeaderLen = 19
+	_logSeparator       = byte('\u0001')
+)
+
+var (
+	// ErrInvalidAddr invalid address.
+	ErrInvalidAddr = errors.New("invalid address")
+	// logMagic log magic.
+	logMagic = []byte{0xAC, 0xBE}
+	local, _ = time.LoadLocation("Local")
+)
+
+func init() {
+	err := input.Register("sock", NewSock)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type Sock struct {
+	c        *Config
+	output   chan<- *event.ProcessorEvent
+	readChan chan *event.ProcessorEvent
+	ctx      context.Context
+	cancel   context.CancelFunc
+	closed   bool
+}
+
+func NewSock(ctx context.Context, config interface{}, output chan<- *event.ProcessorEvent) (input.Input, error) {
+	sock := new(Sock)
+	if c, ok := config.(*Config); !ok {
+		return nil, fmt.Errorf("Error config for Sock Input")
+	} else {
+		if err := c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		sock.c = c
+	}
+	sock.output = output
+	sock.ctx, sock.cancel = context.WithCancel(ctx)
+	sock.readChan = make(chan *event.ProcessorEvent, sock.c.ReadChanSize)
+	sock.closed = false
+
+	return sock, nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}
+
+func (s *Sock) Run() (err error) {
+	s.listen()
+	go s.writetoProcessor()
+	return nil
+}
+
+func (s *Sock) Stop() {
+	s.closed = true
+	s.cancel()
+}
+
+func (s *Sock) Ctx() context.Context {
+	return s.ctx
+}
+
+//listen listen unix socket to buffer
+func (s *Sock) listen() {
+	// SOCK_DGRAM
+	if s.c.UdpAddr != "" {
+		if flag, _ := pathExists(s.c.UdpAddr); flag {
+			os.Remove(s.c.UdpAddr)
+		}
+		if flag, _ := pathExists(filepath.Dir(s.c.UdpAddr)); !flag {
+			os.Mkdir(filepath.Dir(s.c.UdpAddr), os.ModePerm)
+		}
+		udpconn, err := net.ListenPacket("unixgram", s.c.UdpAddr)
+		if err != nil {
+			panic(err)
+		}
+		if err := os.Chmod(s.c.UdpAddr, 0777); err != nil {
+			panic(err)
+		}
+		log.Info("start listen: %s", s.c.UdpAddr)
+		for i := 0; i < runtime.NumCPU(); i++ {
+			go s.udpread(udpconn)
+		}
+	}
+	// SOCK_SEQPACKET
+	if s.c.TcpAddr != "" {
+		if flag, _ := pathExists(s.c.TcpAddr); flag {
+			os.Remove(s.c.TcpAddr)
+		}
+		if flag, _ := pathExists(filepath.Dir(s.c.TcpAddr)); !flag {
+			os.Mkdir(filepath.Dir(s.c.TcpAddr), os.ModePerm)
+		}
+		tcplistener, err := net.Listen("unixpacket", s.c.TcpAddr)
+		if err != nil {
+			panic(err)
+		}
+		if err := os.Chmod(s.c.TcpAddr, 0777); err != nil {
+			panic(err)
+		}
+		log.Info("start listen: %s", s.c.TcpAddr)
+		go s.tcpread(tcplistener)
+	}
+}
+
+func (s *Sock) writeToReadChan(e *event.ProcessorEvent) {
+	lancermonitor.IncreaseLogCount("agent.receive.count", e.LogId)
+	select {
+	case s.readChan <- e:
+	default:
+		event.PutEvent(e)
+		flowmonitor.Fm.AddEvent(e, "log-agent.input.sock", "ERROR", "read chan full")
+		log.Warn("sock read chan full, discard log")
+	}
+}
+
+// tcpread accept tcp connection
+func (s *Sock) tcpread(listener net.Listener) {
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			continue
+		}
+		go s.handleTcpConn(conn)
+	}
+}
+
+// process single tcp connection
+func (s *Sock) handleTcpConn(conn net.Conn) {
+	defer conn.Close()
+	rd := bufio.NewReaderSize(conn, s.c.TcpBatchMaxBytes)
+	for {
+		conn.SetReadDeadline(time.Now().Add(time.Duration(s.c.TcpReadTimeout)))
+		b, err := rd.ReadSlice(_logSeparator)
+
+		if err == nil && len(b) <= _logLancerHeaderLen {
+			continue
+		}
+
+		if err == nil {
+			e := s.preproccess(b[:len(b)-1])
+			if e != nil {
+				s.writeToReadChan(e)
+				flowmonitor.Fm.AddEvent(e, "log-agent.input.sock", "OK", "received")
+				continue
+			}
+		}
+
+		// conn closed and return EOF
+		if err == io.EOF {
+			e := s.preproccess(b)
+			if e != nil {
+				s.writeToReadChan(e)
+				flowmonitor.Fm.AddEvent(e, "log-agent.input.sock", "OK", "received")
+				continue
+			}
+			log.Info("get EOF from conn, close conn")
+			return
+		}
+
+		log.Error("read from tcp conn error(%v). close conn", err)
+		return
+	}
+}
+
+//read read from unix socket conn
+func (s *Sock) udpread(conn net.PacketConn) {
+	b := make([]byte, s.c.UdpPacketMaxSize)
+	for {
+		conn.SetReadDeadline(time.Now().Add(time.Duration(s.c.UdpReadTimeout)))
+		l, _, err := conn.ReadFrom(b)
+
+		if err != nil && !strings.Contains(err.Error(), "i/o timeout") {
+			log.Error("conn.ReadFrom() error(%v)", err)
+			continue
+		}
+
+		e := s.preproccess(b[:l])
+
+		if e != nil {
+			flowmonitor.Fm.AddEvent(e, "log-agent.input.sock", "OK", "received")
+			s.writeToReadChan(e)
+		}
+	}
+}
+
+func (s *Sock) preproccess(b []byte) *event.ProcessorEvent {
+	if len(b) <= _logLancerHeaderLen {
+		return nil
+	}
+	e := event.GetEvent()
+	e.LogId = string(b[:_logIdLen])
+	e.Destination = lancerroute.GetLancerByLogid(e.LogId)
+	e.Write(b[_logLancerHeaderLen:])
+	e.Source = "sock"
+	flowmonitor.Fm.AddEvent(e, "log-agent.input.sock", "OK", "received")
+	return e
+}
+
+// pathExists judge if the file exists
+func pathExists(path string) (bool, error) {
+	_, err := os.Stat(path)
+	if err == nil {
+		return true, nil
+	}
+	if os.IsNotExist(err) {
+		return false, nil
+	}
+	return false, err
+}
+
+func (s *Sock) writetoProcessor() {
+	for {
+		select {
+		case e := <-s.readChan:
+			s.output <- e
+		case <-s.ctx.Done():
+			return
+		}
+	}
+}

app/service/ops/log-agent/output/BUILD

@@ -0,0 +1,43 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "output.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/output",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/output/cache:all-srcs",
+        "//app/service/ops/log-agent/output/lancergrpc:all-srcs",
+        "//app/service/ops/log-agent/output/lancerlogstream:all-srcs",
+        "//app/service/ops/log-agent/output/stdout:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/cache/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["cache.go"],
+    importpath = "go-common/app/service/ops/log-agent/output/cache",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//app/service/ops/log-agent/event:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/output/cache/file:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/cache/cache.go

@@ -0,0 +1,10 @@
+package cache
+
+import (
+	"go-common/app/service/ops/log-agent/event"
+)
+
+type Cahce interface {
+	WriteToCache(e *event.ProcessorEvent)
+	ReadFromCache() (e *event.ProcessorEvent)
+}

app/service/ops/log-agent/output/cache/file/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "file.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/output/cache/file",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/fsnotify/fsnotify:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/cache/file/config.go

@@ -0,0 +1,60 @@
+package file
+
+import (
+	"time"
+	"errors"
+	"path"
+
+	xtime "go-common/library/time"
+)
+
+type Config struct {
+	CacheFlushInterval xtime.Duration `tome:"cacheFlushInterval"`
+	WriteBuffer        int           `tome:"writeBuffer"`
+	Storage            string        `tome:"storage"`
+	StorageMaxMB       int           `tome:"storageMaxMB"`
+	FileBytes          int           `tome:"fileBytes"`
+	Suffix             string        `tome:"suffix"`
+	ReadBuffer         int           `tome:"readBuffer"`
+	Index              string        `tome:"index"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of fileCache is nil")
+	}
+
+	if time.Duration(c.CacheFlushInterval) == 0 {
+		c.CacheFlushInterval = xtime.Duration(time.Second * 5)
+	}
+
+	if c.WriteBuffer == 0 {
+		c.WriteBuffer = 1024 * 1024 * 2 // 2M by default
+	}
+
+	if c.Storage == "" {
+		return errors.New("storage settings for lancer output can't be nil")
+	}
+
+	if c.StorageMaxMB == 0 {
+		c.StorageMaxMB = 5120
+	}
+
+	if c.FileBytes == 0 {
+		c.FileBytes = 1024 * 1024 * 2 // 2M by default
+	}
+
+	if c.Suffix == "" {
+		c.Suffix = ".log"
+	}
+
+	if c.ReadBuffer == 0 {
+		c.ReadBuffer = 1024 * 1024 * 2 // 2M by default
+	}
+
+	if c.Index == "" {
+		c.Index = path.Join(c.Storage, "output.index")
+	}
+
+	return nil
+}

app/service/ops/log-agent/output/cache/file/file.go

@@ -0,0 +1,503 @@
+package file
+
+import (
+	"os"
+	"bufio"
+	"time"
+	"io/ioutil"
+	"math/rand"
+	"path"
+	"strconv"
+	"sync"
+	"fmt"
+	"errors"
+	"encoding/binary"
+	"encoding/json"
+	"sort"
+	"bytes"
+	"io"
+	"strings"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+
+	"github.com/fsnotify/fsnotify"
+)
+
+const (
+	_formatUpdated      = "2006-01-02 15:04:05"
+	_logMagicSize       = 2
+	_logHeadSize        = 6
+	_logLenSize         = 4
+	_logIdSize          = 6
+	_logLancerHeaderLen = 19
+)
+
+var (
+	errLogNotFound  = errors.New("log not found")
+	errMagicInvaild = errors.New("log magic invalid")
+	logMagic        = []byte{0xAC, 0xBE}
+	_logType        = []byte{0, 1}
+	_logLength      = []byte{0, 0, 0, 0}
+	local, _        = time.LoadLocation("Local")
+)
+
+// Index index.
+type Index struct {
+	Name    string `json:"name"`
+	Offset  int64  `json:"offset"`
+	Updated string `json:"updated"`
+}
+
+type FileCache struct {
+	c           *Config
+	next        chan string
+	storageFull bool
+	writeChan   chan *event.ProcessorEvent
+	readChan    chan *event.ProcessorEvent
+	eLock       sync.RWMutex
+	logs        map[string]os.FileInfo
+	wh          *fsnotify.Watcher
+}
+
+func NewFileCache(c *Config) (f *FileCache, err error) {
+	if err = c.ConfigValidate(); err != nil {
+		return nil, err
+	}
+	f = new(FileCache)
+	f.c = c
+	f.storageFull = false
+	f.next = make(chan string, 1)
+	f.writeChan = make(chan *event.ProcessorEvent)
+	f.readChan = make(chan *event.ProcessorEvent)
+	f.logs = make(map[string]os.FileInfo)
+
+	if _, err := os.Stat(f.c.Storage); os.IsNotExist(err) {
+		if err = os.MkdirAll(f.c.Storage, 0755); err != nil {
+			return nil, err
+		}
+	}
+
+	if err = f.nextFile(); err != nil {
+		return nil, err
+	}
+	if err = f.watch(); err != nil {
+		return
+	}
+	if err = f.loadFiles(); err != nil {
+		return
+	}
+
+	go f.watchproc()
+	go f.writeProcess()
+	go f.readProcess()
+	return f, nil
+}
+
+func (f *FileCache) WriteToCache(e *event.ProcessorEvent) {
+	f.writeChan <- e
+}
+
+func (f *FileCache) ReadFromCache() (e *event.ProcessorEvent) {
+	e = <-f.readChan
+	return
+}
+
+// loadFiles loadFiles
+func (f *FileCache) loadFiles() (err error) {
+	var (
+		fi  os.FileInfo
+		fis []os.FileInfo
+	)
+	if fis, err = ioutil.ReadDir(f.c.Storage); err != nil {
+		log.Error("ioutil.ReadDir(%s) error(%v)", f.c.Storage, err)
+		return
+	}
+	for _, fi = range fis {
+		name := path.Join(f.c.Storage, fi.Name())
+		if !fi.IsDir() && strings.HasSuffix(name, f.c.Suffix) {
+			f.eLock.Lock()
+			f.logs[name] = fi
+			f.eLock.Unlock()
+			log.Info("loadFile: %s, size: %d", name, fi.Size())
+		}
+	}
+	return
+}
+
+func (f *FileCache) writeProcess() {
+	var (
+		err       error
+		n, total  int
+		lengthbuf = make([]byte, 4)
+		cur       *os.File
+		wr        = bufio.NewWriterSize(nil, f.c.WriteBuffer)
+		tk        = time.Tick(time.Duration(f.c.CacheFlushInterval))
+		timestamp = []byte(fmt.Sprintf("%d", time.Now().UnixNano()/1e6))
+	)
+	rand.Seed(time.Now().UnixNano())
+	for {
+		select {
+		case next := <-f.next:
+			if cur != nil && wr != nil {
+				wr.Flush()
+				cur.Close()
+			}
+			f, err := os.OpenFile(next, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
+			if err != nil {
+				log.Error("os.OpenFile(%s) error(%v)", next, err)
+				continue
+			}
+			cur = f
+			wr.Reset(f)
+			total = 0
+		case <-tk:
+			if wr != nil && cur != nil {
+				wr.Flush()
+			}
+			f.checkStorageSize()
+		case e := <-f.writeChan:
+			if f.storageFull {
+				flowmonitor.Fm.AddEvent(e, "log-agent.output.lancer", "ERROR", "file cache storgefull")
+				event.PutEvent(e)
+				continue
+			}
+
+			if total > f.c.FileBytes && len(f.next) == 0 {
+				if err := f.nextFile(); err != nil {
+					log.Error("c.nextFile() error(%v)", err)
+				}
+			}
+
+			binary.BigEndian.PutUint32(lengthbuf, uint32(e.Length+_logLancerHeaderLen))
+			// write logMagic
+			if n, err = wr.Write(logMagic); err != nil {
+				goto HERE
+			}
+			total += n
+			// write length
+			if n, err = wr.Write(lengthbuf); err != nil {
+				goto HERE
+			}
+			total += n
+			// write log
+			if n, err = wr.Write([]byte(e.LogId)); err != nil {
+				goto HERE
+			}
+
+			if n, err = wr.Write(timestamp); err != nil {
+				goto HERE
+			}
+
+			if n, err = wr.Write(e.Bytes()); err != nil {
+				goto HERE
+			}
+
+			total += n
+			flowmonitor.Fm.AddEvent(e, "log-agent.output.lancer", "OK", "write file cache ok")
+			event.PutEvent(e)
+			continue
+
+		HERE: // write file cache  error
+			flowmonitor.Fm.AddEvent(e, "log-agent.output.lancer", "ERROR", "write file cache failed")
+			event.PutEvent(e)
+			log.Error("wr.Write() error(%v)", err)
+			if cur != nil && wr != nil {
+				wr.Flush()
+				cur.Close()
+			}
+			name := f.nextFileName()
+			f, err := os.OpenFile(name, os.O_WRONLY|os.O_APPEND|os.O_CREATE, 0666)
+			if err != nil {
+				log.Error("os.OpenFile(%s) error(%v)", name, err)
+				continue
+			}
+			cur = f
+			wr.Reset(f)
+			total = 0
+			continue
+
+		}
+	}
+}
+
+// index index
+func (f *FileCache) index() (idx *Index, err error) {
+	f.eLock.RLock()
+	length := len(f.logs)
+	f.eLock.RUnlock()
+	if length == 0 {
+		err = errLogNotFound
+		return
+	}
+	i, err := os.OpenFile(f.c.Index, os.O_RDONLY, 0666)
+	if err != nil {
+		log.Error("os.OpenFile(%s) error(%v)", f.c.Index, err)
+		return
+	}
+	defer i.Close()
+	b, err := ioutil.ReadAll(i)
+	if err != nil {
+		log.Error("ioutil.ReadAll(%s) error(%v)", f.c.Index, err)
+		return
+	}
+	idx = &Index{}
+	if err = json.Unmarshal(b, idx); err != nil {
+		log.Error("json.Unmarshal(%s) error(%v)", b, err)
+		return
+	}
+	return
+}
+
+// nextFile return first filename.
+// sorted by name.
+func (f *FileCache) nextReadFile() (name string) {
+	var names []string
+	f.eLock.RLock()
+	for name = range f.logs {
+		names = append(names, name)
+	}
+	f.eLock.RUnlock()
+	if len(names) > 0 {
+		sort.Strings(names)
+		name = names[0]
+	}
+	return
+}
+
+// loadRemain loadRemain
+func (f *FileCache) loadRemain() (i *Index, w *os.File, err error) {
+	if i, err = f.index(); err != nil {
+		next := f.nextReadFile()
+		if next == "" {
+			err = errLogNotFound
+			return
+		}
+		i = &Index{
+			Name:    next,
+			Updated: time.Now().Format(_formatUpdated),
+		}
+	}
+	if w, err = f.openLog(i); err != nil {
+		log.Warn("a.openLog(%v) error(%v)", i, err)
+		return
+	}
+	return
+}
+
+// openLog open the log file
+func (f *FileCache) openLog(idx *Index) (w *os.File, err error) {
+	if w, err = os.OpenFile(idx.Name, os.O_RDONLY, 0666); err != nil {
+		log.Error("os.OpenFile(%s) error(%v)", idx.Name, err)
+		return
+	}
+	if _, err = w.Seek(idx.Offset, os.SEEK_SET); err != nil {
+		log.Error("f.Seek(%d) error(%v)", idx.Offset, err)
+		return
+	}
+	return
+}
+
+// watch watch
+func (f *FileCache) watch() (err error) {
+	if f.wh, err = fsnotify.NewWatcher(); err != nil {
+		log.Error("fsnotify.NewWatcher() error(%v)", err)
+		return
+	}
+	if err = f.wh.Add(f.c.Storage); err != nil {
+		log.Error("wh.Watch(%s) error(%v)", err)
+	}
+	return
+}
+
+// watchproc observe the directory file changes
+func (f *FileCache) watchproc() {
+	var evt fsnotify.Event
+	for {
+		evt = <-f.wh.Events
+		if evt.Op&fsnotify.Create == fsnotify.Create {
+			if !strings.HasSuffix(evt.Name, f.c.Suffix) {
+				log.Warn("create invalid file: %s", evt.Name)
+				continue
+			}
+			fi, err := os.Stat(evt.Name)
+			if err != nil {
+				log.Error("os.Stat(%s) error(%v)", evt.Name, err)
+				continue
+			}
+			f.eLock.Lock()
+			f.logs[evt.Name] = fi
+			f.eLock.Unlock()
+			log.Info("create file: %s", evt.Name)
+		}
+		if evt.Op&fsnotify.Remove == fsnotify.Remove {
+			f.eLock.Lock()
+			delete(f.logs, evt.Name)
+			f.eLock.Unlock()
+			log.Info("remove file: %s", evt.Name)
+		}
+	}
+}
+
+// setIndex setIndex
+func (f *FileCache) setIndex(idx *Index) (err error) {
+	w, err := os.OpenFile(f.c.Index, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, 0666)
+	if err != nil {
+		log.Error("os.OpenFile(%s) error(%v)", f.c.Index, err)
+		return
+	}
+	defer w.Close()
+	b, err := json.Marshal(idx)
+	if err != nil {
+		log.Error("json.Marshal(%v)", idx)
+		return
+	}
+	if _, err = w.Write(b); err != nil {
+		log.Error("f.Write(%s) error(%v)", b, err)
+	}
+	return
+}
+
+// tailLog check the log format and get log from reader
+func (f *FileCache) tailLog(rr *bufio.Reader) (b []byte, err error) {
+	var (
+		t []byte
+	)
+	// peek magic
+	for {
+		if b, err = rr.Peek(_logMagicSize); err != nil {
+			return
+		}
+		if bytes.Equal(b, logMagic) {
+			break
+		}
+		rr.Discard(1)
+	}
+	// peek length
+	if t, err = rr.Peek(_logHeadSize); err != nil {
+		if err != io.EOF {
+			log.Error("rr.Peek(len:%d) error(%v)", _logLenSize, err)
+		}
+		return
+	}
+	// peek body
+	l := int(binary.BigEndian.Uint32(t[_logMagicSize:_logHeadSize]))
+	if t, err = rr.Peek(_logHeadSize + l); err != nil {
+		if err != io.EOF {
+			log.Error("rr.Peek(%d) error(%v)", l, err)
+		}
+		return
+	}
+	b = t[_logHeadSize:]
+	rr.Discard(l + _logHeadSize)
+	return
+}
+
+// readproc read data and encapsulation protocol from file
+func (f *FileCache) readProcess() {
+	var (
+		err      error
+		idx      *Index
+		rr       = bufio.NewReaderSize(nil, f.c.ReadBuffer)
+		lastTime int64
+		length   int
+		cur      *os.File
+	)
+	if idx, cur, err = f.loadRemain(); err == nil {
+		rr.Reset(cur)
+	}
+	for {
+		if time.Now().Unix()-lastTime > 5 {
+			if idx != nil {
+				f.setIndex(idx)
+			}
+			lastTime = time.Now().Unix()
+		}
+		f.eLock.RLock()
+		length = len(f.logs)
+		f.eLock.RUnlock()
+		// check is available for observing file
+		if length == 0 {
+			if cur != nil {
+				cur.Close()
+				cur = nil
+			}
+			time.Sleep(time.Second * 1)
+			continue
+		}
+		// read first file from observing logs
+		if cur == nil {
+			next := f.nextReadFile()
+			idx = &Index{
+				Name:    next,
+				Updated: time.Now().Format(_formatUpdated),
+			}
+			if cur, err = f.openLog(idx); err != nil {
+				log.Error("a.openLog(%v) error(%v)", idx, err)
+				continue
+			}
+			rr.Reset(cur)
+			f.setIndex(idx)
+		}
+		// tail a log from thos.OpenFilee buffer
+		b, err := f.tailLog(rr)
+		if err != nil {
+			if err == io.EOF {
+				if length > 1 {
+					cur.Close()
+					cur = nil
+					os.Remove(idx.Name)
+					f.eLock.Lock()
+					delete(f.logs, idx.Name)
+					f.eLock.Unlock()
+				} else {
+					time.Sleep(time.Second * 1)
+				}
+				continue
+			}
+			log.Error("read log error(%v)", err)
+			rr.Discard(1)
+			continue
+		}
+
+		idx.Offset += int64(len(b)) + _logHeadSize
+
+		if len(b) <= _logLancerHeaderLen {
+			continue
+		}
+		e := event.GetEvent()
+		e.Write(b[_logLancerHeaderLen:])
+		e.LogId = string(b[:_logIdSize])
+		f.readChan <- e
+	}
+}
+
+// check storage size
+func (f *FileCache) checkStorageSize() {
+	var size int64
+	if entries, err := ioutil.ReadDir(f.c.Storage); err == nil {
+		for _, entry := range entries {
+			if !entry.IsDir() {
+				size += entry.Size()
+			}
+		}
+	}
+	if size > int64(f.c.StorageMaxMB*1024*1024) {
+		log.Error("storage is full, discard log")
+		flowmonitor.Fm.Add("log-agent", "log-agent.output.file-cache", strconv.FormatInt(time.Now().Unix()/100*100, 10), "ERROR", "storage full")
+		f.storageFull = true
+	} else {
+		f.storageFull = false
+	}
+}
+
+func (f *FileCache) nextFileName() string {
+	return path.Join(f.c.Storage, strconv.FormatInt(time.Now().Unix(), 10)+f.c.Suffix)
+}
+
+// nextFile set first log filename.
+// sorted by name.
+func (f *FileCache) nextFile() (err error) {
+	f.next <- f.nextFileName()
+	return
+}

app/service/ops/log-agent/output/conf.go

@@ -0,0 +1,24 @@
+package output
+
+import (
+	"errors"
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+)
+
+const (
+	lancerConfig = "output.toml"
+)
+
+// ReadConfig read config for default output
+func ReadConfig() (value string, err error) {
+	var ok bool
+
+	// logLevel config
+	if value, ok = configcenter.Client.Value(lancerConfig); !ok {
+		return "", errors.New("failed to get output.toml")
+	} else {
+		return value, nil
+	}
+
+	return value, nil
+}

app/service/ops/log-agent/output/lancergrpc/BUILD

@@ -0,0 +1,51 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aggr.go",
+        "config.go",
+        "lancer.go",
+        "pool.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/output/lancergrpc",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/output:go_default_library",
+        "//app/service/ops/log-agent/output/cache/file:go_default_library",
+        "//app/service/ops/log-agent/output/lancergrpc/lancergateway:go_default_library",
+        "//app/service/ops/log-agent/pkg/bufio:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancermonitor:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_google_grpc//codes:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/output/lancergrpc/lancergateway:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/lancergrpc/aggr.go

@@ -0,0 +1,63 @@
+package lancergrpc
+
+import (
+	"bytes"
+	"time"
+	"go-common/app/service/ops/log-agent/event"
+)
+
+const (
+	_logSeparator       = byte('\u0001')
+	_logLancerHeaderLen = 19
+)
+
+// logAggr aggregates multi logs to one log
+func (l *Lancer) logAggr(e *event.ProcessorEvent) (err error) {
+	logAddrbuf := l.getlogAggrBuf(e.LogId)
+	l.logAggrBufLock.Lock()
+	logAddrbuf.Write(e.Bytes())
+	logAddrbuf.WriteByte(_logSeparator)
+	l.logAggrBufLock.Unlock()
+	if logAddrbuf.Len() > l.c.AggrSize {
+		return l.flushLogAggr(e.LogId)
+	}
+	event.PutEvent(e)
+	return nil
+}
+
+// getlogAggrBuf get logAggrBuf by logId
+func (l *Lancer) getlogAggrBuf(logId string) (*bytes.Buffer) {
+	if _, ok := l.logAggrBuf[logId]; !ok {
+		l.logAggrBuf[logId] = new(bytes.Buffer)
+	}
+	return l.logAggrBuf[logId]
+}
+
+// flushLogAggr write aggregated logs to conn
+func (l *Lancer) flushLogAggr(logId string) (err error) {
+	l.logAggrBufLock.Lock()
+	defer l.logAggrBufLock.Unlock()
+	buf := l.getlogAggrBuf(logId)
+	if buf.Len() > 0 {
+		logDoc := new(logDoc)
+		logDoc.b = make([]byte, buf.Len())
+		copy(logDoc.b, buf.Bytes())
+		logDoc.logId = logId
+		l.sendChan <- logDoc
+	}
+	buf.Reset()
+	return nil
+}
+
+// flushLogAggrPeriodically run flushLogAggr Periodically
+func (l *Lancer) flushLogAggrPeriodically() {
+	tick := time.NewTicker(5 * time.Second)
+	for {
+		select {
+		case <-tick.C:
+			for logid, _ := range l.logAggrBuf {
+				l.flushLogAggr(logid)
+			}
+		}
+	}
+}

app/service/ops/log-agent/output/lancergrpc/config.go

@@ -0,0 +1,90 @@
+package lancergrpc
+
+import (
+	"errors"
+	"time"
+	"go-common/app/service/ops/log-agent/output/cache/file"
+	streamEvent "go-common/app/service/ops/log-agent/output/lancergrpc/lancergateway"
+	xtime "go-common/library/time"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	Local                bool                `tome:"local"`
+	Name                 string              `tome:"name"`
+	AggrSize             int                 `tome:"aggrSize"`
+	SendConcurrency      int                 `tome:"sendConcurrency"`
+	CacheConfig          *file.Config        `tome:"cacheConfig"`
+	LancerGateway        *streamEvent.Config `tome:"lancerGateway"`
+	SendBatchSize        int                 `tome:"sendBatchSize"`
+	SendBatchNum         int                 `tome:"sendBatchNum"`
+	SendBatchTimeout     xtime.Duration      `tome:"sendBatchTimeout"`
+	SendFlushInterval    xtime.Duration      `tome:"sendFlushInterval"`
+	InitialRetryDuration xtime.Duration      `tome:"initialRetryDuration"`
+	MaxRetryDuration     xtime.Duration      `tome:"maxRetryDuration"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of Lancer Output is nil")
+	}
+
+	if c.Name == "" {
+		return errors.New("output Name can't be nil")
+	}
+
+	if c.AggrSize == 0 {
+		c.AggrSize = 819200
+	}
+
+	if c.SendConcurrency == 0 {
+		c.SendConcurrency = 5
+	}
+
+	if err := c.CacheConfig.ConfigValidate(); err != nil {
+		return err
+	}
+
+	if c.SendFlushInterval == 0 {
+		c.SendFlushInterval = xtime.Duration(time.Second * 5)
+	}
+
+	if c.InitialRetryDuration == 0 {
+		c.InitialRetryDuration = xtime.Duration(time.Millisecond * 200)
+	}
+
+	if c.MaxRetryDuration == 0 {
+		c.MaxRetryDuration = xtime.Duration(time.Second * 2)
+	}
+
+	if c.SendBatchNum == 0 {
+		c.SendBatchNum = 3000
+	}
+
+	if c.SendBatchSize == 0 {
+		c.SendBatchSize = 1024 * 1024 * 10
+	}
+
+	if c.SendBatchTimeout == 0 {
+		c.SendBatchTimeout = xtime.Duration(time.Second * 5)
+	}
+
+	if c.LancerGateway == nil {
+		c.LancerGateway = &streamEvent.Config{}
+	}
+
+	if err := c.LancerGateway.ConfigValidate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/output/lancergrpc/lancer.go

@@ -0,0 +1,267 @@
+package lancergrpc
+
+import (
+	"context"
+	"fmt"
+	"bytes"
+	"sync"
+	"strconv"
+	"time"
+	"math"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/output"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+	"go-common/app/service/ops/log-agent/output/cache/file"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/lancermonitor"
+	"google.golang.org/grpc"
+	"google.golang.org/grpc/codes"
+	"go-common/app/service/ops/log-agent/output/lancergrpc/lancergateway"
+)
+
+const (
+	_appIdKey = `"app_id":`
+	_levelKey = `"level":`
+	_logTime  = `"time":`
+)
+
+var (
+	logMagic    = []byte{0xAC, 0xBE}
+	logMagicBuf = []byte{0xAC, 0xBE}
+	_logType    = []byte{0, 1}
+	_logLength  = []byte{0, 0, 0, 0}
+	local, _    = time.LoadLocation("Local")
+)
+
+type logDoc struct {
+	b     []byte
+	logId string
+}
+
+func init() {
+	err := output.Register("lancergrpc", NewLancer)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type Lancer struct {
+	c              *Config
+	next           chan string
+	i              chan *event.ProcessorEvent
+	cache          *file.FileCache
+	logAggrBuf     map[string]*bytes.Buffer
+	logAggrBufLock sync.Mutex
+	sendChan       chan *logDoc
+	lancerClient   lancergateway.Gateway2ServerClient
+	ctx            context.Context
+	cancel         context.CancelFunc
+}
+
+func NewLancer(ctx context.Context, config interface{}) (output.Output, error) {
+	var err error
+
+	lancer := new(Lancer)
+	if c, ok := config.(*Config); !ok {
+		return nil, fmt.Errorf("Error config for Lancer output")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		lancer.c = c
+	}
+	if output.OutputRunning(lancer.c.Name) {
+		return nil, fmt.Errorf("Output %s already running", lancer.c.Name)
+	}
+
+	lancer.i = make(chan *event.ProcessorEvent)
+	lancer.next = make(chan string, 1)
+	lancer.logAggrBuf = make(map[string]*bytes.Buffer)
+	lancer.sendChan = make(chan *logDoc)
+	cache, err := file.NewFileCache(lancer.c.CacheConfig)
+	if err != nil {
+		return nil, err
+	}
+	lancer.cache = cache
+	lancer.lancerClient, err = lancergateway.NewClient(lancer.c.LancerGateway)
+	if err != nil {
+		return nil, err
+	}
+	lancer.ctx, lancer.cancel = context.WithCancel(ctx)
+	return lancer, nil
+}
+
+func (l *Lancer) InputChan() (chan *event.ProcessorEvent) {
+	return l.i
+}
+
+func (l *Lancer) Run() (err error) {
+	go l.readFromProcessor()
+	go l.consumeCache()
+	go l.flushLogAggrPeriodically()
+	for i := 0; i < l.c.SendConcurrency; i++ {
+		go l.sendToLancer()
+	}
+	if l.c.Name != "" {
+		output.RegisterOutput(l.c.Name, l)
+	}
+	return nil
+}
+
+func (l *Lancer) Stop() {
+	l.cancel()
+}
+
+func (l *Lancer) readFromProcessor() {
+	for e := range l.i {
+		// only cache for sock input
+		if e.Source == "sock" {
+			l.cache.WriteToCache(e)
+			continue
+		}
+		// without cache
+		l.preWriteToLancer(e)
+	}
+}
+
+func (l *Lancer) preWriteToLancer(e *event.ProcessorEvent) {
+	flowmonitor.Fm.AddEvent(e, "log-agent.output.lancer", "OK", "write to lancer")
+	lancermonitor.IncreaseLogCount("agent.send.success.count", e.LogId)
+	if l.c.Name == "lancer-ops-log" {
+		l.logAggr(e)
+	} else {
+		l.sendLogDirectToLancer(e)
+	}
+}
+
+// consumeCache consume logs from cache
+func (l *Lancer) consumeCache() {
+	for {
+		e := l.cache.ReadFromCache()
+		if e.Length < _logLancerHeaderLen {
+			event.PutEvent(e)
+			continue
+		}
+		// monitor should be called before event recycle
+		l.parseOpslog(e)
+		l.preWriteToLancer(e)
+	}
+}
+
+func (l *Lancer) parseOpslog(e *event.ProcessorEvent) {
+	if l.c.Name == "lancer-ops-log" {
+		e.AppId, _ = common.SeekValue([]byte(_appIdKey), e.Bytes())
+
+		if timeValue, err := common.SeekValue([]byte(_logTime), e.Bytes()); err == nil {
+			if len(timeValue) >= 19 {
+				// parse time
+				var t time.Time
+				if t, err = time.Parse(time.RFC3339Nano, string(timeValue)); err != nil {
+					if t, err = time.ParseInLocation("2006-01-02T15:04:05", string(timeValue), local); err != nil {
+						if t, err = time.ParseInLocation("2006-01-02T15:04:05", string(timeValue[0:19]), local); err != nil {
+						}
+					}
+				}
+				if !t.IsZero() {
+					e.TimeRangeKey = strconv.FormatInt(t.Unix()/100*100, 10)
+				}
+			}
+		}
+	}
+}
+
+// sendLogDirectToLancer send log direct to lancer without aggr
+func (l *Lancer) sendLogDirectToLancer(e *event.ProcessorEvent) {
+	logDoc := new(logDoc)
+	logDoc.b = make([]byte, e.Length)
+	copy(logDoc.b, e.Bytes())
+	logDoc.logId = e.LogId
+	event.PutEvent(e)
+	l.sendChan <- logDoc
+}
+
+func (l *Lancer) nextRetry(retry int) (time.Duration) {
+	// avoid d too large
+	if retry > 10 {
+		return time.Duration(l.c.MaxRetryDuration)
+	}
+
+	d := time.Duration(math.Pow(2, float64(retry))) * time.Duration(l.c.InitialRetryDuration)
+
+	if d > time.Duration(l.c.MaxRetryDuration) {
+		return time.Duration(l.c.MaxRetryDuration)
+	}
+
+	return d
+}
+
+func (l *Lancer) bulkSendToLancerWithRetry(in *lancergateway.EventList) {
+	retry := 0
+	for {
+		ctx, _ := context.WithTimeout(context.Background(), time.Duration(l.c.SendBatchTimeout))
+		t1 := time.Now()
+		resp, err := l.lancerClient.SendList(ctx, in)
+		if err == nil {
+			if resp.Code == lancergateway.StatusCode_SUCCESS {
+				log.Info("get 200 from lancer gateway: size %d, count %d, cost %s", in.Size(), len(in.Events), time.Since(t1).String())
+				return
+			}
+			flowmonitor.Fm.Add("log-agent", "log-agent.output.lancer", "", "ERROR", fmt.Sprintf("write to lancer None 200: %s", resp.Code))
+			log.Warn("get None 200 from lancer gateway, retry: %s", resp.Code)
+		}
+
+		if err != nil {
+			switch grpc.Code(err) {
+			case codes.Canceled, codes.DeadlineExceeded, codes.Unavailable, codes.ResourceExhausted:
+				flowmonitor.Fm.Add("log-agent", "log-agent.output.lancer", "", "ERROR", fmt.Sprintf("write to lancer failed, retry: %s", err))
+				log.Warn("get error from lancer gateway, retry: %s", err)
+			default:
+				flowmonitor.Fm.Add("log-agent", "log-agent.output.lancer", "", "ERROR", fmt.Sprintf("write to lancer failed, no retry: %s", err))
+				log.Warn("get error from lancer gateway, no retry: %s", err)
+				return
+			}
+		}
+
+		time.Sleep(l.nextRetry(retry))
+		retry ++
+	}
+}
+
+// sendproc send the proc to lancer
+func (l *Lancer) sendToLancer() {
+	eventList := new(lancergateway.EventList)
+	eventListLock := sync.Mutex{}
+	lastSend := time.Now()
+	ticker := time.Tick(time.Second * 1)
+	size := 0
+	for {
+		select {
+		case <-ticker:
+			if lastSend.Add(time.Duration(l.c.SendFlushInterval)).Before(time.Now()) && len(eventList.Events) > 0 {
+				eventListLock.Lock()
+				l.bulkSendToLancerWithRetry(eventList)
+				eventList.Reset()
+				size = 0
+				eventListLock.Unlock()
+				lastSend = time.Now()
+			}
+		case logDoc := <-l.sendChan:
+			event := new(lancergateway.SimpleEvent)
+			event.LogId = logDoc.logId
+			event.Header = map[string]string{"timestamp": strconv.FormatInt(time.Now().Unix()/100*100, 10)}
+			event.Data = logDoc.b
+			size += len(event.Data)
+			eventListLock.Lock()
+			eventList.Events = append(eventList.Events, event)
+			if size > l.c.SendBatchSize || len(eventList.Events) > l.c.SendBatchNum {
+				l.bulkSendToLancerWithRetry(eventList)
+				eventList.Reset()
+				size = 0
+				lastSend = time.Now()
+			}
+			eventListLock.Unlock()
+		}
+	}
+}

app/service/ops/log-agent/output/lancergrpc/lancergateway/BUILD

@@ -0,0 +1,60 @@
+load(
+    "@io_bazel_rules_go//proto:def.bzl",
+    "go_proto_library",
+)
+
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+proto_library(
+    name = "StreamEvent_proto",
+    srcs = ["StreamEvent.proto"],
+    tags = ["automanaged"],
+    deps = ["@gogo_special_proto//github.com/gogo/protobuf/gogoproto"],
+)
+
+go_proto_library(
+    name = "StreamEvent_go_proto",
+    compilers = ["@io_bazel_rules_go//proto:gogofast_grpc"],
+    importpath = "go-common/app/service/ops/log-agent/output/lancergrpc/lancergateway",
+    proto = ":StreamEvent_proto",
+    tags = ["automanaged"],
+    deps = ["@com_github_gogo_protobuf//gogoproto:go_default_library"],
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["client.go"],
+    embed = [":StreamEvent_go_proto"],
+    importpath = "go-common/app/service/ops/log-agent/output/lancergrpc/lancergateway",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//library/naming/discovery:go_default_library",
+        "//library/net/rpc/warden/balancer/wrr:go_default_library",
+        "//library/net/rpc/warden/resolver:go_default_library",
+        "//library/time:go_default_library",
+        "@com_github_gogo_protobuf//gogoproto:go_default_library",
+        "@com_github_gogo_protobuf//proto:go_default_library",
+        "@org_golang_google_grpc//:go_default_library",
+        "@org_golang_x_net//context:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/lancergrpc/lancergateway/StreamEvent.proto

@@ -0,0 +1,36 @@
+syntax = "proto3";
+
+import "github.com/gogo/protobuf/gogoproto/gogo.proto";
+
+option java_multiple_files = true;
+option java_package = "com.bilibili.gateway2.common.protobuf";
+
+service Gateway2Server {
+    rpc sendList (EventList) returns (Response) {
+    }
+
+    rpc send (SimpleEvent) returns (Response) {
+    }
+}
+
+enum StatusCode {
+    NULL = 0;
+    SUCCESS = 200;
+    LOAD_FULL = 429;
+}
+
+message SimpleEvent {
+    string logId = 1;
+    string outerId = 2;
+    map<string, string> header = 3;
+    bytes data = 4;
+}
+
+message EventList {
+    repeated SimpleEvent events = 1;
+}
+
+message Response {
+    StatusCode code = 1;
+    string msg = 2;
+}

app/service/ops/log-agent/output/lancergrpc/lancergateway/client.go

@@ -0,0 +1,61 @@
+package lancergateway
+
+import (
+	"time"
+	"errors"
+	"fmt"
+
+	"go-common/library/net/rpc/warden/resolver"
+	"go-common/library/net/rpc/warden/balancer/wrr"
+	"go-common/library/naming/discovery"
+	xtime "go-common/library/time"
+
+	"google.golang.org/grpc"
+)
+
+type Config struct {
+	AppId   string        `toml:"appId"`
+	Timeout xtime.Duration `toml:"timeout"`
+	Subset  int           `toml:"subset"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of LancerGateway can't be nil")
+	}
+
+	if c.AppId == "" {
+		c.AppId = "datacenter.lancer.gateway2-server"
+	}
+
+	if c.Timeout == 0 {
+		c.Timeout = xtime.Duration(time.Second * 5)
+	}
+
+	if c.Subset == 0 {
+		c.Subset = 5
+	}
+	return nil
+}
+
+func init() {
+	resolver.Register(discovery.Builder())
+}
+
+// NewClient new member grpc client
+func NewClient(c *Config) (Gateway2ServerClient, error) {
+	opts := []grpc.DialOption{
+		grpc.WithInsecure(),
+		grpc.WithBalancerName(wrr.Name),
+	}
+
+	if c.Timeout != 0 {
+		opts = append(opts, grpc.WithTimeout(time.Duration(c.Timeout)))
+	}
+
+	conn, err := grpc.Dial(fmt.Sprintf("discovery://default/%s?subset=%d", c.AppId, c.Subset), opts...)
+	if err != nil {
+		return nil, err
+	}
+	return NewGateway2ServerClient(conn), nil
+}

app/service/ops/log-agent/output/lancergrpc/lancergateway/protoc.sh

@@ -0,0 +1,135 @@
+#!/bin/bash
+DEFAULT_PROTOC_GEN="gogofast"
+DEFAULT_PROTOC="protoc"
+GO_COMMON_DIR_NAME="go-common"
+USR_INCLUDE_DIR="/usr/local/include"
+
+function _install_protoc() {
+    osname=$(uname -s)
+    echo "install protoc ..."
+    case $osname in
+        "Darwin" )
+            brew install protobuf
+            ;;
+        *)
+            echo "unknown operating system, need install protobuf manual see: https://developers.google.com/protocol-buffers"
+            exit 1
+            ;;
+    esac
+}
+
+function _install_protoc_gen() {
+    local protoc_gen=$1
+    case $protoc_gen in
+        "gofast" )
+            echo "install gofast from github.com/gogo/protobuf/protoc-gen-gofast"
+            go get github.com/gogo/protobuf/protoc-gen-gofast
+            ;;
+        "gogofast" )
+            echo "install gogofast from github.com/gogo/protobuf/protoc-gen-gogofast"
+            go get github.com/gogo/protobuf/protoc-gen-gogofast
+            ;;
+        "gogo" )
+            echo "install gogo from github.com/gogo/protobuf/protoc-gen-gogo"
+            go get github.com/gogo/protobuf/protoc-gen-gogo
+            ;;
+        "go" )
+            echo "install protoc-gen-go from github.com/golang/protobuf"
+            go get github.com/golang/protobuf/{proto,protoc-gen-go}
+            ;;
+        *)
+            echo "can't install protoc-gen-${protoc_gen} automatic !"
+            exit 1;
+            ;;
+    esac
+}
+
+function _find_go_common_dir() {
+    local go_common_dir_name=$1
+    local current_dir=$(pwd)
+    while [[ "$(basename $current_dir)" != "$go_common_dir_name" ]]; do
+        current_dir=$(dirname $current_dir)
+        if [[ "$current_dir" == "/" || "$current_dir" == "." || -z "$current_dir" ]]; then
+            return 1
+        fi
+    done
+    echo $current_dir
+}
+
+function _fix_pb_file() {
+    local target_dir=$1
+    echo "fix pb file"
+    local pb_files=$(find $target_dir -name "*.pb.go" -type f)
+    local pkg_name_esc=$(echo "$target_dir" | sed 's_/_\\/_g')
+    for file in $pb_files; do
+        echo "fix pb file $file"
+        if [[ $(uname -s) == 'Darwin' ]]; then
+            sed -i "" -e "s/^import \(.*\) \"app\/\(.*\)\"/import \1 \"go-common\/app\/\2\"/g" $file
+        else
+            sed -i"" -E "s/^import\s*(.*)\s*\"app\/(.*)\"/import\1\"go-common\/app\/\2\"/g" $file
+        fi
+    done
+}
+
+function _esc_string() {
+    echo $(echo "$1" | sed 's_/_\\/_g')
+}
+
+function _run_protoc() {
+    local proto_dir=$1
+    local proto_files=$(find $proto_dir -maxdepth 1 -name "*.proto")
+    if [[ -z $proto_files ]]; then
+        return
+    fi
+    local protoc_cmd="$PROTOC -I$PROTO_PATH --${PROTOC_GEN}_out=plugins=grpc:. ${proto_files}"
+    echo $protoc_cmd
+    $protoc_cmd
+}
+
+if [[ -z $PROTOC ]]; then
+    PROTOC=${DEFAULT_PROTOC}
+    which $PROTOC 
+    if [[ "$?" -ne "0" ]]; then
+        _install_protoc
+    fi
+fi
+if [[ -z $PROTOC_GEN ]]; then
+    PROTOC_GEN=${DEFAULT_PROTOC_GEN}
+    which protoc-gen-$PROTOC_GEN
+    if [[ "$?" -ne "0" ]]; then
+        _install_protoc_gen $PROTOC_GEN
+    fi
+fi
+
+GO_COMMON_DIR=$(_find_go_common_dir $GO_COMMON_DIR_NAME)
+if [[ "$?" != "0" ]]; then
+    echo "can't find go-common directoy"
+    exit 1
+fi
+
+if [[ -z $PROTO_PATH ]]; then
+    PROTO_PATH=$GO_COMMON_DIR:$GO_COMMON_DIR/vendor:$USR_INCLUDE_DIR
+else
+    PROTO_PATH=$PROTO_PATH:$GO_COMMON_DIR:$GO_COMMON_DIR/vendor:$USR_INCLUDE_DIR
+fi
+
+if [[ ! -z $1 ]]; then
+    cd $1
+fi
+TARGET_DIR=$(pwd)
+
+GO_COMMON_DIR_ESC=$(_esc_string "$GO_COMMON_DIR/")
+
+TARGET_DIR=${TARGET_DIR//$GO_COMMON_DIR_ESC/}
+
+# switch to go_common
+cd $GO_COMMON_DIR
+
+DIRS=$(find $TARGET_DIR -type d)
+
+for dir in $DIRS; do
+    echo "run protoc in $dir"
+    _run_protoc $dir
+done
+
+_fix_pb_file $TARGET_DIR

app/service/ops/log-agent/output/lancergrpc/pool.go

@@ -0,0 +1,360 @@
+package lancergrpc
+
+import (
+	"net"
+	"time"
+	"sync"
+	"errors"
+	"math/rand"
+	"expvar"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/bufio"
+)
+
+var (
+	ErrAddrListNil = errors.New("addrList can't be nil")
+	ErrPoolSize    = errors.New("Pool size should be no greater then length of addr list")
+)
+
+type LancerBufConn struct {
+	conn    net.Conn
+	wr      *bufio.Writer
+	enabled bool
+	ctime   time.Time
+}
+
+type connPool struct {
+	c                    *ConnPoolConfig
+	invalidUpstreams     map[string]interface{}
+	invalidUpstreamsLock sync.RWMutex
+	validBufConnChan     chan *LancerBufConn
+	invalidBufConnChan   chan *LancerBufConn
+	connCounter          map[string]int
+	connCounterLock      sync.RWMutex
+	newConnLock          sync.Mutex
+}
+
+type ConnPoolConfig struct {
+	Name        string        `tome:"name"`
+	AddrList    []string      `tome:"addrList"`
+	DialTimeout time.Duration `tome:"dialTimeout"`
+	IdleTimeout time.Duration `tome:"idleTimeout"`
+	BufSize     int           `tome:"bufSize"`
+	PoolSize    int           `tome:"poolSize"`
+}
+
+func (c *ConnPoolConfig) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("Config of pool is nil")
+	}
+
+	if len(c.AddrList) == 0 {
+		return errors.New("pool addr list can't be empty")
+	}
+	if c.DialTimeout.Seconds() == 0 {
+		c.DialTimeout = time.Second * 5
+	}
+
+	if c.IdleTimeout.Seconds() == 0 {
+		c.IdleTimeout = time.Minute * 15
+	}
+
+	if c.BufSize == 0 {
+		c.BufSize = 1024 * 1024 * 2 // 2M by default
+	}
+
+	if c.PoolSize == 0 {
+		c.PoolSize = len(c.AddrList)
+	}
+	return nil
+}
+
+// newConn make a connection to lancer
+func (cp *connPool) newConn() (conn net.Conn, err error) {
+	cp.newConnLock.Lock()
+	defer cp.newConnLock.Unlock()
+	for {
+		if addr, err := cp.randomOneUpstream(); err == nil {
+			if conn, err := net.DialTimeout("tcp", addr, cp.c.DialTimeout); err == nil && conn != nil {
+				log.Info("connect to %s success", addr)
+				cp.connCounterAdd(addr)
+				return conn, nil
+			} else {
+				cp.markUpstreamInvalid(addr)
+				continue
+			}
+		} else {
+			return nil, err
+		}
+	}
+}
+
+// newBufConn 创建一个buf连接, buf连接绑定一个conn(无论连接是否可用)
+func (cp *connPool) newBufConn() (bufConn *LancerBufConn, err error) {
+	bufConn = new(LancerBufConn)
+	bufConn.wr = bufio.NewWriterSize(nil, cp.c.BufSize)
+	if err := cp.setConn(bufConn); err == nil {
+		bufConn.enabled = true
+	} else {
+		bufConn.enabled = false
+	}
+
+	return bufConn, nil
+}
+
+// flushBufConn 定期flush buffer
+func (cp *connPool) flushBufConn() {
+	for {
+		bufConn, _ := cp.getBufConn()
+		bufConn.conn.SetWriteDeadline(time.Now().Add(time.Second * 5))
+		if err := bufConn.wr.Flush(); err != nil {
+			log.Error("Error when flush to %s: %s", bufConn.conn.RemoteAddr().String(), err)
+			bufConn.enabled = false
+		}
+		cp.putBufConn(bufConn)
+		time.Sleep(time.Second * 5)
+	}
+}
+
+// initConnPool 初始化conn pool对象
+func initConnPool(c *ConnPoolConfig) (cp *connPool, err error) {
+	if err = c.ConfigValidate(); err != nil {
+		return nil, err
+	}
+	if len(c.AddrList) == 0 {
+		return nil, ErrAddrListNil
+	}
+
+	if c.PoolSize > len(c.AddrList) {
+		return nil, ErrPoolSize
+	}
+
+	rand.Seed(time.Now().Unix())
+	cp = new(connPool)
+	cp.c = c
+	cp.validBufConnChan = make(chan *LancerBufConn, cp.c.PoolSize)
+	cp.invalidBufConnChan = make(chan *LancerBufConn, cp.c.PoolSize)
+	cp.invalidUpstreams = make(map[string]interface{})
+	cp.connCounter = make(map[string]int)
+	cp.initPool()
+	go cp.maintainUpstream()
+	go cp.flushBufConn()
+	go cp.maintainBufConnPool()
+	expvar.Publish("conn_pool"+cp.c.Name, expvar.Func(cp.connPoolStatus))
+	return cp, nil
+}
+
+// connableUpstreams 返回可以建立连接的upstream列表
+func (cp *connPool) connableUpstreams() ([]string) {
+	list := make([]string, 0)
+	cp.invalidUpstreamsLock.RLock()
+	defer cp.invalidUpstreamsLock.RUnlock()
+	for _, addr := range cp.c.AddrList {
+		if _, ok := cp.invalidUpstreams[addr]; !ok {
+			if count, ok := cp.connCounter[addr]; ok && count == 0 {
+				list = append(list, addr)
+			}
+		}
+	}
+	return list
+}
+
+// write write []byte to BufConn
+func (bc *LancerBufConn) write(p []byte) (int, error) {
+	bc.conn.SetWriteDeadline(time.Now().Add(time.Second * 5))
+	return bc.wr.Write(p)
+}
+
+// randomOneUpstream 随机返回一个可以建立连接的upstream
+func (cp *connPool) randomOneUpstream() (s string, err error) {
+	list := cp.connableUpstreams()
+	if len(list) == 0 {
+		err = errors.New("No valid upstreams")
+		return
+	}
+	return list[rand.Intn(len(list))], nil
+}
+
+// initPool 初始化poolSize个数的bufConn
+func (cp *connPool) initPool() {
+	for _, addr := range cp.c.AddrList {
+		cp.connCounter[addr] = 0
+	}
+	for i := 0; i < cp.c.PoolSize; i++ {
+		if bufConn, err := cp.newBufConn(); err == nil {
+			cp.putBufConn(bufConn)
+		}
+	}
+}
+
+// novalidUpstream check if there is no validUpstream
+func (cp *connPool) novalidUpstream() bool {
+	return len(cp.invalidUpstreams) == len(cp.c.AddrList)
+}
+
+//GetConn 从pool中取一个BufConn
+func (cp *connPool) getBufConn() (*LancerBufConn, error) {
+	for {
+		select {
+		case bufConn := <-cp.validBufConnChan:
+			if !bufConn.enabled {
+				cp.putInvalidBufConn(bufConn)
+				continue
+			}
+			return bufConn, nil
+		case <-time.After(10 * time.Second):
+			log.Warn("timeout when get conn from conn pool")
+			continue
+		}
+	}
+}
+
+// setConn 为bufConn绑定一个新的Conn
+func (cp *connPool) setConn(bufConn *LancerBufConn) (error) {
+	if bufConn.conn != nil {
+		if bufConn.enabled == false {
+			cp.markUpstreamInvalid(bufConn.conn.RemoteAddr().String())
+		}
+		cp.connCounterDel(bufConn.conn.RemoteAddr().String())
+		bufConn.conn.Close()
+		bufConn.conn = nil
+		bufConn.enabled = false
+	}
+	if conn, err := cp.newConn(); err == nil {
+		bufConn.conn = conn
+		bufConn.wr.Reset(conn)
+		bufConn.ctime = time.Now()
+		bufConn.enabled = true
+		return nil
+	} else {
+		bufConn.enabled = false
+		return err
+	}
+}
+
+//putBufConn 把BufConn放回到pool中
+func (cp *connPool) putBufConn(bufConn *LancerBufConn) {
+	if bufConn.enabled == false {
+		cp.putInvalidBufConn(bufConn)
+		return
+	}
+	if bufConn.ctime.Add(cp.c.IdleTimeout).Before(time.Now()) {
+		bufConn.wr.Flush()
+		cp.putInvalidBufConn(bufConn)
+		return
+	}
+	cp.putValidBufConn(bufConn)
+}
+
+// putValidBufConn 把 bufConn放到可用的pool中
+func (cp *connPool) putValidBufConn(bufConn *LancerBufConn) {
+	select {
+	case cp.validBufConnChan <- bufConn:
+		return
+	default:
+		log.Warn("BufConnChan full, discard，this shouldn't happen")
+		return
+	}
+}
+
+// putInvalidBufConn 把bufConn放到不可用的pool中
+func (cp *connPool) putInvalidBufConn(bufConn *LancerBufConn) {
+	select {
+	case cp.invalidBufConnChan <- bufConn:
+		return
+	default:
+		log.Warn("invalidBufConnChan full, discard，this shouldn't happen")
+		return
+	}
+}
+
+// maintainBufConnPool 维护BufConnPool状态
+func (cp *connPool) maintainBufConnPool() {
+	for {
+		select {
+		case bufConn := <-cp.invalidBufConnChan:
+			cp.setConn(bufConn)
+			cp.putBufConn(bufConn)
+		}
+		time.Sleep(time.Second * 1)
+	}
+}
+
+//markConnInvalid会将链接关闭并且将相应upstreamserver设置为不可用
+func (cp *connPool) markUpstreamInvalid(addr string) (err error) {
+	log.Error("mark upstream %s invalid", addr)
+	cp.invalidUpstreamsLock.Lock()
+	cp.invalidUpstreams[addr] = nil
+	cp.invalidUpstreamsLock.Unlock()
+	return
+}
+
+// markUpstreamValid 将某一addr设置为不可用
+func (cp *connPool) markUpstreamValid(addr string) (err error) {
+	log.Info("%s is valid again", addr)
+	cp.invalidUpstreamsLock.Lock()
+	delete(cp.invalidUpstreams, addr)
+	cp.invalidUpstreamsLock.Unlock()
+	return
+}
+
+// connCounterAdd 连接数+1
+func (cp *connPool) connCounterAdd(addr string) {
+	cp.connCounterLock.Lock()
+	defer cp.connCounterLock.Unlock()
+	if _, ok := cp.connCounter[addr]; ok {
+		cp.connCounter[addr] += 1
+	} else {
+		cp.connCounter[addr] = 1
+	}
+	return
+}
+
+//connCounterDel 连接数-1
+func (cp *connPool) connCounterDel(addr string) {
+	cp.connCounterLock.Lock()
+	defer cp.connCounterLock.Unlock()
+	if _, ok := cp.connCounter[addr]; ok {
+		cp.connCounter[addr] -= 1
+	}
+}
+
+// connPoolStatus 返回connPool状态
+func (cp *connPool) connPoolStatus() interface{} {
+	status := make(map[string]interface{})
+	status["conn_num"] = cp.connCounter
+	status["invalidUpstreams"] = cp.invalidUpstreams
+	return status
+}
+
+// maintainUpstream 维护upstream的健康状态
+func (cp *connPool) maintainUpstream() {
+	for {
+		cp.invalidUpstreamsLock.RLock()
+		tryAddrs := make([]string, 0, len(cp.invalidUpstreams))
+		for k := range cp.invalidUpstreams {
+			tryAddrs = append(tryAddrs, k)
+		}
+		cp.invalidUpstreamsLock.RUnlock()
+		for _, addr := range tryAddrs {
+			if conn, err := net.DialTimeout("tcp", addr, cp.c.DialTimeout); err == nil && conn != nil {
+				conn.Close()
+				cp.markUpstreamValid(addr)
+			}
+		}
+		time.Sleep(time.Second * 10)
+	}
+}
+
+//ReleaseConnPool 释放连接池中所有链接
+func (cp *connPool) ReleaseConnPool() {
+	log.Info("Release Conn Pool")
+	close(cp.validBufConnChan)
+	close(cp.invalidBufConnChan)
+	for conn := range cp.validBufConnChan {
+		conn.enabled = false
+		conn.wr.Flush()
+		conn.conn.Close()
+	}
+}

app/service/ops/log-agent/output/lancerlogstream/BUILD

@@ -0,0 +1,45 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "aggr.go",
+        "config.go",
+        "lancer.go",
+        "pool.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/output/lancerlogstream",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/output:go_default_library",
+        "//app/service/ops/log-agent/output/cache/file:go_default_library",
+        "//app/service/ops/log-agent/pkg/bufio:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor:go_default_library",
+        "//app/service/ops/log-agent/pkg/lancermonitor:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/lancerlogstream/aggr.go

@@ -0,0 +1,63 @@
+package lancerlogstream
+
+import (
+	"bytes"
+	"time"
+	"go-common/app/service/ops/log-agent/event"
+)
+
+const (
+	_logSeparator       = byte('\u0001')
+	_logLancerHeaderLen = 19
+)
+
+// logAggr aggregates multi logs to one log
+func (l *Lancer) logAggr(e *event.ProcessorEvent) (err error) {
+	logAddrbuf := l.getlogAggrBuf(e.LogId)
+	l.logAggrBufLock.Lock()
+	logAddrbuf.Write(e.Bytes())
+	logAddrbuf.WriteByte(_logSeparator)
+	l.logAggrBufLock.Unlock()
+	if logAddrbuf.Len() > l.c.AggrSize {
+		return l.flushLogAggr(e.LogId)
+	}
+	event.PutEvent(e)
+	return nil
+}
+
+// getlogAggrBuf get logAggrBuf by logId
+func (l *Lancer) getlogAggrBuf(logId string) (*bytes.Buffer) {
+	if _, ok := l.logAggrBuf[logId]; !ok {
+		l.logAggrBuf[logId] = new(bytes.Buffer)
+	}
+	return l.logAggrBuf[logId]
+}
+
+// flushLogAggr write aggregated logs to conn
+func (l *Lancer) flushLogAggr(logId string) (err error) {
+	l.logAggrBufLock.Lock()
+	defer l.logAggrBufLock.Unlock()
+	buf := l.getlogAggrBuf(logId)
+	if buf.Len() > 0 {
+		logDoc := new(logDoc)
+		logDoc.b = make([]byte, buf.Len())
+		copy(logDoc.b, buf.Bytes())
+		logDoc.logId = logId
+		l.sendChan <- logDoc
+	}
+	buf.Reset()
+	return nil
+}
+
+// flushLogAggrPeriodically run flushLogAggr Periodically
+func (l *Lancer) flushLogAggrPeriodically() {
+	tick := time.NewTicker(5 * time.Second)
+	for {
+		select {
+		case <-tick.C:
+			for logid, _ := range l.logAggrBuf {
+				l.flushLogAggr(logid)
+			}
+		}
+	}
+}

app/service/ops/log-agent/output/lancerlogstream/config.go

@@ -0,0 +1,53 @@
+package lancerlogstream
+
+import (
+	"errors"
+	"go-common/app/service/ops/log-agent/output/cache/file"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	Local           bool            `tome:"local"`
+	Name            string          `tome:"name"`
+	AggrSize        int             `tome:"aggrSize"`
+	SendConcurrency int             `tome:"sendConcurrency"`
+	CacheConfig     *file.Config    `tome:"cacheConfig"`
+	PoolConfig      *ConnPoolConfig `tome:"poolConfig"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of Sock Input is nil")
+	}
+
+	if c.Name == "" {
+		return errors.New("output Name can't be nil")
+	}
+
+	if c.AggrSize == 0 {
+		c.AggrSize = 819200
+	}
+
+	if c.SendConcurrency == 0 {
+		c.SendConcurrency = 5
+	}
+
+	if err := c.CacheConfig.ConfigValidate(); err != nil {
+		return err
+	}
+
+	if err := c.PoolConfig.ConfigValidate(); err != nil {
+		return err
+	}
+
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/output/lancerlogstream/lancer.go

@@ -0,0 +1,228 @@
+package lancerlogstream
+
+import (
+	"context"
+	"fmt"
+	"bytes"
+	"sync"
+	"encoding/binary"
+	"strconv"
+	"time"
+
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/output"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+	"go-common/app/service/ops/log-agent/output/cache/file"
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/lancermonitor"
+)
+
+const (
+	_logLenStart       = 2
+	_logLenEnd         = 6
+	_tokenHeaderFormat = "logId=%s&timestamp=%s&version=1.1"
+	_protocolLen       = 6
+	_appIdKey          = `"app_id":`
+	_levelKey          = `"level":`
+	_logTime           = `"time":`
+)
+
+var (
+	logMagic    = []byte{0xAC, 0xBE}
+	logMagicBuf = []byte{0xAC, 0xBE}
+	_logType    = []byte{0, 1}
+	_logLength  = []byte{0, 0, 0, 0}
+	local, _    = time.LoadLocation("Local")
+)
+
+type logDoc struct {
+	b     []byte
+	logId string
+}
+
+func init() {
+	err := output.Register("lancer", NewLancer)
+	if err != nil {
+		panic(err)
+	}
+}
+
+type Lancer struct {
+	c              *Config
+	next           chan string
+	i              chan *event.ProcessorEvent
+	cache          *file.FileCache
+	logAggrBuf     map[string]*bytes.Buffer
+	logAggrBufLock sync.Mutex
+	sendChan       chan *logDoc
+	connPool       *connPool
+	ctx            context.Context
+	cancel         context.CancelFunc
+}
+
+func NewLancer(ctx context.Context, config interface{}) (output.Output, error) {
+	var err error
+
+	lancer := new(Lancer)
+	if c, ok := config.(*Config); !ok {
+		return nil, fmt.Errorf("Error config for Lancer output")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		lancer.c = c
+	}
+	if output.OutputRunning(lancer.c.Name) {
+		return nil, fmt.Errorf("Output %s already running", lancer.c.Name)
+	}
+
+	lancer.i = make(chan *event.ProcessorEvent)
+	lancer.next = make(chan string, 1)
+	lancer.logAggrBuf = make(map[string]*bytes.Buffer)
+	lancer.sendChan = make(chan *logDoc)
+	cache, err := file.NewFileCache(lancer.c.CacheConfig)
+	if err != nil {
+		return nil, err
+	}
+	lancer.cache = cache
+	lancer.c.PoolConfig.Name = lancer.c.Name
+	lancer.connPool, err = initConnPool(lancer.c.PoolConfig)
+	if err != nil {
+		return nil, err
+	}
+	lancer.ctx, lancer.cancel = context.WithCancel(ctx)
+	return lancer, nil
+}
+
+func (l *Lancer) InputChan() (chan *event.ProcessorEvent) {
+	return l.i
+}
+
+func (l *Lancer) Run() (err error) {
+	go l.writeToCache()
+	go l.readFromCache()
+	go l.flushLogAggrPeriodically()
+	for i := 0; i < l.c.SendConcurrency; i++ {
+		go l.sendToLancer()
+	}
+	output.RegisterOutput(l.c.Name, l)
+	return nil
+}
+
+func (l *Lancer) Stop() {
+	l.cancel()
+}
+
+// writeToCache write the log to cache
+func (l *Lancer) writeToCache() {
+	for e := range l.i {
+		if e.Length < _logLancerHeaderLen {
+			event.PutEvent(e)
+			continue
+		}
+		l.cache.WriteToCache(e)
+	}
+}
+
+func (l *Lancer) readFromCache() {
+	for {
+		e := l.cache.ReadFromCache()
+		if e.Length < _logLancerHeaderLen {
+			event.PutEvent(e)
+			continue
+		}
+		// monitor should be called before event recycle
+		l.parseOpslog(e)
+		flowmonitor.Fm.AddEvent(e, "log-agent.output.lancer", "OK", "write to lancer")
+		lancermonitor.IncreaseLogCount("agent.send.success.count", e.LogId)
+		if l.c.Name == "lancer-ops-log" {
+			l.logAggr(e)
+		} else {
+			l.sendLogDirectToLancer(e)
+		}
+
+	}
+}
+
+func (l *Lancer) parseOpslog(e *event.ProcessorEvent) {
+	if l.c.Name == "lancer-ops-log" && e.Length > _logLancerHeaderLen {
+		logBody := e.Body[(_logLancerHeaderLen):(e.Length)]
+		e.AppId, _ = common.SeekValue([]byte(_appIdKey), logBody)
+
+		if timeValue, err := common.SeekValue([]byte(_logTime), logBody); err == nil {
+			if len(timeValue) >= 19 {
+				// parse time
+				var t time.Time
+				if t, err = time.Parse(time.RFC3339Nano, string(timeValue)); err != nil {
+					if t, err = time.ParseInLocation("2006-01-02T15:04:05", string(timeValue), local); err != nil {
+						if t, err = time.ParseInLocation("2006-01-02T15:04:05", string(timeValue[0:19]), local); err != nil {
+						}
+					}
+				}
+				if !t.IsZero() {
+					e.TimeRangeKey = strconv.FormatInt(t.Unix()/100*100, 10)
+				}
+			}
+		}
+	}
+}
+
+
+// sendLogDirectToLancer send log direct to lancer without aggr
+func (l *Lancer) sendLogDirectToLancer(e *event.ProcessorEvent) {
+	logDoc := new(logDoc)
+	logDoc.b = make([]byte, e.Length)
+	copy(logDoc.b, e.Bytes())
+	logDoc.logId = e.LogId
+	event.PutEvent(e)
+	l.sendChan <- logDoc
+}
+
+// sendproc send the proc to lancer
+func (l *Lancer) sendToLancer() {
+	logSend := new(bytes.Buffer)
+	tokenHeaderLen := []byte{0, 0}
+	for {
+		select {
+		case logDoc := <-l.sendChan:
+			var err error
+			if len(logDoc.b) == 0 {
+				continue
+			}
+			// header
+			logSend.Reset()
+			logSend.Write(logMagicBuf)
+			logSend.Write(_logLength) // placeholder
+			logSend.Write(_logType)
+			// token header
+			tokenheader := []byte(fmt.Sprintf(_tokenHeaderFormat, logDoc.logId, strconv.FormatInt(time.Now().Unix()/100*100, 10)))
+			binary.BigEndian.PutUint16(tokenHeaderLen, uint16(len(tokenheader)))
+			logSend.Write(tokenHeaderLen)
+			logSend.Write(tokenheader)
+			// log body
+			logSend.Write(logDoc.b)
+
+			// set log length
+			bs := logSend.Bytes()
+			binary.BigEndian.PutUint32(bs[_logLenStart:_logLenEnd], uint32(logSend.Len()-_protocolLen))
+
+			// write
+			connBuf, err := l.connPool.getBufConn()
+			if err != nil {
+				flowmonitor.Fm.Add("log-agent", "log-agent.output.lancer", "", "ERROR", "get conn failed")
+				log.Error("get conn error: %v", err)
+				continue
+			}
+			if _, err = connBuf.write(bs); err != nil {
+				log.Error("wr.Write(log) error(%v)", err)
+				connBuf.enabled = false
+				l.connPool.putBufConn(connBuf)
+				flowmonitor.Fm.Add("log-agent", "log-agent.output.lancer", "", "ERROR", "write to lancer failed")
+				continue
+			}
+			l.connPool.putBufConn(connBuf)
+			// TODO: flowmonitor for specific appId
+		}
+	}
+}

app/service/ops/log-agent/output/lancerlogstream/pool.go

@@ -0,0 +1,361 @@
+package lancerlogstream
+
+import (
+	"net"
+	"time"
+	"sync"
+	"errors"
+	"math/rand"
+	"expvar"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/bufio"
+	xtime "go-common/library/time"
+)
+
+var (
+	ErrAddrListNil = errors.New("addrList can't be nil")
+	ErrPoolSize    = errors.New("Pool size should be no greater then length of addr list")
+)
+
+type LancerBufConn struct {
+	conn    net.Conn
+	wr      *bufio.Writer
+	enabled bool
+	ctime   time.Time
+}
+
+type connPool struct {
+	c                    *ConnPoolConfig
+	invalidUpstreams     map[string]interface{}
+	invalidUpstreamsLock sync.RWMutex
+	validBufConnChan     chan *LancerBufConn
+	invalidBufConnChan   chan *LancerBufConn
+	connCounter          map[string]int
+	connCounterLock      sync.RWMutex
+	newConnLock          sync.Mutex
+}
+
+type ConnPoolConfig struct {
+	Name        string         `tome:"name"`
+	AddrList    []string       `tome:"addrList"`
+	DialTimeout xtime.Duration `tome:"dialTimeout"`
+	IdleTimeout xtime.Duration `tome:"idleTimeout"`
+	BufSize     int            `tome:"bufSize"`
+	PoolSize    int            `tome:"poolSize"`
+}
+
+func (c *ConnPoolConfig) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("Config of pool is nil")
+	}
+
+	if len(c.AddrList) == 0 {
+		return errors.New("pool addr list can't be empty")
+	}
+	if time.Duration(c.DialTimeout) == 0 {
+		c.DialTimeout = xtime.Duration(time.Second * 5)
+	}
+
+	if time.Duration(c.IdleTimeout) == 0 {
+		c.IdleTimeout = xtime.Duration(time.Minute * 15)
+	}
+
+	if c.BufSize == 0 {
+		c.BufSize = 1024 * 1024 * 2 // 2M by default
+	}
+
+	if c.PoolSize == 0 {
+		c.PoolSize = len(c.AddrList)
+	}
+	return nil
+}
+
+// newConn make a connection to lancer
+func (cp *connPool) newConn() (conn net.Conn, err error) {
+	cp.newConnLock.Lock()
+	defer cp.newConnLock.Unlock()
+	for {
+		if addr, err := cp.randomOneUpstream(); err == nil {
+			if conn, err := net.DialTimeout("tcp", addr, time.Duration(cp.c.DialTimeout)); err == nil && conn != nil {
+				log.Info("connect to %s success", addr)
+				cp.connCounterAdd(addr)
+				return conn, nil
+			} else {
+				cp.markUpstreamInvalid(addr)
+				continue
+			}
+		} else {
+			return nil, err
+		}
+	}
+}
+
+// newBufConn 创建一个buf连接, buf连接绑定一个conn(无论连接是否可用)
+func (cp *connPool) newBufConn() (bufConn *LancerBufConn, err error) {
+	bufConn = new(LancerBufConn)
+	bufConn.wr = bufio.NewWriterSize(nil, cp.c.BufSize)
+	if err := cp.setConn(bufConn); err == nil {
+		bufConn.enabled = true
+	} else {
+		bufConn.enabled = false
+	}
+
+	return bufConn, nil
+}
+
+// flushBufConn 定期flush buffer
+func (cp *connPool) flushBufConn() {
+	for {
+		bufConn, _ := cp.getBufConn()
+		bufConn.conn.SetWriteDeadline(time.Now().Add(time.Second * 5))
+		if err := bufConn.wr.Flush(); err != nil {
+			log.Error("Error when flush to %s: %s", bufConn.conn.RemoteAddr().String(), err)
+			bufConn.enabled = false
+		}
+		cp.putBufConn(bufConn)
+		time.Sleep(time.Second * 5)
+	}
+}
+
+// initConnPool 初始化conn pool对象
+func initConnPool(c *ConnPoolConfig) (cp *connPool, err error) {
+	if err = c.ConfigValidate(); err != nil {
+		return nil, err
+	}
+	if len(c.AddrList) == 0 {
+		return nil, ErrAddrListNil
+	}
+
+	if c.PoolSize > len(c.AddrList) {
+		return nil, ErrPoolSize
+	}
+
+	rand.Seed(time.Now().Unix())
+	cp = new(connPool)
+	cp.c = c
+	cp.validBufConnChan = make(chan *LancerBufConn, cp.c.PoolSize)
+	cp.invalidBufConnChan = make(chan *LancerBufConn, cp.c.PoolSize)
+	cp.invalidUpstreams = make(map[string]interface{})
+	cp.connCounter = make(map[string]int)
+	cp.initPool()
+	go cp.maintainUpstream()
+	go cp.flushBufConn()
+	go cp.maintainBufConnPool()
+	expvar.Publish("conn_pool"+cp.c.Name, expvar.Func(cp.connPoolStatus))
+	return cp, nil
+}
+
+// connableUpstreams 返回可以建立连接的upstream列表
+func (cp *connPool) connableUpstreams() ([]string) {
+	list := make([]string, 0)
+	cp.invalidUpstreamsLock.RLock()
+	defer cp.invalidUpstreamsLock.RUnlock()
+	for _, addr := range cp.c.AddrList {
+		if _, ok := cp.invalidUpstreams[addr]; !ok {
+			if count, ok := cp.connCounter[addr]; ok && count == 0 {
+				list = append(list, addr)
+			}
+		}
+	}
+	return list
+}
+
+// write write []byte to BufConn
+func (bc *LancerBufConn) write(p []byte) (int, error) {
+	bc.conn.SetWriteDeadline(time.Now().Add(time.Second * 5))
+	return bc.wr.Write(p)
+}
+
+// randomOneUpstream 随机返回一个可以建立连接的upstream
+func (cp *connPool) randomOneUpstream() (s string, err error) {
+	list := cp.connableUpstreams()
+	if len(list) == 0 {
+		err = errors.New("No valid upstreams")
+		return
+	}
+	return list[rand.Intn(len(list))], nil
+}
+
+// initPool 初始化poolSize个数的bufConn
+func (cp *connPool) initPool() {
+	for _, addr := range cp.c.AddrList {
+		cp.connCounter[addr] = 0
+	}
+	for i := 0; i < cp.c.PoolSize; i++ {
+		if bufConn, err := cp.newBufConn(); err == nil {
+			cp.putBufConn(bufConn)
+		}
+	}
+}
+
+// novalidUpstream check if there is no validUpstream
+func (cp *connPool) novalidUpstream() bool {
+	return len(cp.invalidUpstreams) == len(cp.c.AddrList)
+}
+
+//GetConn 从pool中取一个BufConn
+func (cp *connPool) getBufConn() (*LancerBufConn, error) {
+	for {
+		select {
+		case bufConn := <-cp.validBufConnChan:
+			if !bufConn.enabled {
+				cp.putInvalidBufConn(bufConn)
+				continue
+			}
+			return bufConn, nil
+		case <-time.After(10 * time.Second):
+			log.Warn("timeout when get conn from conn pool")
+			continue
+		}
+	}
+}
+
+// setConn 为bufConn绑定一个新的Conn
+func (cp *connPool) setConn(bufConn *LancerBufConn) (error) {
+	if bufConn.conn != nil {
+		if bufConn.enabled == false {
+			cp.markUpstreamInvalid(bufConn.conn.RemoteAddr().String())
+		}
+		cp.connCounterDel(bufConn.conn.RemoteAddr().String())
+		bufConn.conn.Close()
+		bufConn.conn = nil
+		bufConn.enabled = false
+	}
+	if conn, err := cp.newConn(); err == nil {
+		bufConn.conn = conn
+		bufConn.wr.Reset(conn)
+		bufConn.ctime = time.Now()
+		bufConn.enabled = true
+		return nil
+	} else {
+		bufConn.enabled = false
+		return err
+	}
+}
+
+//putBufConn 把BufConn放回到pool中
+func (cp *connPool) putBufConn(bufConn *LancerBufConn) {
+	if bufConn.enabled == false {
+		cp.putInvalidBufConn(bufConn)
+		return
+	}
+	if bufConn.ctime.Add(time.Duration(cp.c.IdleTimeout)).Before(time.Now()) {
+		bufConn.wr.Flush()
+		cp.putInvalidBufConn(bufConn)
+		return
+	}
+	cp.putValidBufConn(bufConn)
+}
+
+// putValidBufConn 把 bufConn放到可用的pool中
+func (cp *connPool) putValidBufConn(bufConn *LancerBufConn) {
+	select {
+	case cp.validBufConnChan <- bufConn:
+		return
+	default:
+		log.Warn("BufConnChan full, discard，this shouldn't happen")
+		return
+	}
+}
+
+// putInvalidBufConn 把bufConn放到不可用的pool中
+func (cp *connPool) putInvalidBufConn(bufConn *LancerBufConn) {
+	select {
+	case cp.invalidBufConnChan <- bufConn:
+		return
+	default:
+		log.Warn("invalidBufConnChan full, discard，this shouldn't happen")
+		return
+	}
+}
+
+// maintainBufConnPool 维护BufConnPool状态
+func (cp *connPool) maintainBufConnPool() {
+	for {
+		select {
+		case bufConn := <-cp.invalidBufConnChan:
+			cp.setConn(bufConn)
+			cp.putBufConn(bufConn)
+		}
+		time.Sleep(time.Second * 1)
+	}
+}
+
+//markConnInvalid会将链接关闭并且将相应upstreamserver设置为不可用
+func (cp *connPool) markUpstreamInvalid(addr string) (err error) {
+	log.Error("mark upstream %s invalid", addr)
+	cp.invalidUpstreamsLock.Lock()
+	cp.invalidUpstreams[addr] = nil
+	cp.invalidUpstreamsLock.Unlock()
+	return
+}
+
+// markUpstreamValid 将某一addr设置为不可用
+func (cp *connPool) markUpstreamValid(addr string) (err error) {
+	log.Info("%s is valid again", addr)
+	cp.invalidUpstreamsLock.Lock()
+	delete(cp.invalidUpstreams, addr)
+	cp.invalidUpstreamsLock.Unlock()
+	return
+}
+
+// connCounterAdd 连接数+1
+func (cp *connPool) connCounterAdd(addr string) {
+	cp.connCounterLock.Lock()
+	defer cp.connCounterLock.Unlock()
+	if _, ok := cp.connCounter[addr]; ok {
+		cp.connCounter[addr] += 1
+	} else {
+		cp.connCounter[addr] = 1
+	}
+	return
+}
+
+//connCounterDel 连接数-1
+func (cp *connPool) connCounterDel(addr string) {
+	cp.connCounterLock.Lock()
+	defer cp.connCounterLock.Unlock()
+	if _, ok := cp.connCounter[addr]; ok {
+		cp.connCounter[addr] -= 1
+	}
+}
+
+// connPoolStatus 返回connPool状态
+func (cp *connPool) connPoolStatus() interface{} {
+	status := make(map[string]interface{})
+	status["conn_num"] = cp.connCounter
+	status["invalidUpstreams"] = cp.invalidUpstreams
+	return status
+}
+
+// maintainUpstream 维护upstream的健康状态
+func (cp *connPool) maintainUpstream() {
+	for {
+		cp.invalidUpstreamsLock.RLock()
+		tryAddrs := make([]string, 0, len(cp.invalidUpstreams))
+		for k := range cp.invalidUpstreams {
+			tryAddrs = append(tryAddrs, k)
+		}
+		cp.invalidUpstreamsLock.RUnlock()
+		for _, addr := range tryAddrs {
+			if conn, err := net.DialTimeout("tcp", addr, time.Duration(cp.c.DialTimeout)); err == nil && conn != nil {
+				conn.Close()
+				cp.markUpstreamValid(addr)
+			}
+		}
+		time.Sleep(time.Second * 10)
+	}
+}
+
+//ReleaseConnPool 释放连接池中所有链接
+func (cp *connPool) ReleaseConnPool() {
+	log.Info("Release Conn Pool")
+	close(cp.validBufConnChan)
+	close(cp.invalidBufConnChan)
+	for conn := range cp.validBufConnChan {
+		conn.enabled = false
+		conn.wr.Flush()
+		conn.conn.Close()
+	}
+}

app/service/ops/log-agent/output/output.go

@@ -0,0 +1,95 @@
+package output
+
+import (
+	"fmt"
+	"context"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+type configDecodeFunc = func(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error)
+
+type Output interface {
+	Run() (err error)
+	Stop()
+	InputChan() (chan *event.ProcessorEvent)
+}
+
+// Factory is used to register functions creating new Input instances.
+type Factory = func(ctx context.Context, config interface{}) (Output, error)
+
+var registry = make(map[string]Factory)
+
+var runningOutput = make(map[string]Output)
+
+func Register(name string, factory Factory) error {
+	log.Info("Registering output factory")
+	if name == "" {
+		return fmt.Errorf("Error registering output: name cannot be empty")
+	}
+	if factory == nil {
+		return fmt.Errorf("Error registering output '%v': factory cannot be empty", name)
+	}
+	if _, exists := registry[name]; exists {
+		return fmt.Errorf("Error registering output '%v': already registered", name)
+	}
+
+	registry[name] = factory
+	log.Info("Successfully registered output")
+
+	return nil
+}
+
+func OutputExists(name string) bool {
+	_, exists := registry[name]
+	return exists
+}
+
+func GetFactory(name string) (Factory, error) {
+	if _, exists := registry[name]; !exists {
+		return nil, fmt.Errorf("Error creating output. No such output type exist: '%v'", name)
+	}
+	return registry[name], nil
+}
+
+func GetOutputChan(name string) (chan *event.ProcessorEvent, error) {
+	if name == "" {
+		name = "lancer-ops-log"
+	}
+	if _, exists := runningOutput[name]; !exists {
+		return nil, fmt.Errorf("Error getting output chan. No such output chan exist: '%v'", name)
+	}
+	return runningOutput[name].InputChan(), nil
+}
+
+func OutputRunning(name string) bool {
+	_, exists := runningOutput[name]
+	return exists
+}
+
+func RegisterOutput(name string, o Output) (error) {
+	if name == "" {
+		return nil
+	}
+	if _, exists := runningOutput[name]; exists {
+		return fmt.Errorf("output %s already running", name)
+	}
+	runningOutput[name] = o
+	return nil
+}
+
+func ChanConnect(ctx context.Context, from <-chan *event.ProcessorEvent, to chan<- *event.ProcessorEvent) {
+	go func() {
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			case e := <-from:
+				to <- e
+			}
+		}
+	}()
+	return
+}

app/service/ops/log-agent/output/stdout/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "stdout.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/output/stdout",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/output:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/output/stdout/config.go

@@ -0,0 +1,27 @@
+package stdout
+
+import (
+	"errors"
+
+	"github.com/BurntSushi/toml"
+)
+
+type Config struct {
+	Name string `tome:"name"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of Stdout Output is nil")
+	}
+
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	c = new(Config)
+	if err = md.PrimitiveDecode(primValue, c); err != nil {
+		return nil, err
+	}
+	return c, nil
+}

app/service/ops/log-agent/output/stdout/stdout.go

@@ -0,0 +1,66 @@
+package stdout
+
+import (
+	"context"
+	"fmt"
+
+	"go-common/app/service/ops/log-agent/output"
+	"go-common/app/service/ops/log-agent/event"
+)
+
+type Stdout struct {
+	c      *Config
+	ctx    context.Context
+	cancel context.CancelFunc
+	i      chan *event.ProcessorEvent
+}
+
+func init() {
+	err := output.Register("stdout", NewStdout)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func NewStdout(ctx context.Context, config interface{}) (output.Output, error) {
+	var err error
+
+	stdout := new(Stdout)
+	if c, ok := config.(*Config); !ok {
+		return nil, fmt.Errorf("Error config for Lancer output")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		stdout.c = c
+	}
+
+	stdout.i = make(chan *event.ProcessorEvent)
+	stdout.ctx, stdout.cancel = context.WithCancel(ctx)
+	return stdout, nil
+}
+
+func (s *Stdout) Run() (err error) {
+	go func() {
+		for {
+			select {
+			case e := <-s.i:
+				fmt.Println(string(e.Body))
+			case <-s.ctx.Done():
+				return
+			}
+		}
+	}()
+	if s.c.Name != "" {
+		output.RegisterOutput(s.c.Name, s)
+	}
+	return nil
+}
+
+func (s *Stdout) Stop() {
+	s.cancel()
+}
+
+func (s *Stdout) InputChan() (chan *event.ProcessorEvent) {
+	return s.i
+}

app/service/ops/log-agent/pipeline/BUILD

@@ -0,0 +1,56 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "management.go",
+        "pipeline.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pipeline",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/input:go_default_library",
+        "//app/service/ops/log-agent/input/file:go_default_library",
+        "//app/service/ops/log-agent/input/sock:go_default_library",
+        "//app/service/ops/log-agent/output:go_default_library",
+        "//app/service/ops/log-agent/output/lancergrpc:go_default_library",
+        "//app/service/ops/log-agent/output/lancerlogstream:go_default_library",
+        "//app/service/ops/log-agent/output/stdout:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//app/service/ops/log-agent/processor/classify:go_default_library",
+        "//app/service/ops/log-agent/processor/fileLog:go_default_library",
+        "//app/service/ops/log-agent/processor/grok:go_default_library",
+        "//app/service/ops/log-agent/processor/httpstream:go_default_library",
+        "//app/service/ops/log-agent/processor/jsonLog:go_default_library",
+        "//app/service/ops/log-agent/processor/lengthCheck:go_default_library",
+        "//app/service/ops/log-agent/processor/sample:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/pipeline/dockerlogcollector:all-srcs",
+        "//app/service/ops/log-agent/pipeline/hostlogcollector:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pipeline/dockerlogcollector/BUILD

@@ -0,0 +1,38 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "file.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pipeline/dockerlogcollector",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/pipeline:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+        "//vendor/github.com/docker/docker/api/types:go_default_library",
+        "//vendor/github.com/docker/docker/client:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pipeline/dockerlogcollector/config.go

@@ -0,0 +1,39 @@
+package dockerlogcollector
+
+import (
+	"errors"
+	"time"
+
+	xtime "go-common/library/time"
+)
+
+type Config struct {
+	ConfigEnv    string        `toml:"configEnv"`
+	ConfigSuffix string        `toml:"configSuffix"`
+	MetaPath     string        `toml:"metaPath"`
+	ScanInterval xtime.Duration `toml:"scanInterval"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of docker log collector can't be nil")
+	}
+
+	if c.ConfigEnv == "" {
+		c.ConfigEnv = "LogCollectorConf"
+	}
+
+	if c.MetaPath == "" {
+		c.MetaPath = "/data/log-agent/meta"
+	}
+
+	if c.ConfigSuffix == "" {
+		c.ConfigSuffix = ".conf"
+	}
+
+	if c.ScanInterval == 0 {
+		c.ScanInterval = xtime.Duration(time.Second * 10)
+	}
+
+	return nil
+}

app/service/ops/log-agent/pipeline/dockerlogcollector/file.go

@@ -0,0 +1,116 @@
+package dockerlogcollector
+
+import (
+	"context"
+	"time"
+	"strings"
+	"path"
+	"io/ioutil"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pipeline"
+
+	"github.com/docker/docker/api/types"
+	"github.com/docker/docker/client"
+)
+
+type DockerLogCollector struct {
+	c      *Config
+	client *client.Client
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+type configItem struct {
+	configPath string
+	MergedDir  string
+}
+
+func InitDockerLogCollector(ctx context.Context, c *Config) (err error) {
+	if err = c.ConfigValidate(); err != nil {
+		return err
+	}
+	collector := new(DockerLogCollector)
+	collector.c = c
+
+	collector.ctx, collector.cancel = context.WithCancel(ctx)
+
+	// init docker client
+	collector.client, err = client.NewEnvClient()
+	if err != nil {
+		return err
+	}
+	go collector.scan()
+
+	return nil
+}
+
+func (collector *DockerLogCollector) getConfigs() ([]*configItem, error) {
+	var (
+		configItems = make([]*configItem, 0)
+		mergedDir   string
+		ok          bool
+	)
+	containers, err := collector.client.ContainerList(collector.ctx, types.ContainerListOptions{})
+	if err != nil {
+		return nil, err
+	}
+
+	for _, container := range containers {
+		info, err := collector.client.ContainerInspect(collector.ctx, container.ID)
+		if err != nil {
+			log.Error("failed to inspect container: %s", container.ID)
+			continue
+		}
+		// get overlay2 info
+		if info.GraphDriver.Name != "overlay2" {
+			log.Error("only overlay2 is supported")
+			continue
+		}
+
+		mergedDir, ok = info.GraphDriver.Data["MergedDir"]
+		if !ok {
+			log.Error("failed to get MergedDir of container:%s", container.ID)
+		}
+
+		for _, env := range info.Config.Env {
+			if strings.HasPrefix(env, collector.c.
+
+				ConfigEnv) {
+				for _, path := range strings.Split(strings.TrimPrefix(env, collector.c.ConfigEnv+"="), ",") {
+					configItems = append(configItems, &configItem{path, mergedDir})
+				}
+			}
+		}
+	}
+
+	return configItems, nil
+}
+
+func (collector *DockerLogCollector) scan() {
+	ticker := time.Tick(time.Duration(collector.c.ScanInterval))
+	for {
+		select {
+		case <-ticker:
+			configItems, err := collector.getConfigs()
+			if err != nil {
+				log.Error("failed to scan hostlogcollector config file list: %s", err)
+				continue
+			}
+			for _, item := range configItems {
+				configPath := path.Join(item.MergedDir, item.configPath)
+				config, err := ioutil.ReadFile(configPath)
+				if err != nil {
+					log.Error("filed to read hostlogcollector config file %s: %s", configPath, err)
+					continue
+				}
+				if !pipeline.PipelineManagement.PipelineExisted(configPath) {
+					ctx := context.WithValue(collector.ctx, "MergedDir", item.MergedDir)
+					go pipeline.PipelineManagement.StartPipeline(ctx, configPath, string(config))
+				}
+			}
+		case <-collector.ctx.Done():
+			return
+		}
+	}
+}

app/service/ops/log-agent/pipeline/hostlogcollector/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "file.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pipeline/hostlogcollector",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/pipeline:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pipeline/hostlogcollector/config.go

@@ -0,0 +1,39 @@
+package hostlogcollector
+
+import (
+	"errors"
+	"time"
+
+	xtime "go-common/library/time"
+)
+
+type Config struct {
+	HostConfigPath string        `toml:"hostConfigPath"`
+	ConfigSuffix   string        `toml:"configSuffix"`
+	MetaPath       string        `toml:"metaPath"`
+	ScanInterval   xtime.Duration `toml:"scanInterval"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of host log collector can't be nil")
+	}
+
+	if c.HostConfigPath == "" {
+		return errors.New("hostConfigPath of host log collector config can't be nil")
+	}
+
+	if c.MetaPath == "" {
+		c.MetaPath = "/data/log-agent/meta"
+	}
+
+	if c.ConfigSuffix == "" {
+		c.ConfigSuffix = ".conf"
+	}
+
+	if c.ScanInterval == 0 {
+		c.ScanInterval = xtime.Duration(time.Second * 10)
+	}
+
+	return nil
+}

app/service/ops/log-agent/pipeline/hostlogcollector/file.go

@@ -0,0 +1,92 @@
+package hostlogcollector
+
+import (
+	"os"
+	"io/ioutil"
+	"path"
+	"strings"
+	"fmt"
+	"time"
+	"context"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pipeline"
+)
+
+type HostLogCollector struct {
+	c      *Config
+	ctx    context.Context
+	cancel context.CancelFunc
+}
+
+func InitHostLogCollector(ctx context.Context, c *Config) (err error) {
+	if err = c.ConfigValidate(); err != nil {
+		return err
+	}
+	collector := new(HostLogCollector)
+	collector.c = c
+
+	collector.ctx, collector.cancel = context.WithCancel(ctx)
+
+	go collector.scan()
+
+	return nil
+}
+
+//
+func (collector *HostLogCollector) scan() {
+	ticker := time.Tick(time.Duration(collector.c.ScanInterval))
+	for {
+		select {
+		case <-ticker:
+			configPaths, err := collector.getConfigs()
+			if err != nil {
+				log.Error("failed to scan hostlogcollector config file list: %s", err)
+				continue
+			}
+			for _, configPath := range configPaths {
+				config, err := ioutil.ReadFile(configPath)
+				if err != nil {
+					log.Error("filed to read hostlogcollector config file %s: %s", configPath, err)
+					continue
+				}
+				if !pipeline.PipelineManagement.PipelineExisted(configPath) {
+					go pipeline.PipelineManagement.StartPipeline(collector.ctx, configPath, string(config))
+				}
+			}
+		case <-collector.ctx.Done():
+			return
+		}
+	}
+}
+
+// HostLogCollector get file collect configs under path
+func (collector *HostLogCollector) getConfigs() ([]string, error) {
+	var (
+		err         error
+		cinfos      []os.FileInfo
+		configFiles = make([]string, 0)
+	)
+
+	dinfo, err := os.Lstat(collector.c.HostConfigPath)
+
+	if err != nil {
+		return nil, fmt.Errorf("lstat(%s) failed: %s", collector.c.HostConfigPath, err)
+	}
+
+	if !dinfo.IsDir() {
+		return nil, fmt.Errorf("file collect config path must be dir")
+	}
+
+	if cinfos, err = ioutil.ReadDir(collector.c.HostConfigPath); err != nil {
+		return nil, fmt.Errorf("ioutil.ReadDir(%s) error(%v)", collector.c.HostConfigPath, err)
+	}
+
+	for _, cinfo := range cinfos {
+		name := path.Join(collector.c.HostConfigPath, cinfo.Name())
+		if !cinfo.IsDir() && strings.HasSuffix(name, collector.c.ConfigSuffix) {
+			configFiles = append(configFiles, name)
+		}
+	}
+	return configFiles, nil
+}

app/service/ops/log-agent/pipeline/management.go

@@ -0,0 +1,394 @@
+package pipeline
+
+import (
+	"sync"
+	"errors"
+	"context"
+	"sort"
+	"time"
+	"os"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/input"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/output"
+	"go-common/app/service/ops/log-agent/pkg/common"
+	"go-common/library/log"
+	"github.com/BurntSushi/toml"
+)
+
+type PipelineMng struct {
+	Pipelines     map[string]*Pipeline
+	PipelinesLock sync.RWMutex
+	ctx           context.Context
+	cancel        context.CancelFunc
+	scanInterval  time.Duration
+}
+
+var PipelineManagement *PipelineMng
+
+const defaultPipeline = "defaultPipeline"
+
+func InitPipelineMng(ctx context.Context) (err error) {
+	m := new(PipelineMng)
+	m.Pipelines = make(map[string]*Pipeline)
+	m.ctx, m.cancel = context.WithCancel(ctx)
+	if err = m.StartDefaultOutput(); err != nil {
+		return err
+	}
+
+	m.scanInterval = time.Second * 10
+
+	go m.scan()
+
+	m.StartDefaultPipeline()
+	// Todo check defaultPipeline
+	//if !m.PipelineExisted(defaultPipeline) {
+	//	return errors.New("failed to start defaultPipeline, see log for more details")
+	//}
+
+	PipelineManagement = m
+
+	return nil
+}
+
+func (m *PipelineMng) RegisterHostFileCollector(configPath string, p *Pipeline) {
+	m.PipelinesLock.Lock()
+	defer m.PipelinesLock.Unlock()
+	m.Pipelines[configPath] = p
+}
+
+func (m *PipelineMng) UnRegisterHostFileCollector(configPath string) {
+	m.PipelinesLock.Lock()
+	defer m.PipelinesLock.Unlock()
+	delete(m.Pipelines, configPath)
+}
+
+func (m *PipelineMng) PipelineExisted(configPath string) bool {
+	m.PipelinesLock.RLock()
+	defer m.PipelinesLock.RUnlock()
+	_, ok := m.Pipelines[configPath]
+	return ok
+}
+
+func (m *PipelineMng) GetPipeline(configPath string) *Pipeline {
+	m.PipelinesLock.RLock()
+	defer m.PipelinesLock.RUnlock()
+	if pipe, ok := m.Pipelines[configPath]; ok {
+		return pipe
+	}
+	return nil
+}
+
+// pipelines get configPath list of registered pipeline
+func (m *PipelineMng) configPaths() []string {
+	m.PipelinesLock.RLock()
+	defer m.PipelinesLock.RUnlock()
+	result := make([]string, 0, len(m.Pipelines))
+	for p, _ := range m.Pipelines {
+		result = append(result, p)
+	}
+	return result
+}
+
+func (m *PipelineMng) scan() {
+	ticker := time.Tick(m.scanInterval)
+	whiteList := make(map[string]struct{})
+	whiteList[defaultPipeline] = struct{}{}
+	for {
+		select {
+		case <-ticker:
+			for _, configPath := range m.configPaths() {
+				if _, ok := whiteList[configPath]; ok {
+					continue
+				}
+				pipe := m.GetPipeline(configPath)
+				// config removed
+				if _, err := os.Stat(configPath); os.IsNotExist(err) {
+					if pipe != nil {
+						log.Info("config file not exist any more, stop pipeline: %s", configPath)
+						pipe.Stop()
+						continue
+					}
+				}
+				// config updated
+				oldMd5 := pipe.configMd5
+				newMd5 := common.FileMd5(configPath)
+				if oldMd5 != newMd5 {
+					log.Info("config file updated, stop old pipeline: %s", configPath)
+					pipe.Stop()
+					continue
+				}
+			}
+		case <-m.ctx.Done():
+			return
+		}
+	}
+}
+
+func (m *PipelineMng) StartPipeline(ctx context.Context, configPath string, config string) () {
+	var err error
+	p := new(Pipeline)
+	p.configPath = configPath
+	p.configMd5 = common.FileMd5(configPath)
+
+	ctx = context.WithValue(ctx, "configPath", configPath)
+
+	p.ctx, p.cancel = context.WithCancel(ctx)
+
+	defer p.Stop()
+
+	var sortedOrder []string
+
+	p.c = new(Config)
+
+	md, err := toml.Decode(config, p.c)
+	if err != nil {
+		p.logError(err)
+		return
+	}
+	inputToProcessor := make(chan *event.ProcessorEvent)
+	// start input
+	inputName := p.c.Input.Name
+	if inputName == "" {
+		p.logError(errors.New("type of Config can't be nil"))
+		return
+	}
+
+	c, err := DecodeInputConfig(inputName, md, p.c.Input.Config)
+	if err != nil {
+		p.logError(err)
+		return
+	}
+
+	InputFactory, err := input.GetFactory(inputName)
+
+	if err != nil {
+		p.logError(err)
+		return
+	}
+
+	i, err := InputFactory(p.ctx, c, inputToProcessor)
+	if err != nil {
+		p.logError(err)
+		return
+	}
+
+	if err = i.Run(); err != nil {
+		p.logError(err)
+		return
+	}
+
+	// start processor
+	var ProcessorConnector chan *event.ProcessorEvent
+	ProcessorConnector = inputToProcessor
+
+	sortedOrder = make([]string, 0)
+	for order, _ := range p.c.Processor {
+		sortedOrder = append(sortedOrder, order)
+	}
+
+	sort.Strings(sortedOrder)
+	for _, order := range sortedOrder {
+		name := p.c.Processor[order].Name
+		if name == "" {
+			p.logError(errors.New("type of Processor can't be nil"))
+			return
+		}
+		c, err := DecodeProcessorConfig(name, md, p.c.Processor[order].Config)
+		if err != nil {
+			p.logError(err)
+			return
+		}
+
+		proc, err := processor.GetFactory(name)
+		if err != nil {
+			p.logError(err)
+			return
+		}
+
+		ProcessorConnector, err = proc(p.ctx, c, ProcessorConnector)
+		if err != nil {
+			p.logError(err)
+			return
+		}
+	}
+
+	// add classify and fileLog processor by default if inputName == "file"
+	if inputName == "file" {
+		config := `
+	[processor]
+	[processor.1]
+	type = "classify"
+	[processor.2]
+	type = "fileLog"
+	`
+		fProcessor := new(Config)
+		md, _ := toml.Decode(config, fProcessor)
+		fsortedOrder := make([]string, 0)
+		for order, _ := range fProcessor.Processor {
+			fsortedOrder = append(fsortedOrder, order)
+		}
+
+		sort.Strings(fsortedOrder)
+		for _, order := range fsortedOrder {
+			name := fProcessor.Processor[order].Name
+			if name == "" {
+				p.logError(errors.New("type of Processor can't be nil"))
+				return
+			}
+			fc, err := DecodeProcessorConfig(name, md, fProcessor.Processor[order].Config)
+			if err != nil {
+				p.logError(err)
+				return
+			}
+
+			proc, err := processor.GetFactory(name)
+			if err != nil {
+				p.logError(err)
+				return
+			}
+
+			ProcessorConnector, err = proc(p.ctx, fc, ProcessorConnector)
+			if err != nil {
+				p.logError(err)
+				return
+			}
+		}
+	}
+
+	// start output
+	if p.c.Output != nil {
+		if len(p.c.Output) > 1 {
+			p.logError(errors.New("only One Output is allowed in One pipeline"))
+			return
+		}
+		var first string
+		for key, _ := range p.c.Output {
+			first = key
+			break
+		}
+
+		o, err := StartOutput(p.ctx, md, p.c.Output[first])
+		if err != nil {
+			p.logError(err)
+			return
+		}
+		// connect processor and output
+		output.ChanConnect(m.ctx, ProcessorConnector, o.InputChan())
+
+	} else {
+		// write to default output
+		if err := processor.WriteToOutput(p.ctx, "", ProcessorConnector); err != nil {
+			p.logError(err)
+			return
+		}
+	}
+	m.RegisterHostFileCollector(configPath, p)
+
+	defer m.UnRegisterHostFileCollector(configPath)
+
+	<-p.ctx.Done()
+}
+
+func (m *PipelineMng) StartDefaultPipeline() {
+	//	config := `
+	//[input]
+	//type = "file"
+	//[input.config]
+	//paths = ["/data/log-agent/log/info.log.2018-11-07.001"]
+	//appId = "ops.billions.test"
+	//[processor]
+	//[output]
+	//[output.1]
+	//type = "stdout"
+	//`
+	//	config := `
+	//[input]
+	//type = "file"
+	//[input.config]
+	//paths = ["/data/log-agent/log/info.log.2018-*"]
+	//appId = "ops.billions.test"
+	//logId = "000069"
+	//[processor]
+	//[processor.1]
+	//type = "fileLog"
+	//`
+
+	config := `
+	[input]
+	type = "sock"
+	[input.config]
+
+	[processor]
+
+	[processor.1]
+	type = "jsonLog"
+
+	[processor.2]
+	type = "lengthCheck"
+
+	[processor.3]
+	type = "httpStream"
+
+	[processor.4]
+	type = "sample"
+
+	[processor.5]
+	type = "classify"
+	`
+	go m.StartPipeline(context.Background(), defaultPipeline, config)
+}
+
+func (m *PipelineMng) StartDefaultOutput() (err error) {
+	var value string
+	if value, err = output.ReadConfig(); err != nil {
+		return err
+	}
+	p := new(Pipeline)
+	p.c = new(Config)
+	md, err := toml.Decode(value, p.c)
+	if err != nil {
+		return err
+	}
+	return StartOutputs(m.ctx, md, p.c.Output)
+}
+
+func StartOutputs(ctx context.Context, md toml.MetaData, config map[string]ConfigItem) (err error) {
+	for _, item := range config {
+		name := item.Name
+		if name == "" {
+			return errors.New("type of Output can't be nil")
+		}
+
+		if _, err = StartOutput(ctx, md, item); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func StartOutput(ctx context.Context, md toml.MetaData, config ConfigItem) (o output.Output, err error) {
+	name := config.Name
+	if name == "" {
+		return nil, errors.New("type of Output can't be nil")
+	}
+	c, err := DecodeOutputConfig(name, md, config.Config)
+	if err != nil {
+		return nil, err
+	}
+	OutputFactory, err := output.GetFactory(name)
+	if err != nil {
+		return nil, err
+	}
+
+	o, err = OutputFactory(ctx, c)
+	if err != nil {
+		return nil, err
+	}
+
+	if err = o.Run(); err != nil {
+		return nil, err
+	}
+
+	return o, nil
+}

app/service/ops/log-agent/pipeline/pipeline.go

@@ -0,0 +1,128 @@
+package pipeline
+
+import (
+	"fmt"
+	"context"
+
+	"go-common/app/service/ops/log-agent/output/lancerlogstream"
+	"go-common/app/service/ops/log-agent/output/lancergrpc"
+	"go-common/app/service/ops/log-agent/input/sock"
+	"go-common/app/service/ops/log-agent/input/file"
+	"go-common/app/service/ops/log-agent/processor/classify"
+	"go-common/app/service/ops/log-agent/processor/jsonLog"
+	"go-common/app/service/ops/log-agent/processor/fileLog"
+	"go-common/app/service/ops/log-agent/processor/lengthCheck"
+	"go-common/app/service/ops/log-agent/processor/sample"
+	"go-common/app/service/ops/log-agent/processor/httpstream"
+	"go-common/app/service/ops/log-agent/processor/grok"
+	"go-common/app/service/ops/log-agent/output/stdout"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+var inputConfigDecodeFactory = make(map[string]configDecodeFunc)
+var processorConfigDecodeFactory = make(map[string]configDecodeFunc)
+var outputConfigDecodeFactory = make(map[string]configDecodeFunc)
+
+func init() {
+	RegisterInputConfigDecodeFunc("sock", sock.DecodeConfig)
+	RegisterInputConfigDecodeFunc("file", file.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("classify", classify.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("jsonLog", jsonLog.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("lengthCheck", lengthCheck.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("sample", sample.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("httpStream", httpstream.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("fileLog", fileLog.DecodeConfig)
+	RegisterProcessorConfigDecodeFunc("grok", grok.DecodeConfig)
+	RegisterOutputConfigDecodeFunc("stdout", stdout.DecodeConfig)
+	RegisterOutputConfigDecodeFunc("lancer", lancerlogstream.DecodeConfig)
+	RegisterOutputConfigDecodeFunc("lancergrpc", lancergrpc.DecodeConfig)
+}
+
+type Pipeline struct {
+	c          *Config
+	ctx        context.Context
+	cancel     context.CancelFunc
+	configPath string
+	configMd5  string
+}
+
+type Config struct {
+	Input     ConfigItem            `toml:"input"`
+	Processor map[string]ConfigItem `toml:"processor"`
+	Output    map[string]ConfigItem `toml:"output"`
+}
+
+type ConfigItem struct {
+	Name   string         `toml:"type"`
+	Config toml.Primitive `toml:"config"`
+}
+
+func (pipe *Pipeline) Stop() {
+	pipe.cancel()
+}
+
+type configDecodeFunc = func(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error)
+
+func RegisterInputConfigDecodeFunc(name string, f configDecodeFunc) {
+	inputConfigDecodeFactory[name] = f
+}
+
+func RegisterProcessorConfigDecodeFunc(name string, f configDecodeFunc) {
+	processorConfigDecodeFactory[name] = f
+}
+
+func GetInputConfigDecodeFunc(name string) (f configDecodeFunc, err error) {
+	if f, exist := inputConfigDecodeFactory[name]; exist {
+		return f, nil
+	}
+	return nil, fmt.Errorf("InputConfigDecodeFunc for %s not exist", name)
+}
+
+func GetProcessorConfigDecodeFunc(name string) (f configDecodeFunc, err error) {
+	if f, exist := processorConfigDecodeFactory[name]; exist {
+		return f, nil
+	}
+	return nil, fmt.Errorf("ProcessorConfigDecodeFunc for %s not exist", name)
+}
+
+func DecodeInputConfig(name string, md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	dFunc, err := GetInputConfigDecodeFunc(name)
+	if err != nil {
+		return nil, err
+	}
+	return dFunc(md, primValue)
+}
+
+func DecodeProcessorConfig(name string, md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	dFunc, err := GetProcessorConfigDecodeFunc(name)
+	if err != nil {
+		return nil, err
+	}
+	return dFunc(md, primValue)
+}
+
+func RegisterOutputConfigDecodeFunc(name string, f configDecodeFunc) {
+	outputConfigDecodeFactory[name] = f
+}
+
+func GetOutputConfigDecodeFunc(name string) (f configDecodeFunc, err error) {
+	if f, exist := outputConfigDecodeFactory[name]; exist {
+		return f, nil
+	}
+	return nil, fmt.Errorf("OutputConfigDecodeFunc for %s not exist", name)
+}
+
+func DecodeOutputConfig(name string, md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	dFunc, err := GetOutputConfigDecodeFunc(name)
+	if err != nil {
+		return nil, err
+	}
+	return dFunc(md, primValue)
+}
+
+func (p *Pipeline) logError(err error) {
+	configPath := p.ctx.Value("configPath")
+	log.Error("failed to run pipeline for %s: %s", configPath, err)
+}

app/service/ops/log-agent/pkg/bufio/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["bufio.go"],
+    importpath = "go-common/app/service/ops/log-agent/pkg/bufio",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/bufio/bufio.go

@@ -0,0 +1,747 @@
+// Copyright 2009 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Package bufio implements buffered I/O. It wraps an io.Reader or io.Writer
+// object, creating another object (Reader or Writer) that also implements
+// the interface but provides buffering and some help for textual I/O.
+package bufio
+
+import (
+	"bytes"
+	"errors"
+	"io"
+	"unicode/utf8"
+)
+
+const (
+	defaultBufSize = 4096
+)
+
+var (
+	ErrInvalidUnreadByte = errors.New("bufio: invalid use of UnreadByte")
+	ErrInvalidUnreadRune = errors.New("bufio: invalid use of UnreadRune")
+	ErrBufferFull        = errors.New("bufio: buffer full")
+	ErrNegativeCount     = errors.New("bufio: negative count")
+)
+
+// Buffered input.
+
+// Reader implements buffering for an io.Reader object.
+type Reader struct {
+	buf          []byte
+	rd           io.Reader // reader provided by the client
+	r, w         int       // buf read and write positions
+	err          error
+	lastByte     int
+	lastRuneSize int
+}
+
+const minReadBufferSize = 16
+const maxConsecutiveEmptyReads = 100
+
+// NewReaderSize returns a new Reader whose buffer has at least the specified
+// size. If the argument io.Reader is already a Reader with large enough
+// size, it returns the underlying Reader.
+func NewReaderSize(rd io.Reader, size int) *Reader {
+	// Is it already a Reader?
+	b, ok := rd.(*Reader)
+	if ok && len(b.buf) >= size {
+		return b
+	}
+	if size < minReadBufferSize {
+		size = minReadBufferSize
+	}
+	r := new(Reader)
+	r.reset(make([]byte, size), rd)
+	return r
+}
+
+// NewReader returns a new Reader whose buffer has the default size.
+func NewReader(rd io.Reader) *Reader {
+	return NewReaderSize(rd, defaultBufSize)
+}
+
+// Size returns the size of the underlying buffer in bytes.
+func (r *Reader) Size() int { return len(r.buf) }
+
+// Reset discards any buffered data, resets all state, and switches
+// the buffered reader to read from r.
+func (b *Reader) Reset(r io.Reader) {
+	b.reset(b.buf, r)
+}
+
+func (b *Reader) reset(buf []byte, r io.Reader) {
+	*b = Reader{
+		buf:          buf,
+		rd:           r,
+		lastByte:     -1,
+		lastRuneSize: -1,
+	}
+}
+
+var errNegativeRead = errors.New("bufio: reader returned negative count from Read")
+
+// fill reads a new chunk into the buffer.
+func (b *Reader) fill() {
+	// Slide existing data to beginning.
+	if b.r > 0 {
+		copy(b.buf, b.buf[b.r:b.w])
+		b.w -= b.r
+		b.r = 0
+	}
+
+	if b.w >= len(b.buf) {
+		panic("bufio: tried to fill full buffer")
+	}
+
+	// Read new data: try a limited number of times.
+	for i := maxConsecutiveEmptyReads; i > 0; i-- {
+		n, err := b.rd.Read(b.buf[b.w:])
+		if n < 0 {
+			panic(errNegativeRead)
+		}
+		b.w += n
+		if err != nil {
+			b.err = err
+			return
+		}
+		if n > 0 {
+			return
+		}
+	}
+	b.err = io.ErrNoProgress
+}
+
+func (b *Reader) readErr() error {
+	err := b.err
+	b.err = nil
+	return err
+}
+
+// Peek returns the next n bytes without advancing the reader. The bytes stop
+// being valid at the next read call. If Peek returns fewer than n bytes, it
+// also returns an error explaining why the read is short. The error is
+// ErrBufferFull if n is larger than b's buffer size.
+func (b *Reader) Peek(n int) ([]byte, error) {
+	if n < 0 {
+		return nil, ErrNegativeCount
+	}
+
+	for b.w-b.r < n && b.w-b.r < len(b.buf) && b.err == nil {
+		b.fill() // b.w-b.r < len(b.buf) => buffer is not full
+	}
+
+	if n > len(b.buf) {
+		return b.buf[b.r:b.w], ErrBufferFull
+	}
+
+	// 0 <= n <= len(b.buf)
+	var err error
+	if avail := b.w - b.r; avail < n {
+		// not enough data in buffer
+		n = avail
+		err = b.readErr()
+		if err == nil {
+			err = ErrBufferFull
+		}
+	}
+	return b.buf[b.r: b.r+n], err
+}
+
+// Discard skips the next n bytes, returning the number of bytes discarded.
+//
+// If Discard skips fewer than n bytes, it also returns an error.
+// If 0 <= n <= b.Buffered(), Discard is guaranteed to succeed without
+// reading from the underlying io.Reader.
+func (b *Reader) Discard(n int) (discarded int, err error) {
+	if n < 0 {
+		return 0, ErrNegativeCount
+	}
+	if n == 0 {
+		return
+	}
+	remain := n
+	for {
+		skip := b.Buffered()
+		if skip == 0 {
+			b.fill()
+			skip = b.Buffered()
+		}
+		if skip > remain {
+			skip = remain
+		}
+		b.r += skip
+		remain -= skip
+		if remain == 0 {
+			return n, nil
+		}
+		if b.err != nil {
+			return n - remain, b.readErr()
+		}
+	}
+}
+
+// Read reads data into p.
+// It returns the number of bytes read into p.
+// The bytes are taken from at most one Read on the underlying Reader,
+// hence n may be less than len(p).
+// At EOF, the count will be zero and err will be io.EOF.
+func (b *Reader) Read(p []byte) (n int, err error) {
+	n = len(p)
+	if n == 0 {
+		return 0, b.readErr()
+	}
+	if b.r == b.w {
+		if b.err != nil {
+			return 0, b.readErr()
+		}
+		if len(p) >= len(b.buf) {
+			// Large read, empty buffer.
+			// Read directly into p to avoid copy.
+			n, b.err = b.rd.Read(p)
+			if n < 0 {
+				panic(errNegativeRead)
+			}
+			if n > 0 {
+				b.lastByte = int(p[n-1])
+				b.lastRuneSize = -1
+			}
+			return n, b.readErr()
+		}
+		// One read.
+		// Do not use b.fill, which will loop.
+		b.r = 0
+		b.w = 0
+		n, b.err = b.rd.Read(b.buf)
+		if n < 0 {
+			panic(errNegativeRead)
+		}
+		if n == 0 {
+			return 0, b.readErr()
+		}
+		b.w += n
+	}
+
+	// copy as much as we can
+	n = copy(p, b.buf[b.r:b.w])
+	b.r += n
+	b.lastByte = int(b.buf[b.r-1])
+	b.lastRuneSize = -1
+	return n, nil
+}
+
+// ReadByte reads and returns a single byte.
+// If no byte is available, returns an error.
+func (b *Reader) ReadByte() (byte, error) {
+	b.lastRuneSize = -1
+	for b.r == b.w {
+		if b.err != nil {
+			return 0, b.readErr()
+		}
+		b.fill() // buffer is empty
+	}
+	c := b.buf[b.r]
+	b.r++
+	b.lastByte = int(c)
+	return c, nil
+}
+
+// UnreadByte unreads the last byte. Only the most recently read byte can be unread.
+func (b *Reader) UnreadByte() error {
+	if b.lastByte < 0 || b.r == 0 && b.w > 0 {
+		return ErrInvalidUnreadByte
+	}
+	// b.r > 0 || b.w == 0
+	if b.r > 0 {
+		b.r--
+	} else {
+		// b.r == 0 && b.w == 0
+		b.w = 1
+	}
+	b.buf[b.r] = byte(b.lastByte)
+	b.lastByte = -1
+	b.lastRuneSize = -1
+	return nil
+}
+
+// ReadRune reads a single UTF-8 encoded Unicode character and returns the
+// rune and its size in bytes. If the encoded rune is invalid, it consumes one byte
+// and returns unicode.ReplacementChar (U+FFFD) with a size of 1.
+func (b *Reader) ReadRune() (r rune, size int, err error) {
+	for b.r+utf8.UTFMax > b.w && !utf8.FullRune(b.buf[b.r:b.w]) && b.err == nil && b.w-b.r < len(b.buf) {
+		b.fill() // b.w-b.r < len(buf) => buffer is not full
+	}
+	b.lastRuneSize = -1
+	if b.r == b.w {
+		return 0, 0, b.readErr()
+	}
+	r, size = rune(b.buf[b.r]), 1
+	if r >= utf8.RuneSelf {
+		r, size = utf8.DecodeRune(b.buf[b.r:b.w])
+	}
+	b.r += size
+	b.lastByte = int(b.buf[b.r-1])
+	b.lastRuneSize = size
+	return r, size, nil
+}
+
+// UnreadRune unreads the last rune. If the most recent read operation on
+// the buffer was not a ReadRune, UnreadRune returns an error.  (In this
+// regard it is stricter than UnreadByte, which will unread the last byte
+// from any read operation.)
+func (b *Reader) UnreadRune() error {
+	if b.lastRuneSize < 0 || b.r < b.lastRuneSize {
+		return ErrInvalidUnreadRune
+	}
+	b.r -= b.lastRuneSize
+	b.lastByte = -1
+	b.lastRuneSize = -1
+	return nil
+}
+
+// Buffered returns the number of bytes that can be read from the current buffer.
+func (b *Reader) Buffered() int { return b.w - b.r }
+
+// ReadSlice reads until the first occurrence of delim in the input,
+// returning a slice pointing at the bytes in the buffer.
+// The bytes stop being valid at the next read.
+// If ReadSlice encounters an error before finding a delimiter,
+// it returns all the data in the buffer and the error itself (often io.EOF).
+// ReadSlice fails with error ErrBufferFull if the buffer fills without a delim.
+// Because the data returned from ReadSlice will be overwritten
+// by the next I/O operation, most clients should use
+// ReadBytes or ReadString instead.
+// ReadSlice returns err != nil if and only if line does not end in delim.
+func (b *Reader) ReadSlice(delim byte) (line []byte, err error) {
+	for {
+		// Search buffer.
+		if i := bytes.IndexByte(b.buf[b.r:b.w], delim); i >= 0 {
+			line = b.buf[b.r: b.r+i+1]
+			b.r += i + 1
+			break
+		}
+
+		// Pending error?
+		if b.err != nil {
+			line = b.buf[b.r:b.w]
+			b.r = b.w
+			err = b.readErr()
+			break
+		}
+
+		// Buffer full?
+		if b.Buffered() >= len(b.buf) {
+			b.r = b.w
+			line = b.buf
+			err = ErrBufferFull
+			break
+		}
+
+		b.fill() // buffer is not full
+	}
+
+	// Handle last byte, if any.
+	if i := len(line) - 1; i >= 0 {
+		b.lastByte = int(line[i])
+		b.lastRuneSize = -1
+	}
+
+	return
+}
+
+// ReadLine is a low-level line-reading primitive. Most callers should use
+// ReadBytes('\n') or ReadString('\n') instead or use a Scanner.
+//
+// ReadLine tries to return a single line, not including the end-of-line bytes.
+// If the line was too long for the buffer then isPrefix is set and the
+// beginning of the line is returned. The rest of the line will be returned
+// from future calls. isPrefix will be false when returning the last fragment
+// of the line. The returned buffer is only valid until the next call to
+// ReadLine. ReadLine either returns a non-nil line or it returns an error,
+// never both.
+//
+// The text returned from ReadLine does not include the line end ("\r\n" or "\n").
+// No indication or error is given if the input ends without a final line end.
+// Calling UnreadByte after ReadLine will always unread the last byte read
+// (possibly a character belonging to the line end) even if that byte is not
+// part of the line returned by ReadLine.
+func (b *Reader) ReadLine() (line []byte, isPrefix bool, err error) {
+	line, err = b.ReadSlice('\n')
+	if err == ErrBufferFull {
+		// Handle the case where "\r\n" straddles the buffer.
+		if len(line) > 0 && line[len(line)-1] == '\r' {
+			// Put the '\r' back on buf and drop it from line.
+			// Let the next call to ReadLine check for "\r\n".
+			if b.r == 0 {
+				// should be unreachable
+				panic("bufio: tried to rewind past start of buffer")
+			}
+			b.r--
+			line = line[:len(line)-1]
+		}
+		return line, true, nil
+	}
+
+	if len(line) == 0 {
+		if err != nil {
+			line = nil
+		}
+		return
+	}
+	err = nil
+
+	if line[len(line)-1] == '\n' {
+		drop := 1
+		if len(line) > 1 && line[len(line)-2] == '\r' {
+			drop = 2
+		}
+		line = line[:len(line)-drop]
+	}
+	return
+}
+
+// ReadBytes reads until the first occurrence of delim in the input,
+// returning a slice containing the data up to and including the delimiter.
+// If ReadBytes encounters an error before finding a delimiter,
+// it returns the data read before the error and the error itself (often io.EOF).
+// ReadBytes returns err != nil if and only if the returned data does not end in
+// delim.
+// For simple uses, a Scanner may be more convenient.
+func (b *Reader) ReadBytes(delim byte) ([]byte, error) {
+	// Use ReadSlice to look for array,
+	// accumulating full buffers.
+	var frag []byte
+	var full [][]byte
+	var err error
+	for {
+		var e error
+		frag, e = b.ReadSlice(delim)
+		if e == nil { // got final fragment
+			break
+		}
+		if e != ErrBufferFull { // unexpected error
+			err = e
+			break
+		}
+
+		// Make a copy of the buffer.
+		buf := make([]byte, len(frag))
+		copy(buf, frag)
+		full = append(full, buf)
+	}
+
+	// Allocate new buffer to hold the full pieces and the fragment.
+	n := 0
+	for i := range full {
+		n += len(full[i])
+	}
+	n += len(frag)
+
+	// Copy full pieces and fragment in.
+	buf := make([]byte, n)
+	n = 0
+	for i := range full {
+		n += copy(buf[n:], full[i])
+	}
+	copy(buf[n:], frag)
+	return buf, err
+}
+
+// ReadString reads until the first occurrence of delim in the input,
+// returning a string containing the data up to and including the delimiter.
+// If ReadString encounters an error before finding a delimiter,
+// it returns the data read before the error and the error itself (often io.EOF).
+// ReadString returns err != nil if and only if the returned data does not end in
+// delim.
+// For simple uses, a Scanner may be more convenient.
+func (b *Reader) ReadString(delim byte) (string, error) {
+	bytes, err := b.ReadBytes(delim)
+	return string(bytes), err
+}
+
+// WriteTo implements io.WriterTo.
+// This may make multiple calls to the Read method of the underlying Reader.
+func (b *Reader) WriteTo(w io.Writer) (n int64, err error) {
+	n, err = b.writeBuf(w)
+	if err != nil {
+		return
+	}
+
+	if r, ok := b.rd.(io.WriterTo); ok {
+		m, err := r.WriteTo(w)
+		n += m
+		return n, err
+	}
+
+	if w, ok := w.(io.ReaderFrom); ok {
+		m, err := w.ReadFrom(b.rd)
+		n += m
+		return n, err
+	}
+
+	if b.w-b.r < len(b.buf) {
+		b.fill() // buffer not full
+	}
+
+	for b.r < b.w {
+		// b.r < b.w => buffer is not empty
+		m, err := b.writeBuf(w)
+		n += m
+		if err != nil {
+			return n, err
+		}
+		b.fill() // buffer is empty
+	}
+
+	if b.err == io.EOF {
+		b.err = nil
+	}
+
+	return n, b.readErr()
+}
+
+var errNegativeWrite = errors.New("bufio: writer returned negative count from Write")
+
+// writeBuf writes the Reader's buffer to the writer.
+func (b *Reader) writeBuf(w io.Writer) (int64, error) {
+	n, err := w.Write(b.buf[b.r:b.w])
+	if n < 0 {
+		panic(errNegativeWrite)
+	}
+	b.r += n
+	return int64(n), err
+}
+
+// buffered output
+
+// Writer implements buffering for an io.Writer object.
+// If an error occurs writing to a Writer, no more data will be
+// accepted and all subsequent writes, and Flush, will return the error.
+// After all data has been written, the client should call the
+// Flush method to guarantee all data has been forwarded to
+// the underlying io.Writer.
+type Writer struct {
+	err error
+	buf []byte
+	n   int
+	wr  io.Writer
+}
+
+// NewWriterSize returns a new Writer whose buffer has at least the specified
+// size. If the argument io.Writer is already a Writer with large enough
+// size, it returns the underlying Writer.
+func NewWriterSize(w io.Writer, size int) *Writer {
+	// Is it already a Writer?
+	b, ok := w.(*Writer)
+	if ok && len(b.buf) >= size {
+		return b
+	}
+	if size <= 0 {
+		size = defaultBufSize
+	}
+	return &Writer{
+		buf: make([]byte, size),
+		wr:  w,
+	}
+}
+
+// NewWriter returns a new Writer whose buffer has the default size.
+func NewWriter(w io.Writer) *Writer {
+	return NewWriterSize(w, defaultBufSize)
+}
+
+// Size returns the size of the underlying buffer in bytes.
+func (b *Writer) Size() int { return len(b.buf) }
+
+// Reset discards any unflushed buffered data, clears any error, and
+// resets b to write its output to w.
+//func (b *Writer) Reset(w io.Writer) {
+//	b.err = nil
+//	b.n = 0
+//	b.wr = w
+//}
+
+// Reset clears any error, keep any unflushed buffered data, resets b to write its output to w.
+func (b *Writer) Reset(w io.Writer) {
+	b.err = nil
+	b.wr = w
+}
+
+// Flush writes any buffered data to the underlying io.Writer.
+func (b *Writer) Flush() error {
+	if b.err != nil {
+		return b.err
+	}
+	if b.n == 0 {
+		return nil
+	}
+	n, err := b.wr.Write(b.buf[0:b.n])
+	if n < b.n && err == nil {
+		err = io.ErrShortWrite
+	}
+	if err != nil {
+		if n > 0 && n < b.n {
+			copy(b.buf[0:b.n-n], b.buf[n:b.n])
+		}
+		b.n -= n
+		b.err = err
+		return err
+	}
+	b.n = 0
+	return nil
+}
+
+// Available returns how many bytes are unused in the buffer.
+func (b *Writer) Available() int { return len(b.buf) - b.n }
+
+// Buffered returns the number of bytes that have been written into the current buffer.
+func (b *Writer) Buffered() int { return b.n }
+
+// Write writes the contents of p into the buffer.
+// It returns the number of bytes written.
+// If nn < len(p), it also returns an error explaining
+// why the write is short.
+func (b *Writer) Write(p []byte) (nn int, err error) {
+	for len(p) > b.Available() && b.err == nil {
+		var n int
+		if b.Buffered() == 0 {
+			// Large write, empty buffer.
+			// Write directly from p to avoid copy.
+			n, b.err = b.wr.Write(p)
+		} else {
+			n = copy(b.buf[b.n:], p)
+			b.n += n
+			b.Flush()
+		}
+		nn += n
+		p = p[n:]
+	}
+	if b.err != nil {
+		return nn, b.err
+	}
+	n := copy(b.buf[b.n:], p)
+	b.n += n
+	nn += n
+	return nn, nil
+}
+
+// WriteByte writes a single byte.
+func (b *Writer) WriteByte(c byte) error {
+	if b.err != nil {
+		return b.err
+	}
+	if b.Available() <= 0 && b.Flush() != nil {
+		return b.err
+	}
+	b.buf[b.n] = c
+	b.n++
+	return nil
+}
+
+// WriteRune writes a single Unicode code point, returning
+// the number of bytes written and any error.
+func (b *Writer) WriteRune(r rune) (size int, err error) {
+	if r < utf8.RuneSelf {
+		err = b.WriteByte(byte(r))
+		if err != nil {
+			return 0, err
+		}
+		return 1, nil
+	}
+	if b.err != nil {
+		return 0, b.err
+	}
+	n := b.Available()
+	if n < utf8.UTFMax {
+		if b.Flush(); b.err != nil {
+			return 0, b.err
+		}
+		n = b.Available()
+		if n < utf8.UTFMax {
+			// Can only happen if buffer is silly small.
+			return b.WriteString(string(r))
+		}
+	}
+	size = utf8.EncodeRune(b.buf[b.n:], r)
+	b.n += size
+	return size, nil
+}
+
+// WriteString writes a string.
+// It returns the number of bytes written.
+// If the count is less than len(s), it also returns an error explaining
+// why the write is short.
+func (b *Writer) WriteString(s string) (int, error) {
+	nn := 0
+	for len(s) > b.Available() && b.err == nil {
+		n := copy(b.buf[b.n:], s)
+		b.n += n
+		nn += n
+		s = s[n:]
+		b.Flush()
+	}
+	if b.err != nil {
+		return nn, b.err
+	}
+	n := copy(b.buf[b.n:], s)
+	b.n += n
+	nn += n
+	return nn, nil
+}
+
+// ReadFrom implements io.ReaderFrom.
+func (b *Writer) ReadFrom(r io.Reader) (n int64, err error) {
+	if b.Buffered() == 0 {
+		if w, ok := b.wr.(io.ReaderFrom); ok {
+			return w.ReadFrom(r)
+		}
+	}
+	var m int
+	for {
+		if b.Available() == 0 {
+			if err1 := b.Flush(); err1 != nil {
+				return n, err1
+			}
+		}
+		nr := 0
+		for nr < maxConsecutiveEmptyReads {
+			m, err = r.Read(b.buf[b.n:])
+			if m != 0 || err != nil {
+				break
+			}
+			nr++
+		}
+		if nr == maxConsecutiveEmptyReads {
+			return n, io.ErrNoProgress
+		}
+		b.n += m
+		n += int64(m)
+		if err != nil {
+			break
+		}
+	}
+	if err == io.EOF {
+		// If we filled the buffer exactly, flush preemptively.
+		if b.Available() == 0 {
+			err = b.Flush()
+		} else {
+			err = nil
+		}
+	}
+	return n, err
+}
+
+// buffered input and output
+
+// ReadWriter stores pointers to a Reader and a Writer.
+// It implements io.ReadWriter.
+type ReadWriter struct {
+	*Reader
+	*Writer
+}
+
+// NewReadWriter allocates a new ReadWriter that dispatches to r and w.
+func NewReadWriter(r *Reader, w *Writer) *ReadWriter {
+	return &ReadWriter{r, w}
+}

app/service/ops/log-agent/pkg/common/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["common.go"],
+    importpath = "go-common/app/service/ops/log-agent/pkg/common",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/common/common.go

@@ -0,0 +1,56 @@
+package common
+
+import (
+	"bytes"
+	"fmt"
+	"crypto/md5"
+	"io/ioutil"
+)
+
+// importantLog check if log level is above ERROR
+func CriticalLog(level []byte) bool {
+	if bytes.Equal(level, []byte("WARN")) || bytes.Equal(level, []byte("ERROR")) || bytes.Equal(level, []byte("FATAL")) {
+		return true
+	}
+	return false
+}
+
+// GetPriority get priority value from json body
+func GetPriority(logBody []byte) (value []byte, err error) {
+	return SeekValue([]byte(`"priority":`), logBody)
+}
+
+// seekValue seek value by key from json
+func SeekValue(key []byte, logBody []byte) (value []byte, err error) {
+	var (
+		b, logLen, begin, end int
+	)
+	b = bytes.Index(logBody, key)
+	if b != -1 {
+		logLen = len(logBody)
+		for begin = b + len(key); begin < logLen && logBody[begin] != byte('"'); begin++ {
+		}
+		if begin >= logLen {
+			err = fmt.Errorf("beginning of value not found by key: %s", string(key))
+			return
+		}
+		begin++ // begin position of value of appid
+		for end = begin; end < logLen && logBody[end] != byte('"'); end++ {
+		}
+		if end >= logLen {
+			err = fmt.Errorf("end of value not found by key: %s", string(key))
+			return
+		}
+		value = logBody[begin:end]
+		return
+	} else {
+		err = fmt.Errorf("key %s not found", string(key))
+		return
+	}
+}
+
+func FileMd5(filePath string) string {
+	data, _ := ioutil.ReadFile(filePath)
+	value := md5.Sum(data)
+	return fmt.Sprintf("%x", value)
+}

app/service/ops/log-agent/pkg/flowmonitor/BUILD

@@ -0,0 +1,40 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "flowmonitor.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pkg/flowmonitor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/flowmonitor/counter:go_default_library",
+        "//library/log:go_default_library",
+        "//library/time:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/pkg/flowmonitor/counter:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/flowmonitor/conf.go

@@ -0,0 +1,26 @@
+package flowmonitor
+
+import (
+	"time"
+	"errors"
+
+	xtime "go-common/library/time"
+)
+
+type Config struct {
+	Interval xtime.Duration
+	Addr     string
+}
+
+func (fm *FlowMonitor) checkConfig() (err error) {
+	if fm.conf == nil {
+		return errors.New("config for flowmonitor is nil")
+	}
+	if fm.conf.Interval == 0 {
+		fm.conf.Interval = xtime.Duration(time.Second * 5)
+	}
+	if fm.conf.Addr == "" {
+		return errors.New("addr of flowmonitor is nil")
+	}
+	return
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/BUILD

@@ -0,0 +1,49 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "collector.go",
+        "counter.go",
+        "desc.go",
+        "fnv.go",
+        "go_collector.go",
+        "metric.go",
+        "observer.go",
+        "process_collector.go",
+        "registry.go",
+        "summary.go",
+        "untyped.go",
+        "value.go",
+        "vec.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pkg/flowmonitor/counter",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/beorn7/perks/quantile:go_default_library",
+        "//vendor/github.com/prometheus/client_model/go:go_default_library",
+        "//vendor/github.com/prometheus/common/model:go_default_library",
+        "//vendor/github.com/prometheus/procfs:go_default_library",
+        "@com_github_golang_protobuf//proto:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/flowmonitor/counter/collector.go

@@ -0,0 +1,75 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+// Collector is the interface implemented by anything that can be used by
+// Prometheus to collect metrics. A Collector has to be registered for
+// collection. See Registerer.Register.
+//
+// The stock metrics provided by this package (Gauge, Counter, Summary,
+// Histogram, Untyped) are also Collectors (which only ever collect one metric,
+// namely itself). An implementer of Collector may, however, collect multiple
+// metrics in a coordinated fashion and/or create metrics on the fly. Examples
+// for collectors already implemented in this library are the metric vectors
+// (i.e. collection of multiple instances of the same Metric but with different
+// label values) like GaugeVec or SummaryVec, and the ExpvarCollector.
+type Collector interface {
+	// Describe sends the super-set of all possible descriptors of metrics
+	// collected by this Collector to the provided channel and returns once
+	// the last descriptor has been sent. The sent descriptors fulfill the
+	// consistency and uniqueness requirements described in the Desc
+	// documentation. (It is valid if one and the same Collector sends
+	// duplicate descriptors. Those duplicates are simply ignored. However,
+	// two different Collectors must not send duplicate descriptors.) This
+	// method idempotently sends the same descriptors throughout the
+	// lifetime of the Collector. If a Collector encounters an error while
+	// executing this method, it must send an invalid descriptor (created
+	// with NewInvalidDesc) to signal the error to the registry.
+	Describe(chan<- *Desc)
+	// Collect is called by the Prometheus registry when collecting
+	// metrics. The implementation sends each collected metric via the
+	// provided channel and returns once the last metric has been sent. The
+	// descriptor of each sent metric is one of those returned by
+	// Describe. Returned metrics that share the same descriptor must differ
+	// in their variable label values. This method may be called
+	// concurrently and must therefore be implemented in a concurrency safe
+	// way. Blocking occurs at the expense of total performance of rendering
+	// all registered metrics. Ideally, Collector implementations support
+	// concurrent readers.
+	Collect(chan<- Metric)
+}
+
+// selfCollector implements Collector for a single Metric so that the Metric
+// collects itself. Add it as an anonymous field to a struct that implements
+// Metric, and call init with the Metric itself as an argument.
+type selfCollector struct {
+	self Metric
+}
+
+// init provides the selfCollector with a reference to the metric it is supposed
+// to collect. It is usually called within the factory function to create a
+// metric. See example.
+func (c *selfCollector) init(self Metric) {
+	c.self = self
+}
+
+// Describe implements Collector.
+func (c *selfCollector) Describe(ch chan<- *Desc) {
+	ch <- c.self.Desc()
+}
+
+// Collect implements Collector.
+func (c *selfCollector) Collect(ch chan<- Metric) {
+	ch <- c.self
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/counter.go

@@ -0,0 +1,180 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"errors"
+	"sync/atomic"
+	"math"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+// Counter is a Metric that represents a single numerical value that only ever
+// goes up. That implies that it cannot be used to count items whose number can
+// also go down, e.g. the number of currently running goroutines. Those
+// "counters" are represented by Gauges.
+//
+// A Counter is typically used to count requests served, tasks completed, errors
+// occurred, etc.
+//
+// To create Counter instances, use NewCounter.
+type Counter interface {
+	Metric
+	Collector
+
+	// Inc increments the counter by 1. Use Add to increment it by arbitrary
+	// non-negative values.
+	Inc()
+	// Add adds the given value to the counter. It panics if the value is <
+	// 0.
+	Add(float64)
+	// Value return value
+	Value() float64
+	// Lables return Lables
+	Lables() []*dto.LabelPair
+}
+
+// CounterOpts is an alias for Opts. See there for doc comments.
+type CounterOpts Opts
+
+// NewCounter creates a new Counter based on the provided CounterOpts.
+func NewCounter(opts CounterOpts) Counter {
+	desc := NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		nil,
+		opts.ConstLabels,
+	)
+	result := &counter{value: value{desc: desc, valType: CounterValue, labelPairs: desc.constLabelPairs}}
+	result.init(result) // Init self-collection.
+	return result
+}
+
+type counter struct {
+	value
+}
+
+func (c *counter) Add(v float64) {
+	if v < 0 {
+		panic(errors.New("counter cannot decrease in value"))
+	}
+	c.value.Add(v)
+}
+
+func (c *counter) Value() float64 {
+	return math.Float64frombits(atomic.LoadUint64(&c.value.valBits))
+}
+
+func (c *counter) Lables() []*dto.LabelPair {
+	return c.labelPairs
+}
+
+// CounterVec is a Collector that bundles a set of Counters that all share the
+// same Desc, but have different values for their variable labels. This is used
+// if you want to count the same thing partitioned by various dimensions
+// (e.g. number of HTTP requests, partitioned by response code and
+// method). Create instances with NewCounterVec.
+//
+// CounterVec embeds MetricVec. See there for a full list of methods with
+// detailed documentation.
+type CounterVec struct {
+	*MetricVec
+}
+
+// NewCounterVec creates a new CounterVec based on the provided CounterOpts and
+// partitioned by the given label names. At least one label name must be
+// provided.
+func NewCounterVec(opts CounterOpts, labelNames []string) *CounterVec {
+	desc := NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		labelNames,
+		opts.ConstLabels,
+	)
+	return &CounterVec{
+		MetricVec: newMetricVec(desc, func(lvs ...string) Metric {
+			result := &counter{value: value{
+				desc:       desc,
+				valType:    CounterValue,
+				labelPairs: makeLabelPairs(desc, lvs),
+			}}
+			result.init(result) // Init self-collection.
+			return result
+		}),
+	}
+}
+
+// GetMetricWithLabelValues replaces the method of the same name in
+// MetricVec. The difference is that this method returns a Counter and not a
+// Metric so that no type conversion is required.
+func (m *CounterVec) GetMetricWithLabelValues(lvs ...string) (Counter, error) {
+	metric, err := m.MetricVec.GetMetricWithLabelValues(lvs...)
+	if metric != nil {
+		return metric.(Counter), err
+	}
+	return nil, err
+}
+
+// GetMetricWith replaces the method of the same name in MetricVec. The
+// difference is that this method returns a Counter and not a Metric so that no
+// type conversion is required.
+func (m *CounterVec) GetMetricWith(labels Labels) (Counter, error) {
+	metric, err := m.MetricVec.GetMetricWith(labels)
+	if metric != nil {
+		return metric.(Counter), err
+	}
+	return nil, err
+}
+
+// WithLabelValues works as GetMetricWithLabelValues, but panics where
+// GetMetricWithLabelValues would have returned an error. By not returning an
+// error, WithLabelValues allows shortcuts like
+//     myVec.WithLabelValues("404", "GET").Add(42)
+func (m *CounterVec) WithLabelValues(lvs ...string) Counter {
+	return m.MetricVec.WithLabelValues(lvs...).(Counter)
+}
+
+// With works as GetMetricWith, but panics where GetMetricWithLabels would have
+// returned an error. By not returning an error, With allows shortcuts like
+//     myVec.With(Labels{"code": "404", "method": "GET"}).Add(42)
+func (m *CounterVec) With(labels Labels) Counter {
+	return m.MetricVec.With(labels).(Counter)
+}
+
+// CounterFunc is a Counter whose value is determined at collect time by calling a
+// provided function.
+//
+// To create CounterFunc instances, use NewCounterFunc.
+type CounterFunc interface {
+	Metric
+	Collector
+}
+
+// NewCounterFunc creates a new CounterFunc based on the provided
+// CounterOpts. The value reported is determined by calling the given function
+// from within the Write method. Take into account that metric collection may
+// happen concurrently. If that results in concurrent calls to Write, like in
+// the case where a CounterFunc is directly registered with Prometheus, the
+// provided function must be concurrency-safe. The function should also honor
+// the contract for a Counter (values only go up, not down), but compliance will
+// not be checked.
+func NewCounterFunc(opts CounterOpts, function func() float64) CounterFunc {
+	return newValueFunc(NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		nil,
+		opts.ConstLabels,
+	), CounterValue, function)
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/desc.go

@@ -0,0 +1,200 @@
+// Copyright 2016 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"errors"
+	"fmt"
+	"sort"
+	"strings"
+
+	"github.com/golang/protobuf/proto"
+	"github.com/prometheus/common/model"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+// reservedLabelPrefix is a prefix which is not legal in user-supplied
+// label names.
+const reservedLabelPrefix = "__"
+
+// Labels represents a collection of label name -> value mappings. This type is
+// commonly used with the With(Labels) and GetMetricWith(Labels) methods of
+// metric vector Collectors, e.g.:
+//     myVec.With(Labels{"code": "404", "method": "GET"}).Add(42)
+//
+// The other use-case is the specification of constant label pairs in Opts or to
+// create a Desc.
+type Labels map[string]string
+
+// Desc is the descriptor used by every Prometheus Metric. It is essentially
+// the immutable meta-data of a Metric. The normal Metric implementations
+// included in this package manage their Desc under the hood. Users only have to
+// deal with Desc if they use advanced features like the ExpvarCollector or
+// custom Collectors and Metrics.
+//
+// Descriptors registered with the same registry have to fulfill certain
+// consistency and uniqueness criteria if they share the same fully-qualified
+// name: They must have the same help string and the same label names (aka label
+// dimensions) in each, constLabels and variableLabels, but they must differ in
+// the values of the constLabels.
+//
+// Descriptors that share the same fully-qualified names and the same label
+// values of their constLabels are considered equal.
+//
+// Use NewDesc to create new Desc instances.
+type Desc struct {
+	// fqName has been built from Namespace, Subsystem, and Name.
+	fqName string
+	// help provides some helpful information about this metric.
+	help string
+	// constLabelPairs contains precalculated DTO label pairs based on
+	// the constant labels.
+	constLabelPairs []*dto.LabelPair
+	// VariableLabels contains names of labels for which the metric
+	// maintains variable values.
+	variableLabels []string
+	// id is a hash of the values of the ConstLabels and fqName. This
+	// must be unique among all registered descriptors and can therefore be
+	// used as an identifier of the descriptor.
+	id uint64
+	// dimHash is a hash of the label names (preset and variable) and the
+	// Help string. Each Desc with the same fqName must have the same
+	// dimHash.
+	dimHash uint64
+	// err is an error that occurred during construction. It is reported on
+	// registration time.
+	err error
+}
+
+// NewDesc allocates and initializes a new Desc. Errors are recorded in the Desc
+// and will be reported on registration time. variableLabels and constLabels can
+// be nil if no such labels should be set. fqName and help must not be empty.
+//
+// variableLabels only contain the label names. Their label values are variable
+// and therefore not part of the Desc. (They are managed within the Metric.)
+//
+// For constLabels, the label values are constant. Therefore, they are fully
+// specified in the Desc. See the Opts documentation for the implications of
+// constant labels.
+func NewDesc(fqName, help string, variableLabels []string, constLabels Labels) *Desc {
+	d := &Desc{
+		fqName:         fqName,
+		help:           help,
+		variableLabels: variableLabels,
+	}
+	if help == "" {
+		d.err = errors.New("empty help string")
+		return d
+	}
+	if !model.IsValidMetricName(model.LabelValue(fqName)) {
+		d.err = fmt.Errorf("%q is not a valid metric name", fqName)
+		return d
+	}
+	// labelValues contains the label values of const labels (in order of
+	// their sorted label names) plus the fqName (at position 0).
+	labelValues := make([]string, 1, len(constLabels)+1)
+	labelValues[0] = fqName
+	labelNames := make([]string, 0, len(constLabels)+len(variableLabels))
+	labelNameSet := map[string]struct{}{}
+	// First add only the const label names and sort them...
+	for labelName := range constLabels {
+		if !checkLabelName(labelName) {
+			d.err = fmt.Errorf("%q is not a valid label name", labelName)
+			return d
+		}
+		labelNames = append(labelNames, labelName)
+		labelNameSet[labelName] = struct{}{}
+	}
+	sort.Strings(labelNames)
+	// ... so that we can now add const label values in the order of their names.
+	for _, labelName := range labelNames {
+		labelValues = append(labelValues, constLabels[labelName])
+	}
+	// Now add the variable label names, but prefix them with something that
+	// cannot be in a regular label name. That prevents matching the label
+	// dimension with a different mix between preset and variable labels.
+	for _, labelName := range variableLabels {
+		if !checkLabelName(labelName) {
+			d.err = fmt.Errorf("%q is not a valid label name", labelName)
+			return d
+		}
+		labelNames = append(labelNames, "$"+labelName)
+		labelNameSet[labelName] = struct{}{}
+	}
+	if len(labelNames) != len(labelNameSet) {
+		d.err = errors.New("duplicate label names")
+		return d
+	}
+	vh := hashNew()
+	for _, val := range labelValues {
+		vh = hashAdd(vh, val)
+		vh = hashAddByte(vh, separatorByte)
+	}
+	d.id = vh
+	// Sort labelNames so that order doesn't matter for the hash.
+	sort.Strings(labelNames)
+	// Now hash together (in this order) the help string and the sorted
+	// label names.
+	lh := hashNew()
+	lh = hashAdd(lh, help)
+	lh = hashAddByte(lh, separatorByte)
+	for _, labelName := range labelNames {
+		lh = hashAdd(lh, labelName)
+		lh = hashAddByte(lh, separatorByte)
+	}
+	d.dimHash = lh
+
+	d.constLabelPairs = make([]*dto.LabelPair, 0, len(constLabels))
+	for n, v := range constLabels {
+		d.constLabelPairs = append(d.constLabelPairs, &dto.LabelPair{
+			Name:  proto.String(n),
+			Value: proto.String(v),
+		})
+	}
+	sort.Sort(LabelPairSorter(d.constLabelPairs))
+	return d
+}
+
+// NewInvalidDesc returns an invalid descriptor, i.e. a descriptor with the
+// provided error set. If a collector returning such a descriptor is registered,
+// registration will fail with the provided error. NewInvalidDesc can be used by
+// a Collector to signal inability to describe itself.
+func NewInvalidDesc(err error) *Desc {
+	return &Desc{
+		err: err,
+	}
+}
+
+func (d *Desc) String() string {
+	lpStrings := make([]string, 0, len(d.constLabelPairs))
+	for _, lp := range d.constLabelPairs {
+		lpStrings = append(
+			lpStrings,
+			fmt.Sprintf("%s=%q", lp.GetName(), lp.GetValue()),
+		)
+	}
+	return fmt.Sprintf(
+		"Desc{fqName: %q, help: %q, constLabels: {%s}, variableLabels: %v}",
+		d.fqName,
+		d.help,
+		strings.Join(lpStrings, ","),
+		d.variableLabels,
+	)
+}
+
+func checkLabelName(l string) bool {
+	return model.LabelName(l).IsValid() &&
+		!strings.HasPrefix(l, reservedLabelPrefix)
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/fnv.go

@@ -0,0 +1,29 @@
+package prometheus
+
+// Inline and byte-free variant of hash/fnv's fnv64a.
+
+const (
+	offset64 = 14695981039346656037
+	prime64  = 1099511628211
+)
+
+// hashNew initializies a new fnv64a hash value.
+func hashNew() uint64 {
+	return offset64
+}
+
+// hashAdd adds a string to a fnv64a hash value, returning the updated hash.
+func hashAdd(h uint64, s string) uint64 {
+	for i := 0; i < len(s); i++ {
+		h ^= uint64(s[i])
+		h *= prime64
+	}
+	return h
+}
+
+// hashAddByte adds a byte to a fnv64a hash value, returning the updated hash.
+func hashAddByte(h uint64, b byte) uint64 {
+	h ^= uint64(b)
+	h *= prime64
+	return h
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/go_collector.go

@@ -0,0 +1,276 @@
+package prometheus
+
+import (
+	"fmt"
+	"runtime"
+	"runtime/debug"
+	"time"
+)
+
+type goCollector struct {
+	goroutinesDesc *Desc
+	threadsDesc    *Desc
+	gcDesc         *Desc
+
+	// metrics to describe and collect
+	metrics memStatsMetrics
+}
+
+// NewGoCollector returns a collector which exports metrics about the current
+// go process.
+func NewGoCollector() Collector {
+	return &goCollector{
+		goroutinesDesc: NewDesc(
+			"go_goroutines",
+			"Number of goroutines that currently exist.",
+			nil, nil),
+		threadsDesc: NewDesc(
+			"go_threads",
+			"Number of OS threads created",
+			nil, nil),
+		gcDesc: NewDesc(
+			"go_gc_duration_seconds",
+			"A summary of the GC invocation durations.",
+			nil, nil),
+		metrics: memStatsMetrics{
+			{
+				desc: NewDesc(
+					memstatNamespace("alloc_bytes"),
+					"Number of bytes allocated and still in use.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Alloc) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("alloc_bytes_total"),
+					"Total number of bytes allocated, even if freed.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.TotalAlloc) },
+				valType: CounterValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("sys_bytes"),
+					"Number of bytes obtained from system.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Sys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("lookups_total"),
+					"Total number of pointer lookups.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Lookups) },
+				valType: CounterValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("mallocs_total"),
+					"Total number of mallocs.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Mallocs) },
+				valType: CounterValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("frees_total"),
+					"Total number of frees.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.Frees) },
+				valType: CounterValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("heap_alloc_bytes"),
+					"Number of heap bytes allocated and still in use.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapAlloc) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("heap_sys_bytes"),
+					"Number of heap bytes obtained from system.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("heap_idle_bytes"),
+					"Number of heap bytes waiting to be used.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapIdle) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("heap_inuse_bytes"),
+					"Number of heap bytes that are in use.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapInuse) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("heap_released_bytes"),
+					"Number of heap bytes released to OS.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapReleased) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("heap_objects"),
+					"Number of allocated objects.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.HeapObjects) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("stack_inuse_bytes"),
+					"Number of bytes in use by the stack allocator.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackInuse) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("stack_sys_bytes"),
+					"Number of bytes obtained from system for stack allocator.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.StackSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("mspan_inuse_bytes"),
+					"Number of bytes in use by mspan structures.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanInuse) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("mspan_sys_bytes"),
+					"Number of bytes used for mspan structures obtained from system.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MSpanSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("mcache_inuse_bytes"),
+					"Number of bytes in use by mcache structures.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheInuse) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("mcache_sys_bytes"),
+					"Number of bytes used for mcache structures obtained from system.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.MCacheSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("buck_hash_sys_bytes"),
+					"Number of bytes used by the profiling bucket hash table.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.BuckHashSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("gc_sys_bytes"),
+					"Number of bytes used for garbage collection system metadata.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.GCSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("other_sys_bytes"),
+					"Number of bytes used for other system allocations.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.OtherSys) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("next_gc_bytes"),
+					"Number of heap bytes when next garbage collection will take place.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.NextGC) },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("last_gc_time_seconds"),
+					"Number of seconds since 1970 of last garbage collection.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return float64(ms.LastGC) / 1e9 },
+				valType: GaugeValue,
+			}, {
+				desc: NewDesc(
+					memstatNamespace("gc_cpu_fraction"),
+					"The fraction of this program's available CPU time used by the GC since the program started.",
+					nil, nil,
+				),
+				eval:    func(ms *runtime.MemStats) float64 { return ms.GCCPUFraction },
+				valType: GaugeValue,
+			},
+		},
+	}
+}
+
+func memstatNamespace(s string) string {
+	return fmt.Sprintf("go_memstats_%s", s)
+}
+
+// Describe returns all descriptions of the collector.
+func (c *goCollector) Describe(ch chan<- *Desc) {
+	ch <- c.goroutinesDesc
+	ch <- c.threadsDesc
+	ch <- c.gcDesc
+	for _, i := range c.metrics {
+		ch <- i.desc
+	}
+}
+
+// Collect returns the current state of all metrics of the collector.
+func (c *goCollector) Collect(ch chan<- Metric) {
+	ch <- MustNewConstMetric(c.goroutinesDesc, GaugeValue, float64(runtime.NumGoroutine()))
+	n, _ := runtime.ThreadCreateProfile(nil)
+	ch <- MustNewConstMetric(c.threadsDesc, GaugeValue, float64(n))
+
+	var stats debug.GCStats
+	stats.PauseQuantiles = make([]time.Duration, 5)
+	debug.ReadGCStats(&stats)
+
+	quantiles := make(map[float64]float64)
+	for idx, pq := range stats.PauseQuantiles[1:] {
+		quantiles[float64(idx+1)/float64(len(stats.PauseQuantiles)-1)] = pq.Seconds()
+	}
+	quantiles[0.0] = stats.PauseQuantiles[0].Seconds()
+	ch <- MustNewConstSummary(c.gcDesc, uint64(stats.NumGC), float64(stats.PauseTotal.Seconds()), quantiles)
+
+	ms := &runtime.MemStats{}
+	runtime.ReadMemStats(ms)
+	for _, i := range c.metrics {
+		ch <- MustNewConstMetric(i.desc, i.valType, i.eval(ms))
+	}
+}
+
+// memStatsMetrics provide description, value, and value type for memstat metrics.
+type memStatsMetrics []struct {
+	desc    *Desc
+	eval    func(*runtime.MemStats) float64
+	valType ValueType
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/metric.go

@@ -0,0 +1,166 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"strings"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+const separatorByte byte = 255
+
+// A Metric models a single sample value with its meta data being exported to
+// Prometheus. Implementations of Metric in this package are Gauge, Counter,
+// Histogram, Summary, and Untyped.
+type Metric interface {
+	// Desc returns the descriptor for the Metric. This method idempotently
+	// returns the same descriptor throughout the lifetime of the
+	// Metric. The returned descriptor is immutable by contract. A Metric
+	// unable to describe itself must return an invalid descriptor (created
+	// with NewInvalidDesc).
+	Desc() *Desc
+	// Write encodes the Metric into a "Metric" Protocol Buffer data
+	// transmission object.
+	//
+	// Metric implementations must observe concurrency safety as reads of
+	// this metric may occur at any time, and any blocking occurs at the
+	// expense of total performance of rendering all registered
+	// metrics. Ideally, Metric implementations should support concurrent
+	// readers.
+	//
+	// While populating dto.Metric, it is the responsibility of the
+	// implementation to ensure validity of the Metric protobuf (like valid
+	// UTF-8 strings or syntactically valid metric and label names). It is
+	// recommended to sort labels lexicographically. (Implementers may find
+	// LabelPairSorter useful for that.) Callers of Write should still make
+	// sure of sorting if they depend on it.
+	Write(*dto.Metric) error
+	// TODO(beorn7): The original rationale of passing in a pre-allocated
+	// dto.Metric protobuf to save allocations has disappeared. The
+	// signature of this method should be changed to "Write() (*dto.Metric,
+	// error)".
+}
+
+// Opts bundles the options for creating most Metric types. Each metric
+// implementation XXX has its own XXXOpts type, but in most cases, it is just be
+// an alias of this type (which might change when the requirement arises.)
+//
+// It is mandatory to set Name and Help to a non-empty string. All other fields
+// are optional and can safely be left at their zero value.
+type Opts struct {
+	// Namespace, Subsystem, and Name are components of the fully-qualified
+	// name of the Metric (created by joining these components with
+	// "_"). Only Name is mandatory, the others merely help structuring the
+	// name. Note that the fully-qualified name of the metric must be a
+	// valid Prometheus metric name.
+	Namespace string
+	Subsystem string
+	Name      string
+
+	// Help provides information about this metric. Mandatory!
+	//
+	// Metrics with the same fully-qualified name must have the same Help
+	// string.
+	Help string
+
+	// ConstLabels are used to attach fixed labels to this metric. Metrics
+	// with the same fully-qualified name must have the same label names in
+	// their ConstLabels.
+	//
+	// Note that in most cases, labels have a value that varies during the
+	// lifetime of a process. Those labels are usually managed with a metric
+	// vector collector (like CounterVec, GaugeVec, UntypedVec). ConstLabels
+	// serve only special purposes. One is for the special case where the
+	// value of a label does not change during the lifetime of a process,
+	// e.g. if the revision of the running binary is put into a
+	// label. Another, more advanced purpose is if more than one Collector
+	// needs to collect Metrics with the same fully-qualified name. In that
+	// case, those Metrics must differ in the values of their
+	// ConstLabels. See the Collector examples.
+	//
+	// If the value of a label never changes (not even between binaries),
+	// that label most likely should not be a label at all (but part of the
+	// metric name).
+	ConstLabels Labels
+}
+
+// BuildFQName joins the given three name components by "_". Empty name
+// components are ignored. If the name parameter itself is empty, an empty
+// string is returned, no matter what. Metric implementations included in this
+// library use this function internally to generate the fully-qualified metric
+// name from the name component in their Opts. Users of the library will only
+// need this function if they implement their own Metric or instantiate a Desc
+// (with NewDesc) directly.
+func BuildFQName(namespace, subsystem, name string) string {
+	if name == "" {
+		return ""
+	}
+	switch {
+	case namespace != "" && subsystem != "":
+		return strings.Join([]string{namespace, subsystem, name}, "_")
+	case namespace != "":
+		return strings.Join([]string{namespace, name}, "_")
+	case subsystem != "":
+		return strings.Join([]string{subsystem, name}, "_")
+	}
+	return name
+}
+
+// LabelPairSorter implements sort.Interface. It is used to sort a slice of
+// dto.LabelPair pointers. This is useful for implementing the Write method of
+// custom metrics.
+type LabelPairSorter []*dto.LabelPair
+
+func (s LabelPairSorter) Len() int {
+	return len(s)
+}
+
+func (s LabelPairSorter) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s LabelPairSorter) Less(i, j int) bool {
+	return s[i].GetName() < s[j].GetName()
+}
+
+type hashSorter []uint64
+
+func (s hashSorter) Len() int {
+	return len(s)
+}
+
+func (s hashSorter) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s hashSorter) Less(i, j int) bool {
+	return s[i] < s[j]
+}
+
+type invalidMetric struct {
+	desc *Desc
+	err  error
+}
+
+// NewInvalidMetric returns a metric whose Write method always returns the
+// provided error. It is useful if a Collector finds itself unable to collect
+// a metric and wishes to report an error to the registry.
+func NewInvalidMetric(desc *Desc, err error) Metric {
+	return &invalidMetric{desc, err}
+}
+
+func (m *invalidMetric) Desc() *Desc { return m.desc }
+
+func (m *invalidMetric) Write(*dto.Metric) error { return m.err }

app/service/ops/log-agent/pkg/flowmonitor/counter/observer.go

@@ -0,0 +1,50 @@
+// Copyright 2017 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+// Observer is the interface that wraps the Observe method, which is used by
+// Histogram and Summary to add observations.
+type Observer interface {
+	Observe(float64)
+}
+
+// The ObserverFunc type is an adapter to allow the use of ordinary
+// functions as Observers. If f is a function with the appropriate
+// signature, ObserverFunc(f) is an Observer that calls f.
+//
+// This adapter is usually used in connection with the Timer type, and there are
+// two general use cases:
+//
+// The most common one is to use a Gauge as the Observer for a Timer.
+// See the "Gauge" Timer example.
+//
+// The more advanced use case is to create a function that dynamically decides
+// which Observer to use for observing the duration. See the "Complex" Timer
+// example.
+type ObserverFunc func(float64)
+
+// Observe calls f(value). It implements Observer.
+func (f ObserverFunc) Observe(value float64) {
+	f(value)
+}
+
+// ObserverVec is an interface implemented by `HistogramVec` and `SummaryVec`.
+type ObserverVec interface {
+	GetMetricWith(Labels) (Observer, error)
+	GetMetricWithLabelValues(lvs ...string) (Observer, error)
+	With(Labels) Observer
+	WithLabelValues(...string) Observer
+
+	Collector
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/process_collector.go

@@ -0,0 +1,140 @@
+// Copyright 2015 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import "github.com/prometheus/procfs"
+
+type processCollector struct {
+	pid             int
+	collectFn       func(chan<- Metric)
+	pidFn           func() (int, error)
+	cpuTotal        *Desc
+	openFDs, maxFDs *Desc
+	vsize, rss      *Desc
+	startTime       *Desc
+}
+
+// NewProcessCollector returns a collector which exports the current state of
+// process metrics including cpu, memory and file descriptor usage as well as
+// the process start time for the given process id under the given namespace.
+func NewProcessCollector(pid int, namespace string) Collector {
+	return NewProcessCollectorPIDFn(
+		func() (int, error) { return pid, nil },
+		namespace,
+	)
+}
+
+// NewProcessCollectorPIDFn returns a collector which exports the current state
+// of process metrics including cpu, memory and file descriptor usage as well
+// as the process start time under the given namespace. The given pidFn is
+// called on each collect and is used to determine the process to export
+// metrics for.
+func NewProcessCollectorPIDFn(
+	pidFn func() (int, error),
+	namespace string,
+) Collector {
+	ns := ""
+	if len(namespace) > 0 {
+		ns = namespace + "_"
+	}
+
+	c := processCollector{
+		pidFn:     pidFn,
+		collectFn: func(chan<- Metric) {},
+
+		cpuTotal: NewDesc(
+			ns+"process_cpu_seconds_total",
+			"Total user and system CPU time spent in seconds.",
+			nil, nil,
+		),
+		openFDs: NewDesc(
+			ns+"process_open_fds",
+			"Number of open file descriptors.",
+			nil, nil,
+		),
+		maxFDs: NewDesc(
+			ns+"process_max_fds",
+			"Maximum number of open file descriptors.",
+			nil, nil,
+		),
+		vsize: NewDesc(
+			ns+"process_virtual_memory_bytes",
+			"Virtual memory size in bytes.",
+			nil, nil,
+		),
+		rss: NewDesc(
+			ns+"process_resident_memory_bytes",
+			"Resident memory size in bytes.",
+			nil, nil,
+		),
+		startTime: NewDesc(
+			ns+"process_start_time_seconds",
+			"Start time of the process since unix epoch in seconds.",
+			nil, nil,
+		),
+	}
+
+	// Set up process metric collection if supported by the runtime.
+	if _, err := procfs.NewStat(); err == nil {
+		c.collectFn = c.processCollect
+	}
+
+	return &c
+}
+
+// Describe returns all descriptions of the collector.
+func (c *processCollector) Describe(ch chan<- *Desc) {
+	ch <- c.cpuTotal
+	ch <- c.openFDs
+	ch <- c.maxFDs
+	ch <- c.vsize
+	ch <- c.rss
+	ch <- c.startTime
+}
+
+// Collect returns the current state of all metrics of the collector.
+func (c *processCollector) Collect(ch chan<- Metric) {
+	c.collectFn(ch)
+}
+
+// TODO(ts): Bring back error reporting by reverting 7faf9e7 as soon as the
+// client allows users to configure the error behavior.
+func (c *processCollector) processCollect(ch chan<- Metric) {
+	pid, err := c.pidFn()
+	if err != nil {
+		return
+	}
+
+	p, err := procfs.NewProc(pid)
+	if err != nil {
+		return
+	}
+
+	if stat, err := p.NewStat(); err == nil {
+		ch <- MustNewConstMetric(c.cpuTotal, CounterValue, stat.CPUTime())
+		ch <- MustNewConstMetric(c.vsize, GaugeValue, float64(stat.VirtualMemory()))
+		ch <- MustNewConstMetric(c.rss, GaugeValue, float64(stat.ResidentMemory()))
+		if startTime, err := stat.StartTime(); err == nil {
+			ch <- MustNewConstMetric(c.startTime, GaugeValue, startTime)
+		}
+	}
+
+	if fds, err := p.FileDescriptorsLen(); err == nil {
+		ch <- MustNewConstMetric(c.openFDs, GaugeValue, float64(fds))
+	}
+
+	if limits, err := p.NewLimits(); err == nil {
+		ch <- MustNewConstMetric(c.maxFDs, GaugeValue, float64(limits.OpenFiles))
+	}
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/registry.go

@@ -0,0 +1,755 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"os"
+	"sort"
+	"sync"
+
+	"github.com/golang/protobuf/proto"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+const (
+	// Capacity for the channel to collect metrics and descriptors.
+	capMetricChan = 1000
+	capDescChan   = 10
+)
+
+// DefaultRegisterer and DefaultGatherer are the implementations of the
+// Registerer and Gatherer interface a number of convenience functions in this
+// package act on. Initially, both variables point to the same Registry, which
+// has a process collector (see NewProcessCollector) and a Go collector (see
+// NewGoCollector) already registered. This approach to keep default instances
+// as global state mirrors the approach of other packages in the Go standard
+// library. Note that there are caveats. Change the variables with caution and
+// only if you understand the consequences. Users who want to avoid global state
+// altogether should not use the convenience function and act on custom
+// instances instead.
+var (
+	defaultRegistry              = NewRegistry()
+	DefaultRegisterer Registerer = defaultRegistry
+	DefaultGatherer   Gatherer   = defaultRegistry
+)
+
+func init() {
+	MustRegister(NewProcessCollector(os.Getpid(), ""))
+	MustRegister(NewGoCollector())
+}
+
+// NewRegistry creates a new vanilla Registry without any Collectors
+// pre-registered.
+func NewRegistry() *Registry {
+	return &Registry{
+		collectorsByID:  map[uint64]Collector{},
+		descIDs:         map[uint64]struct{}{},
+		dimHashesByName: map[string]uint64{},
+	}
+}
+
+// NewPedanticRegistry returns a registry that checks during collection if each
+// collected Metric is consistent with its reported Desc, and if the Desc has
+// actually been registered with the registry.
+//
+// Usually, a Registry will be happy as long as the union of all collected
+// Metrics is consistent and valid even if some metrics are not consistent with
+// their own Desc or a Desc provided by their registered Collector. Well-behaved
+// Collectors and Metrics will only provide consistent Descs. This Registry is
+// useful to test the implementation of Collectors and Metrics.
+func NewPedanticRegistry() *Registry {
+	r := NewRegistry()
+	r.pedanticChecksEnabled = true
+	return r
+}
+
+// Registerer is the interface for the part of a registry in charge of
+// registering and unregistering. Users of custom registries should use
+// Registerer as type for registration purposes (rather then the Registry type
+// directly). In that way, they are free to use custom Registerer implementation
+// (e.g. for testing purposes).
+type Registerer interface {
+	// Register registers a new Collector to be included in metrics
+	// collection. It returns an error if the descriptors provided by the
+	// Collector are invalid or if they — in combination with descriptors of
+	// already registered Collectors — do not fulfill the consistency and
+	// uniqueness criteria described in the documentation of metric.Desc.
+	//
+	// If the provided Collector is equal to a Collector already registered
+	// (which includes the case of re-registering the same Collector), the
+	// returned error is an instance of AlreadyRegisteredError, which
+	// contains the previously registered Collector.
+	//
+	// It is in general not safe to register the same Collector multiple
+	// times concurrently.
+	Register(Collector) error
+	// MustRegister works like Register but registers any number of
+	// Collectors and panics upon the first registration that causes an
+	// error.
+	MustRegister(...Collector)
+	// Unregister unregisters the Collector that equals the Collector passed
+	// in as an argument.  (Two Collectors are considered equal if their
+	// Describe method yields the same set of descriptors.) The function
+	// returns whether a Collector was unregistered.
+	//
+	// Note that even after unregistering, it will not be possible to
+	// register a new Collector that is inconsistent with the unregistered
+	// Collector, e.g. a Collector collecting metrics with the same name but
+	// a different help string. The rationale here is that the same registry
+	// instance must only collect consistent metrics throughout its
+	// lifetime.
+	Unregister(Collector) bool
+}
+
+// Gatherer is the interface for the part of a registry in charge of gathering
+// the collected metrics into a number of MetricFamilies. The Gatherer interface
+// comes with the same general implication as described for the Registerer
+// interface.
+type Gatherer interface {
+	// Gather calls the Collect method of the registered Collectors and then
+	// gathers the collected metrics into a lexicographically sorted slice
+	// of MetricFamily protobufs. Even if an error occurs, Gather attempts
+	// to gather as many metrics as possible. Hence, if a non-nil error is
+	// returned, the returned MetricFamily slice could be nil (in case of a
+	// fatal error that prevented any meaningful metric collection) or
+	// contain a number of MetricFamily protobufs, some of which might be
+	// incomplete, and some might be missing altogether. The returned error
+	// (which might be a MultiError) explains the details. In scenarios
+	// where complete collection is critical, the returned MetricFamily
+	// protobufs should be disregarded if the returned error is non-nil.
+	Gather() ([]*dto.MetricFamily, error)
+}
+
+// Register registers the provided Collector with the DefaultRegisterer.
+//
+// Register is a shortcut for DefaultRegisterer.Register(c). See there for more
+// details.
+func Register(c Collector) error {
+	return DefaultRegisterer.Register(c)
+}
+
+// MustRegister registers the provided Collectors with the DefaultRegisterer and
+// panics if any error occurs.
+//
+// MustRegister is a shortcut for DefaultRegisterer.MustRegister(cs...). See
+// there for more details.
+func MustRegister(cs ...Collector) {
+	DefaultRegisterer.MustRegister(cs...)
+}
+
+// Unregister removes the registration of the provided Collector from the
+// DefaultRegisterer.
+//
+// Unregister is a shortcut for DefaultRegisterer.Unregister(c). See there for
+// more details.
+func Unregister(c Collector) bool {
+	return DefaultRegisterer.Unregister(c)
+}
+
+// GathererFunc turns a function into a Gatherer.
+type GathererFunc func() ([]*dto.MetricFamily, error)
+
+// Gather implements Gatherer.
+func (gf GathererFunc) Gather() ([]*dto.MetricFamily, error) {
+	return gf()
+}
+
+// AlreadyRegisteredError is returned by the Register method if the Collector to
+// be registered has already been registered before, or a different Collector
+// that collects the same metrics has been registered before. Registration fails
+// in that case, but you can detect from the kind of error what has
+// happened. The error contains fields for the existing Collector and the
+// (rejected) new Collector that equals the existing one. This can be used to
+// find out if an equal Collector has been registered before and switch over to
+// using the old one, as demonstrated in the example.
+type AlreadyRegisteredError struct {
+	ExistingCollector, NewCollector Collector
+}
+
+func (err AlreadyRegisteredError) Error() string {
+	return "duplicate metrics collector registration attempted"
+}
+
+// MultiError is a slice of errors implementing the error interface. It is used
+// by a Gatherer to report multiple errors during MetricFamily gathering.
+type MultiError []error
+
+func (errs MultiError) Error() string {
+	if len(errs) == 0 {
+		return ""
+	}
+	buf := &bytes.Buffer{}
+	fmt.Fprintf(buf, "%d error(s) occurred:", len(errs))
+	for _, err := range errs {
+		fmt.Fprintf(buf, "\n* %s", err)
+	}
+	return buf.String()
+}
+
+// MaybeUnwrap returns nil if len(errs) is 0. It returns the first and only
+// contained error as error if len(errs is 1). In all other cases, it returns
+// the MultiError directly. This is helpful for returning a MultiError in a way
+// that only uses the MultiError if needed.
+func (errs MultiError) MaybeUnwrap() error {
+	switch len(errs) {
+	case 0:
+		return nil
+	case 1:
+		return errs[0]
+	default:
+		return errs
+	}
+}
+
+// Registry registers Prometheus collectors, collects their metrics, and gathers
+// them into MetricFamilies for exposition. It implements both Registerer and
+// Gatherer. The zero value is not usable. Create instances with NewRegistry or
+// NewPedanticRegistry.
+type Registry struct {
+	mtx                   sync.RWMutex
+	collectorsByID        map[uint64]Collector // ID is a hash of the descIDs.
+	descIDs               map[uint64]struct{}
+	dimHashesByName       map[string]uint64
+	pedanticChecksEnabled bool
+}
+
+// Register implements Registerer.
+func (r *Registry) Register(c Collector) error {
+	var (
+		descChan           = make(chan *Desc, capDescChan)
+		newDescIDs         = map[uint64]struct{}{}
+		newDimHashesByName = map[string]uint64{}
+		collectorID        uint64 // Just a sum of all desc IDs.
+		duplicateDescErr   error
+	)
+	go func() {
+		c.Describe(descChan)
+		close(descChan)
+	}()
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+	// Conduct various tests...
+	for desc := range descChan {
+
+		// Is the descriptor valid at all?
+		if desc.err != nil {
+			return fmt.Errorf("descriptor %s is invalid: %s", desc, desc.err)
+		}
+
+		// Is the descID unique?
+		// (In other words: Is the fqName + constLabel combination unique?)
+		if _, exists := r.descIDs[desc.id]; exists {
+			duplicateDescErr = fmt.Errorf("descriptor %s already exists with the same fully-qualified name and const label values", desc)
+		}
+		// If it is not a duplicate desc in this collector, add it to
+		// the collectorID.  (We allow duplicate descs within the same
+		// collector, but their existence must be a no-op.)
+		if _, exists := newDescIDs[desc.id]; !exists {
+			newDescIDs[desc.id] = struct{}{}
+			collectorID += desc.id
+		}
+
+		// Are all the label names and the help string consistent with
+		// previous descriptors of the same name?
+		// First check existing descriptors...
+		if dimHash, exists := r.dimHashesByName[desc.fqName]; exists {
+			if dimHash != desc.dimHash {
+				return fmt.Errorf("a previously registered descriptor with the same fully-qualified name as %s has different label names or a different help string", desc)
+			}
+		} else {
+			// ...then check the new descriptors already seen.
+			if dimHash, exists := newDimHashesByName[desc.fqName]; exists {
+				if dimHash != desc.dimHash {
+					return fmt.Errorf("descriptors reported by collector have inconsistent label names or help strings for the same fully-qualified name, offender is %s", desc)
+				}
+			} else {
+				newDimHashesByName[desc.fqName] = desc.dimHash
+			}
+		}
+	}
+	// Did anything happen at all?
+	if len(newDescIDs) == 0 {
+		return errors.New("collector has no descriptors")
+	}
+	if existing, exists := r.collectorsByID[collectorID]; exists {
+		return AlreadyRegisteredError{
+			ExistingCollector: existing,
+			NewCollector:      c,
+		}
+	}
+	// If the collectorID is new, but at least one of the descs existed
+	// before, we are in trouble.
+	if duplicateDescErr != nil {
+		return duplicateDescErr
+	}
+
+	// Only after all tests have passed, actually register.
+	r.collectorsByID[collectorID] = c
+	for hash := range newDescIDs {
+		r.descIDs[hash] = struct{}{}
+	}
+	for name, dimHash := range newDimHashesByName {
+		r.dimHashesByName[name] = dimHash
+	}
+	return nil
+}
+
+// Unregister implements Registerer.
+func (r *Registry) Unregister(c Collector) bool {
+	var (
+		descChan    = make(chan *Desc, capDescChan)
+		descIDs     = map[uint64]struct{}{}
+		collectorID uint64 // Just a sum of the desc IDs.
+	)
+	go func() {
+		c.Describe(descChan)
+		close(descChan)
+	}()
+	for desc := range descChan {
+		if _, exists := descIDs[desc.id]; !exists {
+			collectorID += desc.id
+			descIDs[desc.id] = struct{}{}
+		}
+	}
+
+	r.mtx.RLock()
+	if _, exists := r.collectorsByID[collectorID]; !exists {
+		r.mtx.RUnlock()
+		return false
+	}
+	r.mtx.RUnlock()
+
+	r.mtx.Lock()
+	defer r.mtx.Unlock()
+
+	delete(r.collectorsByID, collectorID)
+	for id := range descIDs {
+		delete(r.descIDs, id)
+	}
+	// dimHashesByName is left untouched as those must be consistent
+	// throughout the lifetime of a program.
+	return true
+}
+
+// MustRegister implements Registerer.
+func (r *Registry) MustRegister(cs ...Collector) {
+	for _, c := range cs {
+		if err := r.Register(c); err != nil {
+			panic(err)
+		}
+	}
+}
+
+// Gather implements Gatherer.
+func (r *Registry) Gather() ([]*dto.MetricFamily, error) {
+	var (
+		metricChan        = make(chan Metric, capMetricChan)
+		metricHashes      = map[uint64]struct{}{}
+		dimHashes         = map[string]uint64{}
+		wg                sync.WaitGroup
+		errs              MultiError          // The collected errors to return in the end.
+		registeredDescIDs map[uint64]struct{} // Only used for pedantic checks
+	)
+
+	r.mtx.RLock()
+	metricFamiliesByName := make(map[string]*dto.MetricFamily, len(r.dimHashesByName))
+
+	// Scatter.
+	// (Collectors could be complex and slow, so we call them all at once.)
+	wg.Add(len(r.collectorsByID))
+	go func() {
+		wg.Wait()
+		close(metricChan)
+	}()
+	for _, collector := range r.collectorsByID {
+		go func(collector Collector) {
+			defer wg.Done()
+			collector.Collect(metricChan)
+		}(collector)
+	}
+
+	// In case pedantic checks are enabled, we have to copy the map before
+	// giving up the RLock.
+	if r.pedanticChecksEnabled {
+		registeredDescIDs = make(map[uint64]struct{}, len(r.descIDs))
+		for id := range r.descIDs {
+			registeredDescIDs[id] = struct{}{}
+		}
+	}
+
+	r.mtx.RUnlock()
+
+	// Drain metricChan in case of premature return.
+	defer func() {
+		for range metricChan {
+		}
+	}()
+
+	// Gather.
+	for metric := range metricChan {
+		// This could be done concurrently, too, but it required locking
+		// of metricFamiliesByName (and of metricHashes if checks are
+		// enabled). Most likely not worth it.
+		desc := metric.Desc()
+		dtoMetric := &dto.Metric{}
+		if err := metric.Write(dtoMetric); err != nil {
+			errs = append(errs, fmt.Errorf(
+				"error collecting metric %v: %s", desc, err,
+			))
+			continue
+		}
+		metricFamily, ok := metricFamiliesByName[desc.fqName]
+		if ok {
+			if metricFamily.GetHelp() != desc.help {
+				errs = append(errs, fmt.Errorf(
+					"collected metric %s %s has help %q but should have %q",
+					desc.fqName, dtoMetric, desc.help, metricFamily.GetHelp(),
+				))
+				continue
+			}
+			// TODO(beorn7): Simplify switch once Desc has type.
+			switch metricFamily.GetType() {
+			case dto.MetricType_COUNTER:
+				if dtoMetric.Counter == nil {
+					errs = append(errs, fmt.Errorf(
+						"collected metric %s %s should be a Counter",
+						desc.fqName, dtoMetric,
+					))
+					continue
+				}
+			case dto.MetricType_GAUGE:
+				if dtoMetric.Gauge == nil {
+					errs = append(errs, fmt.Errorf(
+						"collected metric %s %s should be a Gauge",
+						desc.fqName, dtoMetric,
+					))
+					continue
+				}
+			case dto.MetricType_SUMMARY:
+				if dtoMetric.Summary == nil {
+					errs = append(errs, fmt.Errorf(
+						"collected metric %s %s should be a Summary",
+						desc.fqName, dtoMetric,
+					))
+					continue
+				}
+			case dto.MetricType_UNTYPED:
+				if dtoMetric.Untyped == nil {
+					errs = append(errs, fmt.Errorf(
+						"collected metric %s %s should be Untyped",
+						desc.fqName, dtoMetric,
+					))
+					continue
+				}
+			case dto.MetricType_HISTOGRAM:
+				if dtoMetric.Histogram == nil {
+					errs = append(errs, fmt.Errorf(
+						"collected metric %s %s should be a Histogram",
+						desc.fqName, dtoMetric,
+					))
+					continue
+				}
+			default:
+				panic("encountered MetricFamily with invalid type")
+			}
+		} else {
+			metricFamily = &dto.MetricFamily{}
+			metricFamily.Name = proto.String(desc.fqName)
+			metricFamily.Help = proto.String(desc.help)
+			// TODO(beorn7): Simplify switch once Desc has type.
+			switch {
+			case dtoMetric.Gauge != nil:
+				metricFamily.Type = dto.MetricType_GAUGE.Enum()
+			case dtoMetric.Counter != nil:
+				metricFamily.Type = dto.MetricType_COUNTER.Enum()
+			case dtoMetric.Summary != nil:
+				metricFamily.Type = dto.MetricType_SUMMARY.Enum()
+			case dtoMetric.Untyped != nil:
+				metricFamily.Type = dto.MetricType_UNTYPED.Enum()
+			case dtoMetric.Histogram != nil:
+				metricFamily.Type = dto.MetricType_HISTOGRAM.Enum()
+			default:
+				errs = append(errs, fmt.Errorf(
+					"empty metric collected: %s", dtoMetric,
+				))
+				continue
+			}
+			metricFamiliesByName[desc.fqName] = metricFamily
+		}
+		if err := checkMetricConsistency(metricFamily, dtoMetric, metricHashes, dimHashes); err != nil {
+			errs = append(errs, err)
+			continue
+		}
+		if r.pedanticChecksEnabled {
+			// Is the desc registered at all?
+			if _, exist := registeredDescIDs[desc.id]; !exist {
+				errs = append(errs, fmt.Errorf(
+					"collected metric %s %s with unregistered descriptor %s",
+					metricFamily.GetName(), dtoMetric, desc,
+				))
+				continue
+			}
+			if err := checkDescConsistency(metricFamily, dtoMetric, desc); err != nil {
+				errs = append(errs, err)
+				continue
+			}
+		}
+		metricFamily.Metric = append(metricFamily.Metric, dtoMetric)
+	}
+	return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap()
+}
+
+// Gatherers is a slice of Gatherer instances that implements the Gatherer
+// interface itself. Its Gather method calls Gather on all Gatherers in the
+// slice in order and returns the merged results. Errors returned from the
+// Gather calles are all returned in a flattened MultiError. Duplicate and
+// inconsistent Metrics are skipped (first occurrence in slice order wins) and
+// reported in the returned error.
+//
+// Gatherers can be used to merge the Gather results from multiple
+// Registries. It also provides a way to directly inject existing MetricFamily
+// protobufs into the gathering by creating a custom Gatherer with a Gather
+// method that simply returns the existing MetricFamily protobufs. Note that no
+// registration is involved (in contrast to Collector registration), so
+// obviously registration-time checks cannot happen. Any inconsistencies between
+// the gathered MetricFamilies are reported as errors by the Gather method, and
+// inconsistent Metrics are dropped. Invalid parts of the MetricFamilies
+// (e.g. syntactically invalid metric or label names) will go undetected.
+type Gatherers []Gatherer
+
+// Gather implements Gatherer.
+func (gs Gatherers) Gather() ([]*dto.MetricFamily, error) {
+	var (
+		metricFamiliesByName = map[string]*dto.MetricFamily{}
+		metricHashes         = map[uint64]struct{}{}
+		dimHashes            = map[string]uint64{}
+		errs                 MultiError // The collected errors to return in the end.
+	)
+
+	for i, g := range gs {
+		mfs, err := g.Gather()
+		if err != nil {
+			if multiErr, ok := err.(MultiError); ok {
+				for _, err := range multiErr {
+					errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err))
+				}
+			} else {
+				errs = append(errs, fmt.Errorf("[from Gatherer #%d] %s", i+1, err))
+			}
+		}
+		for _, mf := range mfs {
+			existingMF, exists := metricFamiliesByName[mf.GetName()]
+			if exists {
+				if existingMF.GetHelp() != mf.GetHelp() {
+					errs = append(errs, fmt.Errorf(
+						"gathered metric family %s has help %q but should have %q",
+						mf.GetName(), mf.GetHelp(), existingMF.GetHelp(),
+					))
+					continue
+				}
+				if existingMF.GetType() != mf.GetType() {
+					errs = append(errs, fmt.Errorf(
+						"gathered metric family %s has type %s but should have %s",
+						mf.GetName(), mf.GetType(), existingMF.GetType(),
+					))
+					continue
+				}
+			} else {
+				existingMF = &dto.MetricFamily{}
+				existingMF.Name = mf.Name
+				existingMF.Help = mf.Help
+				existingMF.Type = mf.Type
+				metricFamiliesByName[mf.GetName()] = existingMF
+			}
+			for _, m := range mf.Metric {
+				if err := checkMetricConsistency(existingMF, m, metricHashes, dimHashes); err != nil {
+					errs = append(errs, err)
+					continue
+				}
+				existingMF.Metric = append(existingMF.Metric, m)
+			}
+		}
+	}
+	return normalizeMetricFamilies(metricFamiliesByName), errs.MaybeUnwrap()
+}
+
+// metricSorter is a sortable slice of *dto.Metric.
+type metricSorter []*dto.Metric
+
+func (s metricSorter) Len() int {
+	return len(s)
+}
+
+func (s metricSorter) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s metricSorter) Less(i, j int) bool {
+	if len(s[i].Label) != len(s[j].Label) {
+		// This should not happen. The metrics are
+		// inconsistent. However, we have to deal with the fact, as
+		// people might use custom collectors or metric family injection
+		// to create inconsistent metrics. So let's simply compare the
+		// number of labels in this case. That will still yield
+		// reproducible sorting.
+		return len(s[i].Label) < len(s[j].Label)
+	}
+	for n, lp := range s[i].Label {
+		vi := lp.GetValue()
+		vj := s[j].Label[n].GetValue()
+		if vi != vj {
+			return vi < vj
+		}
+	}
+
+	// We should never arrive here. Multiple metrics with the same
+	// label set in the same scrape will lead to undefined ingestion
+	// behavior. However, as above, we have to provide stable sorting
+	// here, even for inconsistent metrics. So sort equal metrics
+	// by their timestamp, with missing timestamps (implying "now")
+	// coming last.
+	if s[i].TimestampMs == nil {
+		return false
+	}
+	if s[j].TimestampMs == nil {
+		return true
+	}
+	return s[i].GetTimestampMs() < s[j].GetTimestampMs()
+}
+
+// normalizeMetricFamilies returns a MetricFamily slice with empty
+// MetricFamilies pruned and the remaining MetricFamilies sorted by name within
+// the slice, with the contained Metrics sorted within each MetricFamily.
+func normalizeMetricFamilies(metricFamiliesByName map[string]*dto.MetricFamily) []*dto.MetricFamily {
+	for _, mf := range metricFamiliesByName {
+		sort.Sort(metricSorter(mf.Metric))
+	}
+	names := make([]string, 0, len(metricFamiliesByName))
+	for name, mf := range metricFamiliesByName {
+		if len(mf.Metric) > 0 {
+			names = append(names, name)
+		}
+	}
+	sort.Strings(names)
+	result := make([]*dto.MetricFamily, 0, len(names))
+	for _, name := range names {
+		result = append(result, metricFamiliesByName[name])
+	}
+	return result
+}
+
+// checkMetricConsistency checks if the provided Metric is consistent with the
+// provided MetricFamily. It also hashed the Metric labels and the MetricFamily
+// name. If the resulting hash is alread in the provided metricHashes, an error
+// is returned. If not, it is added to metricHashes. The provided dimHashes maps
+// MetricFamily names to their dimHash (hashed sorted label names). If dimHashes
+// doesn't yet contain a hash for the provided MetricFamily, it is
+// added. Otherwise, an error is returned if the existing dimHashes in not equal
+// the calculated dimHash.
+func checkMetricConsistency(
+	metricFamily *dto.MetricFamily,
+	dtoMetric *dto.Metric,
+	metricHashes map[uint64]struct{},
+	dimHashes map[string]uint64,
+) error {
+	// Type consistency with metric family.
+	if metricFamily.GetType() == dto.MetricType_GAUGE && dtoMetric.Gauge == nil ||
+		metricFamily.GetType() == dto.MetricType_COUNTER && dtoMetric.Counter == nil ||
+		metricFamily.GetType() == dto.MetricType_SUMMARY && dtoMetric.Summary == nil ||
+		metricFamily.GetType() == dto.MetricType_HISTOGRAM && dtoMetric.Histogram == nil ||
+		metricFamily.GetType() == dto.MetricType_UNTYPED && dtoMetric.Untyped == nil {
+		return fmt.Errorf(
+			"collected metric %s %s is not a %s",
+			metricFamily.GetName(), dtoMetric, metricFamily.GetType(),
+		)
+	}
+
+	// Is the metric unique (i.e. no other metric with the same name and the same label values)?
+	h := hashNew()
+	h = hashAdd(h, metricFamily.GetName())
+	h = hashAddByte(h, separatorByte)
+	dh := hashNew()
+	// Make sure label pairs are sorted. We depend on it for the consistency
+	// check.
+	sort.Sort(LabelPairSorter(dtoMetric.Label))
+	for _, lp := range dtoMetric.Label {
+		h = hashAdd(h, lp.GetValue())
+		h = hashAddByte(h, separatorByte)
+		dh = hashAdd(dh, lp.GetName())
+		dh = hashAddByte(dh, separatorByte)
+	}
+	if _, exists := metricHashes[h]; exists {
+		return fmt.Errorf(
+			"collected metric %s %s was collected before with the same name and label values",
+			metricFamily.GetName(), dtoMetric,
+		)
+	}
+	if dimHash, ok := dimHashes[metricFamily.GetName()]; ok {
+		if dimHash != dh {
+			return fmt.Errorf(
+				"collected metric %s %s has label dimensions inconsistent with previously collected metrics in the same metric family",
+				metricFamily.GetName(), dtoMetric,
+			)
+		}
+	} else {
+		dimHashes[metricFamily.GetName()] = dh
+	}
+	metricHashes[h] = struct{}{}
+	return nil
+}
+
+func checkDescConsistency(
+	metricFamily *dto.MetricFamily,
+	dtoMetric *dto.Metric,
+	desc *Desc,
+) error {
+	// Desc help consistency with metric family help.
+	if metricFamily.GetHelp() != desc.help {
+		return fmt.Errorf(
+			"collected metric %s %s has help %q but should have %q",
+			metricFamily.GetName(), dtoMetric, metricFamily.GetHelp(), desc.help,
+		)
+	}
+
+	// Is the desc consistent with the content of the metric?
+	lpsFromDesc := make([]*dto.LabelPair, 0, len(dtoMetric.Label))
+	lpsFromDesc = append(lpsFromDesc, desc.constLabelPairs...)
+	for _, l := range desc.variableLabels {
+		lpsFromDesc = append(lpsFromDesc, &dto.LabelPair{
+			Name: proto.String(l),
+		})
+	}
+	if len(lpsFromDesc) != len(dtoMetric.Label) {
+		return fmt.Errorf(
+			"labels in collected metric %s %s are inconsistent with descriptor %s",
+			metricFamily.GetName(), dtoMetric, desc,
+		)
+	}
+	sort.Sort(LabelPairSorter(lpsFromDesc))
+	for i, lpFromDesc := range lpsFromDesc {
+		lpFromMetric := dtoMetric.Label[i]
+		if lpFromDesc.GetName() != lpFromMetric.GetName() ||
+			lpFromDesc.Value != nil && lpFromDesc.GetValue() != lpFromMetric.GetValue() {
+			return fmt.Errorf(
+				"labels in collected metric %s %s are inconsistent with descriptor %s",
+				metricFamily.GetName(), dtoMetric, desc,
+			)
+		}
+	}
+	return nil
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/summary.go

@@ -0,0 +1,543 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"fmt"
+	"math"
+	"sort"
+	"sync"
+	"time"
+
+	"github.com/beorn7/perks/quantile"
+	"github.com/golang/protobuf/proto"
+
+	dto "github.com/prometheus/client_model/go"
+)
+
+// quantileLabel is used for the label that defines the quantile in a
+// summary.
+const quantileLabel = "quantile"
+
+// A Summary captures individual observations from an event or sample stream and
+// summarizes them in a manner similar to traditional summary statistics: 1. sum
+// of observations, 2. observation count, 3. rank estimations.
+//
+// A typical use-case is the observation of request latencies. By default, a
+// Summary provides the median, the 90th and the 99th percentile of the latency
+// as rank estimations.
+//
+// Note that the rank estimations cannot be aggregated in a meaningful way with
+// the Prometheus query language (i.e. you cannot average or add them). If you
+// need aggregatable quantiles (e.g. you want the 99th percentile latency of all
+// queries served across all instances of a service), consider the Histogram
+// metric type. See the Prometheus documentation for more details.
+//
+// To create Summary instances, use NewSummary.
+type Summary interface {
+	Metric
+	Collector
+
+	// Observe adds a single observation to the summary.
+	Observe(float64)
+}
+
+// DefObjectives are the default Summary quantile values.
+//
+// Deprecated: DefObjectives will not be used as the default objectives in
+// v0.10 of the library. The default Summary will have no quantiles then.
+var (
+	DefObjectives = map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}
+
+	errQuantileLabelNotAllowed = fmt.Errorf(
+		"%q is not allowed as label name in summaries", quantileLabel,
+	)
+)
+
+// Default values for SummaryOpts.
+const (
+	// DefMaxAge is the default duration for which observations stay
+	// relevant.
+	DefMaxAge time.Duration = 10 * time.Minute
+	// DefAgeBuckets is the default number of buckets used to calculate the
+	// age of observations.
+	DefAgeBuckets = 5
+	// DefBufCap is the standard buffer size for collecting Summary observations.
+	DefBufCap = 500
+)
+
+// SummaryOpts bundles the options for creating a Summary metric. It is
+// mandatory to set Name and Help to a non-empty string. All other fields are
+// optional and can safely be left at their zero value.
+type SummaryOpts struct {
+	// Namespace, Subsystem, and Name are components of the fully-qualified
+	// name of the Summary (created by joining these components with
+	// "_"). Only Name is mandatory, the others merely help structuring the
+	// name. Note that the fully-qualified name of the Summary must be a
+	// valid Prometheus metric name.
+	Namespace string
+	Subsystem string
+	Name      string
+
+	// Help provides information about this Summary. Mandatory!
+	//
+	// Metrics with the same fully-qualified name must have the same Help
+	// string.
+	Help string
+
+	// ConstLabels are used to attach fixed labels to this
+	// Summary. Summaries with the same fully-qualified name must have the
+	// same label names in their ConstLabels.
+	//
+	// Note that in most cases, labels have a value that varies during the
+	// lifetime of a process. Those labels are usually managed with a
+	// SummaryVec. ConstLabels serve only special purposes. One is for the
+	// special case where the value of a label does not change during the
+	// lifetime of a process, e.g. if the revision of the running binary is
+	// put into a label. Another, more advanced purpose is if more than one
+	// Collector needs to collect Summaries with the same fully-qualified
+	// name. In that case, those Summaries must differ in the values of
+	// their ConstLabels. See the Collector examples.
+	//
+	// If the value of a label never changes (not even between binaries),
+	// that label most likely should not be a label at all (but part of the
+	// metric name).
+	ConstLabels Labels
+
+	// Objectives defines the quantile rank estimates with their respective
+	// absolute error. If Objectives[q] = e, then the value reported for q
+	// will be the φ-quantile value for some φ between q-e and q+e.  The
+	// default value is DefObjectives. It is used if Objectives is left at
+	// its zero value (i.e. nil). To create a Summary without Objectives,
+	// set it to an empty map (i.e. map[float64]float64{}).
+	//
+	// Deprecated: Note that the current value of DefObjectives is
+	// deprecated. It will be replaced by an empty map in v0.10 of the
+	// library. Please explicitly set Objectives to the desired value.
+	Objectives map[float64]float64
+
+	// MaxAge defines the duration for which an observation stays relevant
+	// for the summary. Must be positive. The default value is DefMaxAge.
+	MaxAge time.Duration
+
+	// AgeBuckets is the number of buckets used to exclude observations that
+	// are older than MaxAge from the summary. A higher number has a
+	// resource penalty, so only increase it if the higher resolution is
+	// really required. For very high observation rates, you might want to
+	// reduce the number of age buckets. With only one age bucket, you will
+	// effectively see a complete reset of the summary each time MaxAge has
+	// passed. The default value is DefAgeBuckets.
+	AgeBuckets uint32
+
+	// BufCap defines the default sample stream buffer size.  The default
+	// value of DefBufCap should suffice for most uses. If there is a need
+	// to increase the value, a multiple of 500 is recommended (because that
+	// is the internal buffer size of the underlying package
+	// "github.com/bmizerany/perks/quantile").
+	BufCap uint32
+}
+
+// Great fuck-up with the sliding-window decay algorithm... The Merge method of
+// perk/quantile is actually not working as advertised - and it might be
+// unfixable, as the underlying algorithm is apparently not capable of merging
+// summaries in the first place. To avoid using Merge, we are currently adding
+// observations to _each_ age bucket, i.e. the effort to add a sample is
+// essentially multiplied by the number of age buckets. When rotating age
+// buckets, we empty the previous head stream. On scrape time, we simply take
+// the quantiles from the head stream (no merging required). Result: More effort
+// on observation time, less effort on scrape time, which is exactly the
+// opposite of what we try to accomplish, but at least the results are correct.
+//
+// The quite elegant previous contraption to merge the age buckets efficiently
+// on scrape time (see code up commit 6b9530d72ea715f0ba612c0120e6e09fbf1d49d0)
+// can't be used anymore.
+
+// NewSummary creates a new Summary based on the provided SummaryOpts.
+func NewSummary(opts SummaryOpts) Summary {
+	return newSummary(
+		NewDesc(
+			BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+			opts.Help,
+			nil,
+			opts.ConstLabels,
+		),
+		opts,
+	)
+}
+
+func newSummary(desc *Desc, opts SummaryOpts, labelValues ...string) Summary {
+	if len(desc.variableLabels) != len(labelValues) {
+		panic(errInconsistentCardinality)
+	}
+
+	for _, n := range desc.variableLabels {
+		if n == quantileLabel {
+			panic(errQuantileLabelNotAllowed)
+		}
+	}
+	for _, lp := range desc.constLabelPairs {
+		if lp.GetName() == quantileLabel {
+			panic(errQuantileLabelNotAllowed)
+		}
+	}
+
+	if opts.Objectives == nil {
+		opts.Objectives = DefObjectives
+	}
+
+	if opts.MaxAge < 0 {
+		panic(fmt.Errorf("illegal max age MaxAge=%v", opts.MaxAge))
+	}
+	if opts.MaxAge == 0 {
+		opts.MaxAge = DefMaxAge
+	}
+
+	if opts.AgeBuckets == 0 {
+		opts.AgeBuckets = DefAgeBuckets
+	}
+
+	if opts.BufCap == 0 {
+		opts.BufCap = DefBufCap
+	}
+
+	s := &summary{
+		desc: desc,
+
+		objectives:       opts.Objectives,
+		sortedObjectives: make([]float64, 0, len(opts.Objectives)),
+
+		labelPairs: makeLabelPairs(desc, labelValues),
+
+		hotBuf:         make([]float64, 0, opts.BufCap),
+		coldBuf:        make([]float64, 0, opts.BufCap),
+		streamDuration: opts.MaxAge / time.Duration(opts.AgeBuckets),
+	}
+	s.headStreamExpTime = time.Now().Add(s.streamDuration)
+	s.hotBufExpTime = s.headStreamExpTime
+
+	for i := uint32(0); i < opts.AgeBuckets; i++ {
+		s.streams = append(s.streams, s.newStream())
+	}
+	s.headStream = s.streams[0]
+
+	for qu := range s.objectives {
+		s.sortedObjectives = append(s.sortedObjectives, qu)
+	}
+	sort.Float64s(s.sortedObjectives)
+
+	s.init(s) // Init self-collection.
+	return s
+}
+
+type summary struct {
+	selfCollector
+
+	bufMtx sync.Mutex // Protects hotBuf and hotBufExpTime.
+	mtx    sync.Mutex // Protects every other moving part.
+	// Lock bufMtx before mtx if both are needed.
+
+	desc *Desc
+
+	objectives       map[float64]float64
+	sortedObjectives []float64
+
+	labelPairs []*dto.LabelPair
+
+	sum float64
+	cnt uint64
+
+	hotBuf, coldBuf []float64
+
+	streams                          []*quantile.Stream
+	streamDuration                   time.Duration
+	headStream                       *quantile.Stream
+	headStreamIdx                    int
+	headStreamExpTime, hotBufExpTime time.Time
+}
+
+func (s *summary) Desc() *Desc {
+	return s.desc
+}
+
+func (s *summary) Observe(v float64) {
+	s.bufMtx.Lock()
+	defer s.bufMtx.Unlock()
+
+	now := time.Now()
+	if now.After(s.hotBufExpTime) {
+		s.asyncFlush(now)
+	}
+	s.hotBuf = append(s.hotBuf, v)
+	if len(s.hotBuf) == cap(s.hotBuf) {
+		s.asyncFlush(now)
+	}
+}
+
+func (s *summary) Write(out *dto.Metric) error {
+	sum := &dto.Summary{}
+	qs := make([]*dto.Quantile, 0, len(s.objectives))
+
+	s.bufMtx.Lock()
+	s.mtx.Lock()
+	// Swap bufs even if hotBuf is empty to set new hotBufExpTime.
+	s.swapBufs(time.Now())
+	s.bufMtx.Unlock()
+
+	s.flushColdBuf()
+	sum.SampleCount = proto.Uint64(s.cnt)
+	sum.SampleSum = proto.Float64(s.sum)
+
+	for _, rank := range s.sortedObjectives {
+		var q float64
+		if s.headStream.Count() == 0 {
+			q = math.NaN()
+		} else {
+			q = s.headStream.Query(rank)
+		}
+		qs = append(qs, &dto.Quantile{
+			Quantile: proto.Float64(rank),
+			Value:    proto.Float64(q),
+		})
+	}
+
+	s.mtx.Unlock()
+
+	if len(qs) > 0 {
+		sort.Sort(quantSort(qs))
+	}
+	sum.Quantile = qs
+
+	out.Summary = sum
+	out.Label = s.labelPairs
+	return nil
+}
+
+func (s *summary) newStream() *quantile.Stream {
+	return quantile.NewTargeted(s.objectives)
+}
+
+// asyncFlush needs bufMtx locked.
+func (s *summary) asyncFlush(now time.Time) {
+	s.mtx.Lock()
+	s.swapBufs(now)
+
+	// Unblock the original goroutine that was responsible for the mutation
+	// that triggered the compaction.  But hold onto the global non-buffer
+	// state mutex until the operation finishes.
+	go func() {
+		s.flushColdBuf()
+		s.mtx.Unlock()
+	}()
+}
+
+// rotateStreams needs mtx AND bufMtx locked.
+func (s *summary) maybeRotateStreams() {
+	for !s.hotBufExpTime.Equal(s.headStreamExpTime) {
+		s.headStream.Reset()
+		s.headStreamIdx++
+		if s.headStreamIdx >= len(s.streams) {
+			s.headStreamIdx = 0
+		}
+		s.headStream = s.streams[s.headStreamIdx]
+		s.headStreamExpTime = s.headStreamExpTime.Add(s.streamDuration)
+	}
+}
+
+// flushColdBuf needs mtx locked.
+func (s *summary) flushColdBuf() {
+	for _, v := range s.coldBuf {
+		for _, stream := range s.streams {
+			stream.Insert(v)
+		}
+		s.cnt++
+		s.sum += v
+	}
+	s.coldBuf = s.coldBuf[0:0]
+	s.maybeRotateStreams()
+}
+
+// swapBufs needs mtx AND bufMtx locked, coldBuf must be empty.
+func (s *summary) swapBufs(now time.Time) {
+	if len(s.coldBuf) != 0 {
+		panic("coldBuf is not empty")
+	}
+	s.hotBuf, s.coldBuf = s.coldBuf, s.hotBuf
+	// hotBuf is now empty and gets new expiration set.
+	for now.After(s.hotBufExpTime) {
+		s.hotBufExpTime = s.hotBufExpTime.Add(s.streamDuration)
+	}
+}
+
+type quantSort []*dto.Quantile
+
+func (s quantSort) Len() int {
+	return len(s)
+}
+
+func (s quantSort) Swap(i, j int) {
+	s[i], s[j] = s[j], s[i]
+}
+
+func (s quantSort) Less(i, j int) bool {
+	return s[i].GetQuantile() < s[j].GetQuantile()
+}
+
+// SummaryVec is a Collector that bundles a set of Summaries that all share the
+// same Desc, but have different values for their variable labels. This is used
+// if you want to count the same thing partitioned by various dimensions
+// (e.g. HTTP request latencies, partitioned by status code and method). Create
+// instances with NewSummaryVec.
+type SummaryVec struct {
+	*MetricVec
+}
+
+// NewSummaryVec creates a new SummaryVec based on the provided SummaryOpts and
+// partitioned by the given label names. At least one label name must be
+// provided.
+func NewSummaryVec(opts SummaryOpts, labelNames []string) *SummaryVec {
+	desc := NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		labelNames,
+		opts.ConstLabels,
+	)
+	return &SummaryVec{
+		MetricVec: newMetricVec(desc, func(lvs ...string) Metric {
+			return newSummary(desc, opts, lvs...)
+		}),
+	}
+}
+
+// GetMetricWithLabelValues replaces the method of the same name in MetricVec.
+// The difference is that this method returns an Observer and not a Metric so
+// that no type conversion to an Observer is required.
+func (m *SummaryVec) GetMetricWithLabelValues(lvs ...string) (Observer, error) {
+	metric, err := m.MetricVec.GetMetricWithLabelValues(lvs...)
+	if metric != nil {
+		return metric.(Observer), err
+	}
+	return nil, err
+}
+
+// GetMetricWith replaces the method of the same name in MetricVec. The
+// difference is that this method returns an Observer and not a Metric so that
+// no type conversion to an Observer is required.
+func (m *SummaryVec) GetMetricWith(labels Labels) (Observer, error) {
+	metric, err := m.MetricVec.GetMetricWith(labels)
+	if metric != nil {
+		return metric.(Observer), err
+	}
+	return nil, err
+}
+
+// WithLabelValues works as GetMetricWithLabelValues, but panics where
+// GetMetricWithLabelValues would have returned an error. By not returning an
+// error, WithLabelValues allows shortcuts like
+//     myVec.WithLabelValues("404", "GET").Observe(42.21)
+func (m *SummaryVec) WithLabelValues(lvs ...string) Observer {
+	return m.MetricVec.WithLabelValues(lvs...).(Observer)
+}
+
+// With works as GetMetricWith, but panics where GetMetricWithLabels would have
+// returned an error. By not returning an error, With allows shortcuts like
+//     myVec.With(Labels{"code": "404", "method": "GET"}).Observe(42.21)
+func (m *SummaryVec) With(labels Labels) Observer {
+	return m.MetricVec.With(labels).(Observer)
+}
+
+type constSummary struct {
+	desc       *Desc
+	count      uint64
+	sum        float64
+	quantiles  map[float64]float64
+	labelPairs []*dto.LabelPair
+}
+
+func (s *constSummary) Desc() *Desc {
+	return s.desc
+}
+
+func (s *constSummary) Write(out *dto.Metric) error {
+	sum := &dto.Summary{}
+	qs := make([]*dto.Quantile, 0, len(s.quantiles))
+
+	sum.SampleCount = proto.Uint64(s.count)
+	sum.SampleSum = proto.Float64(s.sum)
+
+	for rank, q := range s.quantiles {
+		qs = append(qs, &dto.Quantile{
+			Quantile: proto.Float64(rank),
+			Value:    proto.Float64(q),
+		})
+	}
+
+	if len(qs) > 0 {
+		sort.Sort(quantSort(qs))
+	}
+	sum.Quantile = qs
+
+	out.Summary = sum
+	out.Label = s.labelPairs
+
+	return nil
+}
+
+// NewConstSummary returns a metric representing a Prometheus summary with fixed
+// values for the count, sum, and quantiles. As those parameters cannot be
+// changed, the returned value does not implement the Summary interface (but
+// only the Metric interface). Users of this package will not have much use for
+// it in regular operations. However, when implementing custom Collectors, it is
+// useful as a throw-away metric that is generated on the fly to send it to
+// Prometheus in the Collect method.
+//
+// quantiles maps ranks to quantile values. For example, a median latency of
+// 0.23s and a 99th percentile latency of 0.56s would be expressed as:
+//     map[float64]float64{0.5: 0.23, 0.99: 0.56}
+//
+// NewConstSummary returns an error if the length of labelValues is not
+// consistent with the variable labels in Desc.
+func NewConstSummary(
+	desc *Desc,
+	count uint64,
+	sum float64,
+	quantiles map[float64]float64,
+	labelValues ...string,
+) (Metric, error) {
+	if len(desc.variableLabels) != len(labelValues) {
+		return nil, errInconsistentCardinality
+	}
+	return &constSummary{
+		desc:       desc,
+		count:      count,
+		sum:        sum,
+		quantiles:  quantiles,
+		labelPairs: makeLabelPairs(desc, labelValues),
+	}, nil
+}
+
+// MustNewConstSummary is a version of NewConstSummary that panics where
+// NewConstMetric would have returned an error.
+func MustNewConstSummary(
+	desc *Desc,
+	count uint64,
+	sum float64,
+	quantiles map[float64]float64,
+	labelValues ...string,
+) Metric {
+	m, err := NewConstSummary(desc, count, sum, quantiles, labelValues...)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/untyped.go

@@ -0,0 +1,143 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+// Untyped is a Metric that represents a single numerical value that can
+// arbitrarily go up and down.
+//
+// An Untyped metric works the same as a Gauge. The only difference is that to
+// no type information is implied.
+//
+// To create Untyped instances, use NewUntyped.
+//
+// Deprecated: The Untyped type is deprecated because it doesn't make sense in
+// direct instrumentation. If you need to mirror an external metric of unknown
+// type (usually while writing exporters), Use MustNewConstMetric to create an
+// untyped metric instance on the fly.
+type Untyped interface {
+	Metric
+	Collector
+
+	// Set sets the Untyped metric to an arbitrary value.
+	Set(float64)
+	// Inc increments the Untyped metric by 1.
+	Inc()
+	// Dec decrements the Untyped metric by 1.
+	Dec()
+	// Add adds the given value to the Untyped metric. (The value can be
+	// negative, resulting in a decrease.)
+	Add(float64)
+	// Sub subtracts the given value from the Untyped metric. (The value can
+	// be negative, resulting in an increase.)
+	Sub(float64)
+}
+
+// UntypedOpts is an alias for Opts. See there for doc comments.
+type UntypedOpts Opts
+
+// NewUntyped creates a new Untyped metric from the provided UntypedOpts.
+func NewUntyped(opts UntypedOpts) Untyped {
+	return newValue(NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		nil,
+		opts.ConstLabels,
+	), UntypedValue, 0)
+}
+
+// UntypedVec is a Collector that bundles a set of Untyped metrics that all
+// share the same Desc, but have different values for their variable
+// labels. This is used if you want to count the same thing partitioned by
+// various dimensions. Create instances with NewUntypedVec.
+type UntypedVec struct {
+	*MetricVec
+}
+
+// NewUntypedVec creates a new UntypedVec based on the provided UntypedOpts and
+// partitioned by the given label names. At least one label name must be
+// provided.
+func NewUntypedVec(opts UntypedOpts, labelNames []string) *UntypedVec {
+	desc := NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		labelNames,
+		opts.ConstLabels,
+	)
+	return &UntypedVec{
+		MetricVec: newMetricVec(desc, func(lvs ...string) Metric {
+			return newValue(desc, UntypedValue, 0, lvs...)
+		}),
+	}
+}
+
+// GetMetricWithLabelValues replaces the method of the same name in
+// MetricVec. The difference is that this method returns an Untyped and not a
+// Metric so that no type conversion is required.
+func (m *UntypedVec) GetMetricWithLabelValues(lvs ...string) (Untyped, error) {
+	metric, err := m.MetricVec.GetMetricWithLabelValues(lvs...)
+	if metric != nil {
+		return metric.(Untyped), err
+	}
+	return nil, err
+}
+
+// GetMetricWith replaces the method of the same name in MetricVec. The
+// difference is that this method returns an Untyped and not a Metric so that no
+// type conversion is required.
+func (m *UntypedVec) GetMetricWith(labels Labels) (Untyped, error) {
+	metric, err := m.MetricVec.GetMetricWith(labels)
+	if metric != nil {
+		return metric.(Untyped), err
+	}
+	return nil, err
+}
+
+// WithLabelValues works as GetMetricWithLabelValues, but panics where
+// GetMetricWithLabelValues would have returned an error. By not returning an
+// error, WithLabelValues allows shortcuts like
+//     myVec.WithLabelValues("404", "GET").Add(42)
+func (m *UntypedVec) WithLabelValues(lvs ...string) Untyped {
+	return m.MetricVec.WithLabelValues(lvs...).(Untyped)
+}
+
+// With works as GetMetricWith, but panics where GetMetricWithLabels would have
+// returned an error. By not returning an error, With allows shortcuts like
+//     myVec.With(Labels{"code": "404", "method": "GET"}).Add(42)
+func (m *UntypedVec) With(labels Labels) Untyped {
+	return m.MetricVec.With(labels).(Untyped)
+}
+
+// UntypedFunc is an Untyped whose value is determined at collect time by
+// calling a provided function.
+//
+// To create UntypedFunc instances, use NewUntypedFunc.
+type UntypedFunc interface {
+	Metric
+	Collector
+}
+
+// NewUntypedFunc creates a new UntypedFunc based on the provided
+// UntypedOpts. The value reported is determined by calling the given function
+// from within the Write method. Take into account that metric collection may
+// happen concurrently. If that results in concurrent calls to Write, like in
+// the case where an UntypedFunc is directly registered with Prometheus, the
+// provided function must be concurrency-safe.
+func NewUntypedFunc(opts UntypedOpts, function func() float64) UntypedFunc {
+	return newValueFunc(NewDesc(
+		BuildFQName(opts.Namespace, opts.Subsystem, opts.Name),
+		opts.Help,
+		nil,
+		opts.ConstLabels,
+	), UntypedValue, function)
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/value.go

@@ -0,0 +1,239 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"errors"
+	"fmt"
+	"math"
+	"sort"
+	"sync/atomic"
+	"time"
+
+	dto "github.com/prometheus/client_model/go"
+
+	"github.com/golang/protobuf/proto"
+)
+
+// ValueType is an enumeration of metric types that represent a simple value.
+type ValueType int
+
+// Possible values for the ValueType enum.
+const (
+	_ ValueType = iota
+	CounterValue
+	GaugeValue
+	UntypedValue
+)
+
+var errInconsistentCardinality = errors.New("inconsistent label cardinality")
+
+// value is a generic metric for simple values. It implements Metric, Collector,
+// Counter, Gauge, and Untyped. Its effective type is determined by
+// ValueType. This is a low-level building block used by the library to back the
+// implementations of Counter, Gauge, and Untyped.
+type value struct {
+	// valBits contains the bits of the represented float64 value. It has
+	// to go first in the struct to guarantee alignment for atomic
+	// operations.  http://golang.org/pkg/sync/atomic/#pkg-note-BUG
+	valBits uint64
+
+	selfCollector
+
+	desc       *Desc
+	valType    ValueType
+	labelPairs []*dto.LabelPair
+}
+
+// newValue returns a newly allocated value with the given Desc, ValueType,
+// sample value and label values. It panics if the number of label
+// values is different from the number of variable labels in Desc.
+func newValue(desc *Desc, valueType ValueType, val float64, labelValues ...string) *value {
+	if len(labelValues) != len(desc.variableLabels) {
+		panic(errInconsistentCardinality)
+	}
+	result := &value{
+		desc:       desc,
+		valType:    valueType,
+		valBits:    math.Float64bits(val),
+		labelPairs: makeLabelPairs(desc, labelValues),
+	}
+	result.init(result)
+	return result
+}
+
+func (v *value) Desc() *Desc {
+	return v.desc
+}
+
+func (v *value) Set(val float64) {
+	atomic.StoreUint64(&v.valBits, math.Float64bits(val))
+}
+
+func (v *value) SetToCurrentTime() {
+	v.Set(float64(time.Now().UnixNano()) / 1e9)
+}
+
+func (v *value) Inc() {
+	v.Add(1)
+}
+
+func (v *value) Dec() {
+	v.Add(-1)
+}
+
+func (v *value) Add(val float64) {
+	for {
+		oldBits := atomic.LoadUint64(&v.valBits)
+		newBits := math.Float64bits(math.Float64frombits(oldBits) + val)
+		if atomic.CompareAndSwapUint64(&v.valBits, oldBits, newBits) {
+			return
+		}
+	}
+}
+
+func (v *value) Sub(val float64) {
+	v.Add(val * -1)
+}
+
+func (v *value) Write(out *dto.Metric) error {
+	val := math.Float64frombits(atomic.LoadUint64(&v.valBits))
+	return populateMetric(v.valType, val, v.labelPairs, out)
+}
+
+// valueFunc is a generic metric for simple values retrieved on collect time
+// from a function. It implements Metric and Collector. Its effective type is
+// determined by ValueType. This is a low-level building block used by the
+// library to back the implementations of CounterFunc, GaugeFunc, and
+// UntypedFunc.
+type valueFunc struct {
+	selfCollector
+
+	desc       *Desc
+	valType    ValueType
+	function   func() float64
+	labelPairs []*dto.LabelPair
+}
+
+// newValueFunc returns a newly allocated valueFunc with the given Desc and
+// ValueType. The value reported is determined by calling the given function
+// from within the Write method. Take into account that metric collection may
+// happen concurrently. If that results in concurrent calls to Write, like in
+// the case where a valueFunc is directly registered with Prometheus, the
+// provided function must be concurrency-safe.
+func newValueFunc(desc *Desc, valueType ValueType, function func() float64) *valueFunc {
+	result := &valueFunc{
+		desc:       desc,
+		valType:    valueType,
+		function:   function,
+		labelPairs: makeLabelPairs(desc, nil),
+	}
+	result.init(result)
+	return result
+}
+
+func (v *valueFunc) Desc() *Desc {
+	return v.desc
+}
+
+func (v *valueFunc) Write(out *dto.Metric) error {
+	return populateMetric(v.valType, v.function(), v.labelPairs, out)
+}
+
+// NewConstMetric returns a metric with one fixed value that cannot be
+// changed. Users of this package will not have much use for it in regular
+// operations. However, when implementing custom Collectors, it is useful as a
+// throw-away metric that is generated on the fly to send it to Prometheus in
+// the Collect method. NewConstMetric returns an error if the length of
+// labelValues is not consistent with the variable labels in Desc.
+func NewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) (Metric, error) {
+	if len(desc.variableLabels) != len(labelValues) {
+		return nil, errInconsistentCardinality
+	}
+	return &constMetric{
+		desc:       desc,
+		valType:    valueType,
+		val:        value,
+		labelPairs: makeLabelPairs(desc, labelValues),
+	}, nil
+}
+
+// MustNewConstMetric is a version of NewConstMetric that panics where
+// NewConstMetric would have returned an error.
+func MustNewConstMetric(desc *Desc, valueType ValueType, value float64, labelValues ...string) Metric {
+	m, err := NewConstMetric(desc, valueType, value, labelValues...)
+	if err != nil {
+		panic(err)
+	}
+	return m
+}
+
+type constMetric struct {
+	desc       *Desc
+	valType    ValueType
+	val        float64
+	labelPairs []*dto.LabelPair
+}
+
+func (m *constMetric) Desc() *Desc {
+	return m.desc
+}
+
+func (m *constMetric) Write(out *dto.Metric) error {
+	return populateMetric(m.valType, m.val, m.labelPairs, out)
+}
+
+func populateMetric(
+	t ValueType,
+	v float64,
+	labelPairs []*dto.LabelPair,
+	m *dto.Metric,
+) error {
+	m.Label = labelPairs
+	switch t {
+	case CounterValue:
+		m.Counter = &dto.Counter{Value: proto.Float64(v)}
+	case GaugeValue:
+		m.Gauge = &dto.Gauge{Value: proto.Float64(v)}
+	case UntypedValue:
+		m.Untyped = &dto.Untyped{Value: proto.Float64(v)}
+	default:
+		return fmt.Errorf("encountered unknown type %v", t)
+	}
+	return nil
+}
+
+func makeLabelPairs(desc *Desc, labelValues []string) []*dto.LabelPair {
+	totalLen := len(desc.variableLabels) + len(desc.constLabelPairs)
+	if totalLen == 0 {
+		// Super fast path.
+		return nil
+	}
+	if len(desc.variableLabels) == 0 {
+		// Moderately fast path.
+		return desc.constLabelPairs
+	}
+	labelPairs := make([]*dto.LabelPair, 0, totalLen)
+	for i, n := range desc.variableLabels {
+		labelPairs = append(labelPairs, &dto.LabelPair{
+			Name:  proto.String(n),
+			Value: proto.String(labelValues[i]),
+		})
+	}
+	for _, lp := range desc.constLabelPairs {
+		labelPairs = append(labelPairs, lp)
+	}
+	sort.Sort(LabelPairSorter(labelPairs))
+	return labelPairs
+}

app/service/ops/log-agent/pkg/flowmonitor/counter/vec.go

@@ -0,0 +1,404 @@
+// Copyright 2014 The Prometheus Authors
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package prometheus
+
+import (
+	"fmt"
+	"sync"
+
+	"github.com/prometheus/common/model"
+)
+
+// MetricVec is a Collector to bundle metrics of the same name that
+// differ in their label values. MetricVec is usually not used directly but as a
+// building block for implementations of vectors of a given metric
+// type. GaugeVec, CounterVec, SummaryVec, and UntypedVec are examples already
+// provided in this package.
+type MetricVec struct {
+	mtx      sync.RWMutex // Protects the children.
+	children map[uint64][]metricWithLabelValues
+	desc     *Desc
+
+	newMetric   func(labelValues ...string) Metric
+	hashAdd     func(h uint64, s string) uint64 // replace hash function for testing collision handling
+	hashAddByte func(h uint64, b byte) uint64
+}
+
+// newMetricVec returns an initialized MetricVec. The concrete value is
+// returned for embedding into another struct.
+func newMetricVec(desc *Desc, newMetric func(lvs ...string) Metric) *MetricVec {
+	return &MetricVec{
+		children:    map[uint64][]metricWithLabelValues{},
+		desc:        desc,
+		newMetric:   newMetric,
+		hashAdd:     hashAdd,
+		hashAddByte: hashAddByte,
+	}
+}
+
+// metricWithLabelValues provides the metric and its label values for
+// disambiguation on hash collision.
+type metricWithLabelValues struct {
+	values []string
+	metric Metric
+}
+
+// Describe implements Collector. The length of the returned slice
+// is always one.
+func (m *MetricVec) Describe(ch chan<- *Desc) {
+	ch <- m.desc
+}
+
+// Collect implements Collector.
+func (m *MetricVec) Collect(ch chan<- Metric) {
+	m.mtx.RLock()
+	defer m.mtx.RUnlock()
+
+	for _, metrics := range m.children {
+		for _, metric := range metrics {
+			ch <- metric.metric
+		}
+	}
+}
+
+// GetMetricWithLabelValues returns the Metric for the given slice of label
+// values (same order as the VariableLabels in Desc). If that combination of
+// label values is accessed for the first time, a new Metric is created.
+//
+// It is possible to call this method without using the returned Metric to only
+// create the new Metric but leave it at its start value (e.g. a Summary or
+// Histogram without any observations). See also the SummaryVec example.
+//
+// Keeping the Metric for later use is possible (and should be considered if
+// performance is critical), but keep in mind that Reset, DeleteLabelValues and
+// Delete can be used to delete the Metric from the MetricVec. In that case, the
+// Metric will still exist, but it will not be exported anymore, even if a
+// Metric with the same label values is created later. See also the CounterVec
+// example.
+//
+// An error is returned if the number of label values is not the same as the
+// number of VariableLabels in Desc.
+//
+// Note that for more than one label value, this method is prone to mistakes
+// caused by an incorrect order of arguments. Consider GetMetricWith(Labels) as
+// an alternative to avoid that type of mistake. For higher label numbers, the
+// latter has a much more readable (albeit more verbose) syntax, but it comes
+// with a performance overhead (for creating and processing the Labels map).
+// See also the GaugeVec example.
+func (m *MetricVec) GetMetricWithLabelValues(lvs ...string) (Metric, error) {
+	h, err := m.hashLabelValues(lvs)
+	if err != nil {
+		return nil, err
+	}
+
+	return m.getOrCreateMetricWithLabelValues(h, lvs), nil
+}
+
+// GetMetricWith returns the Metric for the given Labels map (the label names
+// must match those of the VariableLabels in Desc). If that label map is
+// accessed for the first time, a new Metric is created. Implications of
+// creating a Metric without using it and keeping the Metric for later use are
+// the same as for GetMetricWithLabelValues.
+//
+// An error is returned if the number and names of the Labels are inconsistent
+// with those of the VariableLabels in Desc.
+//
+// This method is used for the same purpose as
+// GetMetricWithLabelValues(...string). See there for pros and cons of the two
+// methods.
+func (m *MetricVec) GetMetricWith(labels Labels) (Metric, error) {
+	h, err := m.hashLabels(labels)
+	if err != nil {
+		return nil, err
+	}
+
+	return m.getOrCreateMetricWithLabels(h, labels), nil
+}
+
+// WithLabelValues works as GetMetricWithLabelValues, but panics if an error
+// occurs. The method allows neat syntax like:
+//     httpReqs.WithLabelValues("404", "POST").Inc()
+func (m *MetricVec) WithLabelValues(lvs ...string) Metric {
+	metric, err := m.GetMetricWithLabelValues(lvs...)
+	if err != nil {
+		panic(err)
+	}
+	return metric
+}
+
+// With works as GetMetricWith, but panics if an error occurs. The method allows
+// neat syntax like:
+//     httpReqs.With(Labels{"status":"404", "method":"POST"}).Inc()
+func (m *MetricVec) With(labels Labels) Metric {
+	metric, err := m.GetMetricWith(labels)
+	if err != nil {
+		panic(err)
+	}
+	return metric
+}
+
+// DeleteLabelValues removes the metric where the variable labels are the same
+// as those passed in as labels (same order as the VariableLabels in Desc). It
+// returns true if a metric was deleted.
+//
+// It is not an error if the number of label values is not the same as the
+// number of VariableLabels in Desc.  However, such inconsistent label count can
+// never match an actual Metric, so the method will always return false in that
+// case.
+//
+// Note that for more than one label value, this method is prone to mistakes
+// caused by an incorrect order of arguments. Consider Delete(Labels) as an
+// alternative to avoid that type of mistake. For higher label numbers, the
+// latter has a much more readable (albeit more verbose) syntax, but it comes
+// with a performance overhead (for creating and processing the Labels map).
+// See also the CounterVec example.
+func (m *MetricVec) DeleteLabelValues(lvs ...string) bool {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+
+	h, err := m.hashLabelValues(lvs)
+	if err != nil {
+		return false
+	}
+	return m.deleteByHashWithLabelValues(h, lvs)
+}
+
+// Delete deletes the metric where the variable labels are the same as those
+// passed in as labels. It returns true if a metric was deleted.
+//
+// It is not an error if the number and names of the Labels are inconsistent
+// with those of the VariableLabels in the Desc of the MetricVec. However, such
+// inconsistent Labels can never match an actual Metric, so the method will
+// always return false in that case.
+//
+// This method is used for the same purpose as DeleteLabelValues(...string). See
+// there for pros and cons of the two methods.
+func (m *MetricVec) Delete(labels Labels) bool {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+
+	h, err := m.hashLabels(labels)
+	if err != nil {
+		return false
+	}
+
+	return m.deleteByHashWithLabels(h, labels)
+}
+
+// deleteByHashWithLabelValues removes the metric from the hash bucket h. If
+// there are multiple matches in the bucket, use lvs to select a metric and
+// remove only that metric.
+func (m *MetricVec) deleteByHashWithLabelValues(h uint64, lvs []string) bool {
+	metrics, ok := m.children[h]
+	if !ok {
+		return false
+	}
+
+	i := m.findMetricWithLabelValues(metrics, lvs)
+	if i >= len(metrics) {
+		return false
+	}
+
+	if len(metrics) > 1 {
+		m.children[h] = append(metrics[:i], metrics[i+1:]...)
+	} else {
+		delete(m.children, h)
+	}
+	return true
+}
+
+// deleteByHashWithLabels removes the metric from the hash bucket h. If there
+// are multiple matches in the bucket, use lvs to select a metric and remove
+// only that metric.
+func (m *MetricVec) deleteByHashWithLabels(h uint64, labels Labels) bool {
+	metrics, ok := m.children[h]
+	if !ok {
+		return false
+	}
+	i := m.findMetricWithLabels(metrics, labels)
+	if i >= len(metrics) {
+		return false
+	}
+
+	if len(metrics) > 1 {
+		m.children[h] = append(metrics[:i], metrics[i+1:]...)
+	} else {
+		delete(m.children, h)
+	}
+	return true
+}
+
+// Reset deletes all metrics in this vector.
+func (m *MetricVec) Reset() {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+
+	for h := range m.children {
+		delete(m.children, h)
+	}
+}
+
+func (m *MetricVec) hashLabelValues(vals []string) (uint64, error) {
+	if len(vals) != len(m.desc.variableLabels) {
+		return 0, errInconsistentCardinality
+	}
+	h := hashNew()
+	for _, val := range vals {
+		h = m.hashAdd(h, val)
+		h = m.hashAddByte(h, model.SeparatorByte)
+	}
+	return h, nil
+}
+
+func (m *MetricVec) hashLabels(labels Labels) (uint64, error) {
+	if len(labels) != len(m.desc.variableLabels) {
+		return 0, errInconsistentCardinality
+	}
+	h := hashNew()
+	for _, label := range m.desc.variableLabels {
+		val, ok := labels[label]
+		if !ok {
+			return 0, fmt.Errorf("label name %q missing in label map", label)
+		}
+		h = m.hashAdd(h, val)
+		h = m.hashAddByte(h, model.SeparatorByte)
+	}
+	return h, nil
+}
+
+// getOrCreateMetricWithLabelValues retrieves the metric by hash and label value
+// or creates it and returns the new one.
+//
+// This function holds the mutex.
+func (m *MetricVec) getOrCreateMetricWithLabelValues(hash uint64, lvs []string) Metric {
+	m.mtx.RLock()
+	metric, ok := m.getMetricWithLabelValues(hash, lvs)
+	m.mtx.RUnlock()
+	if ok {
+		return metric
+	}
+
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	metric, ok = m.getMetricWithLabelValues(hash, lvs)
+	if !ok {
+		// Copy to avoid allocation in case wo don't go down this code path.
+		copiedLVs := make([]string, len(lvs))
+		copy(copiedLVs, lvs)
+		metric = m.newMetric(copiedLVs...)
+		m.children[hash] = append(m.children[hash], metricWithLabelValues{values: copiedLVs, metric: metric})
+	}
+	return metric
+}
+
+// getOrCreateMetricWithLabelValues retrieves the metric by hash and label value
+// or creates it and returns the new one.
+//
+// This function holds the mutex.
+func (m *MetricVec) getOrCreateMetricWithLabels(hash uint64, labels Labels) Metric {
+	m.mtx.RLock()
+	metric, ok := m.getMetricWithLabels(hash, labels)
+	m.mtx.RUnlock()
+	if ok {
+		return metric
+	}
+
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	metric, ok = m.getMetricWithLabels(hash, labels)
+	if !ok {
+		lvs := m.extractLabelValues(labels)
+		metric = m.newMetric(lvs...)
+		m.children[hash] = append(m.children[hash], metricWithLabelValues{values: lvs, metric: metric})
+	}
+	return metric
+}
+
+// getMetricWithLabelValues gets a metric while handling possible collisions in
+// the hash space. Must be called while holding read mutex.
+func (m *MetricVec) getMetricWithLabelValues(h uint64, lvs []string) (Metric, bool) {
+	metrics, ok := m.children[h]
+	if ok {
+		if i := m.findMetricWithLabelValues(metrics, lvs); i < len(metrics) {
+			return metrics[i].metric, true
+		}
+	}
+	return nil, false
+}
+
+// getMetricWithLabels gets a metric while handling possible collisions in
+// the hash space. Must be called while holding read mutex.
+func (m *MetricVec) getMetricWithLabels(h uint64, labels Labels) (Metric, bool) {
+	metrics, ok := m.children[h]
+	if ok {
+		if i := m.findMetricWithLabels(metrics, labels); i < len(metrics) {
+			return metrics[i].metric, true
+		}
+	}
+	return nil, false
+}
+
+// findMetricWithLabelValues returns the index of the matching metric or
+// len(metrics) if not found.
+func (m *MetricVec) findMetricWithLabelValues(metrics []metricWithLabelValues, lvs []string) int {
+	for i, metric := range metrics {
+		if m.matchLabelValues(metric.values, lvs) {
+			return i
+		}
+	}
+	return len(metrics)
+}
+
+// findMetricWithLabels returns the index of the matching metric or len(metrics)
+// if not found.
+func (m *MetricVec) findMetricWithLabels(metrics []metricWithLabelValues, labels Labels) int {
+	for i, metric := range metrics {
+		if m.matchLabels(metric.values, labels) {
+			return i
+		}
+	}
+	return len(metrics)
+}
+
+func (m *MetricVec) matchLabelValues(values []string, lvs []string) bool {
+	if len(values) != len(lvs) {
+		return false
+	}
+	for i, v := range values {
+		if v != lvs[i] {
+			return false
+		}
+	}
+	return true
+}
+
+func (m *MetricVec) matchLabels(values []string, labels Labels) bool {
+	if len(labels) != len(values) {
+		return false
+	}
+	for i, k := range m.desc.variableLabels {
+		if values[i] != labels[k] {
+			return false
+		}
+	}
+	return true
+}
+
+func (m *MetricVec) extractLabelValues(labels Labels) []string {
+	labelValues := make([]string, len(labels))
+	for i, k := range m.desc.variableLabels {
+		labelValues[i] = labels[k]
+	}
+	return labelValues
+}

app/service/ops/log-agent/pkg/flowmonitor/flowmonitor.go

@@ -0,0 +1,171 @@
+package flowmonitor
+
+import (
+	"errors"
+	"time"
+	"os"
+	"strconv"
+	"encoding/json"
+	"net"
+
+	"go-common/library/log"
+	"go-common/app/service/ops/log-agent/pkg/flowmonitor/counter"
+	"go-common/app/service/ops/log-agent/event"
+)
+
+type FlowMonitor struct {
+	conf                *Config
+	monitorLogChanSize  int
+	currentVer          int
+	verMap              map[int]*prometheus.CounterVec
+	flowMonitorThrottle bool
+	indexName           string
+	conn                net.Conn
+}
+
+var argsError = errors.New("appId timeRangeKey status must be specified")
+var throttledError = errors.New("flow monitor is throttled")
+var notInitError = errors.New("flow monitor does not init")
+var Fm *FlowMonitor
+
+// InitFlowMonitor init flow monitor
+func InitFlowMonitor(conf *Config) (err error) {
+	fm := new(FlowMonitor)
+	fm.conf = conf
+	if err = fm.checkConfig(); err != nil {
+		return err
+	}
+	fm.flowMonitorThrottle = false
+	fm.verMap = make(map[int]*prometheus.CounterVec)
+	ver1 := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "FlowMonitor1", Help: "help"}, []string{"appId", "timeRangeKey", "source", "kind", "status"})
+	prometheus.MustRegister(ver1)
+	fm.verMap[0] = ver1
+	ver2 := prometheus.NewCounterVec(prometheus.CounterOpts{Name: "FlowMonitor2", Help: "help"}, []string{"appId", "timeRangeKey", "source", "kind", "status"})
+	prometheus.MustRegister(ver2)
+	fm.verMap[1] = ver2
+	fm.currentVer = 0
+	fm.newConn()
+	go fm.flowmonitorreport()
+	Fm = fm
+	return nil
+}
+
+// getCurrentVer get the ver being used
+func (fm *FlowMonitor) getCurrentVer() *prometheus.CounterVec {
+	return fm.verMap[fm.currentVer]
+}
+
+// rollVer roll to the next ver
+func (fm *FlowMonitor) rollVer() {
+	fm.currentVer = (fm.currentVer + 1) % 2
+}
+
+func (fm *FlowMonitor) AddEvent(e *event.ProcessorEvent, source string, kind string, status string) {
+	if len(e.AppId) != 0 {
+		fm.Add(string(e.AppId), source, e.TimeRangeKey, kind, status)
+	}
+	fm.Add(e.LogId, source, e.TimeRangeKey, kind, status)
+}
+
+// Add do the metric
+func (fm *FlowMonitor) Add(appId string, source string, timeRangeKey string, kind string, status string) (error) {
+	if fm == nil {
+		return notInitError
+	}
+	if fm.flowMonitorThrottle {
+		return throttledError
+	}
+
+	if appId == "" || source == "" || status == "" {
+		return argsError
+	}
+
+	if timeRangeKey == "" {
+		timeRangeKey = strconv.FormatInt(time.Now().Unix()/100*100, 10)
+	}
+
+	if counter, err := fm.getCurrentVer().GetMetricWithLabelValues(appId, timeRangeKey, source, kind, status); err != nil {
+		return err
+	} else {
+		counter.Inc()
+		return nil
+	}
+}
+
+// readVec read metrics from one vec
+func (fm *FlowMonitor) readVec(ver *prometheus.CounterVec) error {
+	if fm.conn == nil {
+		if err := fm.newConn(); err != nil {
+			return err
+		}
+	}
+	metrics := make(chan prometheus.Metric)
+	go func() {
+		ver.Collect(metrics)
+		close(metrics)
+	}()
+
+	hostname, _ := os.Hostname()
+	var ignore_metric bool = false
+	for {
+		select {
+		case metric, ok := <-metrics:
+			if !ok {
+				return nil
+			}
+			if ignore_metric {
+				continue
+			}
+			data := make(map[string]interface{})
+			for _, label := range metric.(prometheus.Counter).Lables() {
+				data[*label.Name] = *label.Value
+			}
+			if timeRangeKey, ok := data["timeRangeKey"]; ok {
+				timeint64, _ := strconv.ParseInt(timeRangeKey.(string), 10, 64)
+				data["time"] = time.Unix(timeint64, 0).UTC().Format("2006-01-02T15:04:05")
+			}
+			data["hostname"] = hostname
+			data["counter"] = metric.(prometheus.Counter).Value()
+			if dataSend, err := json.Marshal(data); err == nil {
+				dataSend = append(dataSend, []byte("\n")...)
+				n, err := fm.conn.Write(dataSend)
+				if err == nil && n < len(dataSend) {
+					log.Error("Error: flow monitor write error: short write")
+				}
+				if err != nil {
+					log.Error("Error: flow monitor write error: %v", err)
+					fm.conn.Close()
+					fm.conn = nil
+					// if conn write error, just ignore. ver.Collect must be finished or RLock will not be released
+					ignore_metric = true
+				}
+			}
+		}
+	}
+}
+
+// linkmonitorreport report link monitor data periodicity
+func (fm *FlowMonitor) flowmonitorreport() {
+	for {
+		time.Sleep(time.Duration(fm.conf.Interval))
+		currentVer := fm.getCurrentVer()
+		fm.rollVer()
+		fm.readVec(currentVer)
+		currentVer.Reset()
+	}
+}
+
+// newConn make a conn to logstash(monitor data receiver)
+func (fm *FlowMonitor) newConn() error {
+	conn, err := net.DialTimeout("tcp", fm.conf.Addr, time.Duration(time.Second*5))
+	if err == nil && conn != nil {
+		fm.conn = conn
+		fm.flowMonitorThrottle = false
+		log.Info("init flow monitor conn to: %s", fm.conf.Addr)
+		return nil
+	} else {
+		log.Error("flow monitor conn failed: %s: %v", fm.conf.Addr, err)
+		fm.flowMonitorThrottle = true
+		return err
+	}
+}

app/service/ops/log-agent/pkg/httpstream/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "httpstream.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pkg/httpstream",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//app/service/ops/log-agent/event:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/httpstream/conf.go

@@ -0,0 +1,20 @@
+package httpstream
+
+import (
+	"errors"
+)
+
+type Config struct {
+	Addr string `toml:"addr"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of Sock Input is nil")
+	}
+
+	if c.Addr == "" {
+		c.Addr = ":18123"
+	}
+	return nil
+}

app/service/ops/log-agent/pkg/httpstream/httpstream.go

@@ -0,0 +1,144 @@
+package httpstream
+
+import (
+	"net/http"
+	"sync"
+	"regexp"
+	"bytes"
+	"strconv"
+	"context"
+	"go-common/app/service/ops/log-agent/event"
+)
+
+type HttpStream struct {
+	c          *Config
+	logstreams map[chan *event.ProcessorEvent]*filterRule
+	l          sync.Mutex
+}
+type filterRule struct {
+	maxLines   int
+	appId      string
+	instanceId string
+	reg        *regexp.Regexp
+}
+
+var LogSourceChan = make(chan *event.ProcessorEvent)
+
+// initlogStream init log stream
+func NewHttpStream(config *Config) (httpStream *HttpStream, err error) {
+	h := new(HttpStream)
+	if err := config.ConfigValidate(); err != nil {
+		return nil, err
+	}
+	h.c = config
+	h.logstreams = make(map[chan *event.ProcessorEvent]*filterRule)
+	http.HandleFunc("/logs", h.LogStreamer())
+	go h.route()
+	go http.ListenAndServe(h.c.Addr, nil)
+	return h, nil
+}
+
+// route 把日志路由到所有注册的logstream
+func (s *HttpStream) route() {
+	for buf := range LogSourceChan {
+		for logstream, _ := range s.logstreams {
+			logstream <- buf
+		}
+	}
+}
+
+// LogStreamer 接收请求
+func (s *HttpStream) LogStreamer() func(w http.ResponseWriter, req *http.Request) {
+	logsHandler := func(w http.ResponseWriter, req *http.Request) {
+		logstream := make(chan *event.ProcessorEvent)
+		f := new(filterRule)
+		// parse params
+		params := req.URL.Query()
+		if appId, ok := params["app_id"]; ok {
+			f.appId = appId[0]
+		} else {
+			w.Write(append([]byte("必须指定app_id"), '\n'))
+			return
+		}
+		if reg, ok := params["regexp"]; ok {
+			if filterReg, err := regexp.Compile(reg[0]); err == nil {
+				f.reg = filterReg
+			} else {
+				w.Write(append([]byte("正则表达式格式错误"), '\n'))
+				return
+			}
+		}
+		if instanceId, ok := params["instance_id"]; ok {
+			f.instanceId = instanceId[0]
+		}
+
+		if maxLines, ok := params["max_lines"]; ok {
+			if n, err := strconv.Atoi(maxLines[0]); err == nil {
+				f.maxLines = n
+			} else {
+				w.Write(append([]byte("max_lines格式错误"), '\n'))
+				return
+			}
+		}
+
+		s.add(logstream, f)
+		go func() {
+			select {
+			case <-req.Context().Done():
+				s.remove(logstream)
+			}
+		}()
+		defer s.httpStreamer(req.Context(), w, req, logstream, f)
+	}
+	return logsHandler
+}
+
+// httpStreamer 过滤并输出日志
+func (s *HttpStream) httpStreamer(ctx context.Context, w http.ResponseWriter, req *http.Request, logstream chan *event.ProcessorEvent, f *filterRule) {
+	w.Header().Add("Content-Type", "text/plain; charset=utf-8")
+	if f.instanceId != "" {
+		//w.Write([]byte(fmt.Sprintf(
+		//	"\x1b[1;31m%s\x1b[0m\n", "注意：caster中，只有(1)app_id满足服务树三级格式 (2)日志包含instance_id且值为实例名称 的情况下，日志才能输出\n")))
+		w.Write([]byte("注意：caster中，只有(1)app_id满足服务树三级格式 (2)日志包含instance_id且值为实例名称 的情况下，日志才能输出                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                     \n"))
+		w.(http.Flusher).Flush()
+	}
+	c := 0
+	for {
+		select {
+		case e := <-logstream:
+			if f.appId != "" && string(e.AppId) != f.appId {
+				continue
+			}
+			if f.reg != nil && !f.reg.Match(e.Bytes()) {
+				continue
+			}
+			if f.instanceId != "" && !bytes.Contains(e.Bytes(), []byte(f.instanceId)) {
+				continue
+			}
+			if f.maxLines != 0 && c >= f.maxLines {
+				s.remove(logstream)
+				return
+			}
+			c += 1
+			// TODO event recycle
+			w.Write(append(e.Bytes(), '\n'))
+			w.(http.Flusher).Flush()
+		case <-ctx.Done():
+			return
+		}
+	}
+}
+
+// add 注册logstream
+func (s *HttpStream) add(logstream chan *event.ProcessorEvent, f *filterRule) {
+	s.l.Lock()
+	defer s.l.Unlock()
+	s.logstreams[logstream] = f
+}
+
+// remove 注销logstream
+func (s *HttpStream) remove(logstream chan *event.ProcessorEvent) {
+	s.l.Lock()
+	defer s.l.Unlock()
+	delete(s.logstreams, logstream)
+}

app/service/ops/log-agent/pkg/lancermonitor/BUILD

@@ -0,0 +1,32 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "conf.go",
+        "lancermonitor.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pkg/lancermonitor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = ["//library/log:go_default_library"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/lancermonitor/conf.go

@@ -0,0 +1,20 @@
+package lancermonitor
+
+import (
+	"errors"
+)
+
+type Config struct {
+	Addr string `toml:"addr"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return errors.New("config of LancerMonitor is nil")
+	}
+
+	if c.Addr == "" {
+		return errors.New("addr of LancerMonitor can't be nil")
+	}
+	return nil
+}

app/service/ops/log-agent/pkg/lancermonitor/lancermonitor.go

@@ -0,0 +1,121 @@
+package lancermonitor
+
+import (
+	"time"
+	"net"
+	"sync"
+	"fmt"
+	"strconv"
+	"strings"
+	"errors"
+	"go-common/library/log"
+)
+
+const (
+	_separator = "####"
+)
+
+var (
+	lm      *LancerMonitor
+	started bool
+)
+
+type LancerMonitor struct {
+	c                *Config
+	logRevStatusLock sync.Mutex
+	logRevStatus     map[string]int64
+	ipAddr           string
+}
+
+func InitLancerMonitor(config *Config) (l *LancerMonitor, err error) {
+	if started {
+		return nil, errors.New("lancer Monitor can only be init Once")
+	}
+	if err = config.ConfigValidate(); err != nil {
+		return nil, err
+	}
+
+	l = new(LancerMonitor)
+	l.c = config
+	l.logRevStatus = make(map[string]int64)
+	l.ipAddr = InternalIP()
+
+	go l.reportStatus()
+	started = true
+	lm = l
+	return l, nil
+}
+
+func (l *LancerMonitor) reportStatus() {
+	reportStatusTk := time.Tick(time.Duration(60 * time.Second))
+	for {
+		select {
+		case <-reportStatusTk:
+			logCount := l.getLogCount()
+			conn, error := net.DialTimeout("tcp", l.c.Addr, time.Second*5)
+			if error != nil {
+				log.Error("failed to connect to lancer when report status")
+			} else {
+				for k, v := range logCount {
+					fields := strings.Split(k, _separator)
+					if len(fields) == 2 {
+						fmt.Fprintf(conn, fields[0]+"\u0001"+strconv.FormatInt(v, 10)+"\u0001"+l.ipAddr+"\u0001"+strconv.FormatInt(time.Now().UnixNano()/int64(time.Millisecond), 10)+"\u0001"+fields[1]+"\u0001\u0001")
+					}
+				}
+				log.Info("report status to lancer")
+				conn.Close()
+			}
+		}
+	}
+}
+
+// InternalIP get internal ip.
+func InternalIP() string {
+	inters, err := net.Interfaces()
+	if err != nil {
+		return ""
+	}
+	for _, inter := range inters {
+		if !strings.HasPrefix(inter.Name, "lo") {
+			addrs, err := inter.Addrs()
+			if err != nil {
+				continue
+			}
+			for _, addr := range addrs {
+				if ipnet, ok := addr.(*net.IPNet); ok && !ipnet.IP.IsLoopback() {
+					if ipnet.IP.To4() != nil {
+						return ipnet.IP.String()
+					}
+				}
+			}
+		}
+	}
+	return ""
+}
+
+//get log count of each logid since last call
+func (l *LancerMonitor) getLogCount() map[string]int64 {
+	l.logRevStatusLock.Lock()
+	defer l.logRevStatusLock.Unlock()
+	logRevSendStatus := make(map[string]int64)
+	for k, v := range l.logRevStatus {
+		logRevSendStatus[k] = v
+	}
+	for k := range l.logRevStatus {
+		delete(l.logRevStatus, k)
+	}
+	return logRevSendStatus
+}
+
+func IncreaseLogCount(name string, logId string) {
+	if lm == nil || !started {
+		return
+	}
+	if name == "" || logId == "" {
+		return
+	}
+	key := name + _separator + logId
+	lm.logRevStatusLock.Lock()
+	defer lm.logRevStatusLock.Unlock()
+	lm.logRevStatus[key] += 1
+}

app/service/ops/log-agent/pkg/lancerroute/BUILD

@@ -0,0 +1,36 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "config.go",
+        "lancerroute.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/pkg/lancerroute",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/lancerroute/config.go

@@ -0,0 +1,60 @@
+package lancerroute
+
+import (
+	"time"
+	"fmt"
+
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	lancerRouteFile = "lancerRoute.toml"
+)
+
+type Config struct {
+	LancerRoute map[string]string `toml:"lancerroute"`
+}
+
+func (l *Lancerroute) InitConfig() (err error) {
+	// read config from config center
+	if err = l.readConfig(); err != nil {
+		return err
+	}
+	// watch update and reload config
+	go func() {
+		currentVersion := configcenter.Version
+		for {
+			if currentVersion != configcenter.Version {
+				log.Info("lancer route config reload")
+				if err := l.readConfig(); err != nil {
+					log.Error("lancer route config reload error (%v", err)
+				}
+				currentVersion = configcenter.Version
+			}
+			time.Sleep(time.Second)
+		}
+	}()
+
+	return nil
+}
+
+func (l *Lancerroute) readConfig() (err error) {
+	var (
+		ok             bool
+		value          string
+		tmpLancerRoute Config
+	)
+
+	// sample config
+	if value, ok = configcenter.Client.Value(lancerRouteFile); !ok {
+		return fmt.Errorf("failed to get %s", lancerRouteFile)
+	}
+	if _, err = toml.Decode(value, &tmpLancerRoute); err != nil {
+		return err
+	}
+	l.c = &tmpLancerRoute
+	return nil
+}

app/service/ops/log-agent/pkg/lancerroute/lancerroute.go

@@ -0,0 +1,20 @@
+package lancerroute
+
+type Lancerroute struct {
+	c *Config
+}
+
+var route *Lancerroute
+
+func InitLancerRoute() error {
+	route = new(Lancerroute)
+	return route.InitConfig()
+}
+
+func GetLancerByLogid(logId string) string {
+	if d, ok := route.c.LancerRoute[logId]; ok {
+		return d
+	}
+	// lancer-common by default
+	return "lancer-common"
+}

app/service/ops/log-agent/pkg/limit/BUILD

@@ -0,0 +1,28 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["limit.go"],
+    importpath = "go-common/app/service/ops/log-agent/pkg/limit",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/pkg/limit/limit.go

@@ -0,0 +1,134 @@
+package limit
+
+import (
+	"errors"
+	"os"
+	"fmt"
+	"io/ioutil"
+	"strconv"
+)
+
+var (
+	LimitConfNil          = errors.New("Config of resource limit is nil")
+	LimitConfError        = errors.New("LimitConfError")
+	CgroupPathNotExist    = errors.New("Cgroup Path Not Exist")
+	MemCgroupPathNotExist = errors.New("Mem subgroup Not Exist")
+	CpuCgroupPathNotExist = errors.New("Cpu subgroup Not Exist")
+)
+
+type LimitConf struct {
+	AppName         string
+	CgroupPath      string
+	LimitMemMB      int
+	LimitMemEnabled bool
+	LimitCpuCore    int
+	LimitCpuEnabled bool
+}
+
+type Limit struct {
+	c *LimitConf
+}
+
+// LimitRes init Limit
+func LimitRes(c *LimitConf) (l *Limit, err error) {
+	if c == nil {
+		return nil, LimitConfError
+	}
+	l = new(Limit)
+	l.c = c
+
+	if c.AppName == "" {
+		return nil, fmt.Errorf("AppName can't be nil")
+	}
+
+	if err = pathExists(l.c.CgroupPath, false); err != nil {
+		return nil, CgroupPathNotExist
+	}
+	if l.c.LimitMemEnabled {
+		if l.c.LimitCpuCore <= 0 {
+			return nil, fmt.Errorf("LimitCpuCore must be greater than 0")
+		}
+		if err = l.limitMem(); err != nil {
+			return nil, err
+		}
+	}
+	if l.c.LimitCpuEnabled {
+		if l.c.LimitMemMB <= 0 {
+			return nil, fmt.Errorf("LimitMemMB must be greater than 0")
+		}
+		if err = l.limitCpu(); err != nil {
+			return nil, err
+		}
+	}
+	return
+}
+
+// limitMem limit memory by memory.limit_in_bytes
+func (l *Limit) limitMem() (err error) {
+	if err = pathExists(fmt.Sprintf("%s/memory", l.c.CgroupPath), false); err != nil {
+		return MemCgroupPathNotExist
+	}
+
+	memPath := fmt.Sprintf("%s/memory/%s/", l.c.CgroupPath, l.c.AppName)
+	if err = pathExists(memPath, true); err != nil {
+		return err
+	}
+	// cgroup.procs
+	pidPath := memPath + "cgroup.procs"
+	if err = pathExists(pidPath, false); err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(pidPath, []byte(strconv.Itoa(os.Getegid())), 0644); err != nil {
+		return err
+	}
+	// memory.limit_in_bytes
+	limitPath := memPath + "memory.limit_in_bytes"
+	if err = pathExists(limitPath, false); err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(limitPath, []byte(fmt.Sprintf("%sM", strconv.Itoa(l.c.LimitMemMB))), 0644); err != nil {
+		return err
+	}
+	return
+}
+
+// limitCpu limit cpu by cpu.cfs_quota_us and cpu.cfs_period_us
+func (l *Limit) limitCpu() (err error) {
+	if err = pathExists(fmt.Sprintf("%s/cpu,cpuacct", l.c.CgroupPath), false); err != nil {
+		return CpuCgroupPathNotExist
+	}
+
+	cpuPath := fmt.Sprintf("%s/cpu,cpuacct/%s/", l.c.CgroupPath, l.c.AppName)
+	if err = pathExists(cpuPath, true); err != nil {
+		return err
+	}
+	// cpu.cfs_quota_us
+	quotaPath := cpuPath + "cpu.cfs_quota_us"
+	if err = pathExists(quotaPath, false); err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(quotaPath, []byte(strconv.Itoa(10000)), 0644); err != nil {
+		return err
+	}
+	// cpu.cfs_period_us
+	periodPath := cpuPath + "cpu.cfs_period_us"
+	if err = pathExists(periodPath, false); err != nil {
+		return err
+	}
+	if err = ioutil.WriteFile(periodPath, []byte(fmt.Sprintf("%s", strconv.Itoa(10000*l.c.LimitCpuCore))), 0644); err != nil {
+		return err
+	}
+	return
+}
+
+// pathExists check if path exist
+func pathExists(path string, create bool) (err error) {
+	if _, err = os.Stat(path); err == nil {
+		return
+	}
+
+	if os.IsNotExist(err) && create == true {
+		return os.MkdirAll(path, os.ModePerm)
+	}
+	return
+}

app/service/ops/log-agent/processor/BUILD

@@ -0,0 +1,42 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = ["processor.go"],
+    importpath = "go-common/app/service/ops/log-agent/processor",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/output:go_default_library",
+        "//library/log:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [
+        ":package-srcs",
+        "//app/service/ops/log-agent/processor/classify:all-srcs",
+        "//app/service/ops/log-agent/processor/fileLog:all-srcs",
+        "//app/service/ops/log-agent/processor/grok:all-srcs",
+        "//app/service/ops/log-agent/processor/httpstream:all-srcs",
+        "//app/service/ops/log-agent/processor/jsonLog:all-srcs",
+        "//app/service/ops/log-agent/processor/lengthCheck:all-srcs",
+        "//app/service/ops/log-agent/processor/sample:all-srcs",
+    ],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/classify/BUILD

@@ -0,0 +1,39 @@
+package(default_visibility = ["//visibility:public"])
+
+load(
+    "@io_bazel_rules_go//go:def.bzl",
+    "go_library",
+)
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "classify.go",
+        "config.go",
+    ],
+    importpath = "go-common/app/service/ops/log-agent/processor/classify",
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+    deps = [
+        "//app/service/ops/log-agent/conf/configcenter:go_default_library",
+        "//app/service/ops/log-agent/event:go_default_library",
+        "//app/service/ops/log-agent/pkg/common:go_default_library",
+        "//app/service/ops/log-agent/processor:go_default_library",
+        "//library/log:go_default_library",
+        "//vendor/github.com/BurntSushi/toml:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

app/service/ops/log-agent/processor/classify/classify.go

@@ -0,0 +1,80 @@
+package classify
+
+import (
+	"strings"
+	"context"
+	"go-common/app/service/ops/log-agent/event"
+	"go-common/app/service/ops/log-agent/processor"
+	"go-common/app/service/ops/log-agent/pkg/common"
+)
+
+type Classify struct {
+	c *Config
+}
+
+func init() {
+	err := processor.Register("classify", Process)
+	if err != nil {
+		panic(err)
+	}
+}
+
+func Process(ctx context.Context, config interface{}, input <-chan *event.ProcessorEvent) (output chan *event.ProcessorEvent, err error) {
+	classify := new(Classify)
+
+	if c, ok := config.(*Config); !ok {
+		panic("Error config for Classify Processor")
+	} else {
+		if err = c.ConfigValidate(); err != nil {
+			return nil, err
+		}
+		classify.c = c
+	}
+
+	output = make(chan *event.ProcessorEvent)
+	go func() {
+		for {
+			select {
+			case e := <-input:
+				// only do classify for ops-log
+				if e.Destination == "lancer-ops-log" {
+					if common.CriticalLog(e.Level) || e.Priority == "high" {
+						e.LogId = classify.getLogIdByLevel("important")
+					} else {
+						e.LogId = classify.getLogIdByAppId(e.AppId)
+					}
+				}
+				output <- e
+			case <-ctx.Done():
+				return
+			}
+		}
+	}()
+	return output, nil
+}
+
+// getLogLevel get logId level by appId
+func (c *Classify) getLogIdByAppId(appId []byte) (logId string) {
+	// get logId by setting
+	if logLevel, ok := c.c.LogLevelMapConfig[string(appId)]; ok {
+		return c.getLogIdByLevel(logLevel)
+	}
+
+	// appId format error, logId 1
+	if len(strings.Split(string(appId), ".")) < 3 {
+		return c.getLogIdByLevel("low") // low level
+	}
+
+	// set logLevel to 2 by default
+	return c.getLogIdByLevel("normal") // normal level
+}
+
+// getLogIdByLevel return logid by level
+func (c *Classify) getLogIdByLevel(level string) (logId string) {
+	if logId, ok := c.c.LogIdMapConfig[level]; ok {
+		return logId
+	} else {
+		// return 000161 by default
+		return "000161"
+	}
+}

app/service/ops/log-agent/processor/classify/config.go

@@ -0,0 +1,101 @@
+package classify
+
+import (
+	"fmt"
+	"errors"
+	"time"
+
+	"go-common/app/service/ops/log-agent/conf/configcenter"
+	"go-common/library/log"
+
+	"github.com/BurntSushi/toml"
+)
+
+const (
+	logLevelMap = "logLevelMap.toml"
+	logIdMap    = "logIdMap.toml"
+)
+
+type Config struct {
+	Local             bool              `toml:"local"`
+	LogLevelMapConfig map[string]string `toml:"logLevelMapConfig"`
+	LogIdMapConfig    map[string]string `toml:"logIdMapConfig"`
+	PriorityBlackList map[string]string `toml:"priorityBlackList"`
+}
+
+func (c *Config) ConfigValidate() (error) {
+	if c == nil {
+		return fmt.Errorf("Error can't be nil")
+	}
+	if c.LogLevelMapConfig == nil {
+		c.LogLevelMapConfig = make(map[string]string)
+	}
+
+	if c.LogIdMapConfig == nil {
+		return fmt.Errorf("LogIdMapConfig of classify can't be nil")
+	}
+
+	if c.PriorityBlackList == nil {
+		c.PriorityBlackList = make(map[string]string)
+	}
+	return nil
+}
+
+func DecodeConfig(md toml.MetaData, primValue toml.Primitive) (c interface{}, err error) {
+	config := new(Config)
+	if err = md.PrimitiveDecode(primValue, config); err != nil {
+		return nil, err
+	}
+
+	// read config from config center
+	if !config.Local {
+		if err = config.readConfig(); err != nil {
+			return nil, err
+		}
+		// watch update and reload config
+		go func() {
+			currentVersion := configcenter.Version
+			for {
+				if currentVersion != configcenter.Version {
+					log.Info("classify config reload")
+					if err := config.readConfig(); err != nil {
+						log.Error("classify config reload error (%v)", err)
+					}
+					currentVersion = configcenter.Version
+				}
+				time.Sleep(time.Second)
+			}
+		}()
+	}
+
+	return config, nil
+}
+
+func (c *Config) readConfig() (err error) {
+	var (
+		ok             bool
+		value          string
+		tmplogLevelMap map[string]string
+		tmplogIdMap    map[string]string
+	)
+
+	// logLevel config
+	if value, ok = configcenter.Client.Value(logLevelMap); !ok {
+		return errors.New("failed to get logLevelMap.toml")
+	}
+	if _, err = toml.Decode(value, &tmplogLevelMap); err != nil {
+		return err
+	}
+	c.LogLevelMapConfig = tmplogLevelMap
+
+	// logIdMap config
+	if value, ok = configcenter.Client.Value(logIdMap); !ok {
+		return errors.New("failed to get logIdMap.toml")
+	}
+	if _, err = toml.Decode(value, &tmplogIdMap); err != nil {
+		return err
+	}
+	c.LogIdMapConfig = tmplogIdMap
+
+	return nil
+}