Initial commit

.rider/.gometalinter.json

@@ -0,0 +1,19 @@
+{
+  "Linters": {},
+  "Vendor": true,
+  "Deadline": "5m",
+  "Cyclo": 50,
+  "Enable": [
+    "gofmt",
+    "vet",
+    "golint",
+    "vetshadow",
+    "gocyclo",
+    "deadcode"
+  ],
+  "Exclude": [
+    ".+\\.pb\\.go",
+    ".+\\.pb\\.gw\\.go"
+  ],
+  "MinConfidence": 0.81
+}

.rider/CHANGELOG.md

@@ -0,0 +1,92 @@
+## .rider
+
+###Version 2.20
+1. 去除megacheck, unused, gosimple三项
+
+###Version 2.19
+1. ut开放library目录执行
+
+###Version 2.18
+1. 更新unit_test.sh的ReadDir()逻辑
+
+###Version 2.17
+1. sagacheck 添加bgr工具作为规则引擎
+
+###Version 2.16
+1. bazel coverage --config=ci 
+
+###Version 2.15
+1. ut增加自定义lint检查
+2. ut 覆盖率文件过滤monkey.go文件
+3. ut magic流程增加评论用户校验
+
+###Version 2.14
+1. ut执行增加 convey json.
+2. ut执行增加过滤器
+
+###Version 2.13
+1. ut脚本增加覆盖率原始文件上传
+
+### Version 2.12
+1. ut脚本去除包含多重子目录测试结果冗余问题
+ 
+### Version 2.11
+1. ut脚本修复未建mr跳过测试
+
+### Version 2.10
+1. ut脚本magic方法bug修复
+
+### Version 2.9
+1. ut脚本bazel test增加超时及测试报告文件检查
+2. ut脚本start方法增加请求commit statues获取实际commit author
+3. ut脚本upload方法增加入参校验
+
+### Version 2.8
+1. ut脚本加入app/(admin|interface)/main的测试
+2. ut脚本加入结束执行后saga评论报告
+
+### Version 2.7
+1. 单元测试改为bazel test
+2. 去除部分debug打印信息
+
+### Version 2.6
+1. 添加跳过check检测开关
+
+### Version 2.5.3
+1. upload接口异常打印日志
+
+### Version 2.5.2
+1. 加入bazel test相关处理
+
+### Version 2.5.1
+1. 优化ut脚本流程控制
+
+### Version 2.5
+1. ut脚本只执行service/main下dao层和library下的单测
+2. 增加达标标准判断
+    2.1 通过率=100% && 覆盖率>=30% && 当前pkg同比上次执行的单测覆盖增长率>=0
+3. 输出当前执行概况日志
+
+### Version 2.4
+1. 删除编译失败的时候打包变化文件的机制
+2. 测试权限
+
+### Version 2.3
+1.修正检测changelog大小写的问题
+2.修改全量编译的job名称
+
+### Version 2.2
+1. 重构pipeline代码
+2. 修改ut为不允许失败
+
+### Version 2.1
+1. cyclo函数复杂度调整到50
+2. gometalinter过滤掉vendor包
+
+### Version 2.0
+1. 合并优化代码到lint.sh
+2. 修改匹配变化文件的逻辑
+
+### Version 1.0
+1. 将compile重试次数改为1次
+2. 修正lint_if_changed.sh匹配文件的bug

.rider/CONTRIBUTORS.md

@@ -0,0 +1,20 @@
+# Owner
+fangrongchang
+tangyongqiang
+chenjianrong
+zhaobingqing
+chengxing
+hedan
+
+# Author 
+muyang
+fangrongchang
+tangyongqiang
+chenjianrong
+zhaobingqing
+chengxing
+fengshanshan
+hedan
+
+# Reviewer
+all 

.rider/OWNERS

@@ -0,0 +1,20 @@
+# See the OWNERS docs at https://go.k8s.io/owners
+
+approvers:
+- chengxing
+- chenjianrong
+- fangrongchang
+- fengshanshan
+- hedan
+- muyang
+- tangyongqiang
+- zhaobingqing
+reviewers:
+- chengxing
+- chenjianrong
+- fangrongchang
+- fengshanshan
+- hedan
+- muyang
+- tangyongqiang
+- zhaobingqing

.rider/bazel_build.sh

@@ -0,0 +1,48 @@
+#!/bin/bash
+
+type=${1}
+
+echo "$(date) build..." >> /data/gitlab-runner/build.log
+
+function compileall()
+{
+    make build-keep-going
+}
+
+function compilepart()
+{
+    pkgs=`.rider/changepkgs.sh`
+    exitCode=$?
+    if [[ ${exitCode} -ne 0 ]]; then
+        echo ".rider/changepkgs.sh fail"
+        exit ${exitCode}
+    fi
+    if [[ "${pkgs}" = "" ]]; then
+        echo "no changepkgs"
+        exit 0
+    fi
+
+    echo -e "change packages:\n${pkgs}\n"
+
+    paths=""
+    for pkg in ${pkgs}
+    do
+        if [[ "${paths}" != "" ]]; then
+            paths="${paths} union allpaths(//app/..., //${pkg}:all)"
+        else
+            paths="allpaths(//app/..., //${pkg}:all)"
+        fi
+    done
+
+    echo "bazel build..."
+    query=`bazel query "${paths}"`
+    echo -e "${query}\n"
+    bazel build --config=ci --watchfs -k $(echo -e "${query}" | grep -v all-srcs | grep -v package-srcs | grep -v _proto)
+    #bazel build $(bazel query "${paths}" |grep -v all-srcs  |grep -v package-srcs)
+}
+
+if [[ "${type}" = "part" ]]; then
+    compilepart
+else
+    compileall
+fi

.rider/changefiles.sh

@@ -0,0 +1,21 @@
+#!/bin/bash
+
+#set -x
+
+suffix=${1}
+
+url="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHA}/merge_requests?private_token=${CI_PRIVATE_TOKEN}"
+json=$(curl -s ${url})
+if [[ "$json" != "[]" && "$json" != "" ]]; then
+    target_branch=$(echo ${json} | jq -r '.[0].target_branch')
+    source_branch=$(echo ${json} | jq -r '.[0].source_branch')
+    files=$(git diff origin/${target_branch}...origin/${source_branch} --name-only  --diff-filter=ACM | grep -E -i "${suffix}")
+else
+#  url="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/pipelines?private_token=${CI_PRIVATE_TOKEN}&status=success&ref=${CI_COMMIT_REF_NAME}"
+#  commit=$(curl -s ${url} | jq -r 'first(.[] | .sha)')
+##  echo "Last green commit is '${commit}'."
+#  files=$(git diff ${commit} --name-only  --diff-filter=ACM | grep -E "${suffix}")
+    files=$(git diff origin/master...origin/${CI_COMMIT_REF_NAME} --name-only  --diff-filter=ACM | grep -E -i "${suffix}")
+fi
+
+echo -e "${files}"

.rider/changepkgs.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+#set -x
+
+files=`.rider/changefiles.sh "*\.go$"`
+
+exitCode=$?
+if [[ ${exitCode} -ne 0 ]]; then
+    echo ".rider/changefiles.sh fail"
+    exit ${exitCode}
+fi
+
+pkgs=""
+for file in ${files}
+do
+    pkg="${file%/*}"
+    if [ $? -eq 0 ]; then
+        if [[ "${pkgs}" = "" ]]; then
+            pkgs="${pkg}"
+        else
+            pkgs="${pkgs}\n${pkg}"
+        fi
+    fi
+done
+echo -e "${pkgs}" > .rider/.pkgs
+pkgs=`sort .rider/.pkgs|uniq`
+
+echo -e "${pkgs}"

.rider/lint.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+#set -x
+
+pkgs=`.rider/changepkgs.sh|grep -v ^vendor/`
+
+exitCode=$?
+if [[ ${exitCode} -ne 0 ]]; then
+    echo ".rider/changepkgs.sh fail"
+    exit ${exitCode}
+fi
+
+if [[ "${pkgs}" = "" ]]; then
+    echo "no changepkgs"
+    exit 0
+fi
+
+echo -e "change packages:\n${pkgs}\n"
+
+if [ ! -d "${CI_PROJECT_DIR}/../src" ];then
+    mkdir ${CI_PROJECT_DIR}/../src
+fi
+ln -fs ${CI_PROJECT_DIR} ${CI_PROJECT_DIR}/../src
+export GOPATH=${CI_PROJECT_DIR}/..
+echo "GOPATH: $GOPATH"
+cd $GOPATH/src/go-common
+
+exitCode=0
+echo -e "\ngometalinter:"
+
+output=`gometalinter --config=.rider/.gometalinter.json ${pkgs}`
+exitCode=$?
+if [[ "${output}" != "" ]]; then
+    exitCode=1
+    echo -e "${output}"
+fi
+
+exit ${exitCode}
+

.rider/make_update.sh

@@ -0,0 +1,24 @@
+#!/bin/bash
+
+#set -x
+
+if [ ! -d "${CI_PROJECT_DIR}/../src" ];then
+  mkdir ${CI_PROJECT_DIR}/../src
+fi
+ln -fs ${CI_PROJECT_DIR} ${CI_PROJECT_DIR}/../src
+export GOPATH=${CI_PROJECT_DIR}/..
+echo "GOPATH: $GOPATH"
+
+vendor=${CI_PROJECT_DIR}/vendor
+for dir in $(ls $vendor)
+do
+  if [ -d $vendor/$dir ]; then
+    ln -fs $vendor/$dir $GOPATH/src
+  fi
+done
+
+sleep $[ ( $RANDOM % 60 )  + 1 ]s
+
+cd ${CI_PROJECT_DIR}/../src/go-common
+#mkdir -p .git/hooks/pre-commit
+make bazel-update

.rider/sagacheck.sh

@@ -0,0 +1,28 @@
+#!/bin/bash
+
+if [ ! -d "${CI_PROJECT_DIR}/../src" ];then
+    mkdir ${CI_PROJECT_DIR}/../src
+fi
+ln -fs ${CI_PROJECT_DIR} ${CI_PROJECT_DIR}/../src
+export GOPATH=${CI_PROJECT_DIR}/..
+exitCode=0
+
+# CHANGELOG check
+echo "====CHANGELOG check:===="
+files=`.rider/changefiles.sh "CHANGELOG.md"`
+if [[ "${files}" = "" ]]; then
+    echo "未发现CHANGELOG.md文件变更，请'添加'或'修改'CHANGELOG.md"
+    exit 1
+else
+    echo -e "变更如下:\n${files}"
+fi
+
+# BGR rule
+echo -e "\n====Bili golang rule check:===="
+diffFiles=`.rider/changefiles.sh`
+cd $GOPATH/src/go-common
+go build ./app/tool/bgr
+./bgr -script=./app/tool/bgr -hit=main -type=file ${diffFiles}
+exitCode=$?
+
+exit ${exitCode}

.rider/unit_test.sh

@@ -0,0 +1,267 @@
+#!/bin/bash
+
+declare -a dirs=(dao)
+declare -a packages
+declare -a projects
+declare mergeUser
+declare commitUser
+declare mergeID
+
+#init env
+function Init(){
+    if [ ! -d "${CI_PROJECT_DIR}/../src" ];then
+        mkdir ${CI_PROJECT_DIR}/../src
+    fi
+    ln -fs ${CI_PROJECT_DIR} ${CI_PROJECT_DIR}/../src
+    export GOPATH=${CI_PROJECT_DIR}/..
+}
+
+function GetPackages(){
+    reg="library/(.*)/(.*).go"
+    for dir in ${dirs[@]}
+    do
+        reg+="|app/(service|interface|admin|job)/main/(.*)/${dir}/(.*).go"
+    done
+    files=`.rider/changefiles.sh ${suffix} | grep  -E "${reg}"`
+    if [[ "${files}" = "" ]]; then
+        echo "shell.GetPackages: no change files"
+        exit 0
+    fi
+    for file in ${files}
+    do
+        # if [[ "${file}" =~ "library"* || "${file}" =~ "/mock" ]]; then
+        if [[ "${file}" =~ "/mock" ]]; then
+            continue
+        fi
+        package="go-common/$(dirname ${file})"
+        if [[ ${packages} =~ ${package} ]]; then
+            continue
+        fi
+        packages+=${package}" "
+        packageSplit=(${package//\// })
+        project=$(printf "%s/" ${packageSplit[@]:0:6})
+        if [[ ${projects} =~ ${project} || ${project} =~ "/library" ]]; then
+            continue
+        fi
+        projects+=${project%*/}" "
+    done
+    if [[ ${packages} = "" || ${projects} = "" ]]; then
+        echo "shell.GetPackages no change packages"
+        exit 0
+    fi
+}
+# GetUserInfo get userinfo by gitlab result.
+function GetUserInfo(){
+    gitMergeRequestUrl="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHA}/merge_requests?private_token=${CI_PRIVATE_TOKEN}"
+    gitCommitUrl="${CI_SERVER_URL}/api/v4/projects/${CI_PROJECT_ID}/repository/commits/${CI_COMMIT_SHA}/statuses?private_token=${CI_PRIVATE_TOKEN}"
+    mergeJson=$(curl -s ${gitMergeRequestUrl})
+    commitJson=$(curl -s ${gitCommitUrl})
+    if [[ "${mergeJson}" = "[]" ]] || [[ "${commitJson}" = "[]" ]]; then
+        echo "Test not run, maybe you should try create a merge request first!"
+        exit 0       
+    fi
+    mergeID=$(echo ${mergeJson} | jq -r '.[0].iid')
+    mergeUser=$(echo ${mergeJson} | jq -r '.[0].author.username')
+    commitUser=$(echo ${commitJson} | jq -r '.[0].author.username')
+}
+
+# Magic ignore method Check()
+function Magic(){
+    url="http://git.bilibili.co/api/v4/projects/${CI_PROJECT_ID}/merge_requests/${mergeID}/notes?private_token=${CI_PRIVATE_TOKEN}"
+    json=$(curl -s ${url})
+    admin="haoguanwei,chenjianrong,hedan,fengshanshan,zhaobingqing"
+    len=$(echo ${json} | jq '.|length')
+    for i in $(seq 0 $len)
+    do
+        comment=$(echo ${json} | jq -r ".[$i].body")
+        user=$(echo ${json} | jq -r ".[$i].author.username")
+        if [[ ${comment} = "+skiput" && ${admin} =~ ${user} ]]; then
+             exit 0
+        fi
+    done
+}
+
+# Check determine whether the standard is up to standard
+#$1: commit_id
+function Check(){
+    curl -s "${CI_UATSVEN_URL}/x/admin/apm/ut/git/report?project_id=${CI_PROJECT_ID}&merge_id=${mergeID}&commit_id=${CI_COMMIT_SHA}"
+    checkURL="${CI_UATSVEN_URL}/x/admin/apm/ut/check?commit_id=${CI_COMMIT_SHA}"
+    json=$(curl -s ${checkURL})
+    code=$(echo ${json} | jq -r '.code')
+    if [[ ${code} -ne 0 ]]; then
+        echo -e "curl ${checkURL} response(${json})"
+        exit 1
+    fi
+    package=$(echo ${json} | jq -r '.data.package')
+    coverage=$(echo ${json} | jq -r '.data.coverage')
+    passRate=$(echo ${json} | jq -r '.data.pass_rate')
+    standard=$(echo ${json} | jq -r '.data.standard')
+    increase=$(echo ${json} | jq -r '.data.increase')
+    tyrant=$(echo ${json} | jq -r '.data.tyrant')
+    lastCID=$(echo ${json} | jq -r '.data.last_cid')
+    if ${tyrant}; then
+        echo -e "\t续命失败!\n\t大佬，本次执行结果未达标哦(灬ꈍ ꈍ灬)，请再次优化ut重新提交🆙"
+        echo -e "\t---------------------------------------------------------------------"
+        printf "\t%-14s %-14s %-14s %-14s\n" "本次覆盖率(%)" "本次通过率(%)" "本次增长量(%)" 执行pkg
+        printf "\t%-13.2f %-13.2f %-13.2f %-12s\n" ${coverage} ${passRate} ${increase} ${package}
+        echo -e "\t(达标标准：覆盖率>=${standard} && 通过率=100% && 同比当前package历史最高覆盖率的增长率>=0)"
+        echo -e "\t---------------------------------------------------------------------"
+        echo -e "本次执行详细结果查询地址请访问：http://sven.bilibili.co/#/ut?merge_id=${mergeID}&&pn=1&ps=20"
+        exit 1
+    else
+        echo -e "\t恭喜你，续命成功，可以请求合并MR了!"
+    fi
+}
+
+function ReadDir(){
+    # get go-common/app all dir path
+    gopath=${GOPATH%..}
+    PathDirs=`find ${gopath}app -maxdepth 3 -type d`
+    value=""
+    for dir in ${PathDirs}
+    do
+        if [[ -d "$dir" ]];then
+            for file in `find ${dir} -maxdepth 1 -type f |grep "OWNERS"`
+            do
+                owner=""
+                substr=${dir#*"go-common"}
+                while read line
+                do
+                    if [[ "${line}" = "#"* ]] || [[ "${line}" = "" ]] || [[ "${line}" = "approvers:" ]];then
+                        continue
+                    elif [[ "${line}" = "labels:"* ]];then
+                        break
+                    else
+                        owner+="${line:1},"
+                    fi
+                done < ${file}
+                value+="{\"path\":\"go-common${substr}\",\"owner\":\"${owner%,}\"},"
+            done
+        fi
+    done
+    # delete "," at the end of value
+    value=${value%,}
+    echo "[${value}]" > path.out
+}
+
+# UTLint check the *_test.go files in the pkg
+# $1: pkg
+function UTLint()
+{   
+    cd $GOPATH
+    declare -i numCase=0
+    declare -i numAssertion=0
+    files=$(ls $1 | grep -E "(.*)_test\.go")
+    if [[ ${#files} -eq 0 ]];then
+        echo "RunPKGUT.UTLint no *_test.go files in pkg($1)"
+        exit 1
+    fi
+    for file in ${files}
+    do
+        numCase+=`grep -c -E "^func Test(.+)\(t \*testing\.T\) \{$" $1/${file}`
+        numAssertion+=`grep -c -E "^(.*)So\((.+)\)$" $1/${file}`
+    done
+    if [[ ${numCase} -eq 0 || ${numAssertion} -eq 0 ]];then
+        echo -e "RunPKGUT.UTLint no test case or assertion in pkg($1)"
+        exit 1
+    fi
+}
+
+# BazelTest execute bazel coverage and go tool
+# $1: pkg
+function BazelTest(){
+    cd $GOPATH/go-common
+    pkg=${1//go-common//}":go_default_test"
+    path=${1//go-common\//}
+
+    bazel coverage --config=ci --instrumentation_filter="//${path}[:],-//${path}/mock[/:]" --test_env=DEPLOY_ENV=uat --test_timeout=60 --test_env=APP_ID=bazel.test --test_output=all --cache_test_results=auto --test_arg=-convey-json ${pkg} > result.out
+    if [[ ! -s result.out ]]; then 
+        echo "==================================WARNING!======================================"
+        echo "No test case found,请完善如下路径测试用例： ${pkg} "
+        exit 1
+    else
+        echo $?
+        cp $GOPATH/go-common/bazel-out/k8-fastbuild/testlogs/${path}/go_default_test/coverage.dat ./
+        go tool cover -html=coverage.dat -o cover.html
+    fi
+}
+
+# BazelTest execute bazel coverage for All files
+# $1: pkg(go-common/app/admin/main/xxx/dao)
+function BazelTestAll(){
+    cd $GOPATH/go-common
+    pkg=${1//go-common//}"/..."
+    path=${1//go-common\//}
+    echo "RunProjUT.BazelTestAll(${1}) pkg(${pkg}) path(${path}) pwd($(pwd))"
+    bazel coverage --config=ci --instrumentation_filter="//${path}[/:],-//${path}/mock[/:]" --test_env=DEPLOY_ENV=uat --test_timeout=60 --test_env=APP_ID=bazel.test --test_output=all --cache_test_results=auto --test_arg=-convey-json ${pkg} > result.out
+    find bazel-out/k8-fastbuild/testlogs/${path} -name "coverage.dat" | xargs cat | sort -nr | rev | uniq -s 1 | rev > coverage.dat
+    coverage=$(cat coverage.dat | awk '{sum += $2;covSum += $2 * $3;} END {print covSum/sum*100}')
+    sed -if "s/coverage: .*%/coverage: ${coverage}%/g" result.out
+    go tool cover -html=coverage.dat -o cover.html
+}
+
+# upload data to apm
+# $1: file result.out path
+function Upload () {
+    if [[ ! -f "result.out" ]] || [[ ! -f "cover.html" ]] || [[ ! -f "coverage.dat" ]]; then
+        echo "==================================WARNING!======================================"
+        echo "No test found!~ 请完善如下路径测试用例： ${1} "
+        exit 1
+    fi
+    url="${CI_UATSVEN_URL}/x/admin/apm/ut/upload?merge_id=${mergeID}&username=${mergeUser}&author=${commitUser}&commit_id=${CI_COMMIT_SHA}&pkg=${1}"
+    json=$(curl -s ${url} -H "Content-type: multipart/form-data" -F "html_file=@cover.html" -F "report_file=@result.out" -F "data_file=@coverage.dat")
+    if [[ "${json}" = "" ]]; then
+        echo "RunPKGUT.Upload curl ${url} fail"
+        exit 1
+    fi
+    msg=$(echo ${json} | jq -r '.message')
+    data=$(echo ${json} | jq -r '.data')
+    code=$(echo ${json} | jq -r '.code')
+    if [[ ${code} -ne 0 ]]; then
+        echo "=============================================================================="
+        echo -e "RunPKGUT.Upload Response. message(${msg})"
+        echo -e "RunPKGUT.Upload Response. data(${data})\n\n"
+        echo -e "RunPKGUT.Upload Upload Fail! status(${code})"
+        exit ${code}
+    fi
+}
+
+function RunPKGUT(){
+    for package in ${packages}
+    do
+        echo "RunPKGUT.UTLint Start. pkg(${package})"
+        UTLint ${package}
+        echo "RunPKGUT.BazelTest Start. pkg(${package})"
+        BazelTest ${package}
+        echo "RunPKGUT.Upload Start. pkg(${package})"
+        Upload ${package}
+    done
+    return 0
+}
+
+function RunProjUT(){
+    for project in ${projects}
+    do
+        echo "RunProjUT.BazelTestAll Start. project(${project})"
+        BazelTestAll ${project}
+        echo "RunProjUT.Upload BazelTest Start. project(${project})"
+        Upload ${project%/*}
+    done
+}
+
+function UploadApp(){
+    ReadDir
+    url="${CI_UATSVEN_URL}/x/admin/apm/ut/upload/app" 
+    curl -s ${url} -H "Content-type: multipart/form-data" -F "path_file=@path.out" > /dev/null
+    echo "UploadApp() UpPath has finshed."
+}
+
+# run
+Init
+GetPackages
+GetUserInfo
+Magic
+RunPKGUT
+RunProjUT
+UploadApp
+Check