Initial commit

vendor/k8s.io/test-infra/prow/clonerefs/BUILD.bazel

@@ -0,0 +1,33 @@
+load("@io_bazel_rules_go//go:def.bzl", "go_library")
+
+go_library(
+    name = "go_default_library",
+    srcs = [
+        "doc.go",
+        "options.go",
+        "parse.go",
+        "run.go",
+    ],
+    importpath = "k8s.io/test-infra/prow/clonerefs",
+    visibility = ["//visibility:public"],
+    deps = [
+        "//vendor/github.com/sirupsen/logrus:go_default_library",
+        "//vendor/k8s.io/apimachinery/pkg/util/sets:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/kube:go_default_library",
+        "//vendor/k8s.io/test-infra/prow/pod-utils/clone:go_default_library",
+    ],
+)
+
+filegroup(
+    name = "package-srcs",
+    srcs = glob(["**"]),
+    tags = ["automanaged"],
+    visibility = ["//visibility:private"],
+)
+
+filegroup(
+    name = "all-srcs",
+    srcs = [":package-srcs"],
+    tags = ["automanaged"],
+    visibility = ["//visibility:public"],
+)

vendor/k8s.io/test-infra/prow/clonerefs/doc.go

@@ -0,0 +1,18 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+// Package clonerefs is a library for cloning references
+package clonerefs

vendor/k8s.io/test-infra/prow/clonerefs/options.go

@@ -0,0 +1,238 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clonerefs
+
+import (
+	"bytes"
+	"encoding/json"
+	"errors"
+	"flag"
+	"fmt"
+	"strings"
+	"text/template"
+
+	"k8s.io/apimachinery/pkg/util/sets"
+	"k8s.io/test-infra/prow/kube"
+)
+
+// Options configures the clonerefs tool
+// completely and may be provided using JSON
+// or user-specified flags, but not both.
+type Options struct {
+	// SrcRoot is the root directory under which
+	// all source code is cloned
+	SrcRoot string `json:"src_root"`
+	// Log is the log file to which clone records are written
+	Log string `json:"log"`
+
+	// GitUserName is an optional field that is used with
+	// `git config user.name`
+	GitUserName string `json:"git_user_name,omitempty"`
+	// GitUserEmail is an optional field that is used with
+	// `git config user.email`
+	GitUserEmail string `json:"git_user_email,omitempty"`
+
+	// GitRefs are the refs to clone
+	GitRefs []kube.Refs `json:"refs"`
+	// KeyFiles are files containing SSH keys to be used
+	// when cloning. Will be added to `ssh-agent`.
+	KeyFiles []string `json:"key_files,omitempty"`
+
+	// HostFingerPrints are ssh-keyscan host fingerprint lines to use
+	// when cloning. Will be added to ~/.ssh/known_hosts
+	HostFingerprints []string `json:"host_fingerprints,omitempty"`
+
+	// MaxParallelWorkers determines how many repositories
+	// can be cloned in parallel. If 0, interpreted as no
+	// limit to parallelism
+	MaxParallelWorkers int `json:"max_parallel_workers,omitempty"`
+
+	// used to hold flag values
+	refs       gitRefs
+	clonePath  orgRepoFormat
+	cloneURI   orgRepoFormat
+	keys       stringSlice
+	CookiePath string `json:"cookie_path,omitempty"`
+}
+
+// Validate ensures that the configuration options are valid
+func (o *Options) Validate() error {
+	if o.SrcRoot == "" {
+		return errors.New("no source root specified")
+	}
+
+	if o.Log == "" {
+		return errors.New("no log file specified")
+	}
+
+	if len(o.GitRefs) == 0 {
+		return errors.New("no refs specified to clone")
+	}
+
+	seen := map[string]sets.String{}
+	for _, ref := range o.GitRefs {
+		if _, seenOrg := seen[ref.Org]; seenOrg {
+			if seen[ref.Org].Has(ref.Repo) {
+				return errors.New("sync config for %s/%s provided more than once")
+			}
+			seen[ref.Org].Insert(ref.Repo)
+		} else {
+			seen[ref.Org] = sets.NewString(ref.Repo)
+		}
+	}
+
+	return nil
+}
+
+const (
+	// JSONConfigEnvVar is the environment variable that
+	// clonerefs expects to find a full JSON configuration
+	// in when run.
+	JSONConfigEnvVar = "CLONEREFS_OPTIONS"
+	// DefaultGitUserName is the default name used in git config
+	DefaultGitUserName = "ci-robot"
+	// DefaultGitUserEmail is the default email used in git config
+	DefaultGitUserEmail = "ci-robot@k8s.io"
+)
+
+// ConfigVar exposes the environment variable used
+// to store serialized configuration
+func (o *Options) ConfigVar() string {
+	return JSONConfigEnvVar
+}
+
+// LoadConfig loads options from serialized config
+func (o *Options) LoadConfig(config string) error {
+	return json.Unmarshal([]byte(config), o)
+}
+
+// Complete internalizes command line arguments
+func (o *Options) Complete(args []string) {
+	o.GitRefs = o.refs.gitRefs
+	o.KeyFiles = o.keys.data
+
+	for _, ref := range o.GitRefs {
+		alias, err := o.clonePath.Execute(OrgRepo{Org: ref.Org, Repo: ref.Repo})
+		if err != nil {
+			panic(err)
+		}
+		ref.PathAlias = alias
+
+		alias, err = o.cloneURI.Execute(OrgRepo{Org: ref.Org, Repo: ref.Repo})
+		if err != nil {
+			panic(err)
+		}
+		ref.CloneURI = alias
+	}
+}
+
+// AddFlags adds flags to the FlagSet that populate
+// the GCS upload options struct given.
+func (o *Options) AddFlags(fs *flag.FlagSet) {
+	fs.StringVar(&o.SrcRoot, "src-root", "", "Where to root source checkouts")
+	fs.StringVar(&o.Log, "log", "", "Where to write logs")
+	fs.StringVar(&o.GitUserName, "git-user-name", DefaultGitUserName, "Username to set in git config")
+	fs.StringVar(&o.GitUserEmail, "git-user-email", DefaultGitUserEmail, "Email to set in git config")
+	fs.Var(&o.refs, "repo", "Mapping of Git URI to refs to check out, can be provided more than once")
+	fs.Var(&o.keys, "ssh-key", "Path to SSH key to enable during cloning, can be provided more than once")
+	fs.Var(&o.clonePath, "clone-alias", "Format string for the path to clone to")
+	fs.Var(&o.cloneURI, "uri-prefix", "Format string for the URI prefix to clone from")
+	fs.IntVar(&o.MaxParallelWorkers, "max-workers", 0, "Maximum number of parallel workers, unset for unlimited.")
+	fs.StringVar(&o.CookiePath, "cookiefile", "", "Path to git http.coookiefile")
+}
+
+type gitRefs struct {
+	gitRefs []kube.Refs
+}
+
+func (r *gitRefs) String() string {
+	representation := bytes.Buffer{}
+	for _, ref := range r.gitRefs {
+		fmt.Fprintf(&representation, "%s,%s=%s", ref.Org, ref.Repo, ref.String())
+	}
+	return representation.String()
+}
+
+// Set parses out a kube.Refs from the user string.
+// The following example shows all possible fields:
+//   org,repo=base-ref:base-sha[,pull-number:pull-sha]...
+// For the base ref and every pull number, the SHAs
+// are optional and any number of them may be set or
+// unset.
+func (r *gitRefs) Set(value string) error {
+	gitRef, err := ParseRefs(value)
+	if err != nil {
+		return err
+	}
+	r.gitRefs = append(r.gitRefs, *gitRef)
+	return nil
+}
+
+type stringSlice struct {
+	data []string
+}
+
+func (r *stringSlice) String() string {
+	return strings.Join(r.data, ",")
+}
+
+// Set records the value passed
+func (r *stringSlice) Set(value string) error {
+	r.data = append(r.data, value)
+	return nil
+}
+
+type orgRepoFormat struct {
+	raw    string
+	format *template.Template
+}
+
+func (a *orgRepoFormat) String() string {
+	return a.raw
+}
+
+// Set parses out overrides from user input
+func (a *orgRepoFormat) Set(value string) error {
+	templ, err := template.New("format").Parse(value)
+	if err != nil {
+		return err
+	}
+	a.raw = value
+	a.format = templ
+	return nil
+}
+
+// OrgRepo hold both an org and repo name.
+type OrgRepo struct {
+	Org, Repo string
+}
+
+func (a *orgRepoFormat) Execute(data OrgRepo) (string, error) {
+	if a.format != nil {
+		output := bytes.Buffer{}
+		err := a.format.Execute(&output, data)
+		return output.String(), err
+	}
+	return "", nil
+}
+
+// Encode will encode the set of options in the format that
+// is expected for the configuration environment variable
+func Encode(options Options) (string, error) {
+	encoded, err := json.Marshal(options)
+	return string(encoded), err
+}

vendor/k8s.io/test-infra/prow/clonerefs/parse.go

@@ -0,0 +1,91 @@
+/*
+Copyright 2017 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clonerefs
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"k8s.io/test-infra/prow/kube"
+)
+
+// ParseRefs parses a human-provided string into the repo
+// that should be cloned and the refs that need to be
+// checked out once it is. The format is:
+//   org,repo=base-ref[:base-sha][,pull-id[:pull-sha[:pull-ref]]]...
+// For the base ref and pull IDs, a SHA may optionally be
+// provided or may be omitted for the latest available SHA.
+// Examples:
+//   kubernetes,test-infra=master
+//   kubernetes,test-infra=master:abcde12
+//   kubernetes,test-infra=master:abcde12,34
+//   kubernetes,test-infra=master:abcde12,34:fghij56
+//   kubernetes,test-infra=master,34:fghij56
+//   kubernetes,test-infra=master:abcde12,34:fghij56,78
+//   gerrit,test-infra=master:abcde12,34:fghij56:refs/changes/00/123/1
+func ParseRefs(value string) (*kube.Refs, error) {
+	gitRef := &kube.Refs{}
+	values := strings.SplitN(value, "=", 2)
+	if len(values) != 2 {
+		return gitRef, fmt.Errorf("refspec %s invalid: does not contain '='", value)
+	}
+	info := values[0]
+	allRefs := values[1]
+
+	infoValues := strings.SplitN(info, ",", 2)
+	if len(infoValues) != 2 {
+		return gitRef, fmt.Errorf("refspec %s invalid: does not contain 'org,repo' as prefix", value)
+	}
+	gitRef.Org = infoValues[0]
+	gitRef.Repo = infoValues[1]
+
+	refValues := strings.Split(allRefs, ",")
+	if len(refValues) == 1 && refValues[0] == "" {
+		return gitRef, fmt.Errorf("refspec %s invalid: does not contain any refs", value)
+	}
+	baseRefParts := strings.Split(refValues[0], ":")
+	if len(baseRefParts) != 1 && len(baseRefParts) != 2 {
+		return gitRef, fmt.Errorf("refspec %s invalid: malformed base ref", refValues[0])
+	}
+	gitRef.BaseRef = baseRefParts[0]
+	if len(baseRefParts) == 2 {
+		gitRef.BaseSHA = baseRefParts[1]
+	}
+	for _, refValue := range refValues[1:] {
+		refParts := strings.Split(refValue, ":")
+		if len(refParts) == 0 || len(refParts) > 3 {
+			return gitRef, fmt.Errorf("refspec %s invalid: malformed pull ref", refValue)
+		}
+		pullNumber, err := strconv.Atoi(refParts[0])
+		if err != nil {
+			return gitRef, fmt.Errorf("refspec %s invalid: pull request identifier not a number: %v", refValue, err)
+		}
+		pullRef := kube.Pull{
+			Number: pullNumber,
+		}
+		if len(refParts) > 1 {
+			pullRef.SHA = refParts[1]
+		}
+		if len(refParts) > 2 {
+			pullRef.Ref = refParts[2]
+		}
+		gitRef.Pulls = append(gitRef.Pulls, pullRef)
+	}
+
+	return gitRef, nil
+}

vendor/k8s.io/test-infra/prow/clonerefs/run.go

@@ -0,0 +1,161 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package clonerefs
+
+import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+	"sync"
+
+	"github.com/sirupsen/logrus"
+	"k8s.io/test-infra/prow/kube"
+	"k8s.io/test-infra/prow/pod-utils/clone"
+)
+
+var cloneFunc = clone.Run
+
+// Run clones the configured refs
+func (o Options) Run() error {
+	var env []string
+	if len(o.KeyFiles) > 0 {
+		var err error
+		env, err = addSSHKeys(o.KeyFiles)
+		if err != nil {
+			logrus.WithError(err).Error("Failed to add SSH keys.")
+			// Continue on error. Clones will fail with an appropriate error message
+			// that initupload can consume whereas quitting without writing the clone
+			// record log is silent and results in an errored prow job instead of a
+			// failed one.
+		}
+	}
+	if len(o.HostFingerprints) > 0 {
+		if err := addHostFingerprints(o.HostFingerprints); err != nil {
+			logrus.WithError(err).Error("failed to add host fingerprints")
+		}
+	}
+
+	var numWorkers int
+	if o.MaxParallelWorkers != 0 {
+		numWorkers = o.MaxParallelWorkers
+	} else {
+		numWorkers = len(o.GitRefs)
+	}
+
+	wg := &sync.WaitGroup{}
+	wg.Add(numWorkers)
+
+	input := make(chan kube.Refs)
+	output := make(chan clone.Record, len(o.GitRefs))
+	for i := 0; i < numWorkers; i++ {
+		go func() {
+			defer wg.Done()
+			for ref := range input {
+				output <- cloneFunc(ref, o.SrcRoot, o.GitUserName, o.GitUserEmail, o.CookiePath, env)
+			}
+		}()
+	}
+
+	for _, ref := range o.GitRefs {
+		input <- ref
+	}
+
+	close(input)
+	wg.Wait()
+	close(output)
+
+	var results []clone.Record
+	for record := range output {
+		results = append(results, record)
+	}
+
+	logData, err := json.Marshal(results)
+	if err != nil {
+		return fmt.Errorf("failed to marshal clone records: %v", err)
+	}
+
+	if err := ioutil.WriteFile(o.Log, logData, 0755); err != nil {
+		return fmt.Errorf("failed to write clone records: %v", err)
+	}
+
+	return nil
+}
+
+func addHostFingerprints(fingerprints []string) error {
+	path := filepath.Join(os.Getenv("HOME"), ".ssh", "known_hosts")
+	f, err := os.OpenFile(path, os.O_APPEND|os.O_CREATE|os.O_WRONLY, 0644)
+	if err != nil {
+		return fmt.Errorf("could not create/append to %s: %v", path, err)
+	}
+	if _, err := f.Write([]byte(strings.Join(fingerprints, "\n"))); err != nil {
+		return fmt.Errorf("failed to write fingerprints to %s: %v", path, err)
+	}
+	if err := f.Close(); err != nil {
+		return fmt.Errorf("failed to close %s: %v", path, err)
+	}
+	return nil
+}
+
+// addSSHKeys will start the ssh-agent and add all the specified
+// keys, returning the ssh-agent environment variables for reuse
+func addSSHKeys(paths []string) ([]string, error) {
+	vars, err := exec.Command("ssh-agent").CombinedOutput()
+	if err != nil {
+		return []string{}, fmt.Errorf("failed to start ssh-agent: %v", err)
+	}
+	logrus.Info("Started SSH agent")
+	// ssh-agent will output three lines of text, in the form:
+	// SSH_AUTH_SOCK=xxx; export SSH_AUTH_SOCK;
+	// SSH_AGENT_PID=xxx; export SSH_AGENT_PID;
+	// echo Agent pid xxx;
+	// We need to parse out the environment variables from that.
+	parts := strings.Split(string(vars), ";")
+	env := []string{strings.TrimSpace(parts[0]), strings.TrimSpace(parts[2])}
+	for _, keyPath := range paths {
+		// we can be given literal paths to keys or paths to dirs
+		// that are mounted from a secret, so we need to check which
+		// we have
+		if err := filepath.Walk(keyPath, func(path string, info os.FileInfo, err error) error {
+			if strings.HasPrefix(info.Name(), "..") {
+				// kubernetes volumes also include files we
+				// should not look be looking into for keys
+				if info.IsDir() {
+					return filepath.SkipDir
+				}
+				return nil
+			}
+			if info.IsDir() {
+				return nil
+			}
+
+			cmd := exec.Command("ssh-add", path)
+			cmd.Env = append(cmd.Env, env...)
+			if output, err := cmd.CombinedOutput(); err != nil {
+				return fmt.Errorf("failed to add ssh key at %s: %v: %s", path, err, output)
+			}
+			logrus.Infof("Added SSH key at %s", path)
+			return nil
+		}); err != nil {
+			return env, fmt.Errorf("error walking path %q: %v", keyPath, err)
+		}
+	}
+	return env, nil
+}