diff --git a/go.mod b/go.mod index e204d4c4d..6c729a4b5 100644 --- a/go.mod +++ b/go.mod @@ -8,13 +8,13 @@ require ( github.com/google/go-cmp v0.5.1 github.com/kelseyhightower/envconfig v1.4.0 go.uber.org/zap v1.15.0 - k8s.io/api v0.18.7-rc.0 - k8s.io/apimachinery v0.18.7-rc.0 + k8s.io/api v0.18.8 + k8s.io/apimachinery v0.18.8 k8s.io/client-go v11.0.1-0.20190805182717-6502b5e7b1b5+incompatible knative.dev/eventing v0.16.1-0.20200806200629-e4bc346017b6 knative.dev/pkg v0.0.0-20200816190606-6d1b0e90624b knative.dev/serving v0.17.1 - knative.dev/test-infra v0.0.0-20200813220834-388e55a496cf + knative.dev/test-infra v0.0.0-20201014021030-ae3984a33f82 ) replace ( diff --git a/go.sum b/go.sum index 8f2ca937f..50624f199 100644 --- a/go.sum +++ b/go.sum @@ -235,6 +235,7 @@ github.com/blang/semver v3.1.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnweb github.com/blang/semver v3.5.0+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= github.com/blang/semver v3.5.1+incompatible/go.mod h1:kRBLl5iJ+tD4TcOOxsy/0fnwebNt5EWlYSAyrTnjyyk= +github.com/blang/semver/v4 v4.0.0/go.mod h1:IbckMUScFkM3pff0VJDNKRiT6TG/YpiHIM2yvyW5YoQ= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b h1:AP/Y7sqYicnjGDfD5VcY4CIfh1hRXBUavxrvELjTiOE= github.com/bmizerany/perks v0.0.0-20141205001514-d9a9656a3a4b/go.mod h1:ac9efd0D1fsDb3EJvhqgXRbFx7bs2wqZ10HQPeU8U/Q= @@ -394,6 +395,10 @@ github.com/globalsign/mgo v0.0.0-20181015135952-eeefdecb41b8/go.mod h1:xkRDCp4j0 github.com/go-bindata/go-bindata/v3 v3.1.3/go.mod h1:1/zrpXsLD8YDIbhZRqXzm1Ghc7NhEvIN9+Z6R5/xH4I= github.com/go-critic/go-critic v0.4.1/go.mod h1:7/14rZGnZbY6E38VEGk2kVhoq6itzc1E68facVDK23g= github.com/go-critic/go-critic v0.4.3/go.mod h1:j4O3D4RoIwRqlZw5jJpx0BNfXWWbpcJoKu5cYSe4YmQ= +github.com/go-git/gcfg v1.5.0/go.mod h1:5m20vg6GwYabIxaOonVkTdrILxQMpEShl1xiMF4ua+E= +github.com/go-git/go-billy/v5 v5.0.0/go.mod h1:pmpqyWchKfYfrkb/UVH4otLvyi/5gJlGI4Hb3ZqZ3W0= +github.com/go-git/go-git-fixtures/v4 v4.0.1/go.mod h1:m+ICp2rF3jDhFgEZ/8yziagdT1C+ZpZcrJjappBCDSw= +github.com/go-git/go-git/v5 v5.1.0/go.mod h1:ZKfuPUoY1ZqIG4QG9BDBh3G4gLM5zvPuSJAozQrZuyM= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -589,6 +594,8 @@ github.com/google/go-containerregistry v0.0.0-20200331213917-3d03ed9b1ca2/go.mod github.com/google/go-containerregistry v0.1.1/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM= github.com/google/go-containerregistry v0.1.2-0.20200717224239-a84993334b27 h1:UE29caIXS8tGBSgWvLOnFdOqV5fv672bfvkCS/9EHo0= github.com/google/go-containerregistry v0.1.2-0.20200717224239-a84993334b27/go.mod h1:npTSyywOeILcgWqd+rvtzGWflIPPcBQhYoOONaY4ltM= +github.com/google/go-containerregistry v0.1.3 h1:vD78UlG9hVJ8DZbUL8uuNco2G1o9eJTA87xR0Y2UX6c= +github.com/google/go-containerregistry v0.1.3/go.mod h1:3Wg/Hjgn/ZDxrYYhtzZJWdThOd8zeI2zAmn4oVfm1Wg= github.com/google/go-github v17.0.0+incompatible h1:N0LgJ1j65A7kfXrZnUDaYCs/Sf4rEjNlfyDHW9dolSY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-github/v27 v27.0.6 h1:oiOZuBmGHvrGM1X9uNUAUlLgp5r1UUO/M/KnbHnLRlQ= @@ -1214,6 +1221,7 @@ github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1: github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= +github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yvasiyarov/go-metrics v0.0.0-20140926110328-57bccd1ccd43/go.mod h1:aX5oPXxHm3bOH+xeAttToC8pqch2ScQN/JoXYupl6xs= github.com/yvasiyarov/gorelic v0.0.0-20141212073537-a9bba5b9ab50/go.mod h1:NUSPSUX/bi6SeDMUh6brw0nXpxHnc96TguQh0+r/ssA= github.com/yvasiyarov/newrelic_platform_go v0.0.0-20140908184405-b21fdbd4370f/go.mod h1:GlGEuHIJweS1mbCqG+7vt2nvWLzLLnRHbXz5JKd/Qbg= @@ -1391,6 +1399,8 @@ golang.org/x/net v0.0.0-20200520182314-0ba52f642ac2/go.mod h1:qpuaurCH72eLCgpAm/ golang.org/x/net v0.0.0-20200625001655-4c5254603344/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20200707034311-ab3426394381 h1:VXak5I6aEWmAXeQjA+QSZzlgNrpq9mjcfDemuexIKsU= golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= +golang.org/x/net v0.0.0-20200822124328-c89045814202 h1:VvcQYSHwXgi7W+TpUR6A9g6Up98WAHf3f/ulnJ62IyA= +golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/oauth2 v0.0.0-20180724155351-3d292e4d0cdc/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -1596,10 +1606,14 @@ golang.org/x/tools v0.0.0-20200725200936-102e7d357031/go.mod h1:njjCfa9FT2d7l9Bc golang.org/x/tools v0.0.0-20200729194436-6467de6f59a7/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6 h1:qKpj8TpV+LEhel7H/fR788J+KvhWZ3o3V6N2fU/iuLU= golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3 h1:DywqrEscRX7O2phNjkT0L6lhHKGBoMLCNX+XcAe7t6s= +golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3/go.mod h1:z6u4i615ZeAfBE4XtMziQW1fSVJXACjjbWkB/mvPzlU= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 h1:go1bK/D/BFZV2I8cIQd1NKEZ+0owSTG1fDTci4IqFcE= +golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= gomodules.xyz/jsonpatch/v2 v2.0.1/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= gomodules.xyz/jsonpatch/v2 v2.1.0 h1:Phva6wqu+xR//Njw6iorylFFgn/z547tw5Ne3HZPQ+k= gomodules.xyz/jsonpatch/v2 v2.1.0/go.mod h1:IhYNNY4jnS53ZnfE4PAmpKtDpTCj1JFXc+3mwe7XcUU= @@ -1805,6 +1819,8 @@ honnef.co/go/tools v0.0.1-2019.2.3/go.mod h1:a3bituU0lyd329TUQxRnasdCoJDkEUEAqEt honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.0.1-2020.1.4 h1:UoveltGrhghAA7ePc+e+QYDHXrBps2PqFZiHkGR/xK8= honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= +honnef.co/go/tools v0.0.1-2020.1.5 h1:nI5egYTGJakVyOryqLs1cQO5dO0ksin5XXs2pspk75k= +honnef.co/go/tools v0.0.1-2020.1.5/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= istio.io/api v0.0.0-20200512234804-e5412c253ffe/go.mod h1:kyq3g5w42zl/AKlbzDGppYpGMQYMYMyZKeq0/eexML8= istio.io/client-go v0.0.0-20200513000250-b1d6e9886b7b/go.mod h1:aUDVNCOKom8n53OPEb7JxKucbKVNveDY4WJj7PGQb14= istio.io/gogo-genproto v0.0.0-20190930162913-45029607206a/go.mod h1:OzpAts7jljZceG4Vqi5/zXy/pOg1b209T3jb7Nv5wIs= @@ -1926,6 +1942,8 @@ knative.dev/test-infra v0.0.0-20200813220306-af5517f4f576 h1:gsZMM8J2MA3iTSMOt5M knative.dev/test-infra v0.0.0-20200813220306-af5517f4f576/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc= knative.dev/test-infra v0.0.0-20200813220834-388e55a496cf h1:GrIpJgHa/iWE1Gk2WKGXSEv1Uw7qPepjCELY+Jjzewo= knative.dev/test-infra v0.0.0-20200813220834-388e55a496cf/go.mod h1:Pmg2c7Z7q7BGFUV/GOpU5BlrD3ePJft4MPqx8AYBplc= +knative.dev/test-infra v0.0.0-20201014021030-ae3984a33f82 h1:/HZKnJdqoGjknSM50+SLHDASF6wVDYOOEXsXXLzD1QU= +knative.dev/test-infra v0.0.0-20201014021030-ae3984a33f82/go.mod h1:lm0U0k6ARXr8Po0LcSTCfETg63LOF3ajUxcoqy5JSH0= modernc.org/cc v1.0.0/go.mod h1:1Sk4//wdnYJiUIxnW8ddKpaOJCF37yAdqYnkxUpaYxw= modernc.org/golex v1.0.0/go.mod h1:b/QX9oBD/LhixY6NDh+IdGv17hgB+51fET1i2kPSmvk= modernc.org/mathutil v1.0.0/go.mod h1:wU0vUrJsVWBZ4P6e7xtFJEhFSNsfRLJ8H458uRjg03k= diff --git a/hack/update-deps.sh b/hack/update-deps.sh index 23ae1d3b7..4f5a5b350 100755 --- a/hack/update-deps.sh +++ b/hack/update-deps.sh @@ -18,48 +18,6 @@ set -o errexit set -o nounset set -o pipefail -export GO111MODULE=on - source $(dirname $0)/../vendor/knative.dev/test-infra/scripts/library.sh -cd ${REPO_ROOT_DIR} - -VERSION="master" - -# The list of dependencies that we track at HEAD and periodically -# float forward in this repository. -FLOATING_DEPS=( - "knative.dev/pkg@${VERSION}" - "knative.dev/eventing@${VERSION}" - "knative.dev/test-infra@${VERSION}" -) - -# Parse flags to determine any we should pass to dep. -GO_GET=0 -while [[ $# -ne 0 ]]; do - parameter=$1 - case ${parameter} in - --upgrade) GO_GET=1 ;; - *) abort "unknown option ${parameter}" ;; - esac - shift -done -readonly GO_GET - -if (( GO_GET )); then - go get -d ${FLOATING_DEPS[@]} -fi - -# Prune modules. -go mod tidy -go mod vendor - -rm -rf $(find vendor/ -name 'OWNERS') -rm -rf $(find vendor/ -name 'OWNERS_ALIASES') -rm -rf $(find vendor/ -name 'BUILD') -rm -rf $(find vendor/ -name 'BUILD.bazel') -rm -rf $(find vendor/ -name '*_test.go') - -export GOFLAGS=-mod=vendor - -update_licenses third_party/VENDOR-LICENSE "./..." +go_update_deps "$@" diff --git a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go index eac9ad148..89c2358d5 100644 --- a/vendor/github.com/google/go-containerregistry/pkg/name/tag.go +++ b/vendor/github.com/google/go-containerregistry/pkg/name/tag.go @@ -69,7 +69,7 @@ func (t Tag) Scope(action string) string { } func checkTag(name string) error { - return checkElement("tag", name, tagChars, 1, 127) + return checkElement("tag", name, tagChars, 1, 128) } // NewTag returns a new Tag representing the given name, according to the given strictness. diff --git a/vendor/golang.org/x/net/http2/transport.go b/vendor/golang.org/x/net/http2/transport.go index 76a92e0ca..2482f7bf9 100644 --- a/vendor/golang.org/x/net/http2/transport.go +++ b/vendor/golang.org/x/net/http2/transport.go @@ -2525,6 +2525,7 @@ func strSliceContains(ss []string, s string) bool { type erringRoundTripper struct{ err error } +func (rt erringRoundTripper) RoundTripErr() error { return rt.err } func (rt erringRoundTripper) RoundTrip(*http.Request) (*http.Response, error) { return nil, rt.err } // gzipReader wraps a response body so it can lazily diff --git a/vendor/golang.org/x/tools/go/analysis/validate.go b/vendor/golang.org/x/tools/go/analysis/validate.go index be9814346..ad0e7276c 100644 --- a/vendor/golang.org/x/tools/go/analysis/validate.go +++ b/vendor/golang.org/x/tools/go/analysis/validate.go @@ -3,6 +3,7 @@ package analysis import ( "fmt" "reflect" + "strings" "unicode" ) @@ -58,14 +59,28 @@ func Validate(analyzers []*Analyzer) error { } // recursion - for i, req := range a.Requires { + for _, req := range a.Requires { if err := visit(req); err != nil { - return fmt.Errorf("%s.Requires[%d]: %v", a.Name, i, err) + return err } } color[a] = black } + if color[a] == grey { + stack := []*Analyzer{a} + inCycle := map[string]bool{} + for len(stack) > 0 { + current := stack[len(stack)-1] + stack = stack[:len(stack)-1] + if color[current] == grey && !inCycle[current.Name] { + inCycle[current.Name] = true + stack = append(stack, current.Requires...) + } + } + return &CycleInRequiresGraphError{AnalyzerNames: inCycle} + } + return nil } for _, a := range analyzers { @@ -95,3 +110,17 @@ func validIdent(name string) bool { } return name != "" } + +type CycleInRequiresGraphError struct { + AnalyzerNames map[string]bool +} + +func (e *CycleInRequiresGraphError) Error() string { + var b strings.Builder + b.WriteString("cycle detected involving the following analyzers:") + for n := range e.AnalyzerNames { + b.WriteByte(' ') + b.WriteString(n) + } + return b.String() +} diff --git a/vendor/golang.org/x/tools/go/packages/golist.go b/vendor/golang.org/x/tools/go/packages/golist.go index 220d40987..bc04503c1 100644 --- a/vendor/golang.org/x/tools/go/packages/golist.go +++ b/vendor/golang.org/x/tools/go/packages/golist.go @@ -89,6 +89,10 @@ type golistState struct { rootDirsError error rootDirs map[string]string + goVersionOnce sync.Once + goVersionError error + goVersion string // third field of 'go version' + // vendorDirs caches the (non)existence of vendor directories. vendorDirs map[string]bool } @@ -242,6 +246,21 @@ extractQueries: } } } + // Add root for any package that matches a pattern. This applies only to + // packages that are modified by overlays, since they are not added as + // roots automatically. + for _, pattern := range restPatterns { + match := matchPattern(pattern) + for _, pkgID := range modifiedPkgs { + pkg, ok := response.seenPackages[pkgID] + if !ok { + continue + } + if match(pkg.PkgPath) { + response.addRoot(pkg.ID) + } + } + } sizeswg.Wait() if sizeserr != nil { @@ -638,7 +657,7 @@ func (state *golistState) createDriverResponse(words ...string) (*driverResponse // Temporary work-around for golang/go#39986. Parse filenames out of // error messages. This happens if there are unrecoverable syntax // errors in the source, so we can't match on a specific error message. - if err := p.Error; err != nil && len(err.ImportStack) == 0 && len(pkg.CompiledGoFiles) == 0 { + if err := p.Error; err != nil && state.shouldAddFilenameFromError(p) { addFilenameFromPos := func(pos string) bool { split := strings.Split(pos, ":") if len(split) < 1 { @@ -697,7 +716,60 @@ func (state *golistState) createDriverResponse(words ...string) (*driverResponse return &response, nil } -// getPkgPath finds the package path of a directory if it's relative to a root directory. +func (state *golistState) shouldAddFilenameFromError(p *jsonPackage) bool { + if len(p.GoFiles) > 0 || len(p.CompiledGoFiles) > 0 { + return false + } + + goV, err := state.getGoVersion() + if err != nil { + return false + } + + // On Go 1.14 and earlier, only add filenames from errors if the import stack is empty. + // The import stack behaves differently for these versions than newer Go versions. + if strings.HasPrefix(goV, "go1.13") || strings.HasPrefix(goV, "go1.14") { + return len(p.Error.ImportStack) == 0 + } + + // On Go 1.15 and later, only parse filenames out of error if there's no import stack, + // or the current package is at the top of the import stack. This is not guaranteed + // to work perfectly, but should avoid some cases where files in errors don't belong to this + // package. + return len(p.Error.ImportStack) == 0 || p.Error.ImportStack[len(p.Error.ImportStack)-1] == p.ImportPath +} + +func (state *golistState) getGoVersion() (string, error) { + state.goVersionOnce.Do(func() { + var b *bytes.Buffer + // Invoke go version. Don't use invokeGo because it will supply build flags, and + // go version doesn't expect build flags. + inv := gocommand.Invocation{ + Verb: "version", + Env: state.cfg.Env, + Logf: state.cfg.Logf, + } + gocmdRunner := state.cfg.gocmdRunner + if gocmdRunner == nil { + gocmdRunner = &gocommand.Runner{} + } + b, _, _, state.goVersionError = gocmdRunner.RunRaw(state.cfg.Context, inv) + if state.goVersionError != nil { + return + } + + sp := strings.Split(b.String(), " ") + if len(sp) < 3 { + state.goVersionError = fmt.Errorf("go version output: expected 'go version ', got '%s'", b.String()) + return + } + state.goVersion = sp[2] + }) + return state.goVersion, state.goVersionError +} + +// getPkgPath finds the package path of a directory if it's relative to a root +// directory. func (state *golistState) getPkgPath(dir string) (string, bool, error) { absDir, err := filepath.Abs(dir) if err != nil { diff --git a/vendor/golang.org/x/tools/go/packages/golist_overlay.go b/vendor/golang.org/x/tools/go/packages/golist_overlay.go index 4eabfd98c..874f90134 100644 --- a/vendor/golang.org/x/tools/go/packages/golist_overlay.go +++ b/vendor/golang.org/x/tools/go/packages/golist_overlay.go @@ -8,9 +8,12 @@ import ( "log" "os" "path/filepath" + "regexp" "sort" "strconv" "strings" + + "golang.org/x/tools/internal/gocommand" ) // processGolistOverlay provides rudimentary support for adding @@ -89,9 +92,19 @@ func (state *golistState) processGolistOverlay(response *responseDeduper) (modif // because the file is generated in another directory. testVariantOf = p continue nextPackage + } else if !isTestFile && hasTestFiles(p) { + // We're examining a test variant, but the overlaid file is + // a non-test file. Because the overlay implementation + // (currently) only adds a file to one package, skip this + // package, so that we can add the file to the production + // variant of the package. (https://golang.org/issue/36857 + // tracks handling overlays on both the production and test + // variant of a package). + continue nextPackage } - // We must have already seen the package of which this is a test variant. if pkg != nil && p != pkg && pkg.PkgPath == p.PkgPath { + // We have already seen the production version of the + // for which p is a test variant. if hasTestFiles(p) { testVariantOf = pkg } @@ -315,24 +328,25 @@ func (state *golistState) determineRootDirs() (map[string]string, error) { } func (state *golistState) determineRootDirsModules() (map[string]string, error) { - // This will only return the root directory for the main module. - // For now we only support overlays in main modules. + // List all of the modules--the first will be the directory for the main + // module. Any replaced modules will also need to be treated as roots. // Editing files in the module cache isn't a great idea, so we don't - // plan to ever support that, but editing files in replaced modules - // is something we may want to support. To do that, we'll want to - // do a go list -m to determine the replaced module's module path and - // directory, and then a go list -m {{with .Replace}}{{.Dir}}{{end}} - // from the main module to determine if that module is actually a replacement. - // See bcmills's comment here: https://github.com/golang/go/issues/37629#issuecomment-594179751 - // for more information. - out, err := state.invokeGo("list", "-m", "-json") + // plan to ever support that. + out, err := state.invokeGo("list", "-m", "-json", "all") if err != nil { - return nil, err + // 'go list all' will fail if we're outside of a module and + // GO111MODULE=on. Try falling back without 'all'. + var innerErr error + out, innerErr = state.invokeGo("list", "-m", "-json") + if innerErr != nil { + return nil, err + } } - m := map[string]string{} - type jsonMod struct{ Path, Dir string } + roots := map[string]string{} + modules := map[string]string{} + var i int for dec := json.NewDecoder(out); dec.More(); { - mod := new(jsonMod) + mod := new(gocommand.ModuleJSON) if err := dec.Decode(mod); err != nil { return nil, err } @@ -342,10 +356,15 @@ func (state *golistState) determineRootDirsModules() (map[string]string, error) if err != nil { return nil, err } - m[absDir] = mod.Path + modules[absDir] = mod.Path + // The first result is the main module. + if i == 0 || mod.Replace != nil && mod.Replace.Path != "" { + roots[absDir] = mod.Path + } } + i++ } - return m, nil + return roots, nil } func (state *golistState) determineRootDirsGOPATH() (map[string]string, error) { @@ -471,3 +490,79 @@ func maybeFixPackageName(newName string, isTestFile bool, pkgsOfDir []*Package) p.Name = newName } } + +// This function is copy-pasted from +// https://github.com/golang/go/blob/9706f510a5e2754595d716bd64be8375997311fb/src/cmd/go/internal/search/search.go#L360. +// It should be deleted when we remove support for overlays from go/packages. +// +// NOTE: This does not handle any ./... or ./ style queries, as this function +// doesn't know the working directory. +// +// matchPattern(pattern)(name) reports whether +// name matches pattern. Pattern is a limited glob +// pattern in which '...' means 'any string' and there +// is no other special syntax. +// Unfortunately, there are two special cases. Quoting "go help packages": +// +// First, /... at the end of the pattern can match an empty string, +// so that net/... matches both net and packages in its subdirectories, like net/http. +// Second, any slash-separated pattern element containing a wildcard never +// participates in a match of the "vendor" element in the path of a vendored +// package, so that ./... does not match packages in subdirectories of +// ./vendor or ./mycode/vendor, but ./vendor/... and ./mycode/vendor/... do. +// Note, however, that a directory named vendor that itself contains code +// is not a vendored package: cmd/vendor would be a command named vendor, +// and the pattern cmd/... matches it. +func matchPattern(pattern string) func(name string) bool { + // Convert pattern to regular expression. + // The strategy for the trailing /... is to nest it in an explicit ? expression. + // The strategy for the vendor exclusion is to change the unmatchable + // vendor strings to a disallowed code point (vendorChar) and to use + // "(anything but that codepoint)*" as the implementation of the ... wildcard. + // This is a bit complicated but the obvious alternative, + // namely a hand-written search like in most shell glob matchers, + // is too easy to make accidentally exponential. + // Using package regexp guarantees linear-time matching. + + const vendorChar = "\x00" + + if strings.Contains(pattern, vendorChar) { + return func(name string) bool { return false } + } + + re := regexp.QuoteMeta(pattern) + re = replaceVendor(re, vendorChar) + switch { + case strings.HasSuffix(re, `/`+vendorChar+`/\.\.\.`): + re = strings.TrimSuffix(re, `/`+vendorChar+`/\.\.\.`) + `(/vendor|/` + vendorChar + `/\.\.\.)` + case re == vendorChar+`/\.\.\.`: + re = `(/vendor|/` + vendorChar + `/\.\.\.)` + case strings.HasSuffix(re, `/\.\.\.`): + re = strings.TrimSuffix(re, `/\.\.\.`) + `(/\.\.\.)?` + } + re = strings.ReplaceAll(re, `\.\.\.`, `[^`+vendorChar+`]*`) + + reg := regexp.MustCompile(`^` + re + `$`) + + return func(name string) bool { + if strings.Contains(name, vendorChar) { + return false + } + return reg.MatchString(replaceVendor(name, vendorChar)) + } +} + +// replaceVendor returns the result of replacing +// non-trailing vendor path elements in x with repl. +func replaceVendor(x, repl string) string { + if !strings.Contains(x, "vendor") { + return x + } + elem := strings.Split(x, "/") + for i := 0; i < len(elem)-1; i++ { + if elem[i] == "vendor" { + elem[i] = repl + } + } + return strings.Join(elem, "/") +} diff --git a/vendor/golang.org/x/tools/internal/analysisinternal/analysis.go b/vendor/golang.org/x/tools/internal/analysisinternal/analysis.go index 9f4c68a18..01f6e829f 100644 --- a/vendor/golang.org/x/tools/internal/analysisinternal/analysis.go +++ b/vendor/golang.org/x/tools/internal/analysisinternal/analysis.go @@ -14,6 +14,7 @@ import ( "strings" "golang.org/x/tools/go/ast/astutil" + "golang.org/x/tools/internal/lsp/fuzzy" ) var ( @@ -50,7 +51,7 @@ func ZeroValue(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.T default: panic("unknown basic type") } - case *types.Chan, *types.Interface, *types.Map, *types.Pointer, *types.Signature, *types.Slice: + case *types.Chan, *types.Interface, *types.Map, *types.Pointer, *types.Signature, *types.Slice, *types.Array: return ast.NewIdent("nil") case *types.Struct: texpr := TypeExpr(fset, f, pkg, typ) // typ because we want the name here. @@ -60,21 +61,23 @@ func ZeroValue(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.T return &ast.CompositeLit{ Type: texpr, } - case *types.Array: - texpr := TypeExpr(fset, f, pkg, u.Elem()) - if texpr == nil { - return nil - } - return &ast.CompositeLit{ - Type: &ast.ArrayType{ - Elt: texpr, - Len: &ast.BasicLit{Kind: token.INT, Value: fmt.Sprintf("%v", u.Len())}, - }, - } } return nil } +// IsZeroValue checks whether the given expression is a 'zero value' (as determined by output of +// analysisinternal.ZeroValue) +func IsZeroValue(expr ast.Expr) bool { + switch e := expr.(type) { + case *ast.BasicLit: + return e.Value == "0" || e.Value == `""` + case *ast.Ident: + return e.Name == "nil" || e.Name == "false" + default: + return false + } +} + func TypeExpr(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.Type) ast.Expr { switch t := typ.(type) { case *types.Basic: @@ -195,8 +198,12 @@ func TypeExpr(fset *token.FileSet, f *ast.File, pkg *types.Package, typ types.Ty X: ast.NewIdent(pkgName), Sel: ast.NewIdent(t.Obj().Name()), } + case *types.Struct: + return ast.NewIdent(t.String()) + case *types.Interface: + return ast.NewIdent(t.String()) default: - return nil // TODO: anonymous structs, but who does that + return nil } } @@ -300,3 +307,119 @@ func WalkASTWithParent(n ast.Node, f func(n ast.Node, parent ast.Node) bool) { return f(n, parent) }) } + +// FindMatchingIdents finds all identifiers in 'node' that match any of the given types. +// 'pos' represents the position at which the identifiers may be inserted. 'pos' must be within +// the scope of each of identifier we select. Otherwise, we will insert a variable at 'pos' that +// is unrecognized. +func FindMatchingIdents(typs []types.Type, node ast.Node, pos token.Pos, info *types.Info, pkg *types.Package) map[types.Type][]*ast.Ident { + matches := map[types.Type][]*ast.Ident{} + // Initialize matches to contain the variable types we are searching for. + for _, typ := range typs { + if typ == nil { + continue + } + matches[typ] = []*ast.Ident{} + } + seen := map[types.Object]struct{}{} + ast.Inspect(node, func(n ast.Node) bool { + if n == nil { + return false + } + // Prevent circular definitions. If 'pos' is within an assignment statement, do not + // allow any identifiers in that assignment statement to be selected. Otherwise, + // we could do the following, where 'x' satisfies the type of 'f0': + // + // x := fakeStruct{f0: x} + // + assignment, ok := n.(*ast.AssignStmt) + if ok && pos > assignment.Pos() && pos <= assignment.End() { + return false + } + if n.End() > pos { + return n.Pos() <= pos + } + ident, ok := n.(*ast.Ident) + if !ok || ident.Name == "_" { + return true + } + obj := info.Defs[ident] + if obj == nil || obj.Type() == nil { + return true + } + if _, ok := obj.(*types.TypeName); ok { + return true + } + // Prevent duplicates in matches' values. + if _, ok = seen[obj]; ok { + return true + } + seen[obj] = struct{}{} + // Find the scope for the given position. Then, check whether the object + // exists within the scope. + innerScope := pkg.Scope().Innermost(pos) + if innerScope == nil { + return true + } + _, foundObj := innerScope.LookupParent(ident.Name, pos) + if foundObj != obj { + return true + } + // The object must match one of the types that we are searching for. + if idents, ok := matches[obj.Type()]; ok { + matches[obj.Type()] = append(idents, ast.NewIdent(ident.Name)) + } + // If the object type does not exactly match any of the target types, greedily + // find the first target type that the object type can satisfy. + for typ := range matches { + if obj.Type() == typ { + continue + } + if equivalentTypes(obj.Type(), typ) { + matches[typ] = append(matches[typ], ast.NewIdent(ident.Name)) + } + } + return true + }) + return matches +} + +func equivalentTypes(want, got types.Type) bool { + if want == got || types.Identical(want, got) { + return true + } + // Code segment to help check for untyped equality from (golang/go#32146). + if rhs, ok := want.(*types.Basic); ok && rhs.Info()&types.IsUntyped > 0 { + if lhs, ok := got.Underlying().(*types.Basic); ok { + return rhs.Info()&types.IsConstType == lhs.Info()&types.IsConstType + } + } + return types.AssignableTo(want, got) +} + +// FindBestMatch employs fuzzy matching to evaluate the similarity of each given identifier to the +// given pattern. We return the identifier whose name is most similar to the pattern. +func FindBestMatch(pattern string, idents []*ast.Ident) ast.Expr { + fuzz := fuzzy.NewMatcher(pattern) + var bestFuzz ast.Expr + highScore := float32(0) // minimum score is 0 (no match) + for _, ident := range idents { + // TODO: Improve scoring algorithm. + score := fuzz.Score(ident.Name) + if score > highScore { + highScore = score + bestFuzz = ident + } else if score == 0 { + // Order matters in the fuzzy matching algorithm. If we find no match + // when matching the target to the identifier, try matching the identifier + // to the target. + revFuzz := fuzzy.NewMatcher(ident.Name) + revScore := revFuzz.Score(pattern) + if revScore > highScore { + highScore = revScore + bestFuzz = ident + } + } + } + return bestFuzz +} diff --git a/vendor/golang.org/x/tools/internal/gopathwalk/walk.go b/vendor/golang.org/x/tools/internal/gopathwalk/walk.go index 390cb9db7..925ff5356 100644 --- a/vendor/golang.org/x/tools/internal/gopathwalk/walk.go +++ b/vendor/golang.org/x/tools/internal/gopathwalk/walk.go @@ -10,7 +10,6 @@ import ( "bufio" "bytes" "fmt" - "go/build" "io/ioutil" "log" "os" @@ -47,16 +46,6 @@ type Root struct { Type RootType } -// SrcDirsRoots returns the roots from build.Default.SrcDirs(). Not modules-compatible. -func SrcDirsRoots(ctx *build.Context) []Root { - var roots []Root - roots = append(roots, Root{filepath.Join(ctx.GOROOT, "src"), RootGOROOT}) - for _, p := range filepath.SplitList(ctx.GOPATH) { - roots = append(roots, Root{filepath.Join(p, "src"), RootGOPATH}) - } - return roots -} - // Walk walks Go source directories ($GOROOT, $GOPATH, etc) to find packages. // For each package found, add will be called (concurrently) with the absolute // paths of the containing source directory and the package directory. diff --git a/vendor/golang.org/x/tools/internal/imports/fix.go b/vendor/golang.org/x/tools/internal/imports/fix.go index ecd13e87a..613afc4d6 100644 --- a/vendor/golang.org/x/tools/internal/imports/fix.go +++ b/vendor/golang.org/x/tools/internal/imports/fix.go @@ -573,7 +573,9 @@ func getFixes(fset *token.FileSet, f *ast.File, filename string, env *ProcessEnv return fixes, nil } - addStdlibCandidates(p, p.missingRefs) + if err := addStdlibCandidates(p, p.missingRefs); err != nil { + return nil, err + } p.assumeSiblingImportsValid() if fixes, done := p.fix(); done { return fixes, nil @@ -601,15 +603,19 @@ func getCandidatePkgs(ctx context.Context, wrappedCallback *scanCallback, filena notSelf := func(p *pkg) bool { return p.packageName != filePkg || p.dir != filepath.Dir(filename) } + goenv, err := env.goEnv() + if err != nil { + return err + } // Start off with the standard library. for importPath, exports := range stdlib { p := &pkg{ - dir: filepath.Join(env.goroot(), "src", importPath), + dir: filepath.Join(goenv["GOROOT"], "src", importPath), importPathShort: importPath, packageName: path.Base(importPath), relevance: MaxRelevance, } - if notSelf(p) && wrappedCallback.packageNameLoaded(p) { + if notSelf(p) && wrappedCallback.dirFound(p) && wrappedCallback.packageNameLoaded(p) { wrappedCallback.exportsLoaded(p, exports) } } @@ -687,8 +693,8 @@ func candidateImportName(pkg *pkg) string { return "" } -// GetAllCandidates gets all of the packages starting with prefix that can be -// imported by filename, sorted by import path. +// GetAllCandidates calls wrapped for each package whose name starts with +// searchPrefix, and can be imported from filename with the package name filePkg. func GetAllCandidates(ctx context.Context, wrapped func(ImportFix), searchPrefix, filename, filePkg string, env *ProcessEnv) error { callback := &scanCallback{ rootFound: func(gopathwalk.Root) bool { @@ -722,6 +728,35 @@ func GetAllCandidates(ctx context.Context, wrapped func(ImportFix), searchPrefix return getCandidatePkgs(ctx, callback, filename, filePkg, env) } +// GetImportPaths calls wrapped for each package whose import path starts with +// searchPrefix, and can be imported from filename with the package name filePkg. +func GetImportPaths(ctx context.Context, wrapped func(ImportFix), searchPrefix, filename, filePkg string, env *ProcessEnv) error { + callback := &scanCallback{ + rootFound: func(gopathwalk.Root) bool { + return true + }, + dirFound: func(pkg *pkg) bool { + if !canUse(filename, pkg.dir) { + return false + } + return strings.HasPrefix(pkg.importPathShort, searchPrefix) + }, + packageNameLoaded: func(pkg *pkg) bool { + wrapped(ImportFix{ + StmtInfo: ImportInfo{ + ImportPath: pkg.importPathShort, + Name: candidateImportName(pkg), + }, + IdentName: pkg.packageName, + FixType: AddImport, + Relevance: pkg.relevance, + }) + return false + }, + } + return getCandidatePkgs(ctx, callback, filename, filePkg, env) +} + // A PackageExport is a package and its exports. type PackageExport struct { Fix *ImportFix @@ -779,33 +814,44 @@ type ProcessEnv struct { // If Logf is non-nil, debug logging is enabled through this function. Logf func(format string, args ...interface{}) - resolver Resolver -} + initialized bool -func (e *ProcessEnv) goroot() string { - return e.mustGetEnv("GOROOT") + resolver Resolver } -func (e *ProcessEnv) gopath() string { - return e.mustGetEnv("GOPATH") +func (e *ProcessEnv) goEnv() (map[string]string, error) { + if err := e.init(); err != nil { + return nil, err + } + return e.Env, nil } -func (e *ProcessEnv) mustGetEnv(k string) string { - v, ok := e.Env[k] - if !ok { - panic(fmt.Sprintf("%v not set in evaluated environment", k)) - } - return v +func (e *ProcessEnv) matchFile(dir, name string) (bool, error) { + return build.Default.MatchFile(dir, name) } // CopyConfig copies the env's configuration into a new env. func (e *ProcessEnv) CopyConfig() *ProcessEnv { - copy := *e - copy.resolver = nil - return © + copy := &ProcessEnv{ + GocmdRunner: e.GocmdRunner, + initialized: e.initialized, + BuildFlags: e.BuildFlags, + Logf: e.Logf, + WorkingDir: e.WorkingDir, + resolver: nil, + Env: map[string]string{}, + } + for k, v := range e.Env { + copy.Env[k] = v + } + return copy } func (e *ProcessEnv) init() error { + if e.initialized { + return nil + } + foundAllRequired := true for _, k := range RequiredGoEnvVars { if _, ok := e.Env[k]; !ok { @@ -814,6 +860,7 @@ func (e *ProcessEnv) init() error { } } if foundAllRequired { + e.initialized = true return nil } @@ -832,6 +879,7 @@ func (e *ProcessEnv) init() error { for k, v := range goEnv { e.Env[k] = v } + e.initialized = true return nil } @@ -858,10 +906,14 @@ func (e *ProcessEnv) GetResolver() (Resolver, error) { return e.resolver, nil } -func (e *ProcessEnv) buildContext() *build.Context { +func (e *ProcessEnv) buildContext() (*build.Context, error) { ctx := build.Default - ctx.GOROOT = e.goroot() - ctx.GOPATH = e.gopath() + goenv, err := e.goEnv() + if err != nil { + return nil, err + } + ctx.GOROOT = goenv["GOROOT"] + ctx.GOPATH = goenv["GOPATH"] // As of Go 1.14, build.Context has a Dir field // (see golang.org/issue/34860). @@ -877,7 +929,7 @@ func (e *ProcessEnv) buildContext() *build.Context { dir.SetString(e.WorkingDir) } - return &ctx + return &ctx, nil } func (e *ProcessEnv) invokeGo(ctx context.Context, verb string, args ...string) (*bytes.Buffer, error) { @@ -892,10 +944,14 @@ func (e *ProcessEnv) invokeGo(ctx context.Context, verb string, args ...string) return e.GocmdRunner.Run(ctx, inv) } -func addStdlibCandidates(pass *pass, refs references) { +func addStdlibCandidates(pass *pass, refs references) error { + goenv, err := pass.env.goEnv() + if err != nil { + return err + } add := func(pkg string) { // Prevent self-imports. - if path.Base(pkg) == pass.f.Name.Name && filepath.Join(pass.env.goroot(), "src", pkg) == pass.srcDir { + if path.Base(pkg) == pass.f.Name.Name && filepath.Join(goenv["GOROOT"], "src", pkg) == pass.srcDir { return } exports := copyExports(stdlib[pkg]) @@ -916,6 +972,7 @@ func addStdlibCandidates(pass *pass, refs references) { } } } + return nil } // A Resolver does the build-system-specific parts of goimports. @@ -1112,21 +1169,24 @@ func (r *gopathResolver) ClearForNewScan() { func (r *gopathResolver) loadPackageNames(importPaths []string, srcDir string) (map[string]string, error) { names := map[string]string{} + bctx, err := r.env.buildContext() + if err != nil { + return nil, err + } for _, path := range importPaths { - names[path] = importPathToName(r.env, path, srcDir) + names[path] = importPathToName(bctx, path, srcDir) } return names, nil } // importPathToName finds out the actual package name, as declared in its .go files. -// If there's a problem, it returns "". -func importPathToName(env *ProcessEnv, importPath, srcDir string) (packageName string) { +func importPathToName(bctx *build.Context, importPath, srcDir string) string { // Fast path for standard library without going to disk. if _, ok := stdlib[importPath]; ok { return path.Base(importPath) // stdlib packages always match their paths. } - buildPkg, err := env.buildContext().Import(importPath, srcDir, build.FindOnly) + buildPkg, err := bctx.Import(importPath, srcDir, build.FindOnly) if err != nil { return "" } @@ -1287,8 +1347,18 @@ func (r *gopathResolver) scan(ctx context.Context, callback *scanCallback) error } stop := r.cache.ScanAndListen(ctx, processDir) defer stop() + + goenv, err := r.env.goEnv() + if err != nil { + return err + } + var roots []gopathwalk.Root + roots = append(roots, gopathwalk.Root{filepath.Join(goenv["GOROOT"], "src"), gopathwalk.RootGOROOT}) + for _, p := range filepath.SplitList(goenv["GOPATH"]) { + roots = append(roots, gopathwalk.Root{filepath.Join(p, "src"), gopathwalk.RootGOPATH}) + } // The callback is not necessarily safe to use in the goroutine below. Process roots eagerly. - roots := filterRoots(gopathwalk.SrcDirsRoots(r.env.buildContext()), callback.rootFound) + roots = filterRoots(roots, callback.rootFound) // We can't cancel walks, because we need them to finish to have a usable // cache. Instead, run them in a separate goroutine and detach. scanDone := make(chan struct{}) @@ -1348,8 +1418,6 @@ func VendorlessPath(ipath string) string { } func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, dir string, includeTest bool) (string, []string, error) { - var exports []string - // Look for non-test, buildable .go files which could provide exports. all, err := ioutil.ReadDir(dir) if err != nil { @@ -1361,7 +1429,7 @@ func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, dir string, incl if !strings.HasSuffix(name, ".go") || (!includeTest && strings.HasSuffix(name, "_test.go")) { continue } - match, err := env.buildContext().MatchFile(dir, fi.Name()) + match, err := env.matchFile(dir, fi.Name()) if err != nil || !match { continue } @@ -1373,6 +1441,7 @@ func loadExportsFromFiles(ctx context.Context, env *ProcessEnv, dir string, incl } var pkgName string + var exports []string fset := token.NewFileSet() for _, fi := range files { select { diff --git a/vendor/golang.org/x/tools/internal/imports/mod.go b/vendor/golang.org/x/tools/internal/imports/mod.go index 664fbbf5b..94880d616 100644 --- a/vendor/golang.org/x/tools/internal/imports/mod.go +++ b/vendor/golang.org/x/tools/internal/imports/mod.go @@ -53,6 +53,10 @@ func (r *ModuleResolver) init() error { return nil } + goenv, err := r.env.goEnv() + if err != nil { + return err + } inv := gocommand.Invocation{ BuildFlags: r.env.BuildFlags, Env: r.env.env(), @@ -82,7 +86,7 @@ func (r *ModuleResolver) init() error { if gmc := r.env.Env["GOMODCACHE"]; gmc != "" { r.moduleCacheDir = gmc } else { - r.moduleCacheDir = filepath.Join(filepath.SplitList(r.env.gopath())[0], "/pkg/mod") + r.moduleCacheDir = filepath.Join(filepath.SplitList(goenv["GOPATH"])[0], "/pkg/mod") } sort.Slice(r.modsByModPath, func(i, j int) bool { @@ -99,7 +103,7 @@ func (r *ModuleResolver) init() error { }) r.roots = []gopathwalk.Root{ - {filepath.Join(r.env.goroot(), "/src"), gopathwalk.RootGOROOT}, + {filepath.Join(goenv["GOROOT"], "/src"), gopathwalk.RootGOROOT}, } if r.main != nil { r.roots = append(r.roots, gopathwalk.Root{r.main.Dir, gopathwalk.RootCurrentModule}) @@ -240,7 +244,7 @@ func (r *ModuleResolver) findPackage(importPath string) (*gocommand.ModuleJSON, // files in that directory. If not, it could be provided by an // outer module. See #29736. for _, fi := range pkgFiles { - if ok, _ := r.env.buildContext().MatchFile(pkgDir, fi.Name()); ok { + if ok, _ := r.env.matchFile(pkgDir, fi.Name()); ok { return m, pkgDir } } diff --git a/vendor/golang.org/x/tools/internal/imports/zstdlib.go b/vendor/golang.org/x/tools/internal/imports/zstdlib.go index 16252111f..7b573b983 100644 --- a/vendor/golang.org/x/tools/internal/imports/zstdlib.go +++ b/vendor/golang.org/x/tools/internal/imports/zstdlib.go @@ -56,6 +56,7 @@ var stdlib = map[string][]string{ }, "bufio": []string{ "ErrAdvanceTooFar", + "ErrBadReadCount", "ErrBufferFull", "ErrFinalToken", "ErrInvalidUnreadByte", @@ -303,7 +304,9 @@ var stdlib = map[string][]string{ "PrivateKey", "PublicKey", "Sign", + "SignASN1", "Verify", + "VerifyASN1", }, "crypto/ed25519": []string{ "GenerateKey", @@ -322,11 +325,13 @@ var stdlib = map[string][]string{ "CurveParams", "GenerateKey", "Marshal", + "MarshalCompressed", "P224", "P256", "P384", "P521", "Unmarshal", + "UnmarshalCompressed", }, "crypto/hmac": []string{ "Equal", @@ -432,6 +437,7 @@ var stdlib = map[string][]string{ "CurveP521", "Dial", "DialWithDialer", + "Dialer", "ECDSAWithP256AndSHA256", "ECDSAWithP384AndSHA384", "ECDSAWithP521AndSHA512", @@ -507,6 +513,7 @@ var stdlib = map[string][]string{ "ConstraintViolationError", "CreateCertificate", "CreateCertificateRequest", + "CreateRevocationList", "DSA", "DSAWithSHA1", "DSAWithSHA256", @@ -581,6 +588,7 @@ var stdlib = map[string][]string{ "PublicKeyAlgorithm", "PureEd25519", "RSA", + "RevocationList", "SHA1WithRSA", "SHA256WithRSA", "SHA256WithRSAPSS", @@ -694,6 +702,7 @@ var stdlib = map[string][]string{ "String", "Tx", "TxOptions", + "Validator", "Value", "ValueConverter", "Valuer", @@ -2349,6 +2358,27 @@ var stdlib = map[string][]string{ "IMAGE_DIRECTORY_ENTRY_RESOURCE", "IMAGE_DIRECTORY_ENTRY_SECURITY", "IMAGE_DIRECTORY_ENTRY_TLS", + "IMAGE_DLLCHARACTERISTICS_APPCONTAINER", + "IMAGE_DLLCHARACTERISTICS_DYNAMIC_BASE", + "IMAGE_DLLCHARACTERISTICS_FORCE_INTEGRITY", + "IMAGE_DLLCHARACTERISTICS_GUARD_CF", + "IMAGE_DLLCHARACTERISTICS_HIGH_ENTROPY_VA", + "IMAGE_DLLCHARACTERISTICS_NO_BIND", + "IMAGE_DLLCHARACTERISTICS_NO_ISOLATION", + "IMAGE_DLLCHARACTERISTICS_NO_SEH", + "IMAGE_DLLCHARACTERISTICS_NX_COMPAT", + "IMAGE_DLLCHARACTERISTICS_TERMINAL_SERVER_AWARE", + "IMAGE_DLLCHARACTERISTICS_WDM_DRIVER", + "IMAGE_FILE_32BIT_MACHINE", + "IMAGE_FILE_AGGRESIVE_WS_TRIM", + "IMAGE_FILE_BYTES_REVERSED_HI", + "IMAGE_FILE_BYTES_REVERSED_LO", + "IMAGE_FILE_DEBUG_STRIPPED", + "IMAGE_FILE_DLL", + "IMAGE_FILE_EXECUTABLE_IMAGE", + "IMAGE_FILE_LARGE_ADDRESS_AWARE", + "IMAGE_FILE_LINE_NUMS_STRIPPED", + "IMAGE_FILE_LOCAL_SYMS_STRIPPED", "IMAGE_FILE_MACHINE_AM33", "IMAGE_FILE_MACHINE_AMD64", "IMAGE_FILE_MACHINE_ARM", @@ -2371,6 +2401,25 @@ var stdlib = map[string][]string{ "IMAGE_FILE_MACHINE_THUMB", "IMAGE_FILE_MACHINE_UNKNOWN", "IMAGE_FILE_MACHINE_WCEMIPSV2", + "IMAGE_FILE_NET_RUN_FROM_SWAP", + "IMAGE_FILE_RELOCS_STRIPPED", + "IMAGE_FILE_REMOVABLE_RUN_FROM_SWAP", + "IMAGE_FILE_SYSTEM", + "IMAGE_FILE_UP_SYSTEM_ONLY", + "IMAGE_SUBSYSTEM_EFI_APPLICATION", + "IMAGE_SUBSYSTEM_EFI_BOOT_SERVICE_DRIVER", + "IMAGE_SUBSYSTEM_EFI_ROM", + "IMAGE_SUBSYSTEM_EFI_RUNTIME_DRIVER", + "IMAGE_SUBSYSTEM_NATIVE", + "IMAGE_SUBSYSTEM_NATIVE_WINDOWS", + "IMAGE_SUBSYSTEM_OS2_CUI", + "IMAGE_SUBSYSTEM_POSIX_CUI", + "IMAGE_SUBSYSTEM_UNKNOWN", + "IMAGE_SUBSYSTEM_WINDOWS_BOOT_APPLICATION", + "IMAGE_SUBSYSTEM_WINDOWS_CE_GUI", + "IMAGE_SUBSYSTEM_WINDOWS_CUI", + "IMAGE_SUBSYSTEM_WINDOWS_GUI", + "IMAGE_SUBSYSTEM_XBOX", "ImportDirectory", "NewFile", "Open", @@ -4188,6 +4237,7 @@ var stdlib = map[string][]string{ "DevNull", "Environ", "ErrClosed", + "ErrDeadlineExceeded", "ErrExist", "ErrInvalid", "ErrNoDeadline", @@ -4646,6 +4696,7 @@ var stdlib = map[string][]string{ "ErrRange", "ErrSyntax", "FormatBool", + "FormatComplex", "FormatFloat", "FormatInt", "FormatUint", @@ -4655,6 +4706,7 @@ var stdlib = map[string][]string{ "Itoa", "NumError", "ParseBool", + "ParseComplex", "ParseFloat", "ParseInt", "ParseUint", diff --git a/vendor/golang.org/x/tools/internal/lsp/fuzzy/input.go b/vendor/golang.org/x/tools/internal/lsp/fuzzy/input.go new file mode 100644 index 000000000..ac377035e --- /dev/null +++ b/vendor/golang.org/x/tools/internal/lsp/fuzzy/input.go @@ -0,0 +1,168 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package fuzzy + +import ( + "unicode" +) + +// RuneRole specifies the role of a rune in the context of an input. +type RuneRole byte + +const ( + // RNone specifies a rune without any role in the input (i.e., whitespace/non-ASCII). + RNone RuneRole = iota + // RSep specifies a rune with the role of segment separator. + RSep + // RTail specifies a rune which is a lower-case tail in a word in the input. + RTail + // RUCTail specifies a rune which is an upper-case tail in a word in the input. + RUCTail + // RHead specifies a rune which is the first character in a word in the input. + RHead +) + +// RuneRoles detects the roles of each byte rune in an input string and stores it in the output +// slice. The rune role depends on the input type. Stops when it parsed all the runes in the string +// or when it filled the output. If output is nil, then it gets created. +func RuneRoles(str string, reuse []RuneRole) []RuneRole { + var output []RuneRole + if cap(reuse) < len(str) { + output = make([]RuneRole, 0, len(str)) + } else { + output = reuse[:0] + } + + prev, prev2 := rtNone, rtNone + for i := 0; i < len(str); i++ { + r := rune(str[i]) + + role := RNone + + curr := rtLower + if str[i] <= unicode.MaxASCII { + curr = runeType(rt[str[i]] - '0') + } + + if curr == rtLower { + if prev == rtNone || prev == rtPunct { + role = RHead + } else { + role = RTail + } + } else if curr == rtUpper { + role = RHead + + if prev == rtUpper { + // This and previous characters are both upper case. + + if i+1 == len(str) { + // This is last character, previous was also uppercase -> this is UCTail + // i.e., (current char is C): aBC / BC / ABC + role = RUCTail + } + } + } else if curr == rtPunct { + switch r { + case '.', ':': + role = RSep + } + } + if curr != rtLower { + if i > 1 && output[i-1] == RHead && prev2 == rtUpper && (output[i-2] == RHead || output[i-2] == RUCTail) { + // The previous two characters were uppercase. The current one is not a lower case, so the + // previous one can't be a HEAD. Make it a UCTail. + // i.e., (last char is current char - B must be a UCTail): ABC / ZABC / AB. + output[i-1] = RUCTail + } + } + + output = append(output, role) + prev2 = prev + prev = curr + } + return output +} + +type runeType byte + +const ( + rtNone runeType = iota + rtPunct + rtLower + rtUpper +) + +const rt = "00000000000000000000000000000000000000000000001122222222221000000333333333333333333333333330000002222222222222222222222222200000" + +// LastSegment returns the substring representing the last segment from the input, where each +// byte has an associated RuneRole in the roles slice. This makes sense only for inputs of Symbol +// or Filename type. +func LastSegment(input string, roles []RuneRole) string { + // Exclude ending separators. + end := len(input) - 1 + for end >= 0 && roles[end] == RSep { + end-- + } + if end < 0 { + return "" + } + + start := end - 1 + for start >= 0 && roles[start] != RSep { + start-- + } + + return input[start+1 : end+1] +} + +// ToLower transforms the input string to lower case, which is stored in the output byte slice. +// The lower casing considers only ASCII values - non ASCII values are left unmodified. +// Stops when parsed all input or when it filled the output slice. If output is nil, then it gets +// created. +func ToLower(input string, reuse []byte) []byte { + output := reuse + if cap(reuse) < len(input) { + output = make([]byte, len(input)) + } + + for i := 0; i < len(input); i++ { + r := rune(input[i]) + if r <= unicode.MaxASCII { + if 'A' <= r && r <= 'Z' { + r += 'a' - 'A' + } + } + output[i] = byte(r) + } + return output[:len(input)] +} + +// WordConsumer defines a consumer for a word delimited by the [start,end) byte offsets in an input +// (start is inclusive, end is exclusive). +type WordConsumer func(start, end int) + +// Words find word delimiters in an input based on its bytes' mappings to rune roles. The offset +// delimiters for each word are fed to the provided consumer function. +func Words(roles []RuneRole, consume WordConsumer) { + var wordStart int + for i, r := range roles { + switch r { + case RUCTail, RTail: + case RHead, RNone, RSep: + if i != wordStart { + consume(wordStart, i) + } + wordStart = i + if r != RHead { + // Skip this character. + wordStart = i + 1 + } + } + } + if wordStart != len(roles) { + consume(wordStart, len(roles)) + } +} diff --git a/vendor/golang.org/x/tools/internal/lsp/fuzzy/matcher.go b/vendor/golang.org/x/tools/internal/lsp/fuzzy/matcher.go new file mode 100644 index 000000000..16a643097 --- /dev/null +++ b/vendor/golang.org/x/tools/internal/lsp/fuzzy/matcher.go @@ -0,0 +1,398 @@ +// Copyright 2019 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +// Package fuzzy implements a fuzzy matching algorithm. +package fuzzy + +import ( + "bytes" + "fmt" +) + +const ( + // MaxInputSize is the maximum size of the input scored against the fuzzy matcher. Longer inputs + // will be truncated to this size. + MaxInputSize = 127 + // MaxPatternSize is the maximum size of the pattern used to construct the fuzzy matcher. Longer + // inputs are truncated to this size. + MaxPatternSize = 63 +) + +type scoreVal int + +func (s scoreVal) val() int { + return int(s) >> 1 +} + +func (s scoreVal) prevK() int { + return int(s) & 1 +} + +func score(val int, prevK int /*0 or 1*/) scoreVal { + return scoreVal(val<<1 + prevK) +} + +// Matcher implements a fuzzy matching algorithm for scoring candidates against a pattern. +// The matcher does not support parallel usage. +type Matcher struct { + pattern string + patternLower []byte // lower-case version of the pattern + patternShort []byte // first characters of the pattern + caseSensitive bool // set if the pattern is mix-cased + + patternRoles []RuneRole // the role of each character in the pattern + roles []RuneRole // the role of each character in the tested string + + scores [MaxInputSize + 1][MaxPatternSize + 1][2]scoreVal + + scoreScale float32 + + lastCandidateLen int // in bytes + lastCandidateMatched bool + + // Here we save the last candidate in lower-case. This is basically a byte slice we reuse for + // performance reasons, so the slice is not reallocated for every candidate. + lowerBuf [MaxInputSize]byte + rolesBuf [MaxInputSize]RuneRole +} + +func (m *Matcher) bestK(i, j int) int { + if m.scores[i][j][0].val() < m.scores[i][j][1].val() { + return 1 + } + return 0 +} + +// NewMatcher returns a new fuzzy matcher for scoring candidates against the provided pattern. +func NewMatcher(pattern string) *Matcher { + if len(pattern) > MaxPatternSize { + pattern = pattern[:MaxPatternSize] + } + + m := &Matcher{ + pattern: pattern, + patternLower: ToLower(pattern, nil), + } + + for i, c := range m.patternLower { + if pattern[i] != c { + m.caseSensitive = true + break + } + } + + if len(pattern) > 3 { + m.patternShort = m.patternLower[:3] + } else { + m.patternShort = m.patternLower + } + + m.patternRoles = RuneRoles(pattern, nil) + + if len(pattern) > 0 { + maxCharScore := 4 + m.scoreScale = 1 / float32(maxCharScore*len(pattern)) + } + + return m +} + +// Score returns the score returned by matching the candidate to the pattern. +// This is not designed for parallel use. Multiple candidates must be scored sequentially. +// Returns a score between 0 and 1 (0 - no match, 1 - perfect match). +func (m *Matcher) Score(candidate string) float32 { + if len(candidate) > MaxInputSize { + candidate = candidate[:MaxInputSize] + } + lower := ToLower(candidate, m.lowerBuf[:]) + m.lastCandidateLen = len(candidate) + + if len(m.pattern) == 0 { + // Empty patterns perfectly match candidates. + return 1 + } + + if m.match(candidate, lower) { + sc := m.computeScore(candidate, lower) + if sc > minScore/2 && !m.poorMatch() { + m.lastCandidateMatched = true + if len(m.pattern) == len(candidate) { + // Perfect match. + return 1 + } + + if sc < 0 { + sc = 0 + } + normalizedScore := float32(sc) * m.scoreScale + if normalizedScore > 1 { + normalizedScore = 1 + } + + return normalizedScore + } + } + + m.lastCandidateMatched = false + return 0 +} + +const minScore = -10000 + +// MatchedRanges returns matches ranges for the last scored string as a flattened array of +// [begin, end) byte offset pairs. +func (m *Matcher) MatchedRanges() []int { + if len(m.pattern) == 0 || !m.lastCandidateMatched { + return nil + } + i, j := m.lastCandidateLen, len(m.pattern) + if m.scores[i][j][0].val() < minScore/2 && m.scores[i][j][1].val() < minScore/2 { + return nil + } + + var ret []int + k := m.bestK(i, j) + for i > 0 { + take := (k == 1) + k = m.scores[i][j][k].prevK() + if take { + if len(ret) == 0 || ret[len(ret)-1] != i { + ret = append(ret, i) + ret = append(ret, i-1) + } else { + ret[len(ret)-1] = i - 1 + } + j-- + } + i-- + } + // Reverse slice. + for i := 0; i < len(ret)/2; i++ { + ret[i], ret[len(ret)-1-i] = ret[len(ret)-1-i], ret[i] + } + return ret +} + +func (m *Matcher) match(candidate string, candidateLower []byte) bool { + i, j := 0, 0 + for ; i < len(candidateLower) && j < len(m.patternLower); i++ { + if candidateLower[i] == m.patternLower[j] { + j++ + } + } + if j != len(m.patternLower) { + return false + } + + // The input passes the simple test against pattern, so it is time to classify its characters. + // Character roles are used below to find the last segment. + m.roles = RuneRoles(candidate, m.rolesBuf[:]) + + return true +} + +func (m *Matcher) computeScore(candidate string, candidateLower []byte) int { + pattLen, candLen := len(m.pattern), len(candidate) + + for j := 0; j <= len(m.pattern); j++ { + m.scores[0][j][0] = minScore << 1 + m.scores[0][j][1] = minScore << 1 + } + m.scores[0][0][0] = score(0, 0) // Start with 0. + + segmentsLeft, lastSegStart := 1, 0 + for i := 0; i < candLen; i++ { + if m.roles[i] == RSep { + segmentsLeft++ + lastSegStart = i + 1 + } + } + + // A per-character bonus for a consecutive match. + consecutiveBonus := 2 + wordIdx := 0 // Word count within segment. + for i := 1; i <= candLen; i++ { + + role := m.roles[i-1] + isHead := role == RHead + + if isHead { + wordIdx++ + } else if role == RSep && segmentsLeft > 1 { + wordIdx = 0 + segmentsLeft-- + } + + var skipPenalty int + if i == 1 || (i-1) == lastSegStart { + // Skipping the start of first or last segment. + skipPenalty++ + } + + for j := 0; j <= pattLen; j++ { + // By default, we don't have a match. Fill in the skip data. + m.scores[i][j][1] = minScore << 1 + + // Compute the skip score. + k := 0 + if m.scores[i-1][j][0].val() < m.scores[i-1][j][1].val() { + k = 1 + } + + skipScore := m.scores[i-1][j][k].val() + // Do not penalize missing characters after the last matched segment. + if j != pattLen { + skipScore -= skipPenalty + } + m.scores[i][j][0] = score(skipScore, k) + + if j == 0 || candidateLower[i-1] != m.patternLower[j-1] { + // Not a match. + continue + } + pRole := m.patternRoles[j-1] + + if role == RTail && pRole == RHead { + if j > 1 { + // Not a match: a head in the pattern matches a tail character in the candidate. + continue + } + // Special treatment for the first character of the pattern. We allow + // matches in the middle of a word if they are long enough, at least + // min(3, pattern.length) characters. + if !bytes.HasPrefix(candidateLower[i-1:], m.patternShort) { + continue + } + } + + // Compute the char score. + var charScore int + // Bonus 1: the char is in the candidate's last segment. + if segmentsLeft <= 1 { + charScore++ + } + // Bonus 2: Case match or a Head in the pattern aligns with one in the word. + // Single-case patterns lack segmentation signals and we assume any character + // can be a head of a segment. + if candidate[i-1] == m.pattern[j-1] || role == RHead && (!m.caseSensitive || pRole == RHead) { + charScore++ + } + + // Penalty 1: pattern char is Head, candidate char is Tail. + if role == RTail && pRole == RHead { + charScore-- + } + // Penalty 2: first pattern character matched in the middle of a word. + if j == 1 && role == RTail { + charScore -= 4 + } + + // Third dimension encodes whether there is a gap between the previous match and the current + // one. + for k := 0; k < 2; k++ { + sc := m.scores[i-1][j-1][k].val() + charScore + + isConsecutive := k == 1 || i-1 == 0 || i-1 == lastSegStart + if isConsecutive { + // Bonus 3: a consecutive match. First character match also gets a bonus to + // ensure prefix final match score normalizes to 1.0. + // Logically, this is a part of charScore, but we have to compute it here because it + // only applies for consecutive matches (k == 1). + sc += consecutiveBonus + } + if k == 0 { + // Penalty 3: Matching inside a segment (and previous char wasn't matched). Penalize for the lack + // of alignment. + if role == RTail || role == RUCTail { + sc -= 3 + } + } + + if sc > m.scores[i][j][1].val() { + m.scores[i][j][1] = score(sc, k) + } + } + } + } + + result := m.scores[len(candidate)][len(m.pattern)][m.bestK(len(candidate), len(m.pattern))].val() + + return result +} + +// ScoreTable returns the score table computed for the provided candidate. Used only for debugging. +func (m *Matcher) ScoreTable(candidate string) string { + var buf bytes.Buffer + + var line1, line2, separator bytes.Buffer + line1.WriteString("\t") + line2.WriteString("\t") + for j := 0; j < len(m.pattern); j++ { + line1.WriteString(fmt.Sprintf("%c\t\t", m.pattern[j])) + separator.WriteString("----------------") + } + + buf.WriteString(line1.String()) + buf.WriteString("\n") + buf.WriteString(separator.String()) + buf.WriteString("\n") + + for i := 1; i <= len(candidate); i++ { + line1.Reset() + line2.Reset() + + line1.WriteString(fmt.Sprintf("%c\t", candidate[i-1])) + line2.WriteString("\t") + + for j := 1; j <= len(m.pattern); j++ { + line1.WriteString(fmt.Sprintf("M%6d(%c)\t", m.scores[i][j][0].val(), dir(m.scores[i][j][0].prevK()))) + line2.WriteString(fmt.Sprintf("H%6d(%c)\t", m.scores[i][j][1].val(), dir(m.scores[i][j][1].prevK()))) + } + buf.WriteString(line1.String()) + buf.WriteString("\n") + buf.WriteString(line2.String()) + buf.WriteString("\n") + buf.WriteString(separator.String()) + buf.WriteString("\n") + } + + return buf.String() +} + +func dir(prevK int) rune { + if prevK == 0 { + return 'M' + } + return 'H' +} + +func (m *Matcher) poorMatch() bool { + if len(m.pattern) < 2 { + return false + } + + i, j := m.lastCandidateLen, len(m.pattern) + k := m.bestK(i, j) + + var counter, len int + for i > 0 { + take := (k == 1) + k = m.scores[i][j][k].prevK() + if take { + len++ + if k == 0 && len < 3 && m.roles[i-1] == RTail { + // Short match in the middle of a word + counter++ + if counter > 1 { + return true + } + } + j-- + } else { + len = 0 + } + i-- + } + return false +} diff --git a/vendor/honnef.co/go/tools/pattern/match.go b/vendor/honnef.co/go/tools/pattern/match.go index ff039baa7..2c036ed19 100644 --- a/vendor/honnef.co/go/tools/pattern/match.go +++ b/vendor/honnef.co/go/tools/pattern/match.go @@ -242,6 +242,28 @@ func match(m *Matcher, l, r interface{}) (interface{}, bool) { } } + { + ln, ok1 := l.([]*ast.Field) + rn, ok2 := r.([]*ast.Field) + if ok1 || ok2 { + if ok1 && !ok2 { + rn = []*ast.Field{r.(*ast.Field)} + } else if !ok1 && ok2 { + ln = []*ast.Field{l.(*ast.Field)} + } + + if len(ln) != len(rn) { + return nil, false + } + for i, ll := range ln { + if _, ok := match(m, ll, rn[i]); !ok { + return nil, false + } + } + return r, true + } + } + panic(fmt.Sprintf("unsupported comparison: %T and %T", l, r)) } diff --git a/vendor/honnef.co/go/tools/staticcheck/lint.go b/vendor/honnef.co/go/tools/staticcheck/lint.go index 16b6dbd84..22b6bbc2d 100644 --- a/vendor/honnef.co/go/tools/staticcheck/lint.go +++ b/vendor/honnef.co/go/tools/staticcheck/lint.go @@ -1721,6 +1721,10 @@ func CheckUnreadVariableValues(pass *analysis.Pass) (interface{}, error) { continue } + if _, ok := val.(*ir.Const); ok { + // a zero-valued constant, for example in 'foo := []string(nil)' + continue + } if !hasUse(val, nil) { report.Report(pass, assign, fmt.Sprintf("this value of %s is never used", lhs)) } diff --git a/vendor/honnef.co/go/tools/version/version.go b/vendor/honnef.co/go/tools/version/version.go index eed7b0def..79066e90a 100644 --- a/vendor/honnef.co/go/tools/version/version.go +++ b/vendor/honnef.co/go/tools/version/version.go @@ -7,7 +7,7 @@ import ( "runtime" ) -const Version = "2020.1.4" +const Version = "2020.1.5" // version returns a version descriptor and reports whether the // version is a known release. diff --git a/vendor/knative.dev/test-infra/scripts/README.md b/vendor/knative.dev/test-infra/scripts/README.md index 32e17ac1f..1f3aa6f70 100644 --- a/vendor/knative.dev/test-infra/scripts/README.md +++ b/vendor/knative.dev/test-infra/scripts/README.md @@ -110,33 +110,17 @@ main "$@" This is a helper script for Knative E2E test scripts. To use it: -1. [optional] Customize the test cluster. Set the following environment - variables if the default values don't fit your needs: - - - `E2E_CLUSTER_REGION`: Cluster region, defaults to `us-central1`. - - `E2E_CLUSTER_BACKUP_REGIONS`: Space-separated list of regions to retry test - cluster creation in case of stockout. Defaults to `us-west1 us-east1`. - - `E2E_CLUSTER_ZONE`: Cluster zone (e.g., `a`), defaults to none (i.e. use a - regional cluster). - - `E2E_CLUSTER_BACKUP_ZONES`: Space-separated list of zones to retry test - cluster creation in case of stockout. If defined, - `E2E_CLUSTER_BACKUP_REGIONS` will be ignored thus it defaults to none. - - `E2E_CLUSTER_MACHINE`: Cluster node machine type, defaults to - `e2-standard-4}`. - - `E2E_MIN_CLUSTER_NODES`: Minimum number of nodes in the cluster when - autoscaling, defaults to 1. - - `E2E_MAX_CLUSTER_NODES`: Maximum number of nodes in the cluster when - autoscaling, defaults to 3. +1. [optional] Customize the test cluster. Pass the flags as described + [here](../kntest/pkg/kubetest2/gke/README.md) to the `initialize` function + call if the default values don't fit your needs. 1. Source the script. 1. [optional] Write the `knative_setup()` function, which will set up your - system under test (e.g., Knative Serving). This function won't be called if - you use the `--skip-knative-setup` flag. + system under test (e.g., Knative Serving). 1. [optional] Write the `knative_teardown()` function, which will tear down your - system under test (e.g., Knative Serving). This function won't be called if - you use the `--skip-knative-setup` flag. + system under test (e.g., Knative Serving). 1. [optional] Write the `test_setup()` function, which will set up the test resources. @@ -160,7 +144,7 @@ This is a helper script for Knative E2E test scripts. To use it: of items to skip in the command line if the flag was parsed successfully. For example, return 1 for a simple flag, and 2 for a flag with a parameter. -1. Call the `initialize()` function passing `$@` (without quotes). +1. Call the `initialize()` function passing `"$@"`. 1. Write logic for the end-to-end tests. Run all go tests using `go_test_e2e()` (or `report_go_test()` if you need a more fine-grained control) and call @@ -170,8 +154,8 @@ This is a helper script for Knative E2E test scripts. To use it: **Notes:** -1. Calling your script without arguments will create a new cluster in the GCP - project `$PROJECT_ID` and run the tests against it. +1. Calling your script without arguments will create a new cluster in your + current GCP project and run the tests against it. 1. Calling your script with `--run-tests` and the variable `KO_DOCKER_REPO` set will immediately start the tests against the cluster currently configured for @@ -183,10 +167,6 @@ This is a helper script for Knative E2E test scripts. To use it: 1. By default Istio is installed on the cluster via Addon, use `--skip-istio-addon` if you choose not to have it preinstalled. -1. You can force running the tests against a specific GKE cluster version by - using the `--cluster-version` flag and passing a full version as the flag - value. - ### Sample end-to-end test script This script will test that the latest Knative Serving nightly release works. It @@ -196,9 +176,6 @@ test cluster is created in a specific region, `us-west2`. ```bash -# This test requires a cluster in LA -E2E_CLUSTER_REGION=us-west2 - source vendor/knative.dev/test-infra/scripts/e2e-tests.sh function knative_setup() { @@ -218,7 +195,8 @@ function parse_flags() { WAIT_FOR_KNATIVE=1 -initialize $@ +# This test requires a cluster in LA +initialize $@ --region=us-west2 # TODO: use go_test_e2e to run the tests. kubectl get pods || fail_test diff --git a/vendor/knative.dev/test-infra/scripts/e2e-tests.sh b/vendor/knative.dev/test-infra/scripts/e2e-tests.sh index a788b6827..19a7b6eec 100644 --- a/vendor/knative.dev/test-infra/scripts/e2e-tests.sh +++ b/vendor/knative.dev/test-infra/scripts/e2e-tests.sh @@ -17,50 +17,9 @@ # This is a helper script for Knative E2E test scripts. # See README.md for instructions on how to use it. -source $(dirname "${BASH_SOURCE[0]}")/library.sh +source $(dirname "${BASH_SOURCE[0]}")/infra-library.sh -# Build a resource name based on $E2E_BASE_NAME, a suffix and $BUILD_NUMBER. -# Restricts the name length to 40 chars (the limit for resource names in GCP). -# Name will have the form $E2E_BASE_NAME-$BUILD_NUMBER. -# Parameters: $1 - name suffix -function build_resource_name() { - local prefix=${E2E_BASE_NAME}-$1 - local suffix=${BUILD_NUMBER} - # Restrict suffix length to 20 chars - if [[ -n "${suffix}" ]]; then - suffix=${suffix:${#suffix}<20?0:-20} - fi - local name="${prefix:0:20}${suffix}" - # Ensure name doesn't end with "-" - echo "${name%-}" -} - -# Test cluster parameters - -# Configurable parameters -# export E2E_CLUSTER_REGION and E2E_CLUSTER_ZONE as they're used in the cluster setup subprocess -export E2E_CLUSTER_REGION=${E2E_CLUSTER_REGION:-us-central1} -# By default we use regional clusters. -export E2E_CLUSTER_ZONE=${E2E_CLUSTER_ZONE:-} - -# Default backup regions in case of stockouts; by default we don't fall back to a different zone in the same region -readonly E2E_CLUSTER_BACKUP_REGIONS=${E2E_CLUSTER_BACKUP_REGIONS:-us-west1 us-east1} -readonly E2E_CLUSTER_BACKUP_ZONES=${E2E_CLUSTER_BACKUP_ZONES:-} - -readonly E2E_GKE_ENVIRONMENT=${E2E_GKE_ENVIRONMENT:-prod} -readonly E2E_GKE_COMMAND_GROUP=${E2E_GKE_COMMAND_GROUP:-beta} - -# Each knative repository may have a different cluster size requirement here, -# so we allow calling code to set these parameters. If they are not set we -# use some sane defaults. -readonly E2E_MIN_CLUSTER_NODES=${E2E_MIN_CLUSTER_NODES:-1} -readonly E2E_MAX_CLUSTER_NODES=${E2E_MAX_CLUSTER_NODES:-3} -readonly E2E_CLUSTER_MACHINE=${E2E_CLUSTER_MACHINE:-e2-standard-4} - -readonly E2E_BASE_NAME="k${REPO_NAME}" -readonly E2E_CLUSTER_NAME=$(build_resource_name e2e-cls) -readonly E2E_NETWORK_NAME=$(build_resource_name e2e-net) -readonly TEST_RESULT_FILE=/tmp/${E2E_BASE_NAME}-e2e-result +readonly TEST_RESULT_FILE=/tmp/${REPO_NAME}-e2e-result # Flag whether test is using a boskos GCP project IS_BOSKOS=0 @@ -71,7 +30,7 @@ function teardown_test_resources() { (( IS_BOSKOS )) && return header "Tearing down test environment" function_exists test_teardown && test_teardown - (( ! SKIP_KNATIVE_SETUP )) && function_exists knative_teardown && knative_teardown + function_exists knative_teardown && knative_teardown } # Run the given E2E tests. Assume tests are tagged e2e, unless `-tags=XXX` is passed. @@ -86,239 +45,32 @@ function go_test_e2e() { report_go_test -race -count=1 "${go_test_args[@]}" } -# Dumps the k8s api server metrics. Spins up a proxy, waits a little bit and -# dumps the metrics to ${ARTIFACTS}/k8s.metrics.txt -function dump_metrics() { - header ">> Starting kube proxy" - kubectl proxy --port=8080 & - local proxy_pid=$! - sleep 5 - header ">> Grabbing k8s metrics" - curl -s http://localhost:8080/metrics > "${ARTIFACTS}"/k8s.metrics.txt - # Clean up proxy so it doesn't interfere with job shutting down - kill $proxy_pid || true -} - -# Dump info about the test cluster. If dump_extra_cluster_info() is defined, calls it too. -# This is intended to be called when a test fails to provide debugging information. -function dump_cluster_state() { - echo "***************************************" - echo "*** E2E TEST FAILED ***" - echo "*** Start of information dump ***" - echo "***************************************" - - local output - output="${ARTIFACTS}/k8s.dump-$(basename "${E2E_SCRIPT}").txt" - echo ">>> The dump is located at ${output}" - - for crd in $(kubectl api-resources --verbs=list -o name | sort); do - local count - count="$(kubectl get "$crd" --all-namespaces --no-headers 2>/dev/null | wc -l)" - echo ">>> ${crd} (${count} objects)" - if [[ "${count}" -gt "0" ]]; then - { - echo ">>> ${crd} (${count} objects)" - - echo ">>> Listing" - kubectl get "${crd}" --all-namespaces - - echo ">>> Details" - if [[ "${crd}" == "secrets" ]]; then - echo "Secrets are ignored for security reasons" - elif [[ "${crd}" == "events" ]]; then - echo "events are ignored as making a lot of noise" - else - kubectl get "${crd}" --all-namespaces -o yaml - fi - } >> "${output}" - fi - done - - if function_exists dump_extra_cluster_state; then - echo ">>> Extra dump" >> "${output}" - dump_extra_cluster_state >> "${output}" - fi - echo "***************************************" - echo "*** E2E TEST FAILED ***" - echo "*** End of information dump ***" - echo "***************************************" -} - -# On a Prow job, save some metadata about the test for Testgrid. -function save_metadata() { - (( ! IS_PROW )) && return - local geo_key="Region" - local geo_value="${E2E_CLUSTER_REGION}" - if [[ -n "${E2E_CLUSTER_ZONE}" ]]; then - geo_key="Zone" - geo_value="${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}" - fi - local cluster_version - cluster_version="$(gcloud container clusters list --project="${E2E_PROJECT_ID}" --format='value(currentMasterVersion)')" - run_kntest metadata set --key="E2E:${geo_key}" --value="${geo_value}" - run_kntest metadata set --key="E2E:Machine" --value="${E2E_CLUSTER_MACHINE}" - run_kntest metadata set --key="E2E:Version" --value="${cluster_version}" - run_kntest metadata set --key="E2E:MinNodes" --value="${E2E_MIN_CLUSTER_NODES}" - run_kntest metadata set --key="E2E:MaxNodes" --value="${E2E_MAX_CLUSTER_NODES}" -} - -# Create a test cluster with kubetest and call the current script again. -function create_test_cluster() { - # Fail fast during setup. - set -o errexit - set -o pipefail - - if function_exists cluster_setup; then - cluster_setup || fail_test "cluster setup failed" - fi - - echo "Cluster will have a minimum of ${E2E_MIN_CLUSTER_NODES} and a maximum of ${E2E_MAX_CLUSTER_NODES} nodes." - local CLUSTER_CREATION_ARGS=( - "gke" - "--create-command=${E2E_GKE_COMMAND_GROUP} container clusters create --quiet --enable-autoscaling - --min-nodes=${E2E_MIN_CLUSTER_NODES} --max-nodes=${E2E_MAX_CLUSTER_NODES} - --cluster-version=${E2E_CLUSTER_VERSION} - --scopes=cloud-platform --enable-basic-auth --no-issue-client-certificate - --no-enable-ip-alias --no-enable-autoupgrade - ${GKE_ADDONS} ${EXTRA_CLUSTER_CREATION_FLAGS[@]}" - "--environment=${E2E_GKE_ENVIRONMENT}" - "--cluster-name=${E2E_CLUSTER_NAME}" - "--num-nodes=${E2E_MIN_CLUSTER_NODES}" - "--machine-type=${E2E_CLUSTER_MACHINE}" - "--network=${E2E_NETWORK_NAME}" - --up - ) - if (( ! IS_BOSKOS )); then - CLUSTER_CREATION_ARGS+=("--gcp-project=${GCP_PROJECT}") - fi - # SSH keys are not used, but kubetest checks for their existence. - # Touch them so if they don't exist, empty files are create to satisfy the check. - mkdir -p "$HOME"/.ssh - touch "$HOME"/.ssh/google_compute_engine.pub - touch "$HOME"/.ssh/google_compute_engine - - local gcloud_project="${GCP_PROJECT}" - [[ -z "${gcloud_project}" ]] && gcloud_project="$(gcloud config get-value project)" - echo "gcloud project is ${gcloud_project}" - echo "gcloud user is $(gcloud config get-value core/account)" - (( IS_BOSKOS )) && echo "Using boskos for the test cluster" - [[ -n "${GCP_PROJECT}" ]] && echo "GCP project for test cluster is ${GCP_PROJECT}" - echo "Test script is ${E2E_SCRIPT}" - # Set arguments for this script again - local test_cmd_args="--run-tests" - (( SKIP_KNATIVE_SETUP )) && test_cmd_args+=" --skip-knative-setup" - [[ -n "${GCP_PROJECT}" ]] && test_cmd_args+=" --project ${GCP_PROJECT}" - [[ -n "${E2E_SCRIPT_CUSTOM_FLAGS[*]}" ]] && test_cmd_args+=" ${E2E_SCRIPT_CUSTOM_FLAGS[*]}" - local extra_flags=() - if (( IS_BOSKOS )); then - # Add arbitrary duration, wait for Boskos projects acquisition before error out - extra_flags+=("--boskos-acquire-timeout-seconds=1200") - elif (( ! SKIP_TEARDOWNS )); then - # Only let kubetest2 tear down the cluster if not using Boskos and teardowns are not expected to be skipped, - # it's done by Janitor if using Boskos - extra_flags+=("--down") - fi - - # Create cluster and run the tests - create_test_cluster_with_retries "${E2E_SCRIPT} ${test_cmd_args}" \ - "${CLUSTER_CREATION_ARGS[@]}" "${extra_flags[@]}" "${EXTRA_KUBETEST2_FLAGS[@]}" - local result="$?" - # Ignore any errors below, this is a best-effort cleanup and shouldn't affect the test result. - set +o errexit - function_exists cluster_teardown && cluster_teardown - echo "Artifacts were written to ${ARTIFACTS}" - echo "Test result code is ${result}" - exit "${result}" -} - -# Retry backup regions/zones if cluster creations failed due to stockout. -# Parameters: $1 - test command to run by the kubetest2 tester -# $2..$n - any other kubetest2 flags other than geo and cluster version flag. -function create_test_cluster_with_retries() { - local tester_command - IFS=' ' read -r -a tester_command <<< "$1" - local kubetest2_flags=( "${@:2}" ) - local cluster_creation_log=/tmp/${E2E_BASE_NAME}-cluster_creation-log - # zone_not_provided is a placeholder for e2e_cluster_zone to make for loop below work - local zone_not_provided="zone_not_provided" - - local e2e_cluster_regions=(${E2E_CLUSTER_REGION}) - local e2e_cluster_zones=(${E2E_CLUSTER_ZONE}) - - if [[ -n "${E2E_CLUSTER_BACKUP_ZONES}" ]]; then - e2e_cluster_zones+=(${E2E_CLUSTER_BACKUP_ZONES}) - elif [[ -n "${E2E_CLUSTER_BACKUP_REGIONS}" ]]; then - e2e_cluster_regions+=(${E2E_CLUSTER_BACKUP_REGIONS}) - e2e_cluster_zones=(${zone_not_provided}) - else - echo "No backup region/zone set, cluster creation will fail in case of stockout" - fi - - for e2e_cluster_region in "${e2e_cluster_regions[@]}"; do - for e2e_cluster_zone in "${e2e_cluster_zones[@]}"; do - E2E_CLUSTER_REGION=${e2e_cluster_region} - E2E_CLUSTER_ZONE=${e2e_cluster_zone} - [[ "${E2E_CLUSTER_ZONE}" == "${zone_not_provided}" ]] && E2E_CLUSTER_ZONE="" - local cluster_creation_zone="${E2E_CLUSTER_REGION}" - [[ -n "${E2E_CLUSTER_ZONE}" ]] && cluster_creation_zone="${E2E_CLUSTER_REGION}-${E2E_CLUSTER_ZONE}" - - header "Creating test cluster ${E2E_CLUSTER_VERSION} in ${cluster_creation_zone}" - if run_go_tool k8s-sigs.io/kubetest2 \ - kubetest2 "${kubetest2_flags[@]}" --region="${cluster_creation_zone}" \ - --test=exec -- "${tester_command[@]}" 2>&1 \ - | tee "${cluster_creation_log}"; then - return 0 - fi - # Retry if cluster creation failed because of: - # - stockout (https://github.com/knative/test-infra/issues/592) - # - latest GKE not available in this region/zone yet (https://github.com/knative/test-infra/issues/694) - [[ -z "$(grep -Fo 'does not have enough resources available to fulfill' "${cluster_creation_log}")" \ - && -z "$(grep -Fo 'ResponseError: code=400, message=No valid versions with the prefix' "${cluster_creation_log}")" \ - && -z "$(grep -Po 'ResponseError: code=400, message=Master version "[0-9a-z\-\.]+" is unsupported' "${cluster_creation_log}")" \ - && -z "$(grep -Po 'only \d+ nodes out of \d+ have registered; this is likely due to Nodes failing to start correctly' "${cluster_creation_log}")" ]] \ - && return 1 - done - done - echo "No more region/zones to try, quitting" - return 1 -} - # Setup the test cluster for running the tests. function setup_test_cluster() { # Fail fast during setup. set -o errexit set -o pipefail - header "Test cluster setup" - kubectl get nodes - header "Setting up test cluster" - + kubectl get nodes # Set the actual project the test cluster resides in # It will be a project assigned by Boskos if test is running on Prow, - # otherwise will be ${GCP_PROJECT} set up by user. + # otherwise will be ${E2E_GCP_PROJECT_ID} set up by user. E2E_PROJECT_ID="$(gcloud config get-value project)" export E2E_PROJECT_ID readonly E2E_PROJECT_ID - # Save some metadata about cluster creation for using in prow and testgrid - save_metadata - - local k8s_user - k8s_user=$(gcloud config get-value core/account) local k8s_cluster k8s_cluster=$(kubectl config current-context) is_protected_cluster "${k8s_cluster}" && \ abort "kubeconfig context set to ${k8s_cluster}, which is forbidden" - # If cluster admin role isn't set, this is a brand new cluster - # Setup the admin role and also KO_DOCKER_REPO if it is a GKE cluster - if [[ -z "$(kubectl get clusterrolebinding cluster-admin-binding 2> /dev/null)" && "${k8s_cluster}" =~ ^gke_.* ]]; then - acquire_cluster_admin_role "${k8s_user}" "${E2E_CLUSTER_NAME}" "${E2E_CLUSTER_REGION}" "${E2E_CLUSTER_ZONE}" - # Incorporate an element of randomness to ensure that each run properly publishes images. - export KO_DOCKER_REPO=gcr.io/${E2E_PROJECT_ID}/${E2E_BASE_NAME}-e2e-img/${RANDOM} - fi + # Acquire cluster admin role for the current user. + acquire_cluster_admin_role "${k8s_cluster}" + + # Setup KO_DOCKER_REPO if it is a GKE cluster. Incorporate an element of randomness to ensure that each run properly publishes images. + [[ "${k8s_cluster}" =~ ^gke_.* ]] && export KO_DOCKER_REPO=gcr.io/${E2E_PROJECT_ID}/${REPO_NAME}-e2e-img/${RANDOM} # Safety checks is_protected_gcr "${KO_DOCKER_REPO}" && \ @@ -327,23 +79,22 @@ function setup_test_cluster() { # Use default namespace for all subsequent kubectl commands in this context kubectl config set-context "${k8s_cluster}" --namespace=default - echo "- gcloud project is ${E2E_PROJECT_ID}" - echo "- gcloud user is ${k8s_user}" echo "- Cluster is ${k8s_cluster}" echo "- Docker is ${KO_DOCKER_REPO}" export KO_DATA_PATH="${REPO_ROOT_DIR}/.git" # Do not run teardowns if we explicitly want to skip them. - (( ! SKIP_TEARDOWNS )) && trap teardown_test_resources EXIT + (( ! SKIP_TEARDOWNS )) && add_trap teardown_test_resources EXIT # Handle failures ourselves, so we can dump useful info. set +o errexit set +o pipefail - if (( ! SKIP_KNATIVE_SETUP )) && function_exists knative_setup; then - # Wait for Istio installation to complete, if necessary, before calling knative_setup. - (( ! SKIP_ISTIO_ADDON )) && (wait_until_batch_job_complete istio-system || return 1) + # Wait for Istio installation to complete, if necessary, before calling knative_setup. + # TODO(chizhg): is it really needed? + (( ! SKIP_ISTIO_ADDON )) && (wait_until_batch_job_complete istio-system || return 1) + if function_exists knative_setup; then knative_setup || fail_test "Knative setup failed" fi if function_exists test_setup; then @@ -369,34 +120,31 @@ function fail_test() { exit 1 } -RUN_TESTS=0 -SKIP_KNATIVE_SETUP=0 -SKIP_ISTIO_ADDON=0 SKIP_TEARDOWNS=0 -GCP_PROJECT="" +SKIP_ISTIO_ADDON=0 E2E_SCRIPT="" -E2E_CLUSTER_VERSION="latest" -GKE_ADDONS="" -EXTRA_CLUSTER_CREATION_FLAGS=() -EXTRA_KUBETEST2_FLAGS=() -E2E_SCRIPT_CUSTOM_FLAGS=() +CLOUD_PROVIDER="gke" # Parse flags and initialize the test cluster. function initialize() { + local run_tests=0 + local custom_flags=() E2E_SCRIPT="$(get_canonical_path "$0")" + local e2e_script_command=( "${E2E_SCRIPT}" "--run-tests" ) cd "${REPO_ROOT_DIR}" while [[ $# -ne 0 ]]; do local parameter=$1 + # TODO(chizhg): remove parse_flags logic if no repos are using it. # Try parsing flag as a custom one. if function_exists parse_flags; then - parse_flags $@ + parse_flags "$@" local skip=$? if [[ ${skip} -ne 0 ]]; then # Skip parsed flag (and possibly argument) and continue # Also save it to it's passed through to the test script for ((i=1;i<=skip;i++)); do - E2E_SCRIPT_CUSTOM_FLAGS+=("$1") + e2e_script_command+=("$1") shift done continue @@ -404,52 +152,34 @@ function initialize() { fi # Try parsing flag as a standard one. case ${parameter} in - --run-tests) RUN_TESTS=1 ;; - --skip-knative-setup) SKIP_KNATIVE_SETUP=1 ;; + --run-tests) run_tests=1 ;; --skip-teardowns) SKIP_TEARDOWNS=1 ;; + # TODO(chizhg): remove this flag once the addons is defined as an env var. --skip-istio-addon) SKIP_ISTIO_ADDON=1 ;; *) - [[ $# -ge 2 ]] || abort "missing parameter after $1" - shift case ${parameter} in - --gcp-project) GCP_PROJECT=$1 ;; - --cluster-version) E2E_CLUSTER_VERSION=$1 ;; - --cluster-creation-flag) EXTRA_CLUSTER_CREATION_FLAGS+=("$1") ;; - --kubetest2-flag) EXTRA_KUBETEST2_FLAGS+=("$1") ;; - *) abort "unknown option ${parameter}" ;; + --cloud-provider) shift; CLOUD_PROVIDER="$1" ;; + *) custom_flags+=("$parameter") ;; esac esac shift done - # Use PROJECT_ID if set, unless --gcp-project was used. - if [[ -n "${PROJECT_ID:-}" && -z "${GCP_PROJECT}" ]]; then - echo "\$PROJECT_ID is set to '${PROJECT_ID}', using it to run the tests" - GCP_PROJECT="${PROJECT_ID}" - fi - if (( ! IS_PROW )) && (( ! RUN_TESTS )) && [[ -z "${GCP_PROJECT}" ]]; then - abort "set \$PROJECT_ID or use --gcp-project to select the GCP project where the tests are run" - fi + (( IS_PROW )) && [[ -z "${GCP_PROJECT_ID:-}" ]] && IS_BOSKOS=1 - (( IS_PROW )) && [[ -z "${GCP_PROJECT}" ]] && IS_BOSKOS=1 - - if (( SKIP_ISTIO_ADDON )); then - GKE_ADDONS="--addons=NodeLocalDNS" - else - GKE_ADDONS="--addons=Istio,NodeLocalDNS" + if [[ "${CLOUD_PROVIDER}" == "gke" ]]; then + if (( SKIP_ISTIO_ADDON )); then + custom_flags+=("--addons=NodeLocalDNS") + else + custom_flags+=("--addons=Istio,NodeLocalDNS") + fi fi - readonly RUN_TESTS - readonly GCP_PROJECT readonly IS_BOSKOS - readonly EXTRA_CLUSTER_CREATION_FLAGS - readonly EXTRA_KUBETEST2_FLAGS - readonly SKIP_KNATIVE_SETUP readonly SKIP_TEARDOWNS - readonly GKE_ADDONS - if (( ! RUN_TESTS )); then - create_test_cluster + if (( ! run_tests )); then + create_test_cluster "${CLOUD_PROVIDER}" custom_flags e2e_script_command else setup_test_cluster fi diff --git a/vendor/knative.dev/test-infra/scripts/infra-library.sh b/vendor/knative.dev/test-infra/scripts/infra-library.sh new file mode 100644 index 000000000..df71bb453 --- /dev/null +++ b/vendor/knative.dev/test-infra/scripts/infra-library.sh @@ -0,0 +1,141 @@ +#!/usr/bin/env bash + +# Copyright 2020 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a collection of functions for infra related setups, mainly +# cluster provisioning. It doesn't do anything when called from command line. + +source $(dirname "${BASH_SOURCE[0]}")/library.sh + +# Dumps the k8s api server metrics. Spins up a proxy, waits a little bit and +# dumps the metrics to ${ARTIFACTS}/k8s.metrics.txt +function dump_metrics() { + header ">> Starting kube proxy" + kubectl proxy --port=8080 & + local proxy_pid=$! + sleep 5 + header ">> Grabbing k8s metrics" + curl -s http://localhost:8080/metrics > "${ARTIFACTS}"/k8s.metrics.txt + # Clean up proxy so it doesn't interfere with job shutting down + kill $proxy_pid || true +} + +# Dump info about the test cluster. If dump_extra_cluster_info() is defined, calls it too. +# This is intended to be called when a test fails to provide debugging information. +function dump_cluster_state() { + echo "***************************************" + echo "*** E2E TEST FAILED ***" + echo "*** Start of information dump ***" + echo "***************************************" + + local output + output="${ARTIFACTS}/k8s.dump-$(basename "${E2E_SCRIPT}").txt" + echo ">>> The dump is located at ${output}" + + for crd in $(kubectl api-resources --verbs=list -o name | sort); do + local count + count="$(kubectl get "$crd" --all-namespaces --no-headers 2>/dev/null | wc -l)" + echo ">>> ${crd} (${count} objects)" + if [[ "${count}" -gt "0" ]]; then + { + echo ">>> ${crd} (${count} objects)" + + echo ">>> Listing" + kubectl get "${crd}" --all-namespaces + + echo ">>> Details" + if [[ "${crd}" == "secrets" ]]; then + echo "Secrets are ignored for security reasons" + elif [[ "${crd}" == "events" ]]; then + echo "events are ignored as making a lot of noise" + else + kubectl get "${crd}" --all-namespaces -o yaml + fi + } >> "${output}" + fi + done + + if function_exists dump_extra_cluster_state; then + echo ">>> Extra dump" >> "${output}" + dump_extra_cluster_state >> "${output}" + fi + echo "***************************************" + echo "*** E2E TEST FAILED ***" + echo "*** End of information dump ***" + echo "***************************************" +} + +# Sets the current user as cluster admin for the given cluster. +# Parameters: $1 - cluster context name +function acquire_cluster_admin_role() { + if [[ -z "$(kubectl get clusterrolebinding cluster-admin-binding 2> /dev/null)" ]]; then + if [[ "$1" =~ ^gke_.* ]]; then + kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole=cluster-admin --user="$(gcloud config get-value core/account)" + else + kubectl create clusterrolebinding cluster-admin-binding \ + --clusterrole=cluster-admin --user="${USER}" + fi + fi +} + +# Create a test cluster and run the tests if provided. +# Parameters: $1 - cluster provider name, e.g. gke +# $2 - custom flags supported by kntest +# $3 - test command to run after cluster is created +function create_test_cluster() { + # Fail fast during setup. + set -o errexit + set -o pipefail + + if function_exists cluster_setup; then + cluster_setup || fail_test "cluster setup failed" + fi + + case "$1" in + gke) create_gke_test_cluster "$2" "$3" "$4" ;; + kind) create_kind_test_cluster "$2" "$3" "$4" ;; + *) echo "unsupported provider: $1"; exit 1 ;; + esac + + local result="$?" + # Ignore any errors below, this is a best-effort cleanup and shouldn't affect the test result. + set +o errexit + set +o pipefail + function_exists cluster_teardown && cluster_teardown + echo "Artifacts were written to ${ARTIFACTS}" + echo "Test result code is ${result}" + exit "${result}" +} + +# Create a KIND test cluster with kubetest2 and run the test command. +# Parameters: $1 - extra cluster creation flags +# $2 - test command to run by the kubetest2 tester +function create_kind_test_cluster() { + local -n _custom_flags=$1 + local -n _test_command=$2 + + kubetest2 kind "${_custom_flags[@]}" --up --down --test=exec -- "${_test_command[@]}" +} + +# Create a GKE test cluster with kubetest2 and run the test command. +# Parameters: $1 - custom flags defined in kntest +# $2 - test command to run after the cluster is created (optional) +function create_gke_test_cluster() { + local -n _custom_flags=$1 + local -n _test_command=$2 + + run_kntest kubetest2 gke "${_custom_flags[@]}" --test-command="${_test_command[*]}" +} diff --git a/vendor/knative.dev/test-infra/scripts/library.sh b/vendor/knative.dev/test-infra/scripts/library.sh index 1267f7986..b11cd33be 100644 --- a/vendor/knative.dev/test-infra/scripts/library.sh +++ b/vendor/knative.dev/test-infra/scripts/library.sh @@ -69,12 +69,12 @@ function abort() { # Parameters: $1 - character to use for the box. # $2 - banner message. function make_banner() { - local msg="$1$1$1$1 $2 $1$1$1$1" - local border="${msg//[-0-9A-Za-z _.,\/()\']/$1}" - echo -e "${border}\n${msg}\n${border}" - # TODO(adrcunha): Remove once logs have timestamps on Prow - # For details, see https://github.com/kubernetes/test-infra/issues/10100 - echo -e "$1$1$1$1 $(TZ='America/Los_Angeles' date)\n${border}" + local msg="$1$1$1$1 $2 $1$1$1$1" + local border="${msg//[-0-9A-Za-z _.,\/()\']/$1}" + echo -e "${border}\n${msg}\n${border}" + # TODO(adrcunha): Remove once logs have timestamps on Prow + # For details, see https://github.com/kubernetes/test-infra/issues/10100 + echo -e "$1$1$1$1 $(TZ='America/Los_Angeles' date)\n${border}" } # Simple header for logging purposes. @@ -176,21 +176,8 @@ function wait_until_pods_running() { # Waits until all batch jobs complete in the given namespace. # Parameters: $1 - namespace. function wait_until_batch_job_complete() { - echo -n "Waiting until all batch jobs in namespace $1 run to completion." - for i in {1..150}; do # timeout after 5 minutes - local jobs=$(kubectl get jobs -n $1 --no-headers \ - -ocustom-columns='n:{.metadata.name},c:{.spec.completions},s:{.status.succeeded}') - # All jobs must be complete - local not_complete=$(echo "${jobs}" | awk '{if ($2!=$3) print $0}' | wc -l) - if [[ ${not_complete} -eq 0 ]]; then - echo -e "\nAll jobs are complete:\n${jobs}" - return 0 - fi - echo -n "." - sleep 2 - done - echo -e "\n\nERROR: timeout waiting for jobs to complete\n${jobs}" - return 1 + echo "Waiting until all batch jobs in namespace $1 run to completion." + kubectl wait job --for=condition=Complete --all -n "$1" --timeout=5m || return 1 } # Waits until the given service has an external address (IP/hostname). @@ -311,44 +298,6 @@ function dump_app_logs() { done } -# Sets the given user as cluster admin. -# Parameters: $1 - user -# $2 - cluster name -# $3 - cluster region -# $4 - cluster zone, optional -function acquire_cluster_admin_role() { - echo "Acquiring cluster-admin role for user '$1'" - local geoflag="--region=$3" - [[ -n $4 ]] && geoflag="--zone=$3-$4" - # Get the password of the admin and use it, as the service account (or the user) - # might not have the necessary permission. - local password=$(gcloud --format="value(masterAuth.password)" \ - container clusters describe $2 ${geoflag}) - if [[ -n "${password}" ]]; then - # Cluster created with basic authentication - kubectl config set-credentials cluster-admin \ - --username=admin --password=${password} - else - local cert=$(mktemp) - local key=$(mktemp) - echo "Certificate in ${cert}, key in ${key}" - gcloud --format="value(masterAuth.clientCertificate)" \ - container clusters describe $2 ${geoflag} | base64 --decode > ${cert} - gcloud --format="value(masterAuth.clientKey)" \ - container clusters describe $2 ${geoflag} | base64 --decode > ${key} - kubectl config set-credentials cluster-admin \ - --client-certificate=${cert} --client-key=${key} - fi - kubectl config set-context $(kubectl config current-context) \ - --user=cluster-admin - kubectl create clusterrolebinding cluster-admin-binding \ - --clusterrole=cluster-admin \ - --user=$1 - # Reset back to the default account - gcloud container clusters get-credentials \ - $2 ${geoflag} --project $(gcloud config get-value project) -} - # Run a command through tee and capture its output. # Parameters: $1 - file where the output will be stored. # $2... - command to run. @@ -564,6 +513,65 @@ function add_trap { done } +# Update go deps. +# Parameters (parsed as flags): +# "--upgrade", bool, do upgrade. +# "--release " used with upgrade. The release version to upgrade +# Knative components. ex: --release v0.18. Defaults to +# "master". +# Additional dependencies can be included in the upgrade by providing them in a +# global env var: FLOATING_DEPS +function go_update_deps() { + cd "${REPO_ROOT_DIR}" || return 1 + + export GO111MODULE=on + export GOFLAGS="" + + echo "=== Update Deps for Golang" + + local UPGRADE=0 + local VERSION="master" + while [[ $# -ne 0 ]]; do + parameter=$1 + case ${parameter} in + --upgrade) UPGRADE=1 ;; + --release) shift; VERSION="$1" ;; + *) abort "unknown option ${parameter}" ;; + esac + shift + done + + if (( UPGRADE )); then + echo "--- Upgrading to ${VERSION}" + FLOATING_DEPS+=( $(run_go_tool knative.dev/test-infra/buoy buoy float ${REPO_ROOT_DIR}/go.mod --release ${VERSION} --domain knative.dev) ) + if (( ${#FLOATING_DEPS[@]} )); then + echo "Floating deps to ${FLOATING_DEPS[@]}" + go get -d ${FLOATING_DEPS[@]} + else + echo "Nothing to upgrade." + fi + fi + + echo "--- Go mod tidy and vendor" + + # Prune modules. + go mod tidy + go mod vendor + + echo "--- Removing unwanted vendor files" + + # Remove unwanted vendor files + find vendor/ \( -name "OWNERS" -o -name "OWNERS_ALIASES" -o -name "BUILD" -o -name "BUILD.bazel" -o -name "*_test.go" \) -print0 | xargs -0 rm -f + + export GOFLAGS=-mod=vendor + + echo "--- Updating licenses" + update_licenses third_party/VENDOR-LICENSE "./..." + + echo "--- Removing broken symlinks" + remove_broken_symlinks ./vendor +} + # Run kntest tool, error out and ask users to install it if it's not currently installed. # Parameters: $1..$n - parameters passed to the tool. function run_kntest() { @@ -602,45 +610,6 @@ function check_licenses() { { echo "--- FAIL: go-licenses failed the license check"; return 1; } } -# Run the given linter on the given files, checking it exists first. -# Parameters: $1 - tool -# $2 - tool purpose (for error message if tool not installed) -# $3 - tool parameters (quote if multiple parameters used) -# $4..$n - files to run linter on -function run_lint_tool() { - local checker=$1 - local params=$3 - if ! hash ${checker} 2>/dev/null; then - warning "${checker} not installed, not $2" - return 127 - fi - shift 3 - local failed=0 - for file in $@; do - ${checker} ${params} ${file} || failed=1 - done - return ${failed} -} - -# Check links in the given markdown files. -# Parameters: $1...$n - files to inspect -function check_links_in_markdown() { - # https://github.com/raviqqe/liche - local config="${REPO_ROOT_DIR}/test/markdown-link-check-config.rc" - [[ ! -e ${config} ]] && config="${_TEST_INFRA_SCRIPTS_DIR}/markdown-link-check-config.rc" - local options="$(grep '^-' ${config} | tr \"\n\" ' ')" - run_lint_tool liche "checking links in markdown files" "-d ${REPO_ROOT_DIR} ${options}" $@ -} - -# Check format of the given markdown files. -# Parameters: $1..$n - files to inspect -function lint_markdown() { - # https://github.com/markdownlint/markdownlint - local config="${REPO_ROOT_DIR}/test/markdown-lint-config.rc" - [[ ! -e ${config} ]] && config="${_TEST_INFRA_SCRIPTS_DIR}/markdown-lint-config.rc" - run_lint_tool mdl "linting markdown files" "-c ${config}" $@ -} - # Return whether the given parameter is an integer. # Parameters: $1 - integer to check function is_int() { diff --git a/vendor/knative.dev/test-infra/scripts/markdown-link-check-config.rc b/vendor/knative.dev/test-infra/scripts/markdown-link-check-config.rc deleted file mode 100644 index 49b042e82..000000000 --- a/vendor/knative.dev/test-infra/scripts/markdown-link-check-config.rc +++ /dev/null @@ -1,5 +0,0 @@ -# For help, see -# https://github.com/raviqqe/liche/blob/master/README.md - -# Don't check localhost links and don't check templated links --x "(^https?://localhost($|[:/].*))|(^https://.*{{.*$)" diff --git a/vendor/knative.dev/test-infra/scripts/markdown-lint-config.rc b/vendor/knative.dev/test-infra/scripts/markdown-lint-config.rc deleted file mode 100644 index 461f891a2..000000000 --- a/vendor/knative.dev/test-infra/scripts/markdown-lint-config.rc +++ /dev/null @@ -1,5 +0,0 @@ -# For help, see -# https://github.com/markdownlint/markdownlint/blob/master/docs/configuration.md - -# Ignore long lines -rules "~MD013" diff --git a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh index a76f890c0..e873736be 100644 --- a/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh +++ b/vendor/knative.dev/test-infra/scripts/presubmit-tests.sh @@ -20,8 +20,6 @@ source $(dirname "${BASH_SOURCE[0]}")/library.sh # Custom configuration of presubmit tests -readonly DISABLE_MD_LINTING=${DISABLE_MD_LINTING:-0} -readonly DISABLE_MD_LINK_CHECK=${DISABLE_MD_LINK_CHECK:-0} readonly PRESUBMIT_TEST_FAIL_FAST=${PRESUBMIT_TEST_FAIL_FAST:-0} # Extensions or file patterns that don't require presubmit tests. @@ -119,36 +117,12 @@ function report_build_test() { [[ -z "${errors}" ]] } -# Perform markdown build tests if necessary, unless disabled. -function markdown_build_tests() { - (( DISABLE_MD_LINTING && DISABLE_MD_LINK_CHECK )) && return 0 - # Get changed markdown files (ignore /vendor, github templates, and deleted files) - local mdfiles="" - for file in $(echo "${CHANGED_FILES}" | grep \\.md$ | grep -v ^vendor/ | grep -v ^.github/); do - [[ -f "${file}" ]] && mdfiles="${mdfiles} ${file}" - done - [[ -z "${mdfiles}" ]] && return 0 - local failed=0 - if (( ! DISABLE_MD_LINTING )); then - subheader "Linting the markdown files" - report_build_test Markdown_Lint lint_markdown "${mdfiles}" || failed=1 - fi - if (( ! DISABLE_MD_LINK_CHECK )); then - subheader "Checking links in the markdown files" - report_build_test Markdown_Link check_links_in_markdown "${mdfiles}" || failed=1 - fi - return ${failed} -} - # Default build test runner that: -# * check markdown files # * run `/hack/verify-codegen.sh` (if it exists) # * `go build` on the entire repo # * check licenses in all go packages function default_build_test_runner() { local failed=0 - # Perform markdown build checks - markdown_build_tests || failed=1 # Run verify-codegen check if [[ -f ./hack/verify-codegen.sh ]]; then subheader "Checking autogenerated code is up-to-date" @@ -272,7 +246,7 @@ function run_integration_tests() { # Default integration test runner that runs all `test/e2e-*tests.sh`. function default_integration_test_runner() { local failed=0 - find test/ ! -name "$(printf "*\n*")" -name "e2e-*tests.sh" > tmp + find test/ ! -name "$(printf "*\n*")" -name "e2e-*tests.sh" -maxdepth 1 > tmp while IFS= read -r e2e_test do echo "Running integration test ${e2e_test}" @@ -322,15 +296,6 @@ function main() { echo ">> docker version" docker version fi - # node/pod names are important for debugging purposes, but they are missing - # after migrating from bootstrap to podutil. - # Report it here with the same logic as in bootstrap until it is fixed. - # (https://github.com/kubernetes/test-infra/blob/09bd4c6709dc64308406443f8996f90cf3b40ed1/jenkins/bootstrap.py#L588) - # TODO(chaodaiG): follow up on https://github.com/kubernetes/test-infra/blob/0fabd2ea816daa8c15d410c77a0c93c0550b283f/prow/initupload/run.go#L49 - echo ">> node name" - curl -H "Metadata-Flavor: Google" 'http://169.254.169.254/computeMetadata/v1/instance/name' 2> /dev/null - echo ">> pod name" - echo "${HOSTNAME}" fi [[ -z $1 ]] && set -- "--all-tests" diff --git a/vendor/knative.dev/test-infra/scripts/release.sh b/vendor/knative.dev/test-infra/scripts/release.sh index a10b842f3..bd05ed8bc 100644 --- a/vendor/knative.dev/test-infra/scripts/release.sh +++ b/vendor/knative.dev/test-infra/scripts/release.sh @@ -94,7 +94,7 @@ RELEASE_NOTES="" RELEASE_BRANCH="" RELEASE_GCS_BUCKET="knative-nightly/${REPO_NAME}" RELEASE_DIR="" -KO_FLAGS="-P" +KO_FLAGS="-P --platform=all" VALIDATION_TESTS="./test/presubmit-tests.sh" ARTIFACTS_TO_PUBLISH="" FROM_NIGHTLY_RELEASE="" diff --git a/vendor/modules.txt b/vendor/modules.txt index 480c05b4c..6176ccc65 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -136,7 +136,7 @@ github.com/google/go-cmp/cmp/internal/diff github.com/google/go-cmp/cmp/internal/flags github.com/google/go-cmp/cmp/internal/function github.com/google/go-cmp/cmp/internal/value -# github.com/google/go-containerregistry v0.1.2-0.20200717224239-a84993334b27 +# github.com/google/go-containerregistry v0.1.3 github.com/google/go-containerregistry/pkg/name # github.com/google/gofuzz v1.1.0 github.com/google/gofuzz @@ -264,7 +264,7 @@ golang.org/x/lint/golint # golang.org/x/mod v0.3.0 golang.org/x/mod/module golang.org/x/mod/semver -# golang.org/x/net v0.0.0-20200707034311-ab3426394381 +# golang.org/x/net v0.0.0-20200822124328-c89045814202 golang.org/x/net/context golang.org/x/net/context/ctxhttp golang.org/x/net/http/httpguts @@ -297,7 +297,7 @@ golang.org/x/text/unicode/norm golang.org/x/text/width # golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e golang.org/x/time/rate -# golang.org/x/tools v0.0.0-20200731060945-b5fad4ed8dd6 +# golang.org/x/tools v0.0.0-20200916195026-c9a70fc28ce3 golang.org/x/tools/go/analysis golang.org/x/tools/go/analysis/passes/inspect golang.org/x/tools/go/ast/astutil @@ -321,9 +321,10 @@ golang.org/x/tools/internal/fastwalk golang.org/x/tools/internal/gocommand golang.org/x/tools/internal/gopathwalk golang.org/x/tools/internal/imports +golang.org/x/tools/internal/lsp/fuzzy golang.org/x/tools/internal/packagesinternal golang.org/x/tools/internal/typesinternal -# golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 +# golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 golang.org/x/xerrors golang.org/x/xerrors/internal # gomodules.xyz/jsonpatch/v2 v2.1.0 @@ -485,7 +486,7 @@ gopkg.in/inf.v0 gopkg.in/yaml.v2 # gopkg.in/yaml.v3 v3.0.0-20191026110619-0b21df46bc1d gopkg.in/yaml.v3 -# honnef.co/go/tools v0.0.1-2020.1.4 +# honnef.co/go/tools v0.0.1-2020.1.5 honnef.co/go/tools/arg honnef.co/go/tools/cmd/staticcheck honnef.co/go/tools/code @@ -515,7 +516,7 @@ honnef.co/go/tools/staticcheck honnef.co/go/tools/stylecheck honnef.co/go/tools/unused honnef.co/go/tools/version -# k8s.io/api v0.18.7-rc.0 => k8s.io/api v0.17.6 +# k8s.io/api v0.18.8 => k8s.io/api v0.17.6 ## explicit k8s.io/api/admission/v1 k8s.io/api/admissionregistration/v1 @@ -561,7 +562,7 @@ k8s.io/api/storage/v1beta1 # k8s.io/apiextensions-apiserver v0.18.4 k8s.io/apiextensions-apiserver/pkg/apis/apiextensions k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1 -# k8s.io/apimachinery v0.18.7-rc.0 => k8s.io/apimachinery v0.17.6 +# k8s.io/apimachinery v0.18.8 => k8s.io/apimachinery v0.17.6 ## explicit k8s.io/apimachinery/pkg/api/apitesting/fuzzer k8s.io/apimachinery/pkg/api/equality @@ -937,7 +938,7 @@ knative.dev/serving/pkg/client/listers/autoscaling/v1alpha1 knative.dev/serving/pkg/client/listers/serving/v1 knative.dev/serving/pkg/client/listers/serving/v1alpha1 knative.dev/serving/pkg/client/listers/serving/v1beta1 -# knative.dev/test-infra v0.0.0-20200813220834-388e55a496cf +# knative.dev/test-infra v0.0.0-20201014021030-ae3984a33f82 ## explicit knative.dev/test-infra/scripts # sigs.k8s.io/yaml v1.2.0