From fc09878b93db35aafc74311f7ea6684ac08a3b83 Mon Sep 17 00:00:00 2001 From: Dmitri Goutnik Date: Mon, 21 Oct 2019 09:41:30 -0500 Subject: [PATCH] support custom Gitlab mirrors --- main.go | 4 +- tuple/mirrors.go | 40 ++++++++-------- tuple/parser.go | 26 +++++----- tuple/source.go | 52 ++++++++++++++++++++ tuple/tuple.go | 112 ++++++++++++++++++++++++++------------------ tuple/tuple_test.go | 1 + tuple/vanity.go | 12 ++--- 7 files changed, 165 insertions(+), 82 deletions(-) create mode 100644 tuple/source.go diff --git a/main.go b/main.go index 4175a60..cde02e6 100644 --- a/main.go +++ b/main.go @@ -27,7 +27,9 @@ func main() { parser := tuple.NewParser(flagPackagePrefix, flagOffline) tuples, errors := parser.Load(args[0]) - fmt.Println(tuples) + if len(tuples) != 0 { + fmt.Println(tuples) + } if errors != nil { fmt.Println(errors) } diff --git a/tuple/mirrors.go b/tuple/mirrors.go index e049692..780113c 100644 --- a/tuple/mirrors.go +++ b/tuple/mirrors.go @@ -1,32 +1,34 @@ package tuple var mirrors = map[string]struct { + source Source account string project string }{ - // Package name GH Account, GH Project - "camlistore.org": {"perkeep", "perkeep"}, - "cloud.google.com/go": {"googleapis", "google-cloud-go"}, - "contrib.go.opencensus.io/exporter/ocagent": {"census-ecosystem", "opencensus-go-exporter-ocagent"}, - "docker.io/go-docker": {"docker", "go-docker"}, - "git.apache.org/thrift.git": {"apache", "thrift"}, - "go.opencensus.io": {"census-instrumentation", "opencensus-go"}, - "go.elastic.co/apm": {"elastic", "apm-agent-go"}, - "go4.org": {"go4org", "go4"}, - "gocloud.dev": {"google", "go-cloud"}, - "google.golang.org/api": {"googleapis", "google-api-go-client"}, - "google.golang.org/appengine": {"golang", "appengine"}, - "google.golang.org/genproto": {"google", "go-genproto"}, - "google.golang.org/grpc": {"grpc", "grpc-go"}, - "gopkg.in/fsnotify.v1": {"fsnotify", "fsnotify"}, // fsnotify is a special case in gopkg.in - "sigs.k8s.io/yaml": {"kubernetes-sigs", "yaml"}, - "go.mongodb.org/mongo-driver": {"mongodb", "mongo-go-driver"}, - "gotest.tools": {"gotestyourself", "gotest.tools"}, + "camlistore.org": {source: GH{}, account: "perkeep", project: "perkeep"}, + "cloud.google.com/go": {source: GH{}, account: "googleapis", project: "google-cloud-go"}, + "contrib.go.opencensus.io/exporter/ocagent": {source: GH{}, account: "census-ecosystem", project: "opencensus-go-exporter-ocagent"}, + "docker.io/go-docker": {source: GH{}, account: "docker", project: "go-docker"}, + "git.apache.org/thrift.git": {source: GH{}, account: "apache", project: "thrift"}, + "go.opencensus.io": {source: GH{}, account: "census-instrumentation", project: "opencensus-go"}, + "go.elastic.co/apm": {source: GH{}, account: "elastic", project: "apm-agent-go"}, + "go.elastic.co/fastjson": {source: GH{}, account: "elastic", project: "go-fastjson"}, + "go4.org": {source: GH{}, account: "go4org", project: "go4"}, + "gocloud.dev": {source: GH{}, account: "google", project: "go-cloud"}, + "google.golang.org/api": {source: GH{}, account: "googleapis", project: "google-api-go-client"}, + "google.golang.org/appengine": {source: GH{}, account: "golang", project: "appengine"}, + "google.golang.org/genproto": {source: GH{}, account: "google", project: "go-genproto"}, + "google.golang.org/grpc": {source: GH{}, account: "grpc", project: "grpc-go"}, + "gopkg.in/fsnotify.v1": {source: GH{}, account: "fsnotify", project: "fsnotify"}, // fsnotify is a special case in gopkg.in + "sigs.k8s.io/yaml": {source: GH{}, account: "kubernetes-sigs", project: "yaml"}, + "go.mongodb.org/mongo-driver": {source: GH{}, account: "mongodb", project: "mongo-go-driver"}, + "gotest.tools": {source: GH{}, account: "gotestyourself", project: "gotest.tools"}, + "howett.net/plist": {source: GL{"https://gitlab.howett.net"}, account: "go", project: "plist"}, } func (t *Tuple) fromMirror() bool { if m, ok := mirrors[t.Package]; ok { - t.setSource(SourceGithub, m.account, m.project) + t.setSource(m.source, m.account, m.project) return true } return false diff --git a/tuple/parser.go b/tuple/parser.go index ca2b64d..130f05e 100644 --- a/tuple/parser.go +++ b/tuple/parser.go @@ -49,15 +49,17 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) { ch <- err return } - // Call Gitlab API to translate go.mod short commit IDs and tags - // to the full 40-character commit IDs as required by bsd.sites.mk - if !p.offline && t.Source == SourceGitlab { - c, err := gitlab.GetCommit(t.Account, t.Project, t.Tag) - if err != nil { - ch <- err - return + if !p.offline { + // Call Gitlab API to translate go.mod short commit IDs and tags + // to the full 40-character commit IDs as required by bsd.sites.mk + if _, ok := t.Source.(GL); ok { + c, err := gitlab.GetCommit(t.Source.Site(), t.Account, t.Project, t.Tag) + if err != nil { + ch <- err + return + } + t.Tag = c.ID } - t.Tag = c.ID } ch <- t }() @@ -73,9 +75,11 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) { if err, ok := res.(error); ok { switch err := err.(type) { case SourceError: - errors.SourceErrors = append(errors.SourceErrors, err) - case ReplacementError: - errors.ReplacementErrors = append(errors.ReplacementErrors, err) + errors.Source = append(errors.Source, err) + case ReplacementMissingCommitError: + errors.ReplacementMissingCommit = append(errors.ReplacementMissingCommit, err) + case ReplacementLocalFilesystemError: + errors.ReplacementLocalFilesystem = append(errors.ReplacementLocalFilesystem, err) default: return nil, err } diff --git a/tuple/source.go b/tuple/source.go new file mode 100644 index 0000000..a897c5a --- /dev/null +++ b/tuple/source.go @@ -0,0 +1,52 @@ +package tuple + +import "fmt" + +type Source interface { + Site() string + IsDefaultSite() bool + VarName() string +} + +type GH struct{} + +func (s GH) Site() string { + return "" +} + +func (s GH) IsDefaultSite() bool { + return true // there's only one Github +} + +func (s GH) VarName() string { + return "GH_TUPLE" +} + +func (s GH) String() string { + return "GH" +} + +type GL struct { + string +} + +const defaultSiteGitlab = "https://gitlab.com" + +func (s GL) Site() string { + if s.string != "" { + return s.string + } + return defaultSiteGitlab +} + +func (s GL) IsDefaultSite() bool { + return s.string == "" || s.string == defaultSiteGitlab +} + +func (s GL) VarName() string { + return "GL_TUPLE" +} + +func (s GL) String() string { + return fmt.Sprintf("GH{%q}", s.string) +} diff --git a/tuple/tuple.go b/tuple/tuple.go index 609de45..230096d 100644 --- a/tuple/tuple.go +++ b/tuple/tuple.go @@ -3,19 +3,12 @@ package tuple import ( "bytes" "fmt" + "reflect" "regexp" "sort" "strings" ) -type Source int - -const ( - SourceUnknown = iota - SourceGithub - SourceGitlab -) - type Tuple struct { Source Source // tuple source Package string // Go package name @@ -27,6 +20,9 @@ type Tuple struct { } func (t *Tuple) String() string { + if t.Source != nil && !t.Source.IsDefaultSite() { + return fmt.Sprintf("%s:%s:%s:%s:%s/%s/%s", t.Source.Site(), t.Account, t.Project, t.Tag, t.Group, t.Prefix, t.Package) + } return fmt.Sprintf("%s:%s:%s:%s/%s/%s", t.Account, t.Project, t.Tag, t.Group, t.Prefix, t.Package) } @@ -71,25 +67,28 @@ func (tt ByAccountAndProject) Less(i, j int) bool { return tt[i].String() < tt[j].String() } -var varName = map[Source]string{ - SourceGithub: "GH_TUPLE", - SourceGitlab: "GL_TUPLE", -} - func (tt Tuples) String() string { - bufs := map[Source]*bytes.Buffer{ - SourceGithub: new(bytes.Buffer), - SourceGitlab: new(bytes.Buffer), + if len(tt) == 0 { + return "" } + bufs := make(map[reflect.Type]*bytes.Buffer) + for _, t := range tt { - b, ok := bufs[t.Source] + st := reflect.TypeOf(t.Source) + if st == nil { + panic(fmt.Sprintf("unknown source in tuple: %v", t)) + } + + buf, ok := bufs[st] if !ok { - panic(fmt.Sprintf("unknown tuple source: %v", t.Source)) + buf = &bytes.Buffer{} + bufs[st] = buf } + var eol string - if b.Len() == 0 { - b.WriteString(fmt.Sprintf("%s=\t", varName[t.Source])) + if buf.Len() == 0 { + buf.WriteString(fmt.Sprintf("%s=\t", t.Source.VarName())) eol = `\` } s := t.String() @@ -98,49 +97,62 @@ func (tt Tuples) String() string { } else if eol == "" { eol = ` \` } - b.WriteString(fmt.Sprintf("%s\n\t\t%s", eol, s)) + buf.WriteString(fmt.Sprintf("%s\n\t\t%s", eol, s)) } - var sb strings.Builder - for _, k := range []Source{SourceGithub, SourceGitlab} { - b := bufs[k].Bytes() - if len(b) > 0 { - sb.Write(bufs[k].Bytes()) - sb.WriteRune('\n') + var ss []string + for _, buf := range bufs { + s := buf.String() + if len(s) > 0 { + ss = append(ss, s) } } + sort.Sort(sort.StringSlice(ss)) - return sb.String() + return fmt.Sprintf("%s\n", strings.Join(ss, "\n\n")) } type Errors struct { - SourceErrors []SourceError - ReplacementErrors []ReplacementError + Source []SourceError + ReplacementLocalFilesystem []ReplacementLocalFilesystemError + ReplacementMissingCommit []ReplacementMissingCommitError } func (ee Errors) Any() bool { - return len(ee.SourceErrors) > 0 || len(ee.ReplacementErrors) > 0 + return len(ee.Source) > 0 || + len(ee.ReplacementLocalFilesystem) > 0 || + len(ee.ReplacementMissingCommit) > 0 } func (ee Errors) Error() string { var buf bytes.Buffer - if len(ee.SourceErrors) > 0 { + if len(ee.Source) > 0 { buf.WriteString("\t\t# Mirrors for the following packages are not currently known, please look them up and handle these tuples manually:\n") - sort.Slice(ee.SourceErrors, func(i, j int) bool { - return ee.SourceErrors[i] < ee.SourceErrors[j] + sort.Slice(ee.Source, func(i, j int) bool { + return ee.Source[i] < ee.Source[j] }) - for _, err := range ee.SourceErrors { + for _, err := range ee.Source { buf.WriteString(fmt.Sprintf("\t\t#\t%s\n", string(err))) } } - if len(ee.ReplacementErrors) > 0 { + if len(ee.ReplacementMissingCommit) > 0 { buf.WriteString("\t\t# The following replacement packages are missing version/commit ID, you may need to symlink them in post-patch:\n") - sort.Slice(ee.ReplacementErrors, func(i, j int) bool { - return ee.ReplacementErrors[i] < ee.ReplacementErrors[j] + sort.Slice(ee.ReplacementMissingCommit, func(i, j int) bool { + return ee.ReplacementMissingCommit[i] < ee.ReplacementMissingCommit[j] }) - for _, err := range ee.ReplacementErrors { + for _, err := range ee.ReplacementMissingCommit { + buf.WriteString(fmt.Sprintf("\t\t#\t%s\n", string(err))) + } + } + + if len(ee.ReplacementLocalFilesystem) > 0 { + buf.WriteString("\t\t# The following replacement packages are referencing a local filesystem path, you may need to symlink them in post-patch:\n") + sort.Slice(ee.ReplacementLocalFilesystem, func(i, j int) bool { + return ee.ReplacementLocalFilesystem[i] < ee.ReplacementLocalFilesystem[j] + }) + for _, err := range ee.ReplacementLocalFilesystem { buf.WriteString(fmt.Sprintf("\t\t#\t%s\n", string(err))) } } @@ -148,9 +160,15 @@ func (ee Errors) Error() string { return buf.String() } -type ReplacementError string +type ReplacementLocalFilesystemError string -func (err ReplacementError) Error() string { +func (err ReplacementLocalFilesystemError) Error() string { + return string(err) +} + +type ReplacementMissingCommitError string + +func (err ReplacementMissingCommitError) Error() string { return string(err) } @@ -194,8 +212,12 @@ func Parse(spec, packagePrefix string) (*Tuple, error) { targetParts := strings.Fields(replaceSpecs[1]) switch len(targetParts) { case 1: + if targetParts[0][0] == '.' || targetParts[0][0] == '/' { + // Target package spec is local filesystem path + return nil, ReplacementLocalFilesystemError(spec) + } // Target package spec is missing commit ID/tag - return nil, ReplacementError(spec) + return nil, ReplacementMissingCommitError(spec) case 2: // OK default: @@ -249,7 +271,7 @@ func Parse(spec, packagePrefix string) (*Tuple, error) { return nil, fmt.Errorf("unexpected version string: %q", version) } - if tuple.Source == SourceUnknown { + if tuple.Source == nil { return nil, SourceError(tuple.String()) } @@ -272,7 +294,7 @@ func (t *Tuple) fromGithub() error { if len(parts) < 3 { return fmt.Errorf("unexpected Github package name: %q", t.Package) } - t.setSource(SourceGithub, parts[1], parts[2]) + t.setSource(GH{}, parts[1], parts[2]) return nil } @@ -281,6 +303,6 @@ func (t *Tuple) fromGitlab() error { if len(parts) < 3 { return fmt.Errorf("unexpected Gitlab package name: %q", t.Package) } - t.setSource(SourceGitlab, parts[1], parts[2]) + t.setSource(GL{}, parts[1], parts[2]) return nil } diff --git a/tuple/tuple_test.go b/tuple/tuple_test.go index 3cc148f..3d5b828 100644 --- a/tuple/tuple_test.go +++ b/tuple/tuple_test.go @@ -185,6 +185,7 @@ gitlab.com/gitlab-org/gitaly-proto/go/gitalypb rogpeppe:go-internal:v1.3.0:rogpeppe_go_internal/vendor/github.com/rogpeppe/go-internal \ ugorji:go:e444a5086c43:ugorji_go/vendor/github.com/ugorji/go \ user:pkg:v3.0.0:user_pkg/vendor/gopkg.in/user/pkg.v3 + GL_TUPLE= \ gitlab-org:gitaly-proto:v1.32.0:gitlab_org_gitaly_proto/vendor/gitlab.com/gitlab-org/gitaly-proto \ gitlab-org:labkit:0c3fc7cdd57c:gitlab_org_labkit/vendor/gitlab.com/gitlab-org/labkit diff --git a/tuple/vanity.go b/tuple/vanity.go index 1fb773e..179ee7f 100644 --- a/tuple/vanity.go +++ b/tuple/vanity.go @@ -32,7 +32,7 @@ func goMozillaOrgParser(t *Tuple) bool { if len(sm) == 0 { return false } - t.setSource(SourceGithub, "mozilla-services", sm[0][1]) + t.setSource(GH{}, "mozilla-services", sm[0][1]) return true } @@ -47,7 +47,7 @@ func goUberOrgParser(t *Tuple) bool { if len(sm) == 0 { return false } - t.setSource(SourceGithub, "uber-go", sm[0][1]) + t.setSource(GH{}, "uber-go", sm[0][1]) return true } @@ -62,7 +62,7 @@ func golangOrgParser(t *Tuple) bool { if len(sm) == 0 { return false } - t.setSource(SourceGithub, "golang", sm[0][1]) + t.setSource(GH{}, "golang", sm[0][1]) return true } @@ -79,9 +79,9 @@ func gopkgInParser(t *Tuple) bool { return false } if sm[0][2] == "" { - t.setSource(SourceGithub, "go-"+sm[0][1], sm[0][1]) + t.setSource(GH{}, "go-"+sm[0][1], sm[0][1]) } else { - t.setSource(SourceGithub, sm[0][1], sm[0][2]) + t.setSource(GH{}, sm[0][1], sm[0][2]) } return true } @@ -97,6 +97,6 @@ func k8sIoParser(t *Tuple) bool { if len(sm) == 0 { return false } - t.setSource(SourceGithub, "kubernetes", sm[0][1]) + t.setSource(GH{}, "kubernetes", sm[0][1]) return true }