support custom Gitlab mirrors

main.go

@@ -27,7 +27,9 @@ func main() {
 
 	parser := tuple.NewParser(flagPackagePrefix, flagOffline)
 	tuples, errors := parser.Load(args[0])
-	fmt.Println(tuples)
+	if len(tuples) != 0 {
+		fmt.Println(tuples)
+	}
 	if errors != nil {
 		fmt.Println(errors)
 	}

tuple/mirrors.go

@@ -1,32 +1,34 @@
 package tuple
 
 var mirrors = map[string]struct {
+	source  Source
 	account string
 	project string
 }{
-	// Package name                              GH Account, GH Project
-	"camlistore.org":                            {"perkeep", "perkeep"},
-	"cloud.google.com/go":                       {"googleapis", "google-cloud-go"},
-	"contrib.go.opencensus.io/exporter/ocagent": {"census-ecosystem", "opencensus-go-exporter-ocagent"},
-	"docker.io/go-docker":                       {"docker", "go-docker"},
-	"git.apache.org/thrift.git":                 {"apache", "thrift"},
-	"go.opencensus.io":                          {"census-instrumentation", "opencensus-go"},
-	"go.elastic.co/apm":                         {"elastic", "apm-agent-go"},
-	"go4.org":                                   {"go4org", "go4"},
-	"gocloud.dev":                               {"google", "go-cloud"},
-	"google.golang.org/api":                     {"googleapis", "google-api-go-client"},
-	"google.golang.org/appengine":               {"golang", "appengine"},
-	"google.golang.org/genproto":                {"google", "go-genproto"},
-	"google.golang.org/grpc":                    {"grpc", "grpc-go"},
-	"gopkg.in/fsnotify.v1":                      {"fsnotify", "fsnotify"}, // fsnotify is a special case in gopkg.in
-	"sigs.k8s.io/yaml":                          {"kubernetes-sigs", "yaml"},
-	"go.mongodb.org/mongo-driver":               {"mongodb", "mongo-go-driver"},
-	"gotest.tools":                              {"gotestyourself", "gotest.tools"},
+	"camlistore.org":                            {source: GH{}, account: "perkeep", project: "perkeep"},
+	"cloud.google.com/go":                       {source: GH{}, account: "googleapis", project: "google-cloud-go"},
+	"contrib.go.opencensus.io/exporter/ocagent": {source: GH{}, account: "census-ecosystem", project: "opencensus-go-exporter-ocagent"},
+	"docker.io/go-docker":                       {source: GH{}, account: "docker", project: "go-docker"},
+	"git.apache.org/thrift.git":                 {source: GH{}, account: "apache", project: "thrift"},
+	"go.opencensus.io":                          {source: GH{}, account: "census-instrumentation", project: "opencensus-go"},
+	"go.elastic.co/apm":                         {source: GH{}, account: "elastic", project: "apm-agent-go"},
+	"go.elastic.co/fastjson":                    {source: GH{}, account: "elastic", project: "go-fastjson"},
+	"go4.org":                                   {source: GH{}, account: "go4org", project: "go4"},
+	"gocloud.dev":                               {source: GH{}, account: "google", project: "go-cloud"},
+	"google.golang.org/api":                     {source: GH{}, account: "googleapis", project: "google-api-go-client"},
+	"google.golang.org/appengine":               {source: GH{}, account: "golang", project: "appengine"},
+	"google.golang.org/genproto":                {source: GH{}, account: "google", project: "go-genproto"},
+	"google.golang.org/grpc":                    {source: GH{}, account: "grpc", project: "grpc-go"},
+	"gopkg.in/fsnotify.v1":                      {source: GH{}, account: "fsnotify", project: "fsnotify"}, // fsnotify is a special case in gopkg.in
+	"sigs.k8s.io/yaml":                          {source: GH{}, account: "kubernetes-sigs", project: "yaml"},
+	"go.mongodb.org/mongo-driver":               {source: GH{}, account: "mongodb", project: "mongo-go-driver"},
+	"gotest.tools":                              {source: GH{}, account: "gotestyourself", project: "gotest.tools"},
+	"howett.net/plist":                          {source: GL{"https://gitlab.howett.net"}, account: "go", project: "plist"},
 }
 
 func (t *Tuple) fromMirror() bool {
 	if m, ok := mirrors[t.Package]; ok {
-		t.setSource(SourceGithub, m.account, m.project)
+		t.setSource(m.source, m.account, m.project)
 		return true
 	}
 	return false

tuple/parser.go

@@ -49,15 +49,17 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) {
 						ch <- err
 						return
 					}
-					// Call Gitlab API to translate go.mod short commit IDs and tags
-					// to the full 40-character commit IDs as required by bsd.sites.mk
-					if !p.offline && t.Source == SourceGitlab {
-						c, err := gitlab.GetCommit(t.Account, t.Project, t.Tag)
-						if err != nil {
-							ch <- err
-							return
+					if !p.offline {
+						// Call Gitlab API to translate go.mod short commit IDs and tags
+						// to the full 40-character commit IDs as required by bsd.sites.mk
+						if _, ok := t.Source.(GL); ok {
+							c, err := gitlab.GetCommit(t.Source.Site(), t.Account, t.Project, t.Tag)
+							if err != nil {
+								ch <- err
+								return
+							}
+							t.Tag = c.ID
 						}
-						t.Tag = c.ID
 					}
 					ch <- t
 				}()
@@ -73,9 +75,11 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) {
 		if err, ok := res.(error); ok {
 			switch err := err.(type) {
 			case SourceError:
-				errors.SourceErrors = append(errors.SourceErrors, err)
-			case ReplacementError:
-				errors.ReplacementErrors = append(errors.ReplacementErrors, err)
+				errors.Source = append(errors.Source, err)
+			case ReplacementMissingCommitError:
+				errors.ReplacementMissingCommit = append(errors.ReplacementMissingCommit, err)
+			case ReplacementLocalFilesystemError:
+				errors.ReplacementLocalFilesystem = append(errors.ReplacementLocalFilesystem, err)
 			default:
 				return nil, err
 			}

tuple/source.go

@@ -0,0 +1,52 @@
+package tuple
+
+import "fmt"
+
+type Source interface {
+	Site() string
+	IsDefaultSite() bool
+	VarName() string
+}
+
+type GH struct{}
+
+func (s GH) Site() string {
+	return ""
+}
+
+func (s GH) IsDefaultSite() bool {
+	return true // there's only one Github
+}
+
+func (s GH) VarName() string {
+	return "GH_TUPLE"
+}
+
+func (s GH) String() string {
+	return "GH"
+}
+
+type GL struct {
+	string
+}
+
+const defaultSiteGitlab = "https://gitlab.com"
+
+func (s GL) Site() string {
+	if s.string != "" {
+		return s.string
+	}
+	return defaultSiteGitlab
+}
+
+func (s GL) IsDefaultSite() bool {
+	return s.string == "" || s.string == defaultSiteGitlab
+}
+
+func (s GL) VarName() string {
+	return "GL_TUPLE"
+}
+
+func (s GL) String() string {
+	return fmt.Sprintf("GH{%q}", s.string)
+}

tuple/tuple.go

@@ -3,19 +3,12 @@ package tuple
 import (
 	"bytes"
 	"fmt"
+	"reflect"
 	"regexp"
 	"sort"
 	"strings"
 )
 
-type Source int
-
-const (
-	SourceUnknown = iota
-	SourceGithub
-	SourceGitlab
-)
-
 type Tuple struct {
 	Source  Source // tuple source
 	Package string // Go package name
@@ -27,6 +20,9 @@ type Tuple struct {
 }
 
 func (t *Tuple) String() string {
+	if t.Source != nil && !t.Source.IsDefaultSite() {
+		return fmt.Sprintf("%s:%s:%s:%s:%s/%s/%s", t.Source.Site(), t.Account, t.Project, t.Tag, t.Group, t.Prefix, t.Package)
+	}
 	return fmt.Sprintf("%s:%s:%s:%s/%s/%s", t.Account, t.Project, t.Tag, t.Group, t.Prefix, t.Package)
 }
 
@@ -71,25 +67,28 @@ func (tt ByAccountAndProject) Less(i, j int) bool {
 	return tt[i].String() < tt[j].String()
 }
 
-var varName = map[Source]string{
-	SourceGithub: "GH_TUPLE",
-	SourceGitlab: "GL_TUPLE",
-}
-
 func (tt Tuples) String() string {
-	bufs := map[Source]*bytes.Buffer{
-		SourceGithub: new(bytes.Buffer),
-		SourceGitlab: new(bytes.Buffer),
+	if len(tt) == 0 {
+		return ""
 	}
 
+	bufs := make(map[reflect.Type]*bytes.Buffer)
+
 	for _, t := range tt {
-		b, ok := bufs[t.Source]
+		st := reflect.TypeOf(t.Source)
+		if st == nil {
+			panic(fmt.Sprintf("unknown source in tuple: %v", t))
+		}
+
+		buf, ok := bufs[st]
 		if !ok {
-			panic(fmt.Sprintf("unknown tuple source: %v", t.Source))
+			buf = &bytes.Buffer{}
+			bufs[st] = buf
 		}
+
 		var eol string
-		if b.Len() == 0 {
-			b.WriteString(fmt.Sprintf("%s=\t", varName[t.Source]))
+		if buf.Len() == 0 {
+			buf.WriteString(fmt.Sprintf("%s=\t", t.Source.VarName()))
 			eol = `\`
 		}
 		s := t.String()
@@ -98,59 +97,78 @@ func (tt Tuples) String() string {
 		} else if eol == "" {
 			eol = ` \`
 		}
-		b.WriteString(fmt.Sprintf("%s\n\t\t%s", eol, s))
+		buf.WriteString(fmt.Sprintf("%s\n\t\t%s", eol, s))
 	}
 
-	var sb strings.Builder
-	for _, k := range []Source{SourceGithub, SourceGitlab} {
-		b := bufs[k].Bytes()
-		if len(b) > 0 {
-			sb.Write(bufs[k].Bytes())
-			sb.WriteRune('\n')
+	var ss []string
+	for _, buf := range bufs {
+		s := buf.String()
+		if len(s) > 0 {
+			ss = append(ss, s)
 		}
 	}
+	sort.Sort(sort.StringSlice(ss))
 
-	return sb.String()
+	return fmt.Sprintf("%s\n", strings.Join(ss, "\n\n"))
 }
 
 type Errors struct {
-	SourceErrors      []SourceError
-	ReplacementErrors []ReplacementError
+	Source                     []SourceError
+	ReplacementLocalFilesystem []ReplacementLocalFilesystemError
+	ReplacementMissingCommit   []ReplacementMissingCommitError
 }
 
 func (ee Errors) Any() bool {
-	return len(ee.SourceErrors) > 0 || len(ee.ReplacementErrors) > 0
+	return len(ee.Source) > 0 ||
+		len(ee.ReplacementLocalFilesystem) > 0 ||
+		len(ee.ReplacementMissingCommit) > 0
 }
 
 func (ee Errors) Error() string {
 	var buf bytes.Buffer
 
-	if len(ee.SourceErrors) > 0 {
+	if len(ee.Source) > 0 {
 		buf.WriteString("\t\t# Mirrors for the following packages are not currently known, please look them up and handle these tuples manually:\n")
-		sort.Slice(ee.SourceErrors, func(i, j int) bool {
-			return ee.SourceErrors[i] < ee.SourceErrors[j]
+		sort.Slice(ee.Source, func(i, j int) bool {
+			return ee.Source[i] < ee.Source[j]
 		})
-		for _, err := range ee.SourceErrors {
+		for _, err := range ee.Source {
 			buf.WriteString(fmt.Sprintf("\t\t#\t%s\n", string(err)))
 		}
 	}
 
-	if len(ee.ReplacementErrors) > 0 {
+	if len(ee.ReplacementMissingCommit) > 0 {
 		buf.WriteString("\t\t# The following replacement packages are missing version/commit ID, you may need to symlink them in post-patch:\n")
-		sort.Slice(ee.ReplacementErrors, func(i, j int) bool {
-			return ee.ReplacementErrors[i] < ee.ReplacementErrors[j]
+		sort.Slice(ee.ReplacementMissingCommit, func(i, j int) bool {
+			return ee.ReplacementMissingCommit[i] < ee.ReplacementMissingCommit[j]
 		})
-		for _, err := range ee.ReplacementErrors {
+		for _, err := range ee.ReplacementMissingCommit {
+			buf.WriteString(fmt.Sprintf("\t\t#\t%s\n", string(err)))
+		}
+	}
+
+	if len(ee.ReplacementLocalFilesystem) > 0 {
+		buf.WriteString("\t\t# The following replacement packages are referencing a local filesystem path, you may need to symlink them in post-patch:\n")
+		sort.Slice(ee.ReplacementLocalFilesystem, func(i, j int) bool {
+			return ee.ReplacementLocalFilesystem[i] < ee.ReplacementLocalFilesystem[j]
+		})
+		for _, err := range ee.ReplacementLocalFilesystem {
 			buf.WriteString(fmt.Sprintf("\t\t#\t%s\n", string(err)))
 		}
 	}
 
 	return buf.String()
 }
 
-type ReplacementError string
+type ReplacementLocalFilesystemError string
 
-func (err ReplacementError) Error() string {
+func (err ReplacementLocalFilesystemError) Error() string {
+	return string(err)
+}
+
+type ReplacementMissingCommitError string
+
+func (err ReplacementMissingCommitError) Error() string {
 	return string(err)
 }
 
@@ -194,8 +212,12 @@ func Parse(spec, packagePrefix string) (*Tuple, error) {
 		targetParts := strings.Fields(replaceSpecs[1])
 		switch len(targetParts) {
 		case 1:
+			if targetParts[0][0] == '.' || targetParts[0][0] == '/' {
+				// Target package spec is local filesystem path
+				return nil, ReplacementLocalFilesystemError(spec)
+			}
 			// Target package spec is missing commit ID/tag
-			return nil, ReplacementError(spec)
+			return nil, ReplacementMissingCommitError(spec)
 		case 2:
 			// OK
 		default:
@@ -249,7 +271,7 @@ func Parse(spec, packagePrefix string) (*Tuple, error) {
 		return nil, fmt.Errorf("unexpected version string: %q", version)
 	}
 
-	if tuple.Source == SourceUnknown {
+	if tuple.Source == nil {
 		return nil, SourceError(tuple.String())
 	}
 
@@ -272,7 +294,7 @@ func (t *Tuple) fromGithub() error {
 	if len(parts) < 3 {
 		return fmt.Errorf("unexpected Github package name: %q", t.Package)
 	}
-	t.setSource(SourceGithub, parts[1], parts[2])
+	t.setSource(GH{}, parts[1], parts[2])
 	return nil
 }
 
@@ -281,6 +303,6 @@ func (t *Tuple) fromGitlab() error {
 	if len(parts) < 3 {
 		return fmt.Errorf("unexpected Gitlab package name: %q", t.Package)
 	}
-	t.setSource(SourceGitlab, parts[1], parts[2])
+	t.setSource(GL{}, parts[1], parts[2])
 	return nil
 }

tuple/tuple_test.go

@@ -185,6 +185,7 @@ gitlab.com/gitlab-org/gitaly-proto/go/gitalypb
 		rogpeppe:go-internal:v1.3.0:rogpeppe_go_internal/vendor/github.com/rogpeppe/go-internal \
 		ugorji:go:e444a5086c43:ugorji_go/vendor/github.com/ugorji/go \
 		user:pkg:v3.0.0:user_pkg/vendor/gopkg.in/user/pkg.v3
+
 GL_TUPLE=	\
 		gitlab-org:gitaly-proto:v1.32.0:gitlab_org_gitaly_proto/vendor/gitlab.com/gitlab-org/gitaly-proto \
 		gitlab-org:labkit:0c3fc7cdd57c:gitlab_org_labkit/vendor/gitlab.com/gitlab-org/labkit