diff --git a/apis/apis.go b/apis/apis.go index d2f0b70..6d4629b 100644 --- a/apis/apis.go +++ b/apis/apis.go @@ -4,10 +4,33 @@ import ( "fmt" "io/ioutil" "net/http" + "os" + "strings" ) -func get(url string) ([]byte, error) { - resp, err := http.Get(url) +const ( + OfflineKey = "M2T_OFFLINE" + GithubCredentialsKey = "M2T_GITHUB" + // GitlabsCredentialsKey = "M2T_GITLAB" +) + +func get(url string, credsKey string) ([]byte, error) { + req, err := http.NewRequest("GET", url, nil) + if err != nil { + return nil, err + } + + if credsKey != "" { + creds := os.Getenv(credsKey) + if creds != "" { + credsSlice := strings.Split(creds, ":") + if len(credsSlice) == 2 { + req.SetBasicAuth(credsSlice[0], credsSlice[1]) + } + } + } + + resp, err := http.DefaultClient.Do(req) if err != nil { return nil, fmt.Errorf("GET %s: %v", url, err) } diff --git a/apis/github.go b/apis/github.go index e986f42..a7010f0 100644 --- a/apis/github.go +++ b/apis/github.go @@ -2,27 +2,71 @@ package apis import ( "encoding/json" + "errors" "fmt" "net/url" + "strings" ) type GithubCommit struct { - ID string `json:"sha"` + SHA string `json:"sha"` } -func GetGithubCommit(account, project, ref string) (*GithubCommit, error) { +type GithubRef struct { + Ref string `json:"ref"` +} + +var githubRateLimitError = fmt.Sprintf(`Github API rate limit exceeded. Please either: +- set %s environment variable to your Github "username:personal_access_token" + to let modules2tuple call Github API using basic authentication. + To create a new token, navigate to https://github.com/settings/tokens/new + (leave all checkboxes unchecked, modules2tuple doesn't need any access to your account) +- set %s=1 or pass "-offline" flag to module2tuple to disable network access`, OfflineKey, GithubCredentialsKey) + +func GetGithubCommit(account, project, tag string) (string, error) { + projectID := fmt.Sprintf("%s/%s", url.PathEscape(account), url.PathEscape(project)) + url := fmt.Sprintf("https://api.github.com/repos/%s/commits/%s", projectID, tag) + + resp, err := get(url, GithubCredentialsKey) + if err != nil { + if strings.Contains(err.Error(), "API rate limit exceeded") { + return "", errors.New(githubRateLimitError) + } + return "", fmt.Errorf("error getting commit %s for %s/%s: %v", tag, account, project, err) + } + + var res GithubCommit + if err := json.Unmarshal(resp, &res); err != nil { + return "", fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp)) + } + + return res.SHA, nil +} + +func LookupGithubTag(account, project, tag string) (string, error) { projectID := fmt.Sprintf("%s/%s", url.PathEscape(account), url.PathEscape(project)) - url := fmt.Sprintf("https://api.github.com/repos/%s/commits/%s", projectID, ref) + url := fmt.Sprintf("https://api.github.com/repos/%s/git/refs/tags", projectID) - resp, err := get(url) + resp, err := get(url, GithubCredentialsKey) if err != nil { - return nil, fmt.Errorf("error getting commit %s for %s/%s: %v", ref, account, project, err) + if strings.Contains(err.Error(), "API rate limit exceeded") { + return "", errors.New(githubRateLimitError) + } + return "", fmt.Errorf("error getting refs for %s/%s: %v", account, project, err) + } + + var res []GithubRef + if err := json.Unmarshal(resp, &res); err != nil { + return "", fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp)) } - var ret GithubCommit - if err := json.Unmarshal(resp, &ret); err != nil { - return nil, fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp)) + // Github API returns tags sorted by creation time, earliest first. + // Iterate through them in reverse order to find the most recent matching tag. + for i := len(res) - 1; i >= 0; i-- { + if strings.HasSuffix(res[i].Ref, "/"+tag) { + return strings.TrimPrefix(res[i].Ref, "refs/tags/"), nil + } } - return &ret, nil + return "", fmt.Errorf("tag %v doesn't seem to exist in %s/%s", tag, account, project) } diff --git a/apis/github_test.go b/apis/github_test.go index c97f97c..940d162 100644 --- a/apis/github_test.go +++ b/apis/github_test.go @@ -6,19 +6,40 @@ import "testing" func TestGetGithubCommit(t *testing.T) { examples := []struct { - account, project, ref, ID string + account, project, ref, hash string }{ {"dmgk", "modules2tuple", "v1.9.0", "fc09878b93db35aafc74311f7ea6684ac08a3b83"}, {"dmgk", "modules2tuple", "a0cdb416ca2c", "a0cdb416ca2cbf6d3dad67a97f4fdcfac954503e"}, } for i, x := range examples { - c, err := GetGithubCommit(x.account, x.project, x.ref) + hash, err := GetGithubCommit(x.account, x.project, x.ref) if err != nil { t.Fatal(err) } - if x.ID != c.ID { - t.Errorf("expected commit ID %s, got %s (example %d)", x.ID, c.ID, i) + if x.hash != hash { + t.Errorf("expected commit hash %s, got %s (example %d)", x.hash, hash, i) + } + } +} + +func TestLookupGithubTag(t *testing.T) { + examples := []struct { + account, project, given, expected string + }{ + {"hashicorp", "vault", "v1.0.4", "api/v1.0.4"}, + {"hashicorp", "vault", "v1.3.4", "v1.3.4"}, + // this repo has earlier mathing tag "codec/codecgen/v1.1.7" + {"ugorji", "go", "v1.1.7", "v1.1.7"}, + } + + for i, x := range examples { + tag, err := LookupGithubTag(x.account, x.project, x.given) + if err != nil { + t.Fatal(err) + } + if x.expected != tag { + t.Errorf("expected tag %s, got %s (example %d)", x.expected, tag, i) } } } diff --git a/apis/gitlab.go b/apis/gitlab.go index 3c44062..8c779b3 100644 --- a/apis/gitlab.go +++ b/apis/gitlab.go @@ -7,22 +7,22 @@ import ( ) type GitlabCommit struct { - ID string `json:"id"` + SHA string `json:"id"` } -func GetGitlabCommit(site, account, project, commit string) (*GitlabCommit, error) { +func GetGitlabCommit(site, account, project, commit string) (string, error) { projectID := url.PathEscape(fmt.Sprintf("%s/%s", account, project)) url := fmt.Sprintf("%s/api/v4/projects/%s/repository/commits/%s", site, projectID, commit) - resp, err := get(url) + resp, err := get(url, "") if err != nil { - return nil, fmt.Errorf("error getting commit %s for %s/%s: %v", commit, account, project, err) + return "", fmt.Errorf("error getting commit %s for %s/%s: %v", commit, account, project, err) } - var ret GitlabCommit - if err := json.Unmarshal(resp, &ret); err != nil { - return nil, fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp)) + var res GitlabCommit + if err := json.Unmarshal(resp, &res); err != nil { + return "", fmt.Errorf("error unmarshalling: %v, resp: %v", err, string(resp)) } - return &ret, nil + return res.SHA, nil } diff --git a/apis/gitlab_test.go b/apis/gitlab_test.go index e67cf5b..e096b4b 100644 --- a/apis/gitlab_test.go +++ b/apis/gitlab_test.go @@ -6,19 +6,19 @@ import "testing" func TestGetGitlabCommit(t *testing.T) { examples := []struct { - site, account, project, ref, ID string + site, account, project, ref, hash string }{ {"https://gitlab.com", "gitlab-org", "gitaly-proto", "v1.32.0", "f4db5d05d437abe1154d7308ca044d3577b5ccba"}, {"https://gitlab.com", "gitlab-org", "labkit", "0c3fc7cdd57c", "0c3fc7cdd57c57da5ab474aa72b6640d2bdc9ebb"}, } for i, x := range examples { - c, err := GetGitlabCommit(x.site, x.account, x.project, x.ref) + hash, err := GetGitlabCommit(x.site, x.account, x.project, x.ref) if err != nil { t.Fatal(err) } - if x.ID != c.ID { - t.Errorf("expected commit ID %s, got %s (example %d)", x.ID, c.ID, i) + if x.hash != hash { + t.Errorf("expected commit hash %s, got %s (example %d)", x.hash, hash, i) } } } diff --git a/main.go b/main.go index 1cea87e..27a94f7 100644 --- a/main.go +++ b/main.go @@ -7,6 +7,7 @@ import ( "path" "text/template" + "github.com/dmgk/modules2tuple/apis" "github.com/dmgk/modules2tuple/tuple" ) @@ -25,14 +26,19 @@ func main() { os.Exit(1) } + var haveTuples bool parser := tuple.NewParser(flagPackagePrefix, flagOffline) tuples, errors := parser.Load(args[0]) if len(tuples) != 0 { fmt.Print(tuples) + haveTuples = true } if errors != nil { - fmt.Println() + if haveTuples { + fmt.Println() + } fmt.Print(errors) + fmt.Println() } } @@ -53,7 +59,7 @@ this commit ID translation can be disabled with -offline flag. `)) var ( - flagOffline = false + flagOffline = os.Getenv(apis.OfflineKey) != "" flagPackagePrefix = "vendor" flagVersion = false ) diff --git a/tuple/parser.go b/tuple/parser.go index 64ec32e..96bbb38 100644 --- a/tuple/parser.go +++ b/tuple/parser.go @@ -50,15 +50,28 @@ func (p *Parser) Read(r io.Reader) (Tuples, error) { return } if !p.offline { - // Call Gitlab API to translate go.mod short commit IDs and tags - // to the full 40-character commit IDs as required by bsd.sites.mk - if _, ok := t.Source.(GL); ok { - c, err := apis.GetGitlabCommit(t.Source.Site(), t.Account, t.Project, t.Tag) + switch t.Source.(type) { + case GH: + if strings.HasPrefix(t.Tag, "v") { + // Call Gihub API to check tags. Go seem to be able to magically + // translate tags like "v1.0.4" to the "api/v1.0.4" which is really used + // by upstream. We'll try to do the same. + tag, err := apis.LookupGithubTag(t.Account, t.Project, t.Tag) + if err != nil { + ch <- err + return + } + t.Tag = tag + } + case GL: + // Call Gitlab API to translate go.mod short commit IDs and tags + // to the full 40-character commit IDs as required by bsd.sites.mk + hash, err := apis.GetGitlabCommit(t.Source.Site(), t.Account, t.Project, t.Tag) if err != nil { ch <- err return } - t.Tag = c.ID + t.Tag = hash } } ch <- t diff --git a/tuple/tuple.go b/tuple/tuple.go index bc55db7..cdbee37 100644 --- a/tuple/tuple.go +++ b/tuple/tuple.go @@ -82,14 +82,14 @@ func (tt Tuples) EnsureUniqueGithubProjectAndTag() error { if t.Account != prevTuple.Account { // different Account, but the same Project and Tag if t.Project == prevTuple.Project && t.Tag == prevTuple.Tag { - c, err := apis.GetGithubCommit(t.Account, t.Project, t.Tag) + hash, err := apis.GetGithubCommit(t.Account, t.Project, t.Tag) if err != nil { return DuplicateProjectAndTag(t.String()) } - if len(c.ID) < 12 { - return errors.New("unexpectedly short commit ID") + if len(hash) < 12 { + return errors.New("unexpectedly short Githib commit hash") } - t.Tag = c.ID[:12] + t.Tag = hash[:12] } } } diff --git a/tuple/tuple_online_test.go b/tuple/tuple_online_test.go index 6e7f994..9bec6c9 100644 --- a/tuple/tuple_online_test.go +++ b/tuple/tuple_online_test.go @@ -22,6 +22,6 @@ func TestUniqueProjectAndTag(t *testing.T) { } out := tt.String() if out != expected { - t.Errorf("expected output %s, got %s", expected, out) + t.Errorf("expected output\n%s, got\n%s", expected, out) } }