From d3b364ff50815b02a30945abfda6372690ff704c Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa Date: Fri, 9 Aug 2024 15:36:38 +0200 Subject: [PATCH 1/2] feat: add support for gitlab public/private blueprints --- go.mod | 7 + go.sum | 14 ++ internal/contentprovider/content_provider.go | 2 + .../contentprovider/content_provider_test.go | 39 +++++ internal/contentprovider/gitlab.go | 161 ++++++++++++++++++ internal/contentprovider/gitlab_internal.go | 20 +++ internal/contentprovider/gitlab_test.go | 21 +++ 7 files changed, 264 insertions(+) create mode 100644 internal/contentprovider/content_provider_test.go create mode 100644 internal/contentprovider/gitlab.go create mode 100644 internal/contentprovider/gitlab_internal.go create mode 100644 internal/contentprovider/gitlab_test.go diff --git a/go.mod b/go.mod index e697929..9ac85ea 100644 --- a/go.mod +++ b/go.mod @@ -104,11 +104,14 @@ require ( github.com/golangci/revgrep v0.5.3 // indirect github.com/golangci/unconvert v0.0.0-20240309020433-c5143eacb3ed // indirect github.com/google/go-cmp v0.6.0 // indirect + github.com/google/go-querystring v1.1.0 // indirect github.com/gordonklaus/ineffassign v0.1.0 // indirect github.com/gostaticanalysis/analysisutil v0.7.1 // indirect github.com/gostaticanalysis/comment v1.4.2 // indirect github.com/gostaticanalysis/forcetypeassert v0.1.0 // indirect github.com/gostaticanalysis/nilerr v0.1.1 // indirect + github.com/hashicorp/go-cleanhttp v0.5.2 // indirect + github.com/hashicorp/go-retryablehttp v0.7.7 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hexops/gotextdiff v1.0.3 // indirect @@ -204,6 +207,7 @@ require ( github.com/ultraware/funlen v0.1.0 // indirect github.com/ultraware/whitespace v0.1.1 // indirect github.com/uudashr/gocognit v1.1.2 // indirect + github.com/xanzy/go-gitlab v0.107.0 // indirect github.com/xanzy/ssh-agent v0.2.1 // indirect github.com/xen0n/gosmopolitan v1.2.2 // indirect github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect @@ -222,9 +226,12 @@ require ( golang.org/x/exp/typeparams v0.0.0-20240314144324-c7f7c6466f7f // indirect golang.org/x/mod v0.17.0 // indirect golang.org/x/net v0.25.0 // indirect + golang.org/x/oauth2 v0.15.0 // indirect golang.org/x/sync v0.7.0 // indirect golang.org/x/sys v0.21.0 // indirect golang.org/x/text v0.16.0 // indirect + golang.org/x/time v0.5.0 // indirect + google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.33.0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/src-d/go-billy.v4 v4.3.2 // indirect diff --git a/go.sum b/go.sum index 1bb511d..8df60b6 100644 --- a/go.sum +++ b/go.sum @@ -305,6 +305,8 @@ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= +github.com/google/go-querystring v1.1.0/go.mod h1:Kcdr2DB4koayq7X8pmAG4sNG59So17icRSOU623lUBU= github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -337,6 +339,10 @@ github.com/gostaticanalysis/testutil v0.4.0 h1:nhdCmubdmDF6VEatUNjgUZBJKWRqugoIS github.com/gostaticanalysis/testutil v0.4.0/go.mod h1:bLIoPefWXrRi/ssLFWX1dx7Repi5x3CuviD3dgAZaBU= github.com/gotesttools/gotestfmt/v2 v2.5.0 h1:fSU3MnR+E+fvuXdw1l8xbufKhDxY3Tfjsjx/I1WerB4= github.com/gotesttools/gotestfmt/v2 v2.5.0/go.mod h1:oQJg2KZ2aGoqEbMC2PDaAeBYm0tOkocgixK9FzsCdp4= +github.com/hashicorp/go-cleanhttp v0.5.2 h1:035FKYIWjmULyFRBKPs8TBQoi0x6d9G4xc9neXJWAZQ= +github.com/hashicorp/go-cleanhttp v0.5.2/go.mod h1:kO/YDlP8L1346E6Sodw+PrpBSV4/SoxCXGY6BqNFT48= +github.com/hashicorp/go-retryablehttp v0.7.7 h1:C8hUCYzor8PIfXHa4UrZkU4VvK8o9ISHxT2Q8+VepXU= +github.com/hashicorp/go-retryablehttp v0.7.7/go.mod h1:pkQpWZeYWskR+D1tR2O5OcBFOxfA7DoAO6xtkuQnHTk= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -655,6 +661,8 @@ github.com/ultraware/whitespace v0.1.1 h1:bTPOGejYFulW3PkcrqkeQwOd6NKOOXvmGD9bo/ github.com/ultraware/whitespace v0.1.1/go.mod h1:XcP1RLD81eV4BW8UhQlpaR+SDc2givTvyI8a586WjW8= github.com/uudashr/gocognit v1.1.2 h1:l6BAEKJqQH2UpKAPKdMfZf5kE4W/2xk8pfU1OVLvniI= github.com/uudashr/gocognit v1.1.2/go.mod h1:aAVdLURqcanke8h3vg35BC++eseDm66Z7KmchI5et4k= +github.com/xanzy/go-gitlab v0.107.0 h1:P2CT9Uy9yN9lJo3FLxpMZ4xj6uWcpnigXsjvqJ6nd2Y= +github.com/xanzy/go-gitlab v0.107.0/go.mod h1:wKNKh3GkYDMOsGmnfuX+ITCmDuSDWFO0G+C4AygL9RY= github.com/xanzy/ssh-agent v0.2.1 h1:TCbipTQL2JiiCprBWx9frJ2eJlCYT00NmctrHxVAr70= github.com/xanzy/ssh-agent v0.2.1/go.mod h1:mLlQY/MoOhWBj+gOGMQkOeiEvkx+8pJSI+0Bx9h2kr4= github.com/xen0n/gosmopolitan v1.2.2 h1:/p2KTnMzwRexIW8GlKawsTWOxn7UHA+jCMF/V8HHtvU= @@ -805,6 +813,8 @@ golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4Iltr golang.org/x/oauth2 v0.0.0-20191202225959-858c2ad4c8b6/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20210514164344-f6687ab2804c/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.15.0 h1:s8pnnxNVzjWyrvYdFUQq5llS1PX2zhPXmccZv99h7uQ= +golang.org/x/oauth2 v0.15.0/go.mod h1:q48ptWNTY5XWf+JNten23lcvHpLJ0ZSxF5ttTHKVCAM= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -903,6 +913,8 @@ golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -993,6 +1005,8 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= +google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= diff --git a/internal/contentprovider/content_provider.go b/internal/contentprovider/content_provider.go index 88e3072..896d483 100644 --- a/internal/contentprovider/content_provider.go +++ b/internal/contentprovider/content_provider.go @@ -11,6 +11,8 @@ func ByURI(uri string) (RemoteContentProvider, error) { switch { case strings.HasPrefix(uri, GitHubPrefix): contentProvider, err = NewGitHub(uri) + case strings.HasPrefix(uri, GitLabPrefix): + contentProvider, err = NewGitLab(uri) case strings.HasPrefix(uri, LocalPathPrefix): contentProvider, err = NewLocalPath(strings.TrimPrefix(uri, LocalPathPrefix)) default: diff --git a/internal/contentprovider/content_provider_test.go b/internal/contentprovider/content_provider_test.go new file mode 100644 index 0000000..9592e79 --- /dev/null +++ b/internal/contentprovider/content_provider_test.go @@ -0,0 +1,39 @@ +package contentprovider + +import ( + "github.com/stretchr/testify/assert" + "reflect" + "testing" +) + +func TestByURI(t *testing.T) { + + var testCases = []struct { + name string + uri string + expectedType RemoteContentProvider + }{ + { + name: "github", + uri: "https://github.com/gchiesa/test", + expectedType: &GitHub{}, + }, + { + name: "gitlab", + uri: "https://gitlab.com/gchiesa/test", + expectedType: &GitLab{}, + }, + { + name: "local", + uri: "file:///home/gchiesa/test", + expectedType: &LocalPath{}, + }, + } + for _, tc := range testCases { + cp, err := ByURI(tc.uri) + assert.Nil(t, err) + cpType := reflect.TypeOf(cp).String() + expType := reflect.TypeOf(tc.expectedType).String() + assert.Equalf(t, expType, cpType, "error in %s test, expected %s, got %s", tc.name, expType, cpType) + } +} diff --git a/internal/contentprovider/gitlab.go b/internal/contentprovider/gitlab.go new file mode 100644 index 0000000..a9f0704 --- /dev/null +++ b/internal/contentprovider/gitlab.go @@ -0,0 +1,161 @@ +package contentprovider + +import ( + "archive/zip" + "fmt" + "github.com/apex/log" + "github.com/xanzy/go-gitlab" + "io" + "os" + "path/filepath" + "strings" +) + +type GitLab struct { + remoteURI string + repositoryURL string + repositoryRef string + projectPath string + workingDir string + log *log.Entry +} + +const GitLabPrefix = "https://gitlab.com/" + +func NewGitLab(remoteURI string) (*GitLab, error) { + logCtx := log.WithFields(log.Fields{ + "pkg": "contentprovider", + "type": "gitlab", + }) + tmpDir, err := os.MkdirTemp("", workingDirPrefix) + if err != nil { + return nil, err + } + return &GitLab{remoteURI: remoteURI, workingDir: tmpDir, log: logCtx}, nil +} + +func (cp *GitLab) WorkingDir() string { + return cp.workingDir +} + +func (cp *GitLab) Cleanup() error { + cp.log.WithFields(log.Fields{"workingDir": cp.workingDir}).Debug("removing working dir.") + err := os.RemoveAll(cp.workingDir) + return err +} + +func (cp *GitLab) RemoteURI() string { + return cp.remoteURI +} + +func (cp *GitLab) downloadRepoZipArchive() (zipArchive string, err error) { + token := os.Getenv("GITLAB_PRIVATE_TOKEN") + + gitlabClient, err := gitlab.NewClient(token) + if err != nil { + return "", err + } + + tmpArchive, err := os.CreateTemp(os.TempDir(), "gitlab-repo-") + defer func(f *os.File) { _ = f.Close() }(tmpArchive) + + if err != nil { + return "", err + } + + var archiveFormat = "zip" + archiveOptions := &gitlab.ArchiveOptions{ + Format: &archiveFormat, + SHA: &cp.repositoryRef, + } + data, resp, err := gitlabClient.Repositories.Archive(cp.projectPath, archiveOptions, gitlab.WithToken(gitlab.PrivateToken, token)) + + if resp.StatusCode == 404 { + return "", fmt.Errorf("repository not found, perhaps you need to setup your private token for GitLab by exporting the environment variable GITLAB_PRIVATE_TOKEN. Status Code: %d", resp.StatusCode) + } + + if err != nil { + return "", err + } + + if resp.StatusCode != 200 { + return "", fmt.Errorf("invalid status code returned from GitLab API. Status Code: %d", resp.StatusCode) + } + + // write data to tmp archive + if _, err := tmpArchive.Write(data); err != nil { + return "", err + } + + return tmpArchive.Name(), nil +} + +func (cp *GitLab) DownloadContent() error { + if err := cp.validateRemoteURI(); err != nil { + return err + } + + tmpArchivePath, err := cp.downloadRepoZipArchive() + if err != nil { + return err + } + + // unzip archive + if err := cp.unzipArchive(cp.workingDir, tmpArchivePath); err != nil { + return err + } + + // remove the temporary archive + if err := os.RemoveAll(tmpArchivePath); err != nil { + return err + } + return nil +} + +func (cp *GitLab) unzipArchive(dst, archivePath string) error { + archive, err := zip.OpenReader(archivePath) + if err != nil { + return err + } + defer func(*zip.ReadCloser) { _ = archive.Close() }(archive) + + rootPath := "" + for _, item := range archive.File { + dstFilePath := filepath.Join(dst, item.Name) //nolint:gosec + + if !strings.HasPrefix(dstFilePath, filepath.Clean(dst)+string(os.PathSeparator)) { + return fmt.Errorf("invalid file path: %s", dstFilePath) + } + if item.FileInfo().IsDir() { + if err := os.MkdirAll(dstFilePath, os.ModePerm); err != nil { + return err + } + if rootPath == "" { + rootPath = item.Name + } + continue + } + if err := os.MkdirAll(filepath.Dir(dstFilePath), os.ModePerm); err != nil { + return err + } + + // open the dstFilePath as file + dstFile, err := os.OpenFile(dstFilePath, os.O_WRONLY|os.O_CREATE|os.O_TRUNC, item.Mode()) + if err != nil { + panic(err) + } + + compressedFile, err := item.Open() + if err != nil { + return err + } + + if _, err := io.Copy(dstFile, compressedFile); err != nil { //nolint:gosec + return err + } + _ = dstFile.Close() + _ = compressedFile.Close() + } + cp.workingDir = filepath.Join(dst, rootPath) + return nil +} diff --git a/internal/contentprovider/gitlab_internal.go b/internal/contentprovider/gitlab_internal.go new file mode 100644 index 0000000..746f043 --- /dev/null +++ b/internal/contentprovider/gitlab_internal.go @@ -0,0 +1,20 @@ +package contentprovider + +import ( + "fmt" + "strings" +) + +func (cp *GitLab) validateRemoteURI() error { + url, ref := parseRemoteURI(cp.remoteURI) + if !strings.HasPrefix(url, GitLabPrefix) { + return fmt.Errorf("invalid github url. The url must start with: %s", GitLabPrefix) + } + if ref == "" { + return fmt.Errorf("invalid github url. The url must contain a reference. Example %s//@", GitLabPrefix) + } + cp.repositoryURL = url + cp.repositoryRef = ref + cp.projectPath = strings.TrimPrefix(cp.repositoryURL, GitLabPrefix) + return nil +} diff --git a/internal/contentprovider/gitlab_test.go b/internal/contentprovider/gitlab_test.go new file mode 100644 index 0000000..c7aeff9 --- /dev/null +++ b/internal/contentprovider/gitlab_test.go @@ -0,0 +1,21 @@ +package contentprovider + +import ( + "github.com/stretchr/testify/assert" + "testing" +) + +// const GitLabTestPublicRepo = "https://gitlab.com/gchiesa/s3vaultlib@master" +const GitLabTestPublicRepo = "https://gitlab.com/gchiesa/test@master" + +func TestGitLab_DownloadPublicRepo(t *testing.T) { + var err error + cp, err := NewGitLab(GitLabTestPublicRepo) + assert.NoErrorf(t, err, "error creating gitlab provider: %s", err) + + err = cp.DownloadContent() + assert.NoErrorf(t, err, "error downloading content: %s", err) + + err = cp.Cleanup() + assert.NoErrorf(t, err, "error cleaning up: %s", err) +} From d136e00768de7b1ce2f2a3e1b77d41dece6d77a7 Mon Sep 17 00:00:00 2001 From: Giuseppe Chiesa Date: Fri, 16 Aug 2024 15:30:41 +0200 Subject: [PATCH 2/2] fix: update url for gitlab test repository --- internal/contentprovider/gitlab_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/internal/contentprovider/gitlab_test.go b/internal/contentprovider/gitlab_test.go index c7aeff9..9975ef3 100644 --- a/internal/contentprovider/gitlab_test.go +++ b/internal/contentprovider/gitlab_test.go @@ -5,7 +5,6 @@ import ( "testing" ) -// const GitLabTestPublicRepo = "https://gitlab.com/gchiesa/s3vaultlib@master" const GitLabTestPublicRepo = "https://gitlab.com/gchiesa/test@master" func TestGitLab_DownloadPublicRepo(t *testing.T) {