From 88e606c9fdc2e8a78b29560c624a667cca55ea8d Mon Sep 17 00:00:00 2001 From: Mohamed Habib Date: Fri, 28 Jun 2024 12:14:06 +0100 Subject: [PATCH] use plan policy provider in spec (#1594) --- cli/cmd/digger/main.go | 9 ++++++-- cli/cmd/digger/run_spec.go | 1 + cli/pkg/core/execution/execution.go | 2 +- cli/pkg/digger/digger.go | 2 +- cli/pkg/github/github.go | 19 ++++++++++++----- cli/pkg/integration/integration_test.go | 9 ++++---- cli/pkg/spec/spec.go | 9 +++++--- cli_e2e/plan_storage_test.go | 8 +++---- ee/cli/cmd/digger/run_spec.go | 1 + libs/spec/providers.go | 14 ++++++++++++- {cli/pkg => libs}/storage/aws_plan_storage.go | 0 .../storage/aws_plan_storage_test.go | 0 {cli/pkg => libs}/storage/gcp_plan_storage.go | 0 {cli/pkg/utils => libs/storage}/io.go | 2 +- {cli/pkg/utils => libs/storage}/io_test.go | 2 +- {cli/pkg => libs}/storage/plan_storage.go | 21 ++++++++----------- .../storage/storage.go | 0 17 files changed, 63 insertions(+), 36 deletions(-) rename {cli/pkg => libs}/storage/aws_plan_storage.go (100%) rename {cli/pkg => libs}/storage/aws_plan_storage_test.go (100%) rename {cli/pkg => libs}/storage/gcp_plan_storage.go (100%) rename {cli/pkg/utils => libs/storage}/io.go (98%) rename {cli/pkg/utils => libs/storage}/io_test.go (99%) rename {cli/pkg => libs}/storage/plan_storage.go (92%) rename cli/pkg/core/storage/plan_storage.go => libs/storage/storage.go (100%) diff --git a/cli/cmd/digger/main.go b/cli/cmd/digger/main.go index be519a4db..15df44255 100644 --- a/cli/cmd/digger/main.go +++ b/cli/cmd/digger/main.go @@ -5,13 +5,13 @@ import ( core_backend "github.com/diggerhq/digger/cli/pkg/core/backend" core_policy "github.com/diggerhq/digger/cli/pkg/core/policy" "github.com/diggerhq/digger/cli/pkg/digger" - "github.com/diggerhq/digger/cli/pkg/storage" "github.com/diggerhq/digger/cli/pkg/usage" "github.com/diggerhq/digger/libs/comment_utils/reporting" "github.com/diggerhq/digger/libs/comment_utils/summary" "github.com/diggerhq/digger/libs/digger_config" core_locking "github.com/diggerhq/digger/libs/locking" orchestrator "github.com/diggerhq/digger/libs/orchestrator" + "github.com/diggerhq/digger/libs/storage" "log" "os" ) @@ -26,7 +26,12 @@ func exec(actor string, projectName string, repoNamespace string, command string } - planStorage := storage.NewPlanStorage("", "", "", actor, nil) + planStorage, err := storage.NewPlanStorage("", "", "", nil) + if err != nil { + + usage.ReportErrorAndExit(actor, fmt.Sprintf("Failed to get plan storage. %s", err), 4) + + } changedFiles, err := prService.GetChangedFiles(prNumber) if err != nil { diff --git a/cli/cmd/digger/run_spec.go b/cli/cmd/digger/run_spec.go index 910ab354f..b6f1d22b0 100644 --- a/cli/cmd/digger/run_spec.go +++ b/cli/cmd/digger/run_spec.go @@ -39,6 +39,7 @@ var runSpecCmd = &cobra.Command{ lib_spec.ReporterProvider{}, lib_spec.BackendApiProvider{}, lib_spec.PolicyProvider{}, + lib_spec.PlanStorageProvider{}, comment_summary.CommentUpdaterProviderBasic{}, ) if err != nil { diff --git a/cli/pkg/core/execution/execution.go b/cli/pkg/core/execution/execution.go index 0a5d80a0b..5637e9360 100644 --- a/cli/pkg/core/execution/execution.go +++ b/cli/pkg/core/execution/execution.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/diggerhq/digger/libs/comment_utils/utils" "github.com/diggerhq/digger/libs/locking" + "github.com/diggerhq/digger/libs/storage" "github.com/diggerhq/digger/libs/terraform_utils" "github.com/samber/lo" "log" @@ -14,7 +15,6 @@ import ( "strings" "github.com/diggerhq/digger/cli/pkg/core/runners" - "github.com/diggerhq/digger/cli/pkg/core/storage" "github.com/diggerhq/digger/cli/pkg/core/terraform" "github.com/diggerhq/digger/libs/comment_utils/reporting" configuration "github.com/diggerhq/digger/libs/digger_config" diff --git a/cli/pkg/digger/digger.go b/cli/pkg/digger/digger.go index 1741cb5b1..88e8b26a7 100644 --- a/cli/pkg/digger/digger.go +++ b/cli/pkg/digger/digger.go @@ -6,6 +6,7 @@ import ( comment_updater "github.com/diggerhq/digger/libs/comment_utils/summary" coreutils "github.com/diggerhq/digger/libs/comment_utils/utils" locking2 "github.com/diggerhq/digger/libs/locking" + "github.com/diggerhq/digger/libs/storage" "log" "os" "path" @@ -17,7 +18,6 @@ import ( "github.com/diggerhq/digger/cli/pkg/core/execution" "github.com/diggerhq/digger/cli/pkg/core/policy" "github.com/diggerhq/digger/cli/pkg/core/runners" - "github.com/diggerhq/digger/cli/pkg/core/storage" "github.com/diggerhq/digger/cli/pkg/core/terraform" "github.com/diggerhq/digger/cli/pkg/usage" utils "github.com/diggerhq/digger/cli/pkg/utils" diff --git a/cli/pkg/github/github.go b/cli/pkg/github/github.go index e61fe4915..c50c63628 100644 --- a/cli/pkg/github/github.go +++ b/cli/pkg/github/github.go @@ -10,7 +10,6 @@ import ( "github.com/diggerhq/digger/cli/pkg/digger" "github.com/diggerhq/digger/cli/pkg/drift" github_models "github.com/diggerhq/digger/cli/pkg/github/models" - "github.com/diggerhq/digger/cli/pkg/storage" "github.com/diggerhq/digger/cli/pkg/usage" "github.com/diggerhq/digger/cli/pkg/utils" "github.com/diggerhq/digger/libs/comment_utils/reporting" @@ -20,6 +19,7 @@ import ( "github.com/diggerhq/digger/libs/orchestrator" dg_github "github.com/diggerhq/digger/libs/orchestrator/github" "github.com/diggerhq/digger/libs/orchestrator/scheduler" + "github.com/diggerhq/digger/libs/storage" "github.com/google/go-github/v61/github" "github.com/samber/lo" "gopkg.in/yaml.v3" @@ -185,7 +185,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh commentUpdater.UpdateComment(serializedBatch.Jobs, serializedBatch.PrNumber, &githubPrService, commentId64) digger.UpdateAggregateStatus(serializedBatch, &githubPrService) - planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, jobSpec.PullRequestNumber) + planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, jobSpec.PullRequestNumber) + if err != nil { + usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4) + } if err != nil { serializedBatch, reportingError := backendApi.ReportProjectJobStatus(repoName, jobSpec.ProjectName, inputs.Id, "failed", time.Now(), nil, "", "") @@ -267,7 +270,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars) - planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, nil) + planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, nil) + if err != nil { + usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4) + } jobs := orchestrator.Job{ ProjectName: project, @@ -285,7 +291,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh StateEnvVars: stateEnvVars, CommandEnvVars: commandEnvVars, } - err := digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir) + err = digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir) if err != nil { usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to run commands. %s", err), 8) } @@ -383,7 +389,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to set job output. Exiting. %s", err), 4) } - planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, &prNumber) + planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, &prNumber) + if err != nil { + usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4) + } reporter := &reporting.CiReporter{ CiService: &githubPrService, diff --git a/cli/pkg/integration/integration_test.go b/cli/pkg/integration/integration_test.go index d1d4fd0bc..956bb79dc 100644 --- a/cli/pkg/integration/integration_test.go +++ b/cli/pkg/integration/integration_test.go @@ -4,6 +4,7 @@ import ( "context" "github.com/diggerhq/digger/libs/locking" "github.com/diggerhq/digger/libs/locking/aws" + "github.com/diggerhq/digger/libs/storage" "log" "math/rand" "os" @@ -15,8 +16,6 @@ import ( "github.com/diggerhq/digger/cli/pkg/core/terraform" "github.com/diggerhq/digger/cli/pkg/digger" "github.com/diggerhq/digger/cli/pkg/github/models" - "github.com/diggerhq/digger/cli/pkg/storage" - "github.com/diggerhq/digger/cli/pkg/utils" "github.com/diggerhq/digger/libs/comment_utils/reporting" configuration "github.com/diggerhq/digger/libs/digger_config" dg_github "github.com/diggerhq/digger/libs/orchestrator/github" @@ -398,7 +397,7 @@ func TestHappyPath(t *testing.T) { event := ghEvent.(github.PullRequestEvent) jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig) assert.NoError(t, err) - zipManager := utils.Zipper{} + zipManager := storage.Zipper{} planStorage := &storage.GithubPlanStorage{ Client: github.NewTokenClient(context.Background(), ghToken), Owner: repoOwner, @@ -556,7 +555,7 @@ func TestMultiEnvHappyPath(t *testing.T) { jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig) assert.NoError(t, err) - zipManager := utils.Zipper{} + zipManager := storage.Zipper{} planStorage := &storage.GithubPlanStorage{ Client: github.NewTokenClient(context.Background(), ghToken), Owner: repoOwner, @@ -777,7 +776,7 @@ workflows: jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig) assert.NoError(t, err) - zipManager := utils.Zipper{} + zipManager := storage.Zipper{} planStorage := &storage.GithubPlanStorage{ Client: github.NewTokenClient(context.Background(), ghToken), Owner: repoOwner, diff --git a/cli/pkg/spec/spec.go b/cli/pkg/spec/spec.go index 53164e5c6..54fd937f8 100644 --- a/cli/pkg/spec/spec.go +++ b/cli/pkg/spec/spec.go @@ -3,7 +3,6 @@ package spec import ( "fmt" "github.com/diggerhq/digger/cli/pkg/digger" - storage2 "github.com/diggerhq/digger/cli/pkg/storage" "github.com/diggerhq/digger/cli/pkg/usage" comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary" "github.com/diggerhq/digger/libs/digger_config" @@ -23,6 +22,7 @@ func RunSpec( reporterProvider spec.ReporterProvider, backedProvider spec.BackendApiProvider, policyProvider spec.PolicyProvider, + PlanStorageProvider spec.PlanStorageProvider, commentUpdaterProvider comment_summary.CommentUpdaterProvider, ) error { @@ -72,8 +72,11 @@ func RunSpec( usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get comment updater: %v", err), 8) } - planStorage := storage2.NewPlanStorage("", "", "", "", nil) - + planStorage, err := PlanStorageProvider.GetPlanStorage(spec.VCS.RepoOwner, spec.VCS.RepoName, *spec.Job.PullRequestNumber) + if err != nil { + usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get plan storage: %v", err), 8) + } + jobs := []orchestrator.Job{job} fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName) diff --git a/cli_e2e/plan_storage_test.go b/cli_e2e/plan_storage_test.go index a28ae4a59..f8601912e 100644 --- a/cli_e2e/plan_storage_test.go +++ b/cli_e2e/plan_storage_test.go @@ -3,12 +3,12 @@ package cli_e2e import ( "fmt" "github.com/diggerhq/digger/libs/locking/gcp" + storage2 "github.com/diggerhq/digger/libs/storage" "log" "os" "strings" "testing" - "github.com/diggerhq/digger/cli/pkg/storage" "github.com/stretchr/testify/assert" ) @@ -28,7 +28,7 @@ func TestGCPPlanStorageStorageAndRetrieval(t *testing.T) { fmt.Printf("getting bucket") bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET")) bucket := client.Bucket(bucketName) - planStorage := &storage.PlanStorageGcp{ + planStorage := &storage2.PlanStorageGcp{ Client: client, Bucket: bucket, Context: ctx, @@ -60,13 +60,13 @@ func TestAWSPlanStorageStorageAndRetrieval(t *testing.T) { defer os.Remove(file.Name()) log.Printf("getting AWS S3 client") - ctx, client, err := storage.GetAWSStorageClient() + ctx, client, err := storage2.GetAWSStorageClient() if err != nil { t.Errorf("failed to get AWS storage client: %v", err) } bucketName := strings.ToLower(os.Getenv("AWS_S3_BUCKET")) log.Printf("using AWS S3 bucket found by env var 'AWS_S3_BUCKET': %s", bucketName) - planStorage := &storage.PlanStorageAWS{ + planStorage := &storage2.PlanStorageAWS{ Client: client, Bucket: bucketName, Context: ctx, diff --git a/ee/cli/cmd/digger/run_spec.go b/ee/cli/cmd/digger/run_spec.go index 910ab354f..b6f1d22b0 100644 --- a/ee/cli/cmd/digger/run_spec.go +++ b/ee/cli/cmd/digger/run_spec.go @@ -39,6 +39,7 @@ var runSpecCmd = &cobra.Command{ lib_spec.ReporterProvider{}, lib_spec.BackendApiProvider{}, lib_spec.PolicyProvider{}, + lib_spec.PlanStorageProvider{}, comment_summary.CommentUpdaterProviderBasic{}, ) if err != nil { diff --git a/libs/spec/providers.go b/libs/spec/providers.go index c76a200c7..d7c718ab9 100644 --- a/libs/spec/providers.go +++ b/libs/spec/providers.go @@ -21,6 +21,7 @@ import ( "github.com/diggerhq/digger/libs/locking/gcp" "github.com/diggerhq/digger/libs/orchestrator" "github.com/diggerhq/digger/libs/orchestrator/github" + storage2 "github.com/diggerhq/digger/libs/storage" "github.com/samber/lo" "log" "net/http" @@ -173,7 +174,7 @@ func (v VCSProvider) GetPrService(vcsSpec VcsSpec) (orchestrator.PullRequestServ case "github": token := os.Getenv("GITHUB_TOKEN") if token == "" { - return nil, fmt.Errorf("failed to get githbu service: GITHUB_TOKEN not specified") + return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified") } return github.GithubServiceProviderBasic{}.NewService(token, vcsSpec.RepoName, vcsSpec.RepoOwner) default: @@ -198,3 +199,14 @@ func (p PolicyProvider) GetPolicyProvider(policySpec PolicySpec, diggerHost stri return nil, fmt.Errorf("could not find ") } } + +type PlanStorageProvider struct{} + +func (p PlanStorageProvider) GetPlanStorage(repoOwner string, repositoryName string, prNumber int) (storage2.PlanStorage, error) { + token := os.Getenv("GITHUB_TOKEN") + if token == "" { + return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified") + } + storage, err := storage2.NewPlanStorage(token, repoOwner, repositoryName, &prNumber) + return storage, err +} diff --git a/cli/pkg/storage/aws_plan_storage.go b/libs/storage/aws_plan_storage.go similarity index 100% rename from cli/pkg/storage/aws_plan_storage.go rename to libs/storage/aws_plan_storage.go diff --git a/cli/pkg/storage/aws_plan_storage_test.go b/libs/storage/aws_plan_storage_test.go similarity index 100% rename from cli/pkg/storage/aws_plan_storage_test.go rename to libs/storage/aws_plan_storage_test.go diff --git a/cli/pkg/storage/gcp_plan_storage.go b/libs/storage/gcp_plan_storage.go similarity index 100% rename from cli/pkg/storage/gcp_plan_storage.go rename to libs/storage/gcp_plan_storage.go diff --git a/cli/pkg/utils/io.go b/libs/storage/io.go similarity index 98% rename from cli/pkg/utils/io.go rename to libs/storage/io.go index 9f3646ee6..4a6c581cc 100644 --- a/cli/pkg/utils/io.go +++ b/libs/storage/io.go @@ -1,4 +1,4 @@ -package utils +package storage import ( "archive/zip" diff --git a/cli/pkg/utils/io_test.go b/libs/storage/io_test.go similarity index 99% rename from cli/pkg/utils/io_test.go rename to libs/storage/io_test.go index 5d2c64299..e34384a82 100644 --- a/cli/pkg/utils/io_test.go +++ b/libs/storage/io_test.go @@ -1,4 +1,4 @@ -package utils +package storage import ( "archive/zip" diff --git a/cli/pkg/storage/plan_storage.go b/libs/storage/plan_storage.go similarity index 92% rename from cli/pkg/storage/plan_storage.go rename to libs/storage/plan_storage.go index b46087e1b..c3d41bb9a 100644 --- a/cli/pkg/storage/plan_storage.go +++ b/libs/storage/plan_storage.go @@ -5,8 +5,6 @@ import ( "context" "encoding/json" "fmt" - "github.com/diggerhq/digger/cli/pkg/core/storage" - "github.com/diggerhq/digger/cli/pkg/usage" "github.com/diggerhq/digger/libs/locking/gcp" "io" "log" @@ -16,8 +14,6 @@ import ( "os/exec" "strings" - "github.com/diggerhq/digger/cli/pkg/utils" - "github.com/google/go-github/v61/github" ) @@ -26,7 +22,7 @@ type GithubPlanStorage struct { Owner string RepoName string PullRequestNumber int - ZipManager utils.Zipper + ZipManager Zipper } func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName string, storedPlanFilePath string) error { @@ -208,13 +204,14 @@ func getLatestArtifactWithName(artifacts []*github.Artifact, name string) *githu return latest } -func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, requestedBy string, prNumber *int) storage.PlanStorage { - var planStorage storage.PlanStorage +// TODO: refactor this function to make it fully parametarised and no reliance on env variables +func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, prNumber *int) (PlanStorage, error) { + var planStorage PlanStorage uploadDestination := strings.ToLower(os.Getenv("PLAN_UPLOAD_DESTINATION")) switch { case uploadDestination == "github": - zipManager := utils.Zipper{} + zipManager := Zipper{} planStorage = &GithubPlanStorage{ Client: github.NewTokenClient(context.Background(), ghToken), Owner: ghRepoOwner, @@ -226,7 +223,7 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, ctx, client := gcp.GetGoogleStorageClient() bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET")) if bucketName == "" { - usage.ReportErrorAndExit(requestedBy, "GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET is not defined", 9) + return nil, fmt.Errorf("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET is not defined") } bucket := client.Bucket(bucketName) planStorage = &PlanStorageGcp{ @@ -237,11 +234,11 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, case uploadDestination == "aws": ctx, client, err := GetAWSStorageClient() if err != nil { - usage.ReportErrorAndExit(requestedBy, fmt.Sprintf("Failed to create AWS storage client: %s", err), 9) + return nil, fmt.Errorf(fmt.Sprintf("Failed to create AWS storage client: %s", err)) } bucketName := strings.ToLower(os.Getenv("AWS_S3_BUCKET")) if bucketName == "" { - usage.ReportErrorAndExit(requestedBy, "AWS_S3_BUCKET is not defined", 9) + return nil, fmt.Errorf("AWS_S3_BUCKET is not defined") } planStorage = &PlanStorageAWS{ Context: ctx, @@ -252,5 +249,5 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, //TODO implement me } - return planStorage + return planStorage, nil } diff --git a/cli/pkg/core/storage/plan_storage.go b/libs/storage/storage.go similarity index 100% rename from cli/pkg/core/storage/plan_storage.go rename to libs/storage/storage.go