use plan policy provider in spec (#1594)

cli/cmd/digger/main.go

@@ -5,13 +5,13 @@ import (
 	core_backend "github.com/diggerhq/digger/cli/pkg/core/backend"
 	core_policy "github.com/diggerhq/digger/cli/pkg/core/policy"
 	"github.com/diggerhq/digger/cli/pkg/digger"
-	"github.com/diggerhq/digger/cli/pkg/storage"
 	"github.com/diggerhq/digger/cli/pkg/usage"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	"github.com/diggerhq/digger/libs/comment_utils/summary"
 	"github.com/diggerhq/digger/libs/digger_config"
 	core_locking "github.com/diggerhq/digger/libs/locking"
 	orchestrator "github.com/diggerhq/digger/libs/orchestrator"
+	"github.com/diggerhq/digger/libs/storage"
 	"log"
 	"os"
 )
@@ -26,7 +26,12 @@ func exec(actor string, projectName string, repoNamespace string, command string
 
 	}
 
-	planStorage := storage.NewPlanStorage("", "", "", actor, nil)
+	planStorage, err := storage.NewPlanStorage("", "", "", nil)
+	if err != nil {
+
+		usage.ReportErrorAndExit(actor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
+
+	}
 
 	changedFiles, err := prService.GetChangedFiles(prNumber)
 	if err != nil {

cli/cmd/digger/run_spec.go

@@ -39,6 +39,7 @@ var runSpecCmd = &cobra.Command{
 			lib_spec.ReporterProvider{},
 			lib_spec.BackendApiProvider{},
 			lib_spec.PolicyProvider{},
+			lib_spec.PlanStorageProvider{},
 			comment_summary.CommentUpdaterProviderBasic{},
 		)
 		if err != nil {

cli/pkg/core/execution/execution.go

@@ -4,6 +4,7 @@ import (
 	"fmt"
 	"github.com/diggerhq/digger/libs/comment_utils/utils"
 	"github.com/diggerhq/digger/libs/locking"
+	"github.com/diggerhq/digger/libs/storage"
 	"github.com/diggerhq/digger/libs/terraform_utils"
 	"github.com/samber/lo"
 	"log"
@@ -14,7 +15,6 @@ import (
 	"strings"
 
 	"github.com/diggerhq/digger/cli/pkg/core/runners"
-	"github.com/diggerhq/digger/cli/pkg/core/storage"
 	"github.com/diggerhq/digger/cli/pkg/core/terraform"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	configuration "github.com/diggerhq/digger/libs/digger_config"

cli/pkg/digger/digger.go

@@ -6,6 +6,7 @@ import (
 	comment_updater "github.com/diggerhq/digger/libs/comment_utils/summary"
 	coreutils "github.com/diggerhq/digger/libs/comment_utils/utils"
 	locking2 "github.com/diggerhq/digger/libs/locking"
+	"github.com/diggerhq/digger/libs/storage"
 	"log"
 	"os"
 	"path"
@@ -17,7 +18,6 @@ import (
 	"github.com/diggerhq/digger/cli/pkg/core/execution"
 	"github.com/diggerhq/digger/cli/pkg/core/policy"
 	"github.com/diggerhq/digger/cli/pkg/core/runners"
-	"github.com/diggerhq/digger/cli/pkg/core/storage"
 	"github.com/diggerhq/digger/cli/pkg/core/terraform"
 	"github.com/diggerhq/digger/cli/pkg/usage"
 	utils "github.com/diggerhq/digger/cli/pkg/utils"

cli/pkg/github/github.go

@@ -10,7 +10,6 @@ import (
 	"github.com/diggerhq/digger/cli/pkg/digger"
 	"github.com/diggerhq/digger/cli/pkg/drift"
 	github_models "github.com/diggerhq/digger/cli/pkg/github/models"
-	"github.com/diggerhq/digger/cli/pkg/storage"
 	"github.com/diggerhq/digger/cli/pkg/usage"
 	"github.com/diggerhq/digger/cli/pkg/utils"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
@@ -20,6 +19,7 @@ import (
 	"github.com/diggerhq/digger/libs/orchestrator"
 	dg_github "github.com/diggerhq/digger/libs/orchestrator/github"
 	"github.com/diggerhq/digger/libs/orchestrator/scheduler"
+	"github.com/diggerhq/digger/libs/storage"
 	"github.com/google/go-github/v61/github"
 	"github.com/samber/lo"
 	"gopkg.in/yaml.v3"
@@ -185,7 +185,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 		commentUpdater.UpdateComment(serializedBatch.Jobs, serializedBatch.PrNumber, &githubPrService, commentId64)
 		digger.UpdateAggregateStatus(serializedBatch, &githubPrService)
 
-		planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, jobSpec.PullRequestNumber)
+		planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, jobSpec.PullRequestNumber)
+		if err != nil {
+			usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
+		}
 
 		if err != nil {
 			serializedBatch, reportingError := backendApi.ReportProjectJobStatus(repoName, jobSpec.ProjectName, inputs.Id, "failed", time.Now(), nil, "", "")
@@ -267,7 +270,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 
 		stateEnvVars, commandEnvVars := digger_config.CollectTerraformEnvConfig(workflow.EnvVars)
 
-		planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, nil)
+		planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, nil)
+		if err != nil {
+			usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
+		}
 
 		jobs := orchestrator.Job{
 			ProjectName:       project,
@@ -285,7 +291,7 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 			StateEnvVars:      stateEnvVars,
 			CommandEnvVars:    commandEnvVars,
 		}
-		err := digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir)
+		err = digger.RunJob(jobs, ghRepository, githubActor, &githubPrService, policyChecker, planStorage, backendApi, nil, currentDir)
 		if err != nil {
 			usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to run commands. %s", err), 8)
 		}
@@ -383,7 +389,10 @@ func GitHubCI(lock core_locking.Lock, policyCheckerProvider core_policy.PolicyCh
 			usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to set job output. Exiting. %s", err), 4)
 		}
 
-		planStorage := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, githubActor, &prNumber)
+		planStorage, err := storage.NewPlanStorage(ghToken, repoOwner, repositoryName, &prNumber)
+		if err != nil {
+			usage.ReportErrorAndExit(githubActor, fmt.Sprintf("Failed to get plan storage. %s", err), 4)
+		}
 
 		reporter := &reporting.CiReporter{
 			CiService:         &githubPrService,

cli/pkg/integration/integration_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"github.com/diggerhq/digger/libs/locking"
 	"github.com/diggerhq/digger/libs/locking/aws"
+	"github.com/diggerhq/digger/libs/storage"
 	"log"
 	"math/rand"
 	"os"
@@ -15,8 +16,6 @@ import (
 	"github.com/diggerhq/digger/cli/pkg/core/terraform"
 	"github.com/diggerhq/digger/cli/pkg/digger"
 	"github.com/diggerhq/digger/cli/pkg/github/models"
-	"github.com/diggerhq/digger/cli/pkg/storage"
-	"github.com/diggerhq/digger/cli/pkg/utils"
 	"github.com/diggerhq/digger/libs/comment_utils/reporting"
 	configuration "github.com/diggerhq/digger/libs/digger_config"
 	dg_github "github.com/diggerhq/digger/libs/orchestrator/github"
@@ -398,7 +397,7 @@ func TestHappyPath(t *testing.T) {
 	event := ghEvent.(github.PullRequestEvent)
 	jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&event, impactedProjects, requestedProject, *diggerConfig)
 	assert.NoError(t, err)
-	zipManager := utils.Zipper{}
+	zipManager := storage.Zipper{}
 	planStorage := &storage.GithubPlanStorage{
 		Client:            github.NewTokenClient(context.Background(), ghToken),
 		Owner:             repoOwner,
@@ -556,7 +555,7 @@ func TestMultiEnvHappyPath(t *testing.T) {
 	jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
 	assert.NoError(t, err)
 
-	zipManager := utils.Zipper{}
+	zipManager := storage.Zipper{}
 	planStorage := &storage.GithubPlanStorage{
 		Client:            github.NewTokenClient(context.Background(), ghToken),
 		Owner:             repoOwner,
@@ -777,7 +776,7 @@ workflows:
 	jobs, _, err := dg_github.ConvertGithubPullRequestEventToJobs(&pEvent, impactedProjects, requestedProject, *diggerConfig)
 	assert.NoError(t, err)
 
-	zipManager := utils.Zipper{}
+	zipManager := storage.Zipper{}
 	planStorage := &storage.GithubPlanStorage{
 		Client:            github.NewTokenClient(context.Background(), ghToken),
 		Owner:             repoOwner,

cli/pkg/spec/spec.go

@@ -3,7 +3,6 @@ package spec
 import (
 	"fmt"
 	"github.com/diggerhq/digger/cli/pkg/digger"
-	storage2 "github.com/diggerhq/digger/cli/pkg/storage"
 	"github.com/diggerhq/digger/cli/pkg/usage"
 	comment_summary "github.com/diggerhq/digger/libs/comment_utils/summary"
 	"github.com/diggerhq/digger/libs/digger_config"
@@ -23,6 +22,7 @@ func RunSpec(
 	reporterProvider spec.ReporterProvider,
 	backedProvider spec.BackendApiProvider,
 	policyProvider spec.PolicyProvider,
+	PlanStorageProvider spec.PlanStorageProvider,
 	commentUpdaterProvider comment_summary.CommentUpdaterProvider,
 ) error {
 
@@ -72,8 +72,11 @@ func RunSpec(
 		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get comment updater: %v", err), 8)
 	}
 
-	planStorage := storage2.NewPlanStorage("", "", "", "", nil)
-
+	planStorage, err := PlanStorageProvider.GetPlanStorage(spec.VCS.RepoOwner, spec.VCS.RepoName, *spec.Job.PullRequestNumber)
+	if err != nil {
+		usage.ReportErrorAndExit(spec.VCS.Actor, fmt.Sprintf("could not get plan storage: %v", err), 8)
+	}
+
 	jobs := []orchestrator.Job{job}
 
 	fullRepoName := fmt.Sprintf("%v-%v", spec.VCS.RepoOwner, spec.VCS.RepoName)

cli_e2e/plan_storage_test.go

@@ -3,12 +3,12 @@ package cli_e2e
 import (
 	"fmt"
 	"github.com/diggerhq/digger/libs/locking/gcp"
+	storage2 "github.com/diggerhq/digger/libs/storage"
 	"log"
 	"os"
 	"strings"
 	"testing"
 
-	"github.com/diggerhq/digger/cli/pkg/storage"
 	"github.com/stretchr/testify/assert"
 )
 
@@ -28,7 +28,7 @@ func TestGCPPlanStorageStorageAndRetrieval(t *testing.T) {
 	fmt.Printf("getting bucket")
 	bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET"))
 	bucket := client.Bucket(bucketName)
-	planStorage := &storage.PlanStorageGcp{
+	planStorage := &storage2.PlanStorageGcp{
 		Client:  client,
 		Bucket:  bucket,
 		Context: ctx,
@@ -60,13 +60,13 @@ func TestAWSPlanStorageStorageAndRetrieval(t *testing.T) {
 	defer os.Remove(file.Name())
 
 	log.Printf("getting AWS S3 client")
-	ctx, client, err := storage.GetAWSStorageClient()
+	ctx, client, err := storage2.GetAWSStorageClient()
 	if err != nil {
 		t.Errorf("failed to get AWS storage client: %v", err)
 	}
 	bucketName := strings.ToLower(os.Getenv("AWS_S3_BUCKET"))
 	log.Printf("using AWS S3 bucket found by env var 'AWS_S3_BUCKET': %s", bucketName)
-	planStorage := &storage.PlanStorageAWS{
+	planStorage := &storage2.PlanStorageAWS{
 		Client:  client,
 		Bucket:  bucketName,
 		Context: ctx,

ee/cli/cmd/digger/run_spec.go

@@ -39,6 +39,7 @@ var runSpecCmd = &cobra.Command{
 			lib_spec.ReporterProvider{},
 			lib_spec.BackendApiProvider{},
 			lib_spec.PolicyProvider{},
+			lib_spec.PlanStorageProvider{},
 			comment_summary.CommentUpdaterProviderBasic{},
 		)
 		if err != nil {

libs/spec/providers.go

@@ -21,6 +21,7 @@ import (
 	"github.com/diggerhq/digger/libs/locking/gcp"
 	"github.com/diggerhq/digger/libs/orchestrator"
 	"github.com/diggerhq/digger/libs/orchestrator/github"
+	storage2 "github.com/diggerhq/digger/libs/storage"
 	"github.com/samber/lo"
 	"log"
 	"net/http"
@@ -173,7 +174,7 @@ func (v VCSProvider) GetPrService(vcsSpec VcsSpec) (orchestrator.PullRequestServ
 	case "github":
 		token := os.Getenv("GITHUB_TOKEN")
 		if token == "" {
-			return nil, fmt.Errorf("failed to get githbu service: GITHUB_TOKEN not specified")
+			return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified")
 		}
 		return github.GithubServiceProviderBasic{}.NewService(token, vcsSpec.RepoName, vcsSpec.RepoOwner)
 	default:
@@ -198,3 +199,14 @@ func (p PolicyProvider) GetPolicyProvider(policySpec PolicySpec, diggerHost stri
 		return nil, fmt.Errorf("could not find ")
 	}
 }
+
+type PlanStorageProvider struct{}
+
+func (p PlanStorageProvider) GetPlanStorage(repoOwner string, repositoryName string, prNumber int) (storage2.PlanStorage, error) {
+	token := os.Getenv("GITHUB_TOKEN")
+	if token == "" {
+		return nil, fmt.Errorf("failed to get github service: GITHUB_TOKEN not specified")
+	}
+	storage, err := storage2.NewPlanStorage(token, repoOwner, repositoryName, &prNumber)
+	return storage, err
+}

cli/pkg/utils/io.go → libs/storage/io.go

@@ -1,4 +1,4 @@
-package utils
+package storage
 
 import (
 	"archive/zip"

cli/pkg/utils/io_test.go → libs/storage/io_test.go

@@ -1,4 +1,4 @@
-package utils
+package storage
 
 import (
 	"archive/zip"

cli/pkg/storage/plan_storage.go → libs/storage/plan_storage.go

@@ -5,8 +5,6 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
-	"github.com/diggerhq/digger/cli/pkg/core/storage"
-	"github.com/diggerhq/digger/cli/pkg/usage"
 	"github.com/diggerhq/digger/libs/locking/gcp"
 	"io"
 	"log"
@@ -16,8 +14,6 @@ import (
 	"os/exec"
 	"strings"
 
-	"github.com/diggerhq/digger/cli/pkg/utils"
-
 	"github.com/google/go-github/v61/github"
 )
 
@@ -26,7 +22,7 @@ type GithubPlanStorage struct {
 	Owner             string
 	RepoName          string
 	PullRequestNumber int
-	ZipManager        utils.Zipper
+	ZipManager        Zipper
 }
 
 func (gps *GithubPlanStorage) StorePlanFile(fileContents []byte, artifactName string, storedPlanFilePath string) error {
@@ -208,13 +204,14 @@ func getLatestArtifactWithName(artifacts []*github.Artifact, name string) *githu
 	return latest
 }
 
-func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, requestedBy string, prNumber *int) storage.PlanStorage {
-	var planStorage storage.PlanStorage
+// TODO: refactor this function to make it fully parametarised and no reliance on env variables
+func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string, prNumber *int) (PlanStorage, error) {
+	var planStorage PlanStorage
 
 	uploadDestination := strings.ToLower(os.Getenv("PLAN_UPLOAD_DESTINATION"))
 	switch {
 	case uploadDestination == "github":
-		zipManager := utils.Zipper{}
+		zipManager := Zipper{}
 		planStorage = &GithubPlanStorage{
 			Client:            github.NewTokenClient(context.Background(), ghToken),
 			Owner:             ghRepoOwner,
@@ -226,7 +223,7 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string,
 		ctx, client := gcp.GetGoogleStorageClient()
 		bucketName := strings.ToLower(os.Getenv("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET"))
 		if bucketName == "" {
-			usage.ReportErrorAndExit(requestedBy, "GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET is not defined", 9)
+			return nil, fmt.Errorf("GOOGLE_STORAGE_PLAN_ARTEFACT_BUCKET is not defined")
 		}
 		bucket := client.Bucket(bucketName)
 		planStorage = &PlanStorageGcp{
@@ -237,11 +234,11 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string,
 	case uploadDestination == "aws":
 		ctx, client, err := GetAWSStorageClient()
 		if err != nil {
-			usage.ReportErrorAndExit(requestedBy, fmt.Sprintf("Failed to create AWS storage client: %s", err), 9)
+			return nil, fmt.Errorf(fmt.Sprintf("Failed to create AWS storage client: %s", err))
 		}
 		bucketName := strings.ToLower(os.Getenv("AWS_S3_BUCKET"))
 		if bucketName == "" {
-			usage.ReportErrorAndExit(requestedBy, "AWS_S3_BUCKET is not defined", 9)
+			return nil, fmt.Errorf("AWS_S3_BUCKET is not defined")
 		}
 		planStorage = &PlanStorageAWS{
 			Context: ctx,
@@ -252,5 +249,5 @@ func NewPlanStorage(ghToken string, ghRepoOwner string, ghRepositoryName string,
 		//TODO implement me
 	}
 
-	return planStorage
+	return planStorage, nil
 }