Feat/use digger spec (#1597)

* Use digger spec with the orchestrator

action.yml

@@ -155,6 +155,11 @@ inputs:
     description: "Leverage actions/cache to cache dependencies to speed up execution"
     required: false
     default: 'false'
+  digger-spec:
+    description: "(orchestrator only) the spec to pass onto digger cli"
+    required: false
+    default: ''
+
 outputs:
   output:
     value: ${{ steps.digger.outputs.output }}
@@ -348,6 +353,7 @@ runs:
         TF_PLUGIN_CACHE_DIR: ${{ github.workspace }}/cache
         TERRAGRUNT_PROVIDER_CACHE: ${{ inputs.cache-dependencies == 'true' && 1 || 0 }}
         TERRAGRUNT_PROVIDER_CACHE_DIR: ${{ github.workspace }}/cache
+        DIGGER_RUN_SPEC: ${{inputs.digger-spec}}
       run: |
           if [[ ${{ inputs.ee }} == "true" ]]; then
             cd $GITHUB_ACTION_PATH/ee/cli
@@ -384,6 +390,7 @@ runs:
         TF_PLUGIN_CACHE_DIR: ${{ github.workspace }}/cache
         TERRAGRUNT_PROVIDER_CACHE: ${{ inputs.cache-dependencies == 'true' && 1 || 0 }}
         TERRAGRUNT_PROVIDER_CACHE_DIR: ${{ github.workspace }}/cache
+        DIGGER_RUN_SPEC: ${{inputs.digger-spec}}
       id: digger
       shell: bash
       run: |

backend/ci_backends/ci_backends.go

@@ -1,12 +1,12 @@
 package ci_backends
 
 import (
-	"github.com/diggerhq/digger/backend/models"
 	"github.com/diggerhq/digger/backend/utils"
+	"github.com/diggerhq/digger/libs/spec"
 )
 
 type CiBackend interface {
-	TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error
+	TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error
 }
 
 type JenkinsCi struct{}

backend/ci_backends/github_actions.go

@@ -3,43 +3,28 @@ package ci_backends
 import (
 	"context"
 	"encoding/json"
-	"fmt"
-	"github.com/diggerhq/digger/backend/models"
-	"github.com/diggerhq/digger/libs/orchestrator"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
+	"github.com/diggerhq/digger/libs/spec"
 	"github.com/google/go-github/v61/github"
 	"log"
-	"strconv"
 )
 
 type GithubActionCi struct {
 	Client *github.Client
 }
 
-func (g GithubActionCi) TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error {
-	log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", repoOwner, repoName, commentId)
+func (g GithubActionCi) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error {
+	log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", spec.VCS.RepoOwner, spec.VCS.RepoName, spec.CommentId)
 	client := g.Client
-	var jobSpec orchestrator.JobJson
-	err := json.Unmarshal([]byte(jobString), &jobSpec)
-	if err != nil {
-		log.Printf("could not unmarshal job string: %v", err)
-		return fmt.Errorf("could not marshal json string: %v", err)
-	}
+	specBytes, err := json.Marshal(spec)
 
-	batchIdShort := job.Batch.ID.String()[:8]
-	diggerCommand := fmt.Sprintf("digger %v", job.Batch.BatchType)
-	projectName := jobSpec.ProjectName
-	requestedBy := jobSpec.RequestedBy
-	prNumber := *jobSpec.PullRequestNumber
 	inputs := orchestrator_scheduler.WorkflowInput{
-		Id:        job.DiggerJobID,
-		JobString: jobString,
-		CommentId: strconv.FormatInt(commentId, 10),
-		RunName:   fmt.Sprintf("[%v] %v %v By: %v PR: %v", batchIdShort, diggerCommand, projectName, requestedBy, prNumber),
+		Spec:    string(specBytes),
+		RunName: runName,
 	}
 
-	_, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, job.WorkflowFile, github.CreateWorkflowDispatchEventRequest{
-		Ref:    job.Batch.BranchName,
+	_, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), spec.VCS.RepoOwner, spec.VCS.RepoName, spec.VCS.WorkflowFile, github.CreateWorkflowDispatchEventRequest{
+		Ref:    spec.Job.Branch,
 		Inputs: inputs.ToMap(),
 	})
 

backend/services/scheduler.go

@@ -92,20 +92,32 @@ func ScheduleJob(ciBackend ci_backends.CiBackend, repoOwner string, repoName str
 func TriggerJob(ciBackend ci_backends.CiBackend, repoOwner string, repoName string, batchId *uuid.UUID, job *models.DiggerJob) error {
 	log.Printf("TriggerJob jobId: %v", job.DiggerJobID)
 
-	batch, err := models.DB.GetDiggerBatch(batchId)
-	if err != nil {
-		log.Printf("TriggerJob err: %v\n", err)
-		return err
-	}
-
 	if job.SerializedJobSpec == nil {
-		log.Printf("GitHub job can't be nil")
+		log.Printf("Jobspec can't be nil")
 		return fmt.Errorf("JobSpec is nil, skipping")
 	}
 	jobString := string(job.SerializedJobSpec)
 	log.Printf("jobString: %v \n", jobString)
 
-	err = ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobString, *batch.CommentId)
+	runName, err := GetRunNameFromJob(*job)
+	if err != nil {
+		log.Printf("could not get run name: %v", err)
+		return fmt.Errorf("coult not get run name %v", err)
+	}
+
+	spec, err := GetSpecFromJob(*job)
+	if err != nil {
+		log.Printf("could not get spec: %v", err)
+		return fmt.Errorf("coult not get spec %v", err)
+	}
+
+	vcsToken, err := GetVCSTokenFromJob(*job)
+	if err != nil {
+		log.Printf("could not get vcs token: %v", err)
+		return fmt.Errorf("coult not get vcs token: %v", err)
+	}
+
+	err = ciBackend.TriggerWorkflow(*spec, *runName, *vcsToken)
 	if err != nil {
 		log.Printf("TriggerJob err: %v\n", err)
 		return err

backend/services/spec.go

@@ -0,0 +1,87 @@
+package services
+
+import (
+	"encoding/json"
+	"fmt"
+	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
+	"github.com/diggerhq/digger/libs/orchestrator"
+	"github.com/diggerhq/digger/libs/spec"
+	"log"
+	"strconv"
+)
+
+func GetVCSTokenFromJob(job models.DiggerJob) (*string, error) {
+	// TODO: make it VCS generic
+	_, ghToken, err := utils.GetGithubService(
+		utils.DiggerGithubRealClientProvider{},
+		job.Batch.GithubInstallationId,
+		job.Batch.RepoFullName,
+		job.Batch.RepoOwner,
+		job.Batch.RepoName,
+	)
+	if err != nil {
+		return nil, fmt.Errorf("TriggerWorkflow: could not retrieve token: %v", err)
+	}
+	return ghToken, nil
+}
+
+func GetRunNameFromJob(job models.DiggerJob) (*string, error) {
+	var jobSpec orchestrator.JobJson
+	err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec)
+	if err != nil {
+		log.Printf("could not unmarshal job string: %v", err)
+		return nil, fmt.Errorf("could not marshal json string: %v", err)
+	}
+
+	batch := job.Batch
+	batchIdShort := batch.ID.String()[:8]
+	diggerCommand := fmt.Sprintf("digger %v", batch.BatchType)
+	projectName := jobSpec.ProjectName
+	requestedBy := jobSpec.RequestedBy
+	prNumber := *jobSpec.PullRequestNumber
+
+	runName := fmt.Sprintf("[%v] %v %v By: %v PR: %v", batchIdShort, diggerCommand, projectName, requestedBy, prNumber)
+	return &runName, nil
+}
+
+func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
+	var jobSpec orchestrator.JobJson
+	err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec)
+	if err != nil {
+		log.Printf("could not unmarshal job string: %v", err)
+		return nil, fmt.Errorf("could not marshal json string: %v", err)
+	}
+
+	batch := job.Batch
+
+	spec := spec.Spec{
+		JobId:     job.DiggerJobID,
+		CommentId: strconv.FormatInt(*batch.CommentId, 10),
+		Job:       jobSpec,
+		Reporter: spec.ReporterSpec{
+			ReportingStrategy: "comments_per_run",
+			ReporterType:      "lazy",
+		},
+		Lock: spec.LockSpec{
+			LockType: "noop",
+		},
+		Backend: spec.BackendSpec{
+			BackendHostname:         jobSpec.BackendHostname,
+			BackendOrganisationName: jobSpec.BackendOrganisationName,
+			BackendJobToken:         jobSpec.BackendJobToken,
+			BackendType:             "backend",
+		},
+		VCS: spec.VcsSpec{
+			VcsType:      "github",
+			Actor:        jobSpec.RequestedBy,
+			RepoOwner:    batch.RepoOwner,
+			RepoName:     batch.RepoName,
+			WorkflowFile: job.WorkflowFile,
+		},
+		Policy: spec.PolicySpec{
+			PolicyType: "http",
+		},
+	}
+	return &spec, nil
+}

backend/tasks/runs.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/diggerhq/digger/backend/ci_backends"
 	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/services"
 	"github.com/diggerhq/digger/libs/orchestrator"
 	"github.com/diggerhq/digger/libs/orchestrator/github"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
@@ -14,15 +15,28 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, service orchest
 	switch queueItem.DiggerRun.Status {
 	case models.RunQueued:
 		// trigger plan workflow (trigger the batch)
-
-		repoOwner := dr.Repo.RepoOrganisation
-		repoName := dr.Repo.RepoName
 		job, err := models.DB.GetDiggerJobFromRunStage(dr.PlanStage)
-		jobSpec := string(job.SerializedJobSpec)
-		commentId := int64(2037675659)
 		client := service.(*github.GithubService).Client
 		ciBackend := ci_backends.GithubActionCi{Client: client}
-		ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobSpec, commentId)
+		runName, err := services.GetRunNameFromJob(*job)
+		if err != nil {
+			log.Printf("could not get run name: %v", err)
+			return
+		}
+
+		spec, err := services.GetSpecFromJob(*job)
+		if err != nil {
+			log.Printf("could not get spec: %v", err)
+			return
+		}
+
+		vcsToken, err := services.GetVCSTokenFromJob(*job)
+		if err != nil {
+			log.Printf("could not get vcs token: %v", err)
+			return
+		}
+
+		ciBackend.TriggerWorkflow(*spec, *runName, *vcsToken)
 
 		// change status to RunPendingPlan
 		log.Printf("Updating run queueItem item to planning state")
@@ -70,14 +84,32 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, service orchest
 		// do nothing
 	case models.RunApproved:
 		// trigger apply stage workflow
-		repoOwner := dr.Repo.RepoOrganisation
-		repoName := dr.Repo.RepoName
 		job, err := models.DB.GetDiggerJobFromRunStage(dr.ApplyStage)
-		jobSpec := string(job.SerializedJobSpec)
-		commentId := int64(2037675659)
 		client := service.(*github.GithubService).Client
 		ciBackend := ci_backends.GithubActionCi{Client: client}
-		ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobSpec, commentId)
+		if err != nil {
+			log.Printf("could not get run name: %v", err)
+			return
+		}
+		runName, err := services.GetRunNameFromJob(*job)
+		if err != nil {
+			log.Printf("could not get run name: %v", err)
+			return
+		}
+
+		spec, err := services.GetSpecFromJob(*job)
+		if err != nil {
+			log.Printf("could not get spec: %v", err)
+			return
+		}
+
+		vcsToken, err := services.GetVCSTokenFromJob(*job)
+		if err != nil {
+			log.Printf("could not get vcs token: %v", err)
+			return
+		}
+
+		ciBackend.TriggerWorkflow(*spec, *runName, *vcsToken)
 
 		dr.Status = models.RunApplying
 		err = models.DB.UpdateDiggerRun(&dr)

backend/tasks/runs_test.go

@@ -5,6 +5,7 @@ import (
 	"github.com/diggerhq/digger/backend/models"
 	github2 "github.com/diggerhq/digger/libs/orchestrator/github"
 	orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
+	"github.com/diggerhq/digger/libs/spec"
 	"github.com/stretchr/testify/assert"
 	"gorm.io/driver/sqlite"
 	"gorm.io/gorm"
@@ -22,7 +23,7 @@ func init() {
 type MockCiBackend struct {
 }
 
-func (m MockCiBackend) TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error {
+func (m MockCiBackend) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error {
 	return nil
 }