Skip to content

Commit

Permalink
Feat/use digger spec (#1597)
Browse files Browse the repository at this point in the history
* Use digger spec with the orchestrator
  • Loading branch information
motatoes authored Jul 1, 2024
1 parent 1c1af6b commit 522b8d6
Show file tree
Hide file tree
Showing 22 changed files with 280 additions and 511 deletions.
7 changes: 7 additions & 0 deletions action.yml
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,11 @@ inputs:
description: "Leverage actions/cache to cache dependencies to speed up execution"
required: false
default: 'false'
digger-spec:
description: "(orchestrator only) the spec to pass onto digger cli"
required: false
default: ''

outputs:
output:
value: ${{ steps.digger.outputs.output }}
Expand Down Expand Up @@ -348,6 +353,7 @@ runs:
TF_PLUGIN_CACHE_DIR: ${{ github.workspace }}/cache
TERRAGRUNT_PROVIDER_CACHE: ${{ inputs.cache-dependencies == 'true' && 1 || 0 }}
TERRAGRUNT_PROVIDER_CACHE_DIR: ${{ github.workspace }}/cache
DIGGER_RUN_SPEC: ${{inputs.digger-spec}}
run: |
if [[ ${{ inputs.ee }} == "true" ]]; then
cd $GITHUB_ACTION_PATH/ee/cli
Expand Down Expand Up @@ -384,6 +390,7 @@ runs:
TF_PLUGIN_CACHE_DIR: ${{ github.workspace }}/cache
TERRAGRUNT_PROVIDER_CACHE: ${{ inputs.cache-dependencies == 'true' && 1 || 0 }}
TERRAGRUNT_PROVIDER_CACHE_DIR: ${{ github.workspace }}/cache
DIGGER_RUN_SPEC: ${{inputs.digger-spec}}
id: digger
shell: bash
run: |
Expand Down
4 changes: 2 additions & 2 deletions backend/ci_backends/ci_backends.go
Original file line number Diff line number Diff line change
@@ -1,12 +1,12 @@
package ci_backends

import (
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/backend/utils"
"github.com/diggerhq/digger/libs/spec"
)

type CiBackend interface {
TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error
TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error
}

type JenkinsCi struct{}
Expand Down
31 changes: 8 additions & 23 deletions backend/ci_backends/github_actions.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,43 +3,28 @@ package ci_backends
import (
"context"
"encoding/json"
"fmt"
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/libs/orchestrator"
orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
"github.com/diggerhq/digger/libs/spec"
"github.com/google/go-github/v61/github"
"log"
"strconv"
)

type GithubActionCi struct {
Client *github.Client
}

func (g GithubActionCi) TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error {
log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", repoOwner, repoName, commentId)
func (g GithubActionCi) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error {
log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", spec.VCS.RepoOwner, spec.VCS.RepoName, spec.CommentId)
client := g.Client
var jobSpec orchestrator.JobJson
err := json.Unmarshal([]byte(jobString), &jobSpec)
if err != nil {
log.Printf("could not unmarshal job string: %v", err)
return fmt.Errorf("could not marshal json string: %v", err)
}
specBytes, err := json.Marshal(spec)

batchIdShort := job.Batch.ID.String()[:8]
diggerCommand := fmt.Sprintf("digger %v", job.Batch.BatchType)
projectName := jobSpec.ProjectName
requestedBy := jobSpec.RequestedBy
prNumber := *jobSpec.PullRequestNumber
inputs := orchestrator_scheduler.WorkflowInput{
Id: job.DiggerJobID,
JobString: jobString,
CommentId: strconv.FormatInt(commentId, 10),
RunName: fmt.Sprintf("[%v] %v %v By: %v PR: %v", batchIdShort, diggerCommand, projectName, requestedBy, prNumber),
Spec: string(specBytes),
RunName: runName,
}

_, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, job.WorkflowFile, github.CreateWorkflowDispatchEventRequest{
Ref: job.Batch.BranchName,
_, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), spec.VCS.RepoOwner, spec.VCS.RepoName, spec.VCS.WorkflowFile, github.CreateWorkflowDispatchEventRequest{
Ref: spec.Job.Branch,
Inputs: inputs.ToMap(),
})

Expand Down
28 changes: 20 additions & 8 deletions backend/services/scheduler.go
Original file line number Diff line number Diff line change
Expand Up @@ -92,20 +92,32 @@ func ScheduleJob(ciBackend ci_backends.CiBackend, repoOwner string, repoName str
func TriggerJob(ciBackend ci_backends.CiBackend, repoOwner string, repoName string, batchId *uuid.UUID, job *models.DiggerJob) error {
log.Printf("TriggerJob jobId: %v", job.DiggerJobID)

batch, err := models.DB.GetDiggerBatch(batchId)
if err != nil {
log.Printf("TriggerJob err: %v\n", err)
return err
}

if job.SerializedJobSpec == nil {
log.Printf("GitHub job can't be nil")
log.Printf("Jobspec can't be nil")
return fmt.Errorf("JobSpec is nil, skipping")
}
jobString := string(job.SerializedJobSpec)
log.Printf("jobString: %v \n", jobString)

err = ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobString, *batch.CommentId)
runName, err := GetRunNameFromJob(*job)
if err != nil {
log.Printf("could not get run name: %v", err)
return fmt.Errorf("coult not get run name %v", err)
}

spec, err := GetSpecFromJob(*job)
if err != nil {
log.Printf("could not get spec: %v", err)
return fmt.Errorf("coult not get spec %v", err)
}

vcsToken, err := GetVCSTokenFromJob(*job)
if err != nil {
log.Printf("could not get vcs token: %v", err)
return fmt.Errorf("coult not get vcs token: %v", err)
}

err = ciBackend.TriggerWorkflow(*spec, *runName, *vcsToken)
if err != nil {
log.Printf("TriggerJob err: %v\n", err)
return err
Expand Down
87 changes: 87 additions & 0 deletions backend/services/spec.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,87 @@
package services

import (
"encoding/json"
"fmt"
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/backend/utils"
"github.com/diggerhq/digger/libs/orchestrator"
"github.com/diggerhq/digger/libs/spec"
"log"
"strconv"
)

func GetVCSTokenFromJob(job models.DiggerJob) (*string, error) {
// TODO: make it VCS generic
_, ghToken, err := utils.GetGithubService(
utils.DiggerGithubRealClientProvider{},
job.Batch.GithubInstallationId,
job.Batch.RepoFullName,
job.Batch.RepoOwner,
job.Batch.RepoName,
)
if err != nil {
return nil, fmt.Errorf("TriggerWorkflow: could not retrieve token: %v", err)
}
return ghToken, nil
}

func GetRunNameFromJob(job models.DiggerJob) (*string, error) {
var jobSpec orchestrator.JobJson
err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec)
if err != nil {
log.Printf("could not unmarshal job string: %v", err)
return nil, fmt.Errorf("could not marshal json string: %v", err)
}

batch := job.Batch
batchIdShort := batch.ID.String()[:8]
diggerCommand := fmt.Sprintf("digger %v", batch.BatchType)
projectName := jobSpec.ProjectName
requestedBy := jobSpec.RequestedBy
prNumber := *jobSpec.PullRequestNumber

runName := fmt.Sprintf("[%v] %v %v By: %v PR: %v", batchIdShort, diggerCommand, projectName, requestedBy, prNumber)
return &runName, nil
}

func GetSpecFromJob(job models.DiggerJob) (*spec.Spec, error) {
var jobSpec orchestrator.JobJson
err := json.Unmarshal([]byte(job.SerializedJobSpec), &jobSpec)
if err != nil {
log.Printf("could not unmarshal job string: %v", err)
return nil, fmt.Errorf("could not marshal json string: %v", err)
}

batch := job.Batch

spec := spec.Spec{
JobId: job.DiggerJobID,
CommentId: strconv.FormatInt(*batch.CommentId, 10),
Job: jobSpec,
Reporter: spec.ReporterSpec{
ReportingStrategy: "comments_per_run",
ReporterType: "lazy",
},
Lock: spec.LockSpec{
LockType: "noop",
},
Backend: spec.BackendSpec{
BackendHostname: jobSpec.BackendHostname,
BackendOrganisationName: jobSpec.BackendOrganisationName,
BackendJobToken: jobSpec.BackendJobToken,
BackendType: "backend",
},
VCS: spec.VcsSpec{
VcsType: "github",
Actor: jobSpec.RequestedBy,
RepoOwner: batch.RepoOwner,
RepoName: batch.RepoName,
WorkflowFile: job.WorkflowFile,
},
Policy: spec.PolicySpec{
PolicyType: "http",
},
}
return &spec, nil
}
54 changes: 43 additions & 11 deletions backend/tasks/runs.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package main
import (
"github.com/diggerhq/digger/backend/ci_backends"
"github.com/diggerhq/digger/backend/models"
"github.com/diggerhq/digger/backend/services"
"github.com/diggerhq/digger/libs/orchestrator"
"github.com/diggerhq/digger/libs/orchestrator/github"
orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
Expand All @@ -14,15 +15,28 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, service orchest
switch queueItem.DiggerRun.Status {
case models.RunQueued:
// trigger plan workflow (trigger the batch)

repoOwner := dr.Repo.RepoOrganisation
repoName := dr.Repo.RepoName
job, err := models.DB.GetDiggerJobFromRunStage(dr.PlanStage)
jobSpec := string(job.SerializedJobSpec)
commentId := int64(2037675659)
client := service.(*github.GithubService).Client
ciBackend := ci_backends.GithubActionCi{Client: client}
ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobSpec, commentId)
runName, err := services.GetRunNameFromJob(*job)
if err != nil {
log.Printf("could not get run name: %v", err)
return
}

spec, err := services.GetSpecFromJob(*job)
if err != nil {
log.Printf("could not get spec: %v", err)
return
}

vcsToken, err := services.GetVCSTokenFromJob(*job)
if err != nil {
log.Printf("could not get vcs token: %v", err)
return
}

ciBackend.TriggerWorkflow(*spec, *runName, *vcsToken)

// change status to RunPendingPlan
log.Printf("Updating run queueItem item to planning state")
Expand Down Expand Up @@ -70,14 +84,32 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, service orchest
// do nothing
case models.RunApproved:
// trigger apply stage workflow
repoOwner := dr.Repo.RepoOrganisation
repoName := dr.Repo.RepoName
job, err := models.DB.GetDiggerJobFromRunStage(dr.ApplyStage)
jobSpec := string(job.SerializedJobSpec)
commentId := int64(2037675659)
client := service.(*github.GithubService).Client
ciBackend := ci_backends.GithubActionCi{Client: client}
ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobSpec, commentId)
if err != nil {
log.Printf("could not get run name: %v", err)
return
}
runName, err := services.GetRunNameFromJob(*job)
if err != nil {
log.Printf("could not get run name: %v", err)
return
}

spec, err := services.GetSpecFromJob(*job)
if err != nil {
log.Printf("could not get spec: %v", err)
return
}

vcsToken, err := services.GetVCSTokenFromJob(*job)
if err != nil {
log.Printf("could not get vcs token: %v", err)
return
}

ciBackend.TriggerWorkflow(*spec, *runName, *vcsToken)

dr.Status = models.RunApplying
err = models.DB.UpdateDiggerRun(&dr)
Expand Down
3 changes: 2 additions & 1 deletion backend/tasks/runs_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@ import (
"github.com/diggerhq/digger/backend/models"
github2 "github.com/diggerhq/digger/libs/orchestrator/github"
orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler"
"github.com/diggerhq/digger/libs/spec"
"github.com/stretchr/testify/assert"
"gorm.io/driver/sqlite"
"gorm.io/gorm"
Expand All @@ -22,7 +23,7 @@ func init() {
type MockCiBackend struct {
}

func (m MockCiBackend) TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error {
func (m MockCiBackend) TriggerWorkflow(spec spec.Spec, runName string, vcsToken string) error {
return nil
}

Expand Down
61 changes: 0 additions & 61 deletions cli/cmd/digger/apply.go

This file was deleted.

Loading

0 comments on commit 522b8d6

Please sign in to comment.