From 50bc84a6807e4d709f6cc3c556d182a001befe29 Mon Sep 17 00:00:00 2001 From: Mohamed Habib Date: Tue, 4 Jun 2024 14:42:26 +0100 Subject: [PATCH] use the common ci backend interface (#1559) --- backend/ci_backends/github_actions.go | 22 ++++++++++++++--- backend/controllers/github.go | 12 ++++++---- backend/services/scheduler.go | 15 ++++++------ backend/tasks/runs.go | 10 +++++--- backend/tasks/tasks.go | 5 +++- backend/utils/github.go | 34 --------------------------- go.work.sum | 5 ++-- 7 files changed, 49 insertions(+), 54 deletions(-) diff --git a/backend/ci_backends/github_actions.go b/backend/ci_backends/github_actions.go index f4b5574ca..722405db3 100644 --- a/backend/ci_backends/github_actions.go +++ b/backend/ci_backends/github_actions.go @@ -2,7 +2,10 @@ package ci_backends import ( "context" + "encoding/json" + "fmt" "github.com/diggerhq/digger/backend/models" + "github.com/diggerhq/digger/libs/orchestrator" orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler" "github.com/google/go-github/v61/github" "log" @@ -14,18 +17,31 @@ type GithubActionCi struct { } func (g GithubActionCi) TriggerWorkflow(repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error { - client := g.Client log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", repoOwner, repoName, commentId) + client := g.Client + var jobSpec orchestrator.JobJson + err := json.Unmarshal([]byte(jobString), &jobSpec) + if err != nil { + log.Printf("could not unmarshal job string: %v", err) + return fmt.Errorf("could not marshal json string: %v", err) + } + + batchIdShort := job.Batch.ID.String()[:8] + diggerCommand := fmt.Sprintf("digger %v", job.Batch.BatchType) + projectName := jobSpec.ProjectName + requestedBy := jobSpec.RequestedBy + prNumber := *jobSpec.PullRequestNumber inputs := orchestrator_scheduler.WorkflowInput{ Id: job.DiggerJobID, JobString: jobString, CommentId: strconv.FormatInt(commentId, 10), + RunName: fmt.Sprintf("[%v] %v %v By: %v PR: %v", batchIdShort, diggerCommand, projectName, requestedBy, prNumber), } - _, err := client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, "digger_workflow.yml", github.CreateWorkflowDispatchEventRequest{ + + _, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, job.WorkflowFile, github.CreateWorkflowDispatchEventRequest{ Ref: job.Batch.BranchName, Inputs: inputs.ToMap(), }) return err - } diff --git a/backend/controllers/github.go b/backend/controllers/github.go index ee7ef5d2c..864f34cc0 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/locking" "github.com/diggerhq/digger/backend/segment" "github.com/diggerhq/digger/backend/services" @@ -574,7 +575,8 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR } segment.Track(strconv.Itoa(int(organisationId)), "backend_trigger_job") - err = TriggerDiggerJobs(ghService.Client, repoOwner, repoName, batchId, prNumber, ghService) + ciBackend := ci_backends.GithubActionCi{Client: ghService.Client} + err = TriggerDiggerJobs(ciBackend, repoOwner, repoName, batchId, prNumber, ghService) if err != nil { log.Printf("TriggerDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err)) @@ -839,7 +841,9 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu } segment.Track(strconv.Itoa(int(orgId)), "backend_trigger_job") - err = TriggerDiggerJobs(ghService.Client, repoOwner, repoName, batchId, issueNumber, ghService) + + ciBackend := ci_backends.GithubActionCi{Client: ghService.Client} + err = TriggerDiggerJobs(ciBackend, repoOwner, repoName, batchId, issueNumber, ghService) if err != nil { log.Printf("TriggerDiggerJobs error: %v", err) utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err)) @@ -875,7 +879,7 @@ func PerformLockingActionFromCommand(prLock dg_locking.PullRequestLock, command return err } -func TriggerDiggerJobs(client *github.Client, repoOwner string, repoName string, batchId *uuid.UUID, prNumber int, prService *dg_github.GithubService) error { +func TriggerDiggerJobs(ciBackend ci_backends.CiBackend, repoOwner string, repoName string, batchId *uuid.UUID, prNumber int, prService *dg_github.GithubService) error { _, err := models.DB.GetDiggerBatch(batchId) if err != nil { log.Printf("failed to get digger batch, %v\n", err) @@ -898,7 +902,7 @@ func TriggerDiggerJobs(client *github.Client, repoOwner string, repoName string, log.Printf("jobString: %v \n", jobString) // TODO: make workflow file name configurable - err = services.ScheduleJob(client, repoOwner, repoName, batchId, &job) + err = services.ScheduleJob(ciBackend, repoOwner, repoName, batchId, &job) if err != nil { log.Printf("failed to trigger github workflow, %v\n", err) return fmt.Errorf("failed to trigger github workflow, %v\n", err) diff --git a/backend/services/scheduler.go b/backend/services/scheduler.go index 680e2e2be..c3848ecef 100644 --- a/backend/services/scheduler.go +++ b/backend/services/scheduler.go @@ -2,9 +2,9 @@ package services import ( "fmt" + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/config" "github.com/diggerhq/digger/backend/models" - "github.com/diggerhq/digger/backend/utils" orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler" "github.com/google/go-github/v61/github" "github.com/google/uuid" @@ -44,18 +44,19 @@ func DiggerJobCompleted(client *github.Client, batchId *uuid.UUID, parentJob *mo if err != nil { return err } - ScheduleJob(client, repoOwner, repoName, batchId, job) + ciBackend := ci_backends.GithubActionCi{Client: client} + ScheduleJob(ciBackend, repoOwner, repoName, batchId, job) } } return nil } -func ScheduleJob(client *github.Client, repoOwner string, repoName string, batchId *uuid.UUID, job *models.DiggerJob) error { +func ScheduleJob(ciBackend ci_backends.CiBackend, repoOwner string, repoName string, batchId *uuid.UUID, job *models.DiggerJob) error { maxConcurrencyForBatch := config.DiggerConfig.GetInt("max_concurrency_per_batch") if maxConcurrencyForBatch == 0 { // concurrency limits not set - err := TriggerJob(client, repoOwner, repoName, batchId, job) + err := TriggerJob(ciBackend, repoOwner, repoName, batchId, job) if err != nil { log.Printf("Could not trigger job: %v", err) return err @@ -78,7 +79,7 @@ func ScheduleJob(client *github.Client, repoOwner string, repoName string, batch models.DB.UpdateDiggerJob(job) return nil } else { - err := TriggerJob(client, repoOwner, repoName, batchId, job) + err := TriggerJob(ciBackend, repoOwner, repoName, batchId, job) if err != nil { log.Printf("Could not trigger job: %v", err) return err @@ -88,7 +89,7 @@ func ScheduleJob(client *github.Client, repoOwner string, repoName string, batch return nil } -func TriggerJob(client *github.Client, repoOwner string, repoName string, batchId *uuid.UUID, job *models.DiggerJob) error { +func TriggerJob(ciBackend ci_backends.CiBackend, repoOwner string, repoName string, batchId *uuid.UUID, job *models.DiggerJob) error { log.Printf("TriggerJob jobId: %v", job.DiggerJobID) batch, err := models.DB.GetDiggerBatch(batchId) @@ -104,7 +105,7 @@ func TriggerJob(client *github.Client, repoOwner string, repoName string, batchI jobString := string(job.SerializedJobSpec) log.Printf("jobString: %v \n", jobString) - err = utils.TriggerGithubWorkflow(client, repoOwner, repoName, *job, jobString, *batch.CommentId) + err = ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobString, *batch.CommentId) if err != nil { log.Printf("TriggerJob err: %v\n", err) return err diff --git a/backend/tasks/runs.go b/backend/tasks/runs.go index ad5d60171..1f74af61c 100644 --- a/backend/tasks/runs.go +++ b/backend/tasks/runs.go @@ -1,8 +1,8 @@ package main import ( + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/models" - "github.com/diggerhq/digger/backend/utils" "github.com/diggerhq/digger/libs/orchestrator" "github.com/diggerhq/digger/libs/orchestrator/github" orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler" @@ -20,7 +20,9 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, service orchest job, err := models.DB.GetDiggerJobFromRunStage(dr.PlanStage) jobSpec := string(job.SerializedJobSpec) commentId := int64(2037675659) - utils.TriggerGithubWorkflow(service.(*github.GithubService).Client, repoOwner, repoName, *job, jobSpec, commentId) + client := service.(*github.GithubService).Client + ciBackend := ci_backends.GithubActionCi{Client: client} + ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobSpec, commentId) // change status to RunPendingPlan log.Printf("Updating run queueItem item to planning state") @@ -73,7 +75,9 @@ func RunQueuesStateMachine(queueItem *models.DiggerRunQueueItem, service orchest job, err := models.DB.GetDiggerJobFromRunStage(dr.ApplyStage) jobSpec := string(job.SerializedJobSpec) commentId := int64(2037675659) - utils.TriggerGithubWorkflow(service.(*github.GithubService).Client, repoOwner, repoName, *job, jobSpec, commentId) + client := service.(*github.GithubService).Client + ciBackend := ci_backends.GithubActionCi{Client: client} + ciBackend.TriggerWorkflow(repoOwner, repoName, *job, jobSpec, commentId) dr.Status = models.RunApplying err = models.DB.UpdateDiggerRun(&dr) diff --git a/backend/tasks/tasks.go b/backend/tasks/tasks.go index 29498df85..715829286 100644 --- a/backend/tasks/tasks.go +++ b/backend/tasks/tasks.go @@ -1,6 +1,7 @@ package main import ( + "github.com/diggerhq/digger/backend/ci_backends" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/backend/services" "github.com/diggerhq/digger/backend/utils" @@ -61,7 +62,9 @@ func main() { if err != nil { log.Printf("Failed to get github service: %v", err) } - services.ScheduleJob(service.Client, repoOwner, repoName, &batch.ID, &job) + + ciBackend := ci_backends.GithubActionCi{Client: service.Client} + services.ScheduleJob(ciBackend, repoOwner, repoName, &batch.ID, &job) } }) diff --git a/backend/utils/github.go b/backend/utils/github.go index 71ae5c153..a5a7665e9 100644 --- a/backend/utils/github.go +++ b/backend/utils/github.go @@ -3,13 +3,11 @@ package utils import ( "context" "encoding/base64" - "encoding/json" "fmt" "github.com/bradleyfalzon/ghinstallation/v2" "github.com/diggerhq/digger/backend/models" "github.com/diggerhq/digger/libs/orchestrator" github2 "github.com/diggerhq/digger/libs/orchestrator/github" - orchestrator_scheduler "github.com/diggerhq/digger/libs/orchestrator/scheduler" "github.com/go-git/go-git/v5" "github.com/go-git/go-git/v5/plumbing" "github.com/go-git/go-git/v5/plumbing/transport/http" @@ -17,7 +15,6 @@ import ( "log" net "net/http" "os" - "strconv" "strings" "time" ) @@ -220,34 +217,3 @@ func GetWorkflowIdAndUrlFromDiggerJobId(client *github.Client, repoOwner string, } return 0, "#", fmt.Errorf("workflow not found") } - -func TriggerGithubWorkflow(client *github.Client, repoOwner string, repoName string, job models.DiggerJob, jobString string, commentId int64) error { - log.Printf("TriggerGithubWorkflow: repoOwner: %v, repoName: %v, commentId: %v", repoOwner, repoName, commentId) - - var jobSpec orchestrator.JobJson - err := json.Unmarshal([]byte(jobString), &jobSpec) - if err != nil { - log.Printf("could not unmarshal job string: %v", err) - return fmt.Errorf("could not marshal json string: %v", err) - } - - batchIdShort := job.Batch.ID.String()[:8] - diggerCommand := fmt.Sprintf("digger %v", job.Batch.BatchType) - projectName := jobSpec.ProjectName - requestedBy := jobSpec.RequestedBy - prNumber := *jobSpec.PullRequestNumber - inputs := orchestrator_scheduler.WorkflowInput{ - Id: job.DiggerJobID, - JobString: jobString, - CommentId: strconv.FormatInt(commentId, 10), - RunName: fmt.Sprintf("[%v] %v %v By: %v PR: %v", batchIdShort, diggerCommand, projectName, requestedBy, prNumber), - } - - _, err = client.Actions.CreateWorkflowDispatchEventByFileName(context.Background(), repoOwner, repoName, job.WorkflowFile, github.CreateWorkflowDispatchEventRequest{ - Ref: job.Batch.BranchName, - Inputs: inputs.ToMap(), - }) - - return err - -} diff --git a/go.work.sum b/go.work.sum index 53b1f001b..692189c55 100644 --- a/go.work.sum +++ b/go.work.sum @@ -500,7 +500,6 @@ github.com/alecthomas/chroma v0.10.0 h1:7XDcGkCQopCNKjZHfYrNLraA+M7e0fMiJ/Mfikbf github.com/alecthomas/chroma v0.10.0/go.mod h1:jtJATyUxlIORhUOFNA9NZDWGAQ8wpxQQqNSB4rjA/1s= github.com/alecthomas/kingpin/v2 v2.4.0 h1:f48lwail6p8zpO1bC4TxtqACaGqHYA22qkHjHpqDjYY= github.com/alecthomas/kingpin/v2 v2.4.0/go.mod h1:0gyi0zQnjuFk8xrkNKamJoyUo382HRL7ATRpFZCw6tE= -github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= @@ -586,6 +585,7 @@ github.com/cncf/udpa/go v0.0.0-20220112060539-c52dc94e7fbe/go.mod h1:6pvJx4me5XP github.com/cncf/xds/go v0.0.0-20231109132714-523115ebc101/go.mod h1:eXthEFrGJvWHgFFCl3hGmgk+/aYT6PnTQLykKQRLhEs= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa h1:jQCWAUqqlij9Pgj2i/PB79y4KOPYVyFYdROxgaCwdTQ= github.com/cncf/xds/go v0.0.0-20231128003011-0fa0005c9caa/go.mod h1:x/1Gn8zydmfq8dk6e9PdstVsDgu9RuyIIJqAaF//0IM= +github.com/cncf/xds/go v0.0.0-20240318125728-8a4994d93e50/go.mod h1:5e1+Vvlzido69INQaVO6d87Qn543Xr6nooe9Kz7oBFM= github.com/cockroachdb/apd v1.1.0 h1:3LFP3629v+1aKXU5Q37mxmRxX/pIu1nijXydLShEq5I= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0 h1:sDMmm+q/3+BukdIpxwO365v/Rbspp2Nt5XntgQRXq8Q= github.com/codegangsta/inject v0.0.0-20150114235600-33e0aa1cb7c0/go.mod h1:4Zcjuz89kmFXt9morQgcfYZAYZ5n8WHjt81YYWIwtTM= @@ -673,7 +673,6 @@ github.com/google/go-github/v29 v29.0.2 h1:opYN6Wc7DOz7Ku3Oh4l7prmkOMwEcQxpFtxdU github.com/google/go-github/v29 v29.0.2/go.mod h1:CHKiKKPHJ0REzfwc14QMklvtHwCveD0PxlMjLlzAM5E= github.com/google/go-github/v44 v44.1.0 h1:shWPaufgdhr+Ad4eo/pZv9ORTxFpsxPEPEuuXAKIQGA= github.com/google/go-github/v44 v44.1.0/go.mod h1:iWn00mWcP6PRWHhXm0zuFJ8wbEjE5AGO5D5HXYM4zgw= -github.com/google/go-github/v62 v62.0.0 h1:/6mGCaRywZz9MuHyw9gD1CwsbmBX8GWsbFkwMmHdhl4= github.com/google/go-jsonnet v0.18.0 h1:/6pTy6g+Jh1a1I2UMoAODkqELFiVIdOxbNwv0DDzoOg= github.com/google/go-jsonnet v0.18.0/go.mod h1:C3fTzyVJDslXdiTqw/bTFk7vSGyCtH3MGRbDfvEwGd0= github.com/google/go-pkcs11 v0.2.1-0.20230907215043-c6f79328ddf9 h1:OF1IPgv+F4NmqmJ98KTjdN97Vs1JxDPB3vbmYzV2dpk= @@ -1029,6 +1028,7 @@ google.golang.org/genproto/googleapis/api v0.0.0-20240213162025-012b6fc9bca9/go. google.golang.org/genproto/googleapis/api v0.0.0-20240221002015-b0ce06bbee7c/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= google.golang.org/genproto/googleapis/api v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:5iCWqnniDlqZHrd3neWVTOwvh/v6s3232omMecelax8= google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/api v0.0.0-20240318140521-94a12d6c2237/go.mod h1:Z5Iiy3jtmioajWHDGFk7CeugTyHtPvMHA4UTmUkyalE= google.golang.org/genproto/googleapis/api v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:K4kfzHtI0kqWA79gecJarFtDn/Mls+GxQcg3Zox91Ac= google.golang.org/genproto/googleapis/api v0.0.0-20240415180920-8c6c420018be/go.mod h1:dvdCTIoAGbkWbcIKBniID56/7XHTt6WfxXNMxuziJ+w= google.golang.org/genproto/googleapis/api v0.0.0-20240429193739-8cf5692501f6/go.mod h1:10yRODfgim2/T8csjQsMPgZOMvtytXKTDRzH6HRGzRw= @@ -1048,6 +1048,7 @@ google.golang.org/genproto/googleapis/rpc v0.0.0-20240221002015-b0ce06bbee7c/go. google.golang.org/genproto/googleapis/rpc v0.0.0-20240227224415-6ceb2ff114de/go.mod h1:H4O17MA/PE9BsGx3w+a+W2VOLLD1Qf7oJneAoU6WktY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240304161311-37d4d3c04a78/go.mod h1:UCOku4NytXMJuLQE5VuqA5lX3PcHCBo8pxNyvkf4xBs= google.golang.org/genproto/googleapis/rpc v0.0.0-20240314234333-6e1732d8331c/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240325203815-454cdb8f5daa/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/genproto/googleapis/rpc v0.0.0-20240415180920-8c6c420018be/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.61.0/go.mod h1:VUbo7IFqmF1QtCAstipjG0GIoq49KvMe9+h1jFLBNJs=