buildkite ci backend in ee (#1563)

* [EE] supporting buildkite as a ci_backend

backend/bootstrap/main.go

@@ -24,7 +24,7 @@ import (
 // based on https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-version-information-for-go-applications
 var Version = "dev"
 
-func Bootstrap(templates embed.FS) *gin.Engine {
+func Bootstrap(templates embed.FS, githubController controllers.GithubController) *gin.Engine {
 	defer segment.CloseClient()
 	initLogging()
 	cfg := config.DiggerConfig
@@ -79,7 +79,7 @@ func Bootstrap(templates embed.FS) *gin.Engine {
 		r.LoadHTMLGlob("templates/*.tmpl")
 	}
 
-	r.POST("/github-app-webhook", controllers.GithubAppWebHook)
+	r.POST("/github-app-webhook", githubController.GithubAppWebHook)
 	r.POST("/github-app-webhook/aam", controllers.GithubAppWebHookAfterMerge)
 
 	tenantActionsGroup := r.Group("/api/tenants")

backend/ci_backends/interface.go → backend/ci_backends/ci_backends.go

@@ -2,7 +2,6 @@ package ci_backends
 
 import (
 	"github.com/diggerhq/digger/backend/models"
-	"os"
 )
 
 type CiBackend interface {
@@ -11,11 +10,9 @@ type CiBackend interface {
 
 type JenkinsCi struct{}
 
-func GetCiBackend() {
-	ciBackend := os.Getenv("CI_BACKEND")
-	switch ciBackend {
-
-	default:
-	}
-
+type CiBackendOptions struct {
+	GithubInstallationId int64
+	RepoFullName         string
+	RepoOwner            string
+	RepoName             string
 }

backend/ci_backends/provider.go

@@ -0,0 +1,25 @@
+package ci_backends
+
+import (
+	"fmt"
+	"github.com/diggerhq/digger/backend/utils"
+	"log"
+)
+
+type CiBackendProvider interface {
+	GetCiBackend(options CiBackendOptions) (CiBackend, error)
+}
+
+type DefaultBackendProvider struct{}
+
+func (d DefaultBackendProvider) GetCiBackend(options CiBackendOptions) (CiBackend, error) {
+	client, _, err := utils.GetGithubClient(&utils.DiggerGithubRealClientProvider{}, options.GithubInstallationId, options.RepoFullName)
+	if err != nil {
+		log.Printf("GetCiBackend: could not get github client: %v", err)
+		return nil, fmt.Errorf("could not get github client: %v", err)
+	}
+	backend := &GithubActionCi{
+		Client: client,
+	}
+	return backend, nil
+}

backend/controllers/github.go

@@ -36,7 +36,11 @@ import (
 	"golang.org/x/oauth2"
 )
 
-func GithubAppWebHook(c *gin.Context) {
+type GithubController struct {
+	CiBackendProvider ci_backends.CiBackendProvider
+}
+
+func (g GithubController) GithubAppWebHook(c *gin.Context) {
 	c.Header("Content-Type", "application/json")
 	gh := &utils.DiggerGithubRealClientProvider{}
 	log.Printf("GithubAppWebHook")
@@ -96,15 +100,15 @@ func GithubAppWebHook(c *gin.Context) {
 			c.String(http.StatusOK, "OK")
 			return
 		}
-		err := handleIssueCommentEvent(gh, event)
+		err := handleIssueCommentEvent(gh, event, g.CiBackendProvider)
 		if err != nil {
 			log.Printf("handleIssueCommentEvent error: %v", err)
 			c.String(http.StatusInternalServerError, err.Error())
 			return
 		}
 	case *github.PullRequestEvent:
 		log.Printf("Got pull request event for %d", *event.PullRequest.ID)
-		err := handlePullRequestEvent(gh, event)
+		err := handlePullRequestEvent(gh, event, g.CiBackendProvider)
 		if err != nil {
 			log.Printf("handlePullRequestEvent error: %v", err)
 			c.String(http.StatusInternalServerError, err.Error())
@@ -422,14 +426,16 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) e
 	return nil
 }
 
-func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent) error {
+func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullRequestEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
 	installationId := *payload.Installation.ID
 	repoName := *payload.Repo.Name
 	repoOwner := *payload.Repo.Owner.Login
 	repoFullName := *payload.Repo.FullName
 	cloneURL := *payload.Repo.CloneURL
 	prNumber := *payload.PullRequest.Number
 	isDraft := payload.PullRequest.GetDraft()
+	commitSha := payload.PullRequest.Head.GetSHA()
+	branch := payload.PullRequest.Head.GetRef()
 
 	link, err := models.DB.GetGithubAppInstallationLink(installationId)
 	if err != nil {
@@ -438,7 +444,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	}
 	organisationId := link.OrganisationId
 
-	diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
+	diggerYmlStr, ghService, config, projectsGraph, _, _, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, prNumber)
 	if err != nil {
 		log.Printf("getDiggerConfigForPR error: %v", err)
 		return fmt.Errorf("error getting digger config")
@@ -540,7 +546,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 		impactedJobsMap[j.ProjectName] = j
 	}
 
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, *branch, prNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, organisationId, impactedJobsMap, impactedProjectsMap, projectsGraph, installationId, branch, prNumber, repoOwner, repoName, repoFullName, commitSha, commentReporter.CommentId, diggerYmlStr)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -575,12 +581,26 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 	}
 
 	segment.Track(strconv.Itoa(int(organisationId)), "backend_trigger_job")
-	ciBackend := ci_backends.GithubActionCi{Client: ghService.Client}
+
+	ciBackend, err := ciBackendProvider.GetCiBackend(
+		ci_backends.CiBackendOptions{
+			GithubInstallationId: installationId,
+			RepoName:             repoName,
+			RepoOwner:            repoOwner,
+			RepoFullName:         repoFullName,
+		},
+	)
+	if err != nil {
+		log.Printf("GetCiBackend error: %v", err)
+		utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: GetCiBackend error: %v", err))
+		return fmt.Errorf("error fetching ci backed %v", err)
+	}
+
 	err = TriggerDiggerJobs(ciBackend, repoOwner, repoName, batchId, prNumber, ghService)
 	if err != nil {
 		log.Printf("TriggerDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, prNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err))
-		return fmt.Errorf("error triggerring GitHub Actions for Digger Jobs")
+		return fmt.Errorf("error triggerring Digger Jobs")
 	}
 
 	return nil
@@ -615,28 +635,28 @@ func getDiggerConfigForBranch(gh utils.GithubClientProvider, installationId int6
 	return diggerYmlStr, ghService, config, dependencyGraph, nil
 }
 
-func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, error) {
+func getDiggerConfigForPR(gh utils.GithubClientProvider, installationId int64, repoFullName string, repoOwner string, repoName string, cloneUrl string, prNumber int) (string, *dg_github.GithubService, *dg_configuration.DiggerConfig, graph.Graph[string, dg_configuration.Project], *string, *string, error) {
 	ghService, _, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName)
 	if err != nil {
 		log.Printf("Error getting github service: %v", err)
-		return "", nil, nil, nil, nil, fmt.Errorf("error getting github service")
+		return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting github service")
 	}
 
 	var prBranch string
-	prBranch, err = ghService.GetBranchName(prNumber)
+	prBranch, prCommitSha, err := ghService.GetBranchName(prNumber)
 	if err != nil {
 		log.Printf("Error getting branch name: %v", err)
-		return "", nil, nil, nil, nil, fmt.Errorf("error getting branch name")
+		return "", nil, nil, nil, nil, nil, fmt.Errorf("error getting branch name")
 	}
 
 	diggerYmlStr, ghService, config, dependencyGraph, err := getDiggerConfigForBranch(gh, installationId, repoFullName, repoOwner, repoName, cloneUrl, prBranch)
 	if err != nil {
 		log.Printf("Error loading digger.yml: %v", err)
-		return "", nil, nil, nil, nil, fmt.Errorf("error loading digger.yml")
+		return "", nil, nil, nil, nil, nil, fmt.Errorf("error loading digger.yml")
 	}
 
 	log.Printf("Digger config loadded successfully\n")
-	return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, nil
+	return diggerYmlStr, ghService, config, dependencyGraph, &prBranch, &prCommitSha, nil
 }
 
 func GetRepoByInstllationId(installationId int64, repoOwner string, repoName string) (*models.Repo, error) {
@@ -667,7 +687,7 @@ func getBatchType(jobs []orchestrator.Job) orchestrator_scheduler.DiggerBatchTyp
 	}
 }
 
-func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent) error {
+func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.IssueCommentEvent, ciBackendProvider ci_backends.CiBackendProvider) error {
 	installationId := *payload.Installation.ID
 	repoName := *payload.Repo.Name
 	repoOwner := *payload.Repo.Owner.Login
@@ -694,7 +714,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		return nil
 	}
 
-	diggerYmlStr, ghService, config, projectsGraph, branch, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
+	diggerYmlStr, ghService, config, projectsGraph, branch, commitSha, err := getDiggerConfigForPR(gh, installationId, repoFullName, repoOwner, repoName, cloneURL, issueNumber)
 	if err != nil {
 		log.Printf("getDiggerConfigForPR error: %v", err)
 		return fmt.Errorf("error getting digger config")
@@ -723,7 +743,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		return fmt.Errorf("unkown digger command in comment %v", err)
 	}
 
-	prBranchName, err := ghService.GetBranchName(issueNumber)
+	prBranchName, _, err := ghService.GetBranchName(issueNumber)
 	if err != nil {
 		log.Printf("GetBranchName error: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: GetBranchName error: %v", err))
@@ -803,7 +823,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 		impactedProjectsJobMap[j.ProjectName] = j
 	}
 
-	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, commentReporter.CommentId, diggerYmlStr)
+	batchId, _, err := utils.ConvertJobsToDiggerJobs(*diggerCommand, orgId, impactedProjectsJobMap, impactedProjectsMap, projectsGraph, installationId, *branch, issueNumber, repoOwner, repoName, repoFullName, *commitSha, commentReporter.CommentId, diggerYmlStr)
 	if err != nil {
 		log.Printf("ConvertJobsToDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: ConvertJobsToDiggerJobs error: %v", err))
@@ -842,12 +862,24 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 
 	segment.Track(strconv.Itoa(int(orgId)), "backend_trigger_job")
 
-	ciBackend := ci_backends.GithubActionCi{Client: ghService.Client}
+	ciBackend, err := ciBackendProvider.GetCiBackend(
+		ci_backends.CiBackendOptions{
+			GithubInstallationId: installationId,
+			RepoName:             repoName,
+			RepoOwner:            repoOwner,
+			RepoFullName:         repoFullName,
+		},
+	)
+	if err != nil {
+		log.Printf("GetCiBackend error: %v", err)
+		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: GetCiBackend error: %v", err))
+		return fmt.Errorf("error fetching ci backed %v", err)
+	}
 	err = TriggerDiggerJobs(ciBackend, repoOwner, repoName, batchId, issueNumber, ghService)
 	if err != nil {
 		log.Printf("TriggerDiggerJobs error: %v", err)
 		utils.InitCommentReporter(ghService, issueNumber, fmt.Sprintf(":x: TriggerDiggerJobs error: %v", err))
-		return fmt.Errorf("error triggerring GitHub Actions for Digger Jobs")
+		return fmt.Errorf("error triggerring Digger Jobs")
 	}
 	return nil
 }

backend/controllers/github_after_merge.go

@@ -87,7 +87,7 @@ func GithubAppWebHookAfterMerge(c *gin.Context) {
 	//	}
 	case *github.PullRequestEvent:
 		log.Printf("Got pull request event for %d  IN APPLY AFTER MERGE", *event.PullRequest.ID)
-		err := handlePullRequestEvent(gh, event)
+		err := handlePullRequestEvent(gh, event, nil)
 		if err != nil {
 			log.Printf("handlePullRequestEvent error: %v", err)
 			c.String(http.StatusInternalServerError, err.Error())
@@ -216,7 +216,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 				return fmt.Errorf("error creating job token")
 			}
 
-			planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, orchestrator.DiggerCommandPlan, orgName, planJobToken.Value, backendHostName, impactedProjects[i]))
+			planJobSpec, err := json.Marshal(orchestrator.JobToJson(planJob, orchestrator.DiggerCommandPlan, orgName, "", "", planJobToken.Value, backendHostName, impactedProjects[i]))
 			if err != nil {
 				log.Printf("Error creating jobspec: %v %v", projectName, err)
 				return fmt.Errorf("error creating jobspec")
@@ -229,7 +229,7 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith
 				return fmt.Errorf("error creating job token")
 			}
 
-			applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, orchestrator.DiggerCommandApply, orgName, applyJobToken.Value, backendHostName, impactedProjects[i]))
+			applyJobSpec, err := json.Marshal(orchestrator.JobToJson(applyJob, orchestrator.DiggerCommandApply, orgName, "", "", applyJobToken.Value, backendHostName, impactedProjects[i]))
 			if err != nil {
 				log.Printf("Error creating jobs: %v %v", projectName, err)
 				return fmt.Errorf("error creating jobs")

backend/controllers/github_test.go

@@ -706,7 +706,7 @@ func TestGithubHandleIssueCommentEvent(t *testing.T) {
 	var payload github.IssueCommentEvent
 	err := json.Unmarshal([]byte(issueCommentPayload), &payload)
 	assert.NoError(t, err)
-	err = handleIssueCommentEvent(gh, &payload)
+	err = handleIssueCommentEvent(gh, &payload, nil)
 	assert.NoError(t, err)
 
 	jobs, err := models.DB.GetPendingParentDiggerJobs(nil)
@@ -731,7 +731,7 @@ func TestJobsTreeWithOneJobsAndTwoProjects(t *testing.T) {
 	graph, err := configuration.CreateProjectDependencyGraph(projects)
 	assert.NoError(t, err)
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", 123, "test")
+	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 41584295, "", 2, "diggerhq", "parallel_jobs_demo", "diggerhq/parallel_jobs_demo", "", 123, "test")
 	assert.NoError(t, err)
 	assert.Equal(t, 1, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -760,7 +760,7 @@ func TestJobsTreeWithTwoDependantJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test")
+	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test")
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 
@@ -793,7 +793,7 @@ func TestJobsTreeWithTwoIndependentJobs(t *testing.T) {
 	projectMap["dev"] = project1
 	projectMap["prod"] = project2
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test")
+	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test")
 	assert.NoError(t, err)
 	assert.Equal(t, 2, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["dev"].DiggerJobID)
@@ -838,7 +838,7 @@ func TestJobsTreeWithThreeLevels(t *testing.T) {
 	projectMap["555"] = project5
 	projectMap["666"] = project6
 
-	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", 123, "test")
+	_, result, err := utils.ConvertJobsToDiggerJobs("", 1, jobs, projectMap, graph, 123, "", 2, "", "", "test", "", 123, "test")
 	assert.NoError(t, err)
 	assert.Equal(t, 6, len(result))
 	parentLinks, err := models.DB.GetDiggerJobParentLinksChildId(&result["111"].DiggerJobID)

backend/go.mod

@@ -10,6 +10,7 @@ replace github.com/ugorji/go => github.com/ugorji/go v1.2.12
 require (
 	ariga.io/atlas-provider-gorm v0.3.4
 	github.com/bradleyfalzon/ghinstallation/v2 v2.11.0
+	github.com/buildkite/go-buildkite/v3 v3.11.0
 	github.com/dchest/uniuri v1.2.0
 	github.com/diggerhq/digger/libs v0.4.15
 	github.com/dominikbraun/graph v0.23.0
@@ -92,6 +93,7 @@ require (
 	github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 // indirect
 	github.com/bytedance/sonic v1.11.6 // indirect
 	github.com/bytedance/sonic/loader v0.1.1 // indirect
+	github.com/cenkalti/backoff v2.2.1+incompatible // indirect
 	github.com/cenkalti/backoff/v3 v3.2.2 // indirect
 	github.com/cloudflare/circl v1.3.7 // indirect
 	github.com/cloudwego/base64x v0.1.4 // indirect
@@ -117,7 +119,7 @@ require (
 	github.com/go-playground/universal-translator v0.18.1 // indirect
 	github.com/go-playground/validator/v10 v10.20.0 // indirect
 	github.com/go-sql-driver/mysql v1.7.0 // indirect
-	github.com/goccy/go-json v0.10.2 // indirect
+	github.com/goccy/go-json v0.10.3 // indirect
 	github.com/golang-jwt/jwt/v4 v4.5.0 // indirect
 	github.com/golang-jwt/jwt/v5 v5.2.1 // indirect
 	github.com/golang-sql/civil v0.0.0-20220223132316-b832511892a9 // indirect
@@ -128,7 +130,6 @@ require (
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/google/go-github/v35 v35.3.0 // indirect
 	github.com/google/go-github/v59 v59.0.0 // indirect
-	github.com/google/go-github/v60 v60.0.0 // indirect
 	github.com/google/go-github/v62 v62.0.0 // indirect
 	github.com/google/go-querystring v1.1.0 // indirect
 	github.com/google/s2a-go v0.1.7 // indirect