Merge pull request #1582 from diggerhq/feat/github-private-vcs-suppor…

…t-ee

GitHub EE private vcs support

backend/bootstrap/main.go

@@ -24,7 +24,7 @@ import (
 // based on https://www.digitalocean.com/community/tutorials/using-ldflags-to-set-version-information-for-go-applications
 var Version = "dev"
 
-func Bootstrap(templates embed.FS, githubController controllers.GithubController) *gin.Engine {
+func Bootstrap(templates embed.FS, diggerController controllers.DiggerController) *gin.Engine {
 	defer segment.CloseClient()
 	initLogging()
 	cfg := config.DiggerConfig
@@ -79,19 +79,19 @@ func Bootstrap(templates embed.FS, githubController controllers.GithubController
 		r.LoadHTMLGlob("templates/*.tmpl")
 	}
 
-	r.POST("/github-app-webhook", githubController.GithubAppWebHook)
-	r.POST("/github-app-webhook/aam", controllers.GithubAppWebHookAfterMerge)
+	r.POST("/github-app-webhook", diggerController.GithubAppWebHook)
+	r.POST("/github-app-webhook/aam", diggerController.GithubAppWebHookAfterMerge)
 
 	tenantActionsGroup := r.Group("/api/tenants")
 	tenantActionsGroup.Use(middleware.CORSMiddleware())
 	tenantActionsGroup.Any("/associateTenantIdToDiggerOrg", controllers.AssociateTenantIdToDiggerOrg)
 
 	githubGroup := r.Group("/github")
 	githubGroup.Use(middleware.GetWebMiddleware())
-	githubGroup.GET("/callback", controllers.GithubAppCallbackPage)
-	githubGroup.GET("/repos", controllers.GithubReposPage)
+	githubGroup.GET("/callback", diggerController.GithubAppCallbackPage)
+	githubGroup.GET("/repos", diggerController.GithubReposPage)
 	githubGroup.GET("/setup", controllers.GithubAppSetup)
-	githubGroup.GET("/exchange-code", controllers.GithubSetupExchangeCode)
+	githubGroup.GET("/exchange-code", diggerController.GithubSetupExchangeCode)
 
 	authorized := r.Group("/")
 	authorized.Use(middleware.GetApiMiddleware(), middleware.AccessLevel(models.CliJobAccessType, models.AccessPolicyType, models.AdminPolicyType))
@@ -114,7 +114,7 @@ func Bootstrap(templates embed.FS, githubController controllers.GithubController
 	authorized.GET("/repos/:repo/projects/:projectName/runs", controllers.RunHistoryForProject)
 	authorized.POST("/repos/:repo/projects/:projectName/runs", controllers.CreateRunForProject)
 
-	authorized.POST("/repos/:repo/projects/:projectName/jobs/:jobId/set-status", controllers.SetJobStatusForProject)
+	authorized.POST("/repos/:repo/projects/:projectName/jobs/:jobId/set-status", diggerController.SetJobStatusForProject)
 
 	authorized.GET("/repos/:repo/projects", controllers.FindProjectsForRepo)
 	authorized.POST("/repos/:repo/report-projects", controllers.ReportProjectsForRepo)

backend/ci_backends/ci_backends.go

@@ -2,6 +2,7 @@ package ci_backends
 
 import (
 	"github.com/diggerhq/digger/backend/models"
+	"github.com/diggerhq/digger/backend/utils"
 )
 
 type CiBackend interface {
@@ -11,6 +12,7 @@ type CiBackend interface {
 type JenkinsCi struct{}
 
 type CiBackendOptions struct {
+	GithubClientProvider utils.GithubClientProvider
 	GithubInstallationId int64
 	RepoFullName         string
 	RepoOwner            string

backend/ci_backends/provider.go

@@ -13,7 +13,7 @@ type CiBackendProvider interface {
 type DefaultBackendProvider struct{}
 
 func (d DefaultBackendProvider) GetCiBackend(options CiBackendOptions) (CiBackend, error) {
-	client, _, err := utils.GetGithubClient(&utils.DiggerGithubRealClientProvider{}, options.GithubInstallationId, options.RepoFullName)
+	client, _, err := utils.GetGithubClient(options.GithubClientProvider, options.GithubInstallationId, options.RepoFullName)
 	if err != nil {
 		log.Printf("GetCiBackend: could not get github client: %v", err)
 		return nil, fmt.Errorf("could not get github client: %v", err)

backend/controllers/github.go

@@ -36,13 +36,14 @@ import (
 	"golang.org/x/oauth2"
 )
 
-type GithubController struct {
-	CiBackendProvider ci_backends.CiBackendProvider
+type DiggerController struct {
+	CiBackendProvider    ci_backends.CiBackendProvider
+	GithubClientProvider utils.GithubClientProvider
 }
 
-func (g GithubController) GithubAppWebHook(c *gin.Context) {
+func (d DiggerController) GithubAppWebHook(c *gin.Context) {
 	c.Header("Content-Type", "application/json")
-	gh := &utils.DiggerGithubRealClientProvider{}
+	gh := d.GithubClientProvider
 	log.Printf("GithubAppWebHook")
 
 	payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")))
@@ -100,15 +101,15 @@ func (g GithubController) GithubAppWebHook(c *gin.Context) {
 			c.String(http.StatusOK, "OK")
 			return
 		}
-		err := handleIssueCommentEvent(gh, event, g.CiBackendProvider)
+		err := handleIssueCommentEvent(gh, event, d.CiBackendProvider)
 		if err != nil {
 			log.Printf("handleIssueCommentEvent error: %v", err)
 			c.String(http.StatusInternalServerError, err.Error())
 			return
 		}
 	case *github.PullRequestEvent:
 		log.Printf("Got pull request event for %d", *event.PullRequest.ID)
-		err := handlePullRequestEvent(gh, event, g.CiBackendProvider)
+		err := handlePullRequestEvent(gh, event, d.CiBackendProvider)
 		if err != nil {
 			log.Printf("handlePullRequestEvent error: %v", err)
 			c.String(http.StatusInternalServerError, err.Error())
@@ -191,9 +192,10 @@ func GithubAppSetup(c *gin.Context) {
 		},
 	}
 
+	githubHostname := getGithubHostname()
 	url := &url.URL{
 		Scheme: "https",
-		Host:   "github.com",
+		Host:   githubHostname,
 		Path:   "/settings/apps/new",
 	}
 
@@ -212,16 +214,36 @@ func GithubAppSetup(c *gin.Context) {
 	c.HTML(http.StatusOK, "github_setup.tmpl", gin.H{"Target": url.String(), "Manifest": string(jsonManifest)})
 }
 
+func getGithubHostname() string {
+	githubHostname := os.Getenv("DIGGER_GITHUB_HOSTNAME")
+	if githubHostname == "" {
+		githubHostname = "github.com"
+	}
+	return githubHostname
+}
+
 // GithubSetupExchangeCode handles the user coming back from creating their app
 // A code query parameter is exchanged for this app's ID, key, and webhook_secret
 // Implements https://developer.github.com/apps/building-github-apps/creating-github-apps-from-a-manifest/#implementing-the-github-app-manifest-flow
-func GithubSetupExchangeCode(c *gin.Context) {
+func (d DiggerController) GithubSetupExchangeCode(c *gin.Context) {
 	code := c.Query("code")
 	if code == "" {
 		c.Error(fmt.Errorf("Ignoring callback, missing code query parameter"))
 	}
 
-	client := github.NewClient(nil)
+	// TODO: to make tls verification configurable for debug purposes
+	//var transport *http.Transport = nil
+	//_, exists := os.LookupEnv("DIGGER_GITHUB_SKIP_TLS")
+	//if exists {
+	//	transport = &http.Transport{
+	//		TLSClientConfig: &tls.Config{InsecureSkipVerify: true},
+	//	}
+	//}
+
+	client, err := d.GithubClientProvider.NewClient(nil)
+	if err != nil {
+		c.Error(fmt.Errorf("could not create github client: %v", err))
+	}
 	cfg, _, err := client.Apps.CompleteAppManifest(context.Background(), code)
 	if err != nil {
 		c.Error(fmt.Errorf("Failed to exchange code for github app: %s", err))
@@ -584,6 +606,7 @@ func handlePullRequestEvent(gh utils.GithubClientProvider, payload *github.PullR
 
 	ciBackend, err := ciBackendProvider.GetCiBackend(
 		ci_backends.CiBackendOptions{
+			GithubClientProvider: gh,
 			GithubInstallationId: installationId,
 			RepoName:             repoName,
 			RepoOwner:            repoOwner,
@@ -870,6 +893,7 @@ func handleIssueCommentEvent(gh utils.GithubClientProvider, payload *github.Issu
 
 	ciBackend, err := ciBackendProvider.GetCiBackend(
 		ci_backends.CiBackendOptions{
+			GithubClientProvider: gh,
 			GithubInstallationId: installationId,
 			RepoName:             repoName,
 			RepoOwner:            repoOwner,
@@ -1057,7 +1081,7 @@ jobs:
 	return nil
 }
 
-func GithubAppCallbackPage(c *gin.Context) {
+func (d DiggerController) GithubAppCallbackPage(c *gin.Context) {
 	installationId := c.Request.URL.Query()["installation_id"][0]
 	//setupAction := c.Request.URL.Query()["setup_action"][0]
 	code := c.Request.URL.Query()["code"][0]
@@ -1077,7 +1101,7 @@ func GithubAppCallbackPage(c *gin.Context) {
 		return
 	}
 
-	result, err := validateGithubCallback(clientId, clientSecret, code, installationId64)
+	result, err := validateGithubCallback(d.GithubClientProvider, clientId, clientSecret, code, installationId64)
 	if !result {
 		log.Printf("Failed to validated installation id, %v\n", err)
 		c.String(http.StatusInternalServerError, "Failed to validate installation_id.")
@@ -1100,7 +1124,7 @@ func GithubAppCallbackPage(c *gin.Context) {
 	c.HTML(http.StatusOK, "github_success.tmpl", gin.H{})
 }
 
-func GithubReposPage(c *gin.Context) {
+func (d DiggerController) GithubReposPage(c *gin.Context) {
 	orgId, exists := c.Get(middleware.ORGANISATION_ID_KEY)
 	if !exists {
 		log.Printf("Organisation ID not found in context")
@@ -1127,7 +1151,7 @@ func GithubReposPage(c *gin.Context) {
 		return
 	}
 
-	gh := &utils.DiggerGithubRealClientProvider{}
+	gh := d.GithubClientProvider
 	client, _, err := gh.Get(installations[0].GithubAppId, installations[0].GithubInstallationId)
 	if err != nil {
 		log.Printf("failed to create github client, %v", err)
@@ -1147,14 +1171,15 @@ func GithubReposPage(c *gin.Context) {
 
 // why this validation is needed: https://roadie.io/blog/avoid-leaking-github-org-data/
 // validation based on https://docs.github.com/en/apps/creating-github-apps/authenticating-with-a-github-app/generating-a-user-access-token-for-a-github-app , step 3
-func validateGithubCallback(clientId string, clientSecret string, code string, installationId int64) (bool, error) {
+func validateGithubCallback(githubClientProvider utils.GithubClientProvider, clientId string, clientSecret string, code string, installationId int64) (bool, error) {
 	ctx := context.Background()
 	type OAuthAccessResponse struct {
 		AccessToken string `json:"access_token"`
 	}
 	httpClient := http.Client{}
 
-	reqURL := fmt.Sprintf("https://github.com/login/oauth/access_token?client_id=%s&client_secret=%s&code=%s", clientId, clientSecret, code)
+	githubHostname := getGithubHostname()
+	reqURL := fmt.Sprintf("https://%v/login/oauth/access_token?client_id=%s&client_secret=%s&code=%s", githubHostname, clientId, clientSecret, code)
 	req, err := http.NewRequest(http.MethodPost, reqURL, nil)
 	if err != nil {
 		return false, fmt.Errorf("could not create HTTP request: %v\n", err)
@@ -1179,11 +1204,27 @@ func validateGithubCallback(clientId string, clientSecret string, code string, i
 		&oauth2.Token{AccessToken: t.AccessToken},
 	)
 	tc := oauth2.NewClient(ctx, ts)
-	client := github.NewClient(tc)
+	//tc := &http.Client{
+	//	Transport: &oauth2.Transport{
+	//		Base:   httpClient.Transport,
+	//		Source: oauth2.ReuseTokenSource(nil, ts),
+	//	},
+	//}
+
+	client, err := githubClientProvider.NewClient(tc)
+	if err != nil {
+		log.Printf("could create github client: %v", err)
+		return false, fmt.Errorf("could not create github client: %v", err)
+	}
 
 	installationIdMatch := false
 	// list all installations for the user
 	installations, _, err := client.Apps.ListUserInstallations(ctx, nil)
+	if err != nil {
+		log.Printf("could not retrieve installations: %v", err)
+		return false, fmt.Errorf("could not retrieve installations: %v", installationId)
+	}
+	log.Printf("installations %v", installations)
 	for _, v := range installations {
 		log.Printf("installation id: %v\n", *v.ID)
 		if *v.ID == installationId {

backend/controllers/github_after_merge.go

@@ -18,9 +18,9 @@ import (
 	"strings"
 )
 
-func GithubAppWebHookAfterMerge(c *gin.Context) {
+func (d DiggerController) GithubAppWebHookAfterMerge(c *gin.Context) {
 	c.Header("Content-Type", "application/json")
-	gh := &utils.DiggerGithubRealClientProvider{}
+	gh := d.GithubClientProvider
 	log.Printf("GithubAppWebHook")
 
 	payload, err := github.ValidatePayload(c.Request, []byte(os.Getenv("GITHUB_WEBHOOK_SECRET")))

backend/controllers/projects.go

@@ -321,7 +321,7 @@ type SetJobStatusRequest struct {
 	TerraformOutput string                                  `json:"terraform_output""`
 }
 
-func SetJobStatusForProject(c *gin.Context) {
+func (d DiggerController) SetJobStatusForProject(c *gin.Context) {
 	jobId := c.Param("jobId")
 
 	orgId, exists := c.Get(middleware.ORGANISATION_ID_KEY)
@@ -359,7 +359,7 @@ func SetJobStatusForProject(c *gin.Context) {
 			return
 		}
 
-		client, _, err := utils.GetGithubClient(&utils.DiggerGithubRealClientProvider{}, job.Batch.GithubInstallationId, job.Batch.RepoFullName)
+		client, _, err := utils.GetGithubClient(d.GithubClientProvider, job.Batch.GithubInstallationId, job.Batch.RepoFullName)
 		if err != nil {
 			log.Printf("Error Creating github client: %v", err)
 		} else {
@@ -397,7 +397,7 @@ func SetJobStatusForProject(c *gin.Context) {
 					log.Printf("Recovered from panic while executing goroutine dispatching digger jobs: %v ", r)
 				}
 			}()
-			ghClientProvider := &utils.DiggerGithubRealClientProvider{}
+			ghClientProvider := d.GithubClientProvider
 			installationLink, err := models.DB.GetGithubInstallationLinkForOrg(orgId)
 			if err != nil {
 				log.Printf("Error fetching installation link: %v", err)
@@ -477,7 +477,7 @@ func SetJobStatusForProject(c *gin.Context) {
 		return
 	}
 
-	err = AutomergePRforBatchIfEnabled(&utils.DiggerGithubRealClientProvider{}, batch)
+	err = AutomergePRforBatchIfEnabled(d.GithubClientProvider, batch)
 	if err != nil {
 		log.Printf("Error merging PR with automerge option: %v", err)
 		c.JSON(http.StatusInternalServerError, gin.H{"error": "Error merging PR with automerge option"})
@@ -491,7 +491,7 @@ func SetJobStatusForProject(c *gin.Context) {
 
 	}
 
-	UpdateCommentsForBatchGroup(&utils.DiggerGithubRealClientProvider{}, batch, res.Jobs)
+	UpdateCommentsForBatchGroup(d.GithubClientProvider, batch, res.Jobs)
 
 	c.JSON(http.StatusOK, res)
 }

backend/main.go

@@ -7,14 +7,16 @@ import (
 	"github.com/diggerhq/digger/backend/ci_backends"
 	"github.com/diggerhq/digger/backend/config"
 	"github.com/diggerhq/digger/backend/controllers"
+	"github.com/diggerhq/digger/backend/utils"
 )
 
 //go:embed templates
 var templates embed.FS
 
 func main() {
-	ghController := controllers.GithubController{
-		CiBackendProvider: ci_backends.DefaultBackendProvider{},
+	ghController := controllers.DiggerController{
+		CiBackendProvider:    ci_backends.DefaultBackendProvider{},
+		GithubClientProvider: utils.DiggerGithubRealClientProvider{},
 	}
 	r := bootstrap.Bootstrap(templates, ghController)
 	r.GET("/", controllers.Home)

backend/utils/github.go

@@ -71,9 +71,15 @@ type DiggerGithubClientMockProvider struct {
 }
 
 type GithubClientProvider interface {
+	NewClient(netClient *net.Client) (*github.Client, error)
 	Get(githubAppId int64, installationId int64) (*github.Client, *string, error)
 }
 
+func (gh DiggerGithubRealClientProvider) NewClient(netClient *net.Client) (*github.Client, error) {
+	ghClient := github.NewClient(netClient)
+	return ghClient, nil
+}
+
 func (gh DiggerGithubRealClientProvider) Get(githubAppId int64, installationId int64) (*github.Client, *string, error) {
 	githubAppPrivateKey := ""
 	githubAppPrivateKeyB64 := os.Getenv("GITHUB_APP_PRIVATE_KEY_BASE64")
@@ -103,12 +109,20 @@ func (gh DiggerGithubRealClientProvider) Get(githubAppId int64, installationId i
 	if err != nil {
 		return nil, nil, fmt.Errorf("error initialising git app token: %v\n", err)
 	}
-	ghClient := github.NewClient(&net.Client{Transport: itr})
+	ghClient, err := gh.NewClient(&net.Client{Transport: itr})
+	if err != nil {
+		log.Printf("error creating new client: %v", err)
+	}
 	return ghClient, &token, nil
 }
 
-func (gh *DiggerGithubClientMockProvider) Get(githubAppId int64, installationId int64) (*github.Client, *string, error) {
+func (gh DiggerGithubClientMockProvider) NewClient(netClient *net.Client) (*github.Client, error) {
 	ghClient := github.NewClient(gh.MockedHTTPClient)
+	return ghClient, nil
+}
+
+func (gh *DiggerGithubClientMockProvider) Get(githubAppId int64, installationId int64) (*github.Client, *string, error) {
+	ghClient, _ := gh.NewClient(gh.MockedHTTPClient)
 	token := "token"
 	return ghClient, &token, nil
 }

ee/backend/ci_backends/provider.go

@@ -15,7 +15,7 @@ func (b EEBackendProvider) GetCiBackend(options ci_backends.CiBackendOptions) (c
 	ciBackendType := os.Getenv("CI_BACKEND")
 	switch ciBackendType {
 	case "github_actions", "":
-		client, _, err := utils.GetGithubClient(&utils.DiggerGithubRealClientProvider{}, options.GithubInstallationId, options.RepoFullName)
+		client, _, err := utils.GetGithubClient(options.GithubClientProvider, options.GithubInstallationId, options.RepoFullName)
 		if err != nil {
 			log.Printf("GetCiBackend: could not get github client: %v", err)
 			return nil, fmt.Errorf("could not get github client: %v", err)

ee/backend/main.go

@@ -9,6 +9,7 @@ import (
 	"github.com/diggerhq/digger/backend/middleware"
 	ci_backends2 "github.com/diggerhq/digger/ee/backend/ci_backends"
 	"github.com/diggerhq/digger/ee/backend/controllers"
+	"github.com/diggerhq/digger/ee/backend/providers/github"
 	"github.com/diggerhq/digger/libs/license"
 	"github.com/gin-gonic/gin"
 	"log"
@@ -28,8 +29,9 @@ func main() {
 		log.Printf("error checking license %v", err)
 		os.Exit(1)
 	}
-	ghController := ce_controllers.GithubController{
-		CiBackendProvider: ci_backends2.EEBackendProvider{},
+	ghController := ce_controllers.DiggerController{
+		CiBackendProvider:    ci_backends2.EEBackendProvider{},
+		GithubClientProvider: github.DiggerGithubEEClientProvider{},
 	}
 
 	r := bootstrap.Bootstrap(templates, ghController)