diff --git a/backend/controllers/github.go b/backend/controllers/github.go index 187143ad2..8b8f9c32f 100644 --- a/backend/controllers/github.go +++ b/backend/controllers/github.go @@ -410,39 +410,48 @@ func handlePushEvent(gh utils.GithubClientProvider, payload *github.PushEvent) e ref := *payload.Ref defaultBranch := *payload.Repo.DefaultBranch - if strings.HasSuffix(ref, defaultBranch) { - link, err := models.DB.GetGithubAppInstallationLink(installationId) - if err != nil { - log.Printf("Error getting GetGithubAppInstallationLink: %v", err) - return fmt.Errorf("error getting github app link") - } + link, err := models.DB.GetGithubAppInstallationLink(installationId) + if err != nil { + log.Printf("Error getting GetGithubAppInstallationLink: %v", err) + return fmt.Errorf("error getting github app link") + } - orgId := link.OrganisationId - diggerRepoName := strings.ReplaceAll(repoFullName, "/", "-") - repo, err := models.DB.GetRepo(orgId, diggerRepoName) - if err != nil { - log.Printf("Error getting Repo: %v", err) - return fmt.Errorf("error getting github app link") - } - if repo == nil { - log.Printf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) - return fmt.Errorf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) - } + orgId := link.OrganisationId + diggerRepoName := strings.ReplaceAll(repoFullName, "/", "-") + repo, err := models.DB.GetRepo(orgId, diggerRepoName) + if err != nil { + log.Printf("Error getting Repo: %v", err) + return fmt.Errorf("error getting github app link") + } + if repo == nil { + log.Printf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) + return fmt.Errorf("Repo not found: Org: %v | repo: %v", orgId, diggerRepoName) + } + + _, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) + if err != nil { + log.Printf("Error getting github service: %v", err) + return fmt.Errorf("error getting github service") + } - _, token, err := utils.GetGithubService(gh, installationId, repoFullName, repoOwner, repoName) + var isMainBranch bool + if strings.HasSuffix(ref, defaultBranch) { + isMainBranch = true + } else { + isMainBranch = false + } + + err = utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error { + config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) if err != nil { - log.Printf("Error getting github service: %v", err) - return fmt.Errorf("error getting github service") + log.Printf("ERROR load digger.yml: %v", err) + return fmt.Errorf("error loading digger.yml %v", err) } - utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error { - dat, err := os.ReadFile(path.Join(dir, "digger.yml")) - //TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that - if err != nil { - log.Printf("ERROR fetching digger.yml file: %v", err) - } - models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo) - return nil - }) + models.DB.UpdateRepoDiggerConfig(link.OrganisationId, *config, repo, isMainBranch) + return nil + }) + if err != nil { + return fmt.Errorf("error while cloning repo: %v", err) } return nil diff --git a/backend/controllers/github_after_merge.go b/backend/controllers/github_after_merge.go index 701c62370..854e537d0 100644 --- a/backend/controllers/github_after_merge.go +++ b/backend/controllers/github_after_merge.go @@ -13,7 +13,6 @@ import ( "log" "net/http" "os" - "path" "reflect" "strings" ) @@ -146,13 +145,13 @@ func handlePushEventApplyAfterMerge(gh utils.GithubClientProvider, payload *gith log.Printf("Error getting github service: %v", err) return fmt.Errorf("error getting github service") } - utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error { - dat, err := os.ReadFile(path.Join(dir, "digger.yml")) - //TODO: fail here and return failure to main fn (need to refactor CloneGitRepoAndDoAction for that + err = utils.CloneGitRepoAndDoAction(cloneURL, defaultBranch, *token, func(dir string) error { + config, err := dg_configuration.LoadDiggerConfigYaml(dir, true, nil) if err != nil { - log.Printf("ERROR fetching digger.yml file: %v", err) + log.Printf("ERROR load digger.yml: %v", err) + return fmt.Errorf("error loading digger.yml %v", err) } - models.DB.UpdateRepoDiggerConfig(link.OrganisationId, string(dat), repo) + models.DB.UpdateRepoDiggerConfig(link.OrganisationId, *config, repo, true) return nil }) diff --git a/backend/controllers/github_test.go b/backend/controllers/github_test.go index d257e8fe2..e34e576d7 100644 --- a/backend/controllers/github_test.go +++ b/backend/controllers/github_test.go @@ -616,7 +616,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) { // create test project projectName := "test project" - _, err = database.CreateProject(projectName, org, repo) + _, err = database.CreateProject(projectName, org, repo, false, false) if err != nil { log.Fatal(err) } diff --git a/backend/controllers/projects.go b/backend/controllers/projects.go index ed2130dc4..7000b606d 100644 --- a/backend/controllers/projects.go +++ b/backend/controllers/projects.go @@ -97,7 +97,11 @@ func FindProjectsForOrg(c *gin.Context) { err = models.DB.GormDB.Preload("Organisation").Preload("Repo"). Joins("LEFT JOIN repos ON projects.repo_id = repos.id"). Joins("LEFT JOIN organisations ON projects.organisation_id = organisations.id"). - Where("projects.organisation_id = ?", org.ID).Find(&projects).Error + Where("projects.organisation_id = ?", org.ID). + Order("is_in_main_branch"). + Order("repos.repo_full_name"). + Order("name"). + Find(&projects).Error if err != nil { c.String(http.StatusInternalServerError, "Unknown error occurred while fetching database") diff --git a/backend/go.sum b/backend/go.sum index e38217f52..035674271 100644 --- a/backend/go.sum +++ b/backend/go.sum @@ -35,6 +35,7 @@ cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+ cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= cloud.google.com/go v0.112.2 h1:ZaGT6LiG7dBzi6zNOvVZwacaXlmf3lRqnC4DQzqyRQw= +cloud.google.com/go v0.112.2/go.mod h1:iEqjp//KquGIJV/m+Pk3xecgKNhV+ry+vVTsy4TbDms= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -50,7 +51,9 @@ cloud.google.com/go/assuredworkloads v1.5.0/go.mod h1:n8HOZ6pff6re5KYfBXcFvSViQj cloud.google.com/go/assuredworkloads v1.6.0/go.mod h1:yo2YOk37Yc89Rsd5QMVECvjaMKymF9OP+QXWlKXUkXw= cloud.google.com/go/assuredworkloads v1.7.0/go.mod h1:z/736/oNmtGAyU47reJgGN+KVoYoxeLBoj4XkKYscNI= cloud.google.com/go/auth v0.3.0 h1:PRyzEpGfx/Z9e8+lHsbkoUVXD0gnu4MNmm7Gp8TQNIs= +cloud.google.com/go/auth v0.3.0/go.mod h1:lBv6NKTWp8E3LPzmO1TbiiRKc4drLOfHsgmlH9ogv5w= cloud.google.com/go/auth/oauth2adapt v0.2.2 h1:+TTV8aXpjeChS9M+aTtN/TjdQnzJvmzKFt//oWu7HX4= +cloud.google.com/go/auth/oauth2adapt v0.2.2/go.mod h1:wcYjgpZI9+Yu7LyYBg4pqSiaRkfEK3GQcpb7C/uyF1Q= cloud.google.com/go/automl v1.5.0/go.mod h1:34EjfoFGMZ5sgJ9EoLsRtdPSNZLcfflJR39VbVNS2M0= cloud.google.com/go/automl v1.6.0/go.mod h1:ugf8a6Fx+zP0D59WLhqgTDsQI9w07o64uf/Is3Nh5p8= cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o= @@ -113,6 +116,7 @@ cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= cloud.google.com/go/iam v1.1.8 h1:r7umDwhj+BQyz0ScZMp4QrGXjSTI3ZINnpgU2nlB/K0= +cloud.google.com/go/iam v1.1.8/go.mod h1:GvE6lyMmfxXauzNq8NbgJbeVQNspG+tcdL/W8QO1+zE= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -174,6 +178,7 @@ cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= cloud.google.com/go/storage v1.41.0 h1:RusiwatSu6lHeEXe3kglxakAmAbfV+rhtPqA6i8RBx0= +cloud.google.com/go/storage v1.41.0/go.mod h1:J1WCa/Z2FcgdEDuPUY8DxT5I+d9mFKsCepp5vR6Sq80= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -288,6 +293,8 @@ github.com/agext/levenshtein v1.2.1/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki github.com/agext/levenshtein v1.2.2/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= github.com/agext/levenshtein v1.2.3 h1:YB2fHEn0UJagG8T1rrWknE3ZQzWM06O8AMAatNn7lmo= github.com/agext/levenshtein v1.2.3/go.mod h1:JEDfjyjHDjOF/1e4FlBE/PkbqA9OfWu2ki2W0IB5558= +github.com/alecthomas/kong v0.7.1 h1:azoTh0IOfwlAX3qN9sHWTxACE2oV8Bg2gAwBsMwDQY4= +github.com/alecthomas/kong v0.7.1/go.mod h1:n1iCIO2xS46oE8ZfYCNDqdR0b0wZNrXAIAqro/2132U= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -336,7 +343,9 @@ github.com/aws/aws-sdk-go-v2/config v1.27.16/go.mod h1:vutqgRhDUktwSge3hrC3nkuir github.com/aws/aws-sdk-go-v2/credentials v1.17.16 h1:7d2QxY83uYl0l58ceyiSpxg9bSbStqBC6BeEeHEchwo= github.com/aws/aws-sdk-go-v2/credentials v1.17.16/go.mod h1:Ae6li/6Yc6eMzysRL2BXlPYvnrLLBg3D11/AmOjw50k= github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.20 h1:Tb9z3/GkyjD16ngZBZjOAsOXvKSkBKahQm37SCxOXhY= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/attributevalue v1.13.20/go.mod h1:43wfYl5jBLYjUoZcmW4OzbXKe38VvaMYNXp2+oIwREg= github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.20 h1:AvAKoZa3S2K/Z/H1wC3Qjfuk8r0wYybPppahzoDfD4s= +github.com/aws/aws-sdk-go-v2/feature/dynamodb/expression v1.7.20/go.mod h1:Yx3vUgyvGqLzyKvViwxwOHHnO5/8r0UYjgIHToEY+Ys= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3 h1:dQLK4TjtnlRGb0czOht2CevZ5l6RSyRWAnKeGd7VAFE= github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.16.3/go.mod h1:TL79f2P6+8Q7dTsILpiVST+AL9lkF6PPGI167Ny0Cjw= github.com/aws/aws-sdk-go-v2/internal/configsources v1.3.7 h1:lf/8VTF2cM+N4SLzaYJERKEWAXq8MOMpZfU6wEPWsPk= @@ -346,10 +355,13 @@ github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.6.7/go.mod h1:vd7ESTEvI76T github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0 h1:hT8rVHwugYE2lEfdFE0QWVo81lF7jMrYJVDWI+f+VxU= github.com/aws/aws-sdk-go-v2/internal/ini v1.8.0/go.mod h1:8tu/lYfQfFe6IGnaOdrpVgEL2IrrDOf6/m9RQum4NkY= github.com/aws/aws-sdk-go-v2/service/dynamodb v1.32.6 h1:170E8A7abwLNy8wF53Wu496IaIlQ+DYQLgCbTqhYf/M= +github.com/aws/aws-sdk-go-v2/service/dynamodb v1.32.6/go.mod h1:uNhUf9Z3MT6Ex+u0ADa8r3MKK5zjuActEfXQPo4YqEI= github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.8 h1:PapW7iWHqua6Gk+qRjgXpM3fNqUxY3N+1WURHPcmKhc= +github.com/aws/aws-sdk-go-v2/service/dynamodbstreams v1.20.8/go.mod h1:IL6qnQxrc/qIjwzeg7USP3P7ySEehOPpXJslRbXNYJ4= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2 h1:Ji0DY1xUsUr3I8cHps0G+XM3WWU16lP6yG8qu1GAZAs= github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.11.2/go.mod h1:5CsjAbs3NlGQyZNFACh+zztPDI7fU6eW9QsxjfnuBKg= github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.8 h1:yEeIld7Fh/2iM4pYeQw8a3kH6OYcyIn6lwKlUFiVk7Y= +github.com/aws/aws-sdk-go-v2/service/internal/endpoint-discovery v1.9.8/go.mod h1:lZJMX2Z5/rQ6OlSbBnW1WWScK6ngLt43xtqM8voMm2w= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9 h1:Wx0rlZoEJR7JwlSZcHnEa7CNjrSIyVxMFWGAaXy4fJY= github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.11.9/go.mod h1:aVMHdE0aHO3v+f/iw01fmXV/5DbfQ3Bi9nN7nd9bE9Y= github.com/aws/aws-sdk-go-v2/service/sso v1.20.9 h1:aD7AGQhvPuAxlSUfo0CWU7s6FpkbyykMhGYMvlqTjVs= @@ -534,6 +546,7 @@ github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG github.com/go-logr/logr v0.1.0/go.mod h1:ixOQHD9gLJUVQQ2ZOR7zLEifBX6tGkNJF4QyIY7sIas= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.4.2 h1:6pFjapn8bFcIbiKo3XT4j/BhANplGihG6tvd+8rYgrY= +github.com/go-logr/logr v1.4.2/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-openapi/jsonpointer v0.0.0-20160704185906-46af16f9f7b1/go.mod h1:+35s3my2LFTysnkMfxsJBAMHj/DoqoB9knIWoYG/Vk0= @@ -700,6 +713,7 @@ github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= github.com/googleapis/gax-go/v2 v2.12.4 h1:9gWcmF85Wvq4ryPFvGFaOgPIs1AQX0d0bcbGw4Z96qg= +github.com/googleapis/gax-go/v2 v2.12.4/go.mod h1:KYEYLorsnIGDi/rPC8b5TdlB9kbKoFubselGIoBMCwI= github.com/googleapis/gnostic v0.0.0-20170729233727-0c5108395e2d/go.mod h1:sJBsCZ4ayReDTBIg8b9dl28c5xFWyhBTVRp3pOg5EKY= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/gophercloud/gophercloud v0.6.1-0.20191122030953-d8ac278c1c9d/go.mod h1:ozGNgr9KYOVATV5jsgHl/ceCDXGuguqOZAzoQ/2vcNM= @@ -1088,6 +1102,7 @@ github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3ev github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0 h1:8SG7/vwALn54lVB/0yZ/MMwhFrPYtpEHQb2IpWsCzug= +github.com/opencontainers/image-spec v1.1.0/go.mod h1:W4s4sFTMaBeK1BQLXbG4AdM2szdn85PY75RI83NrTrM= github.com/opencontainers/runc v1.1.9 h1:XR0VIHTGce5eWPkaPesqTBrhW2yAcaraWfsEalNwQLM= github.com/opencontainers/runc v1.1.9/go.mod h1:CbUumNnWCuTGFukNXahoo/RFBZvDAgRh/smNYNOhA50= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= @@ -1147,6 +1162,7 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8= +github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -1379,6 +1395,7 @@ golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc h1:O9NuF4s+E/PvMIy+9IUZB9znFwUIXEWSstNjek6VpVg= +golang.org/x/exp v0.0.0-20240531132922-fd00a4e0eefc/go.mod h1:XtvwrStGgqGPLc4cjQfWqZHG1YFdYs6swckp8vpsjnc= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1778,6 +1795,7 @@ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= google.golang.org/api v0.178.0 h1:yoW/QMI4bRVCHF+NWOTa4cL8MoWL3Jnuc7FlcFF91Ok= +google.golang.org/api v0.178.0/go.mod h1:84/k2v8DFpDRebpGcooklv/lais3MEfqpaBLA12gl2U= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -1888,8 +1906,11 @@ google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda h1:wu/KJm9KJwpfHWhkkZGohVC6KRrc1oJNr4jwtQMOQXw= +google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda/go.mod h1:g2LLCvCeCSir/JJSWosk19BR4NVxGqHUC6rxIRsd7Aw= google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae h1:AH34z6WAGVNkllnKs5raNq3yRq93VnjBG6rpfub/jYk= +google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae/go.mod h1:FfiGhwUm6CJviekPrc0oJ+7h29e+DmWU6UtjX0ZvI7Y= google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6 h1:DujSIu+2tC9Ht0aPNA7jgj23Iq8Ewi5sgkQ++wdvonE= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240429193739-8cf5692501f6/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.8.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= @@ -1928,6 +1949,7 @@ google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCD google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.64.0 h1:KH3VH9y/MgNQg1dE7b3XfVK0GsPSIzJwdF617gUSbvY= +google.golang.org/grpc v1.64.0/go.mod h1:oxjF8E3FBnjp+/gVFYdWacaLDx9na1aqy9oovLpxQYg= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/backend/migrations/20240703121051.sql b/backend/migrations/20240703121051.sql new file mode 100644 index 000000000..29ce30163 --- /dev/null +++ b/backend/migrations/20240703121051.sql @@ -0,0 +1,2 @@ +-- Modify "projects" table +ALTER TABLE "public"."projects" ADD COLUMN "is_generated" boolean NULL, ADD COLUMN "is_in_main_branch" boolean NULL; diff --git a/backend/migrations/atlas.sum b/backend/migrations/atlas.sum index cca1b984a..f80612fef 100644 --- a/backend/migrations/atlas.sum +++ b/backend/migrations/atlas.sum @@ -1,4 +1,4 @@ -h1:3BCSPujef/yl7UJpcpT3jUVmRffXWWbUrXYr1AiR+HY= +h1:yBMerqbjtxt3ZYodZ92oPL+leVDGGb20eiC4jMN0JwU= 20231227132525.sql h1:43xn7XC0GoJsCnXIMczGXWis9d504FAWi4F1gViTIcw= 20240115170600.sql h1:IW8fF/8vc40+eWqP/xDK+R4K9jHJ9QBSGO6rN9LtfSA= 20240116123649.sql h1:R1JlUIgxxF6Cyob9HdtMqiKmx/BfnsctTl5rvOqssQw= @@ -23,3 +23,4 @@ h1:3BCSPujef/yl7UJpcpT3jUVmRffXWWbUrXYr1AiR+HY= 20240524110010.sql h1:tJ4SceBrjNekJtKXzY6IDHM6HZhTLYY0SHWci2znAfE= 20240527112209.sql h1:vuz1G8P1uoo4xYddKnT8tzTmtYcq9ThT4xLERnutERo= 20240530074832.sql h1:uyXvPgFxTfO2QAW2bhXSxJJQLbpr2zCfrlg1ycD8BSU= +20240703121051.sql h1:ntF++QXJ3LYQP3ZFvumMvg28FFr8vAnDReFgvgzhOcg= diff --git a/backend/models/orgs.go b/backend/models/orgs.go index 78f1300ef..7d081ecf6 100644 --- a/backend/models/orgs.go +++ b/backend/models/orgs.go @@ -73,10 +73,12 @@ type Project struct { Repo *Repo ConfigurationYaml string // TODO: probably needs to be deleted Status ProjectStatus + IsGenerated bool + IsInMainBranch bool } func (p *Project) MapToJsonStruct() interface{} { - lastRun, _ := DB.GetLastDiggerRunForProject(p.ID) + lastRun, _ := DB.GetLastDiggerRunForProject(p.Name) status := RunSucceeded if lastRun != nil { status = lastRun.Status @@ -92,6 +94,8 @@ func (p *Project) MapToJsonStruct() interface{} { RepoName string `json:"repo_name"` RepoOrg string `json:"repo_org"` RepoUrl string `json:"repo_url"` + IsInMainBranch bool `json:"is_in_main_branch"` + IsGenerated bool `json:"is_generated"` LastActivityTimestamp string `json:"last_activity_timestamp"` LastActivityAuthor string `json:"last_activity_author"` LastActivityStatus string `json:"last_activity_status"` @@ -108,6 +112,8 @@ func (p *Project) MapToJsonStruct() interface{} { LastActivityTimestamp: p.UpdatedAt.String(), LastActivityAuthor: "unknown", LastActivityStatus: string(status), + IsGenerated: p.IsGenerated, + IsInMainBranch: p.IsInMainBranch, } } diff --git a/backend/models/scheduler_test.go b/backend/models/scheduler_test.go index 0eb7770d2..84ebbe5f9 100644 --- a/backend/models/scheduler_test.go +++ b/backend/models/scheduler_test.go @@ -61,7 +61,7 @@ func setupSuiteScheduler(tb testing.TB) (func(tb testing.TB), *Database) { } projectName := "test project" - _, err = database.CreateProject(projectName, org, repo) + _, err = database.CreateProject(projectName, org, repo, false, false) if err != nil { log.Fatal(err) } diff --git a/backend/models/storage.go b/backend/models/storage.go index a1d935b4d..e6d164cab 100644 --- a/backend/models/storage.go +++ b/backend/models/storage.go @@ -724,9 +724,9 @@ func (db *Database) CreateDiggerRunStage(batchId string) (*DiggerRunStage, error return drs, nil } -func (db *Database) GetLastDiggerRunForProject(projectId uint) (*DiggerRun, error) { +func (db *Database) GetLastDiggerRunForProject(projectName string) (*DiggerRun, error) { diggerRun := &DiggerRun{} - result := db.GormDB.Where("project_id = ? AND status <> ?", projectId, RunQueued).Order("created_at Desc").First(diggerRun) + result := db.GormDB.Where("project_name = ? AND status <> ?", projectName, RunQueued).Order("created_at Desc").First(diggerRun) if result.Error != nil { log.Printf("error while fetching last digger run: %v", result.Error) return nil, result.Error @@ -1013,8 +1013,15 @@ func (db *Database) CreateOrganisation(name string, externalSource string, tenan return org, nil } -func (db *Database) CreateProject(name string, org *Organisation, repo *Repo) (*Project, error) { - project := &Project{Name: name, Organisation: org, Repo: repo, Status: ProjectActive} +func (db *Database) CreateProject(name string, org *Organisation, repo *Repo, isGenerated bool, isInMainBranch bool) (*Project, error) { + project := &Project{ + Name: name, + Organisation: org, + Repo: repo, + Status: ProjectActive, + IsGenerated: isGenerated, + IsInMainBranch: isInMainBranch, + } result := db.GormDB.Save(project) if result.Error != nil { log.Printf("Failed to create project: %v, error: %v\n", name, result.Error) @@ -1024,6 +1031,15 @@ func (db *Database) CreateProject(name string, org *Organisation, repo *Repo) (* return project, nil } +func (db *Database) UpdateProject(project *Project) error { + result := db.GormDB.Save(project) + if result.Error != nil { + return result.Error + } + log.Printf("project %v has been updated successfully\n", project.ID) + return nil +} + func (db *Database) CreateRepo(name string, repoFullName string, repoOrganisation string, repoName string, repoUrl string, org *Organisation, diggerConfig string) (*Repo, error) { var repo Repo // check if repo exist already, do nothing in this case @@ -1126,69 +1142,55 @@ func validateDiggerConfigYaml(configYaml string) (*configuration.DiggerConfig, e return diggerConfig, nil } -func (db *Database) UpdateRepoDiggerConfig(orgId any, diggerConfigYaml string, repo *Repo) ([]string, error) { - messages := make([]string, 0) - if diggerConfigYaml == "" { - return nil, fmt.Errorf("digger config can't be empty") - } - +func (db *Database) UpdateRepoDiggerConfig(orgId any, config configuration.DiggerConfigYaml, repo *Repo, isMainBranch bool) error { log.Printf("UpdateRepoDiggerConfig, repo: %v\n", repo) org, err := db.GetOrganisationById(orgId) if err != nil { - return nil, err - } - - diggerConfig, err := validateDiggerConfigYaml(diggerConfigYaml) - if err != nil { - return nil, err - } - - repo.DiggerConfig = diggerConfigYaml - tx := db.GormDB.Save(&repo) - if tx.Error != nil { - return nil, fmt.Errorf("failed to save digger config to database, %v", err) + return err } - - for _, dc := range diggerConfig.Projects { - projectName := dc.Name - p, err := db.GetProjectByName(orgId, repo, projectName) - if err != nil { - return nil, err - } - if p == nil { - _, err := db.CreateProject(projectName, org, repo) + err = db.GormDB.Transaction(func(tx *gorm.DB) error { + if isMainBranch { + // we reset all projects already in main branch to create new projects + repoProjects, err := db.GetProjectByRepo(orgId, repo) if err != nil { - return nil, err + return fmt.Errorf("could not get repo projects: %v", err) + } + for _, rp := range repoProjects { + rp.IsInMainBranch = false + err = db.UpdateProject(&rp) + if err != nil { + return fmt.Errorf("could not update existing main branch projects: %v", err) + } } - messages = append(messages, fmt.Sprintf("Project %s has been created\n", projectName)) - } else { - messages = append(messages, fmt.Sprintf("Project %s already exist\n", projectName)) } - } - // check if there are any projects in this repo that are not in the config anymore, - repoProjects, err := db.GetProjectByRepo(orgId, repo) - if err != nil { - return nil, err - } - for _, rp := range repoProjects { - projectFound := false - for _, cp := range diggerConfig.Projects { - if cp.Name == rp.Name { - projectFound = true + for _, dc := range config.Projects { + projectName := dc.Name + p, err := db.GetProjectByName(orgId, repo, projectName) + if err != nil { + return fmt.Errorf("error retriving project by name: %v", err) } - } - if !projectFound { - log.Printf("Project %v is not in a config anymore\n", rp.Name) - rp.Status = ProjectInactive - result := db.GormDB.Save(&rp) - if result.Error != nil { - return nil, result.Error + if p == nil { + _, err := db.CreateProject(projectName, org, repo, dc.Generated, isMainBranch) + if err != nil { + return fmt.Errorf("could not create project: %v", err) + } + } else { + if isMainBranch == true { + p.IsInMainBranch = isMainBranch + } + p.IsGenerated = dc.Generated + db.UpdateProject(p) } } + return nil + }) + + if err != nil { + return fmt.Errorf("error while updating projects from config: %v", err) } - return messages, nil + return nil } func (db *Database) CreateDiggerLock(resource string, lockId int, orgId uint) (*DiggerLock, error) { diff --git a/backend/tasks/runs_test.go b/backend/tasks/runs_test.go index 41d95c256..13fc614f8 100644 --- a/backend/tasks/runs_test.go +++ b/backend/tasks/runs_test.go @@ -76,7 +76,7 @@ func setupSuite(tb testing.TB) (func(tb testing.TB), *models.Database) { // create test project projectName := "test project" - _, err = database.CreateProject(projectName, org, repo) + _, err = database.CreateProject(projectName, org, repo, false, false) if err != nil { log.Fatal(err) } @@ -143,7 +143,7 @@ func TestThatRunQueueItemMovesFromQueuedToPlanningAfterPickup(t *testing.T) { for i, testParam := range testParameters { ciService := github2.MockCiService{} batch, _ := models.DB.CreateDiggerBatch(123, "", "", "", 22, "", "", "", nil) - project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, nil) + project, _ := models.DB.CreateProject(fmt.Sprintf("test%v", i), nil, nil, false, false) planStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) applyStage, _ := models.DB.CreateDiggerRunStage(batch.ID.String()) diggerRun, _ := models.DB.CreateDiggerRun("", 1, testParam.InitialStatus, "sha", "", 123, 1, project.Name, "apply", &planStage.ID, &applyStage.ID) diff --git a/ee/backend/controllers/web.go b/ee/backend/controllers/web.go index f7ec63ea5..dd18ecf85 100644 --- a/ee/backend/controllers/web.go +++ b/ee/backend/controllers/web.go @@ -1,7 +1,6 @@ package controllers import ( - "errors" "html/template" "log" "net/http" @@ -289,71 +288,3 @@ func (web *WebController) PolicyDetailsUpdatePage(c *gin.Context) { }) c.HTML(http.StatusOK, "policy_details.tmpl", pageContext) } - -func (web *WebController) UpdateRepoPage(c *gin.Context) { - repoId := c.Param("repoid") - if repoId == "" { - c.String(http.StatusInternalServerError, "Repo ID can't be empty") - return - } - orgId, exists := c.Get(middleware.ORGANISATION_ID_KEY) - if !exists { - log.Printf("Org %v not found in the context\n", middleware.ORGANISATION_ID_KEY) - c.String(http.StatusInternalServerError, "Not allowed to access this resource") - return - } - - repo, err := models.DB.GetRepoById(orgId, repoId) - if err != nil { - c.String(http.StatusForbidden, "Failed to find repo") - return - } - - if c.Request.Method == "GET" { - pageContext := services.GetMessages(c) - maps.Copy(pageContext, gin.H{ - "Repo": repo, - }) - c.HTML(http.StatusOK, "repo_add.tmpl", pageContext) - return - } else if c.Request.Method == "POST" { - diggerConfigYaml := c.PostForm("diggerconfig") - if diggerConfigYaml == "" { - services.AddWarning(c, "Digger config can't be empty") - - pageContext := services.GetMessages(c) - maps.Copy(pageContext, gin.H{ - "Repo": repo, - }) - c.HTML(http.StatusOK, "repo_add.tmpl", pageContext) - return - } - - messages, err := models.DB.UpdateRepoDiggerConfig(orgId, diggerConfigYaml, repo) - if err != nil { - if strings.HasPrefix(err.Error(), "validation error, ") { - services.AddError(c, errors.Unwrap(err).Error()) - - pageContext := services.GetMessages(c) - maps.Copy(pageContext, gin.H{ - "Repo": repo, - }) - c.HTML(http.StatusOK, "repo_add.tmpl", pageContext) - return - } - log.Printf("failed to updated repo %v, %v", repoId, err) - services.AddError(c, "failed to update repo") - - pageContext := services.GetMessages(c) - maps.Copy(pageContext, gin.H{ - "Repo": repo, - }) - c.HTML(http.StatusOK, "repo_add.tmpl", pageContext) - return - } - for _, m := range messages { - services.AddMessage(c, m) - } - c.Redirect(http.StatusFound, "/repos") - } -} diff --git a/ee/backend/main.go b/ee/backend/main.go index b12a9d19d..fe7d84d88 100644 --- a/ee/backend/main.go +++ b/ee/backend/main.go @@ -61,8 +61,6 @@ func main() { repoGroup := r.Group("/repo") repoGroup.Use(middleware.GetWebMiddleware()) repoGroup.GET("/", web.ReposPage) - repoGroup.GET("/:repoid/", web.UpdateRepoPage) - repoGroup.POST("/:repoid/", web.UpdateRepoPage) policiesGroup := r.Group("/policies") policiesGroup.Use(middleware.GetWebMiddleware()) diff --git a/ee/backend/templates/repos.tmpl b/ee/backend/templates/repos.tmpl index 51c3abf91..6882f6b04 100644 --- a/ee/backend/templates/repos.tmpl +++ b/ee/backend/templates/repos.tmpl @@ -15,7 +15,6 @@ Name Organisation - Details @@ -23,16 +22,6 @@ {{ .Name}} {{.Organisation.Name}} - - - Details - {{if eq .DiggerConfig "" }} - - config is missing - - {{ end }} - - {{ end }} diff --git a/libs/digger_config/config.go b/libs/digger_config/config.go index e876bd608..3299c2aa5 100644 --- a/libs/digger_config/config.go +++ b/libs/digger_config/config.go @@ -40,6 +40,7 @@ type Project struct { DependencyProjects []string DriftDetection bool AwsRoleToAssume *AssumeRoleForProject + Generated bool } type Workflow struct { diff --git a/libs/digger_config/converters.go b/libs/digger_config/converters.go index eb237a6db..5f779b624 100644 --- a/libs/digger_config/converters.go +++ b/libs/digger_config/converters.go @@ -64,6 +64,7 @@ func copyProjects(projects []*ProjectYaml) []Project { p.DependencyProjects, driftDetection, roleToAssume, + p.Generated, } result[i] = item } diff --git a/libs/digger_config/digger_config.go b/libs/digger_config/digger_config.go index 952a30b3f..2617477df 100644 --- a/libs/digger_config/digger_config.go +++ b/libs/digger_config/digger_config.go @@ -241,7 +241,14 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string, for _, dir := range dirs { if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) { projectName := strings.ReplaceAll(dir, "/", "_") - project := ProjectYaml{Name: projectName, Dir: dir, Workflow: defaultWorkflowName, Workspace: "default", AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume} + project := ProjectYaml{ + Name: projectName, + Dir: dir, + Workflow: defaultWorkflowName, + Workspace: "default", + AwsRoleToAssume: config.GenerateProjectsConfig.AwsRoleToAssume, + Generated: true, + } config.Projects = append(config.Projects, &project) } } @@ -283,7 +290,14 @@ func HandleYamlProjectGeneration(config *DiggerConfigYaml, terraformDir string, for _, dir := range dirs { if MatchIncludeExcludePatternsToFile(dir, includePatterns, excludePatterns) { projectName := strings.ReplaceAll(dir, "/", "_") - project := ProjectYaml{Name: projectName, Dir: dir, Workflow: workflow, Workspace: "default", AwsRoleToAssume: b.AwsRoleToAssume} + project := ProjectYaml{ + Name: projectName, + Dir: dir, + Workflow: workflow, + Workspace: "default", + AwsRoleToAssume: b.AwsRoleToAssume, + Generated: true, + } config.Projects = append(config.Projects, &project) } } @@ -485,6 +499,7 @@ func hydrateDiggerConfigYamlWithTerragrunt(configYaml *DiggerConfigYaml, parsing Workflow: atlantisProject.Workflow, WorkflowFile: &workflowFile, IncludePatterns: atlantisProject.Autoplan.WhenModified, + Generated: true, }) } return nil diff --git a/libs/digger_config/yaml.go b/libs/digger_config/yaml.go index f1d2a1b2a..7f84cc954 100644 --- a/libs/digger_config/yaml.go +++ b/libs/digger_config/yaml.go @@ -38,6 +38,7 @@ type ProjectYaml struct { DependencyProjects []string `yaml:"depends_on,omitempty"` DriftDetection *bool `yaml:"drift_detection,omitempty"` AwsRoleToAssume *AssumeRoleForProjectConfig `yaml:"aws_role_to_assume,omitempty"` + Generated bool `yaml:"generated"` } type WorkflowYaml struct {