Add S3 bucket upload functionality

cmd/api/config.sample.yml

@@ -13,4 +13,11 @@ docker:
   auth: true
   username: <registry-username>
   password: <registry-password>
-  registryUrl: <registry-url>
+  registryUrl: <registry-url>
+bucket:
+  name: "test"
+  accessKey: ""
+  secretKey: ""
+  endpoint: ""
+  region: ""
+  ssl: false

cmd/api/config/app_config.go

@@ -3,14 +3,17 @@ package config
 import (
 	"github.com/drzombey/aur-package-builder-api/pkg/docker"
 	"github.com/drzombey/aur-package-builder-api/pkg/mongo"
+	"github.com/drzombey/aur-package-builder-api/pkg/storage"
 )
 
 type AppConfig struct {
-	WebserverPort int
-	Debug         bool
-	LogLevel      string
-	JaegerURL     string
-	PackagePath   string
-	Database      mongo.MongoDbConfig
-	Docker        docker.DockerConfig
+	WebserverPort   int
+	Debug           bool
+	LogLevel        string
+	JaegerURL       string
+	PackagePath     string
+	StorageProvider string
+	Database        mongo.MongoDbConfig
+	Docker          docker.DockerConfig
+	Bucket          storage.S3Config
 }

cmd/api/handler/package.go

@@ -1,6 +1,7 @@
 package handler
 
 import (
+	"github.com/drzombey/aur-package-builder-api/pkg/storage"
 	"net/http"
 
 	"github.com/drzombey/aur-package-builder-api/cmd/api/config"
@@ -12,10 +13,14 @@ import (
 	"github.com/sirupsen/logrus"
 )
 
-var app *config.AppConfig
+var (
+	app             *config.AppConfig
+	storageProvider storage.Provider
+)
 
-func InitHandlers(a *config.AppConfig) {
+func InitHandlers(a *config.AppConfig, sp storage.Provider) {
 	app = a
+	storageProvider = sp
 }
 
 func HandleGetAlreadyBuildPackages(c *gin.Context) {
@@ -96,7 +101,6 @@ func HandleBuildPackage(c *gin.Context) {
 	}
 
 	result, err := repo.GetAlreadyBuildPackageByAurIdAndVersion(newPackage.ID, newPackage.Version)
-
 	if err != nil {
 		handleError(c, err)
 		return
@@ -114,7 +118,6 @@ func HandleBuildPackage(c *gin.Context) {
 	}
 
 	err = repo.AddAurPackage(newPackage)
-
 	if err != nil {
 		logrus.Errorf("Error occured [error: %s]", err)
 		c.JSON(http.StatusInternalServerError, gin.H{
@@ -132,20 +135,18 @@ func HandleBuildPackage(c *gin.Context) {
 	}
 
 	containerController, err := docker.NewContainerController(&registryData)
-
 	if err != nil {
 		handleError(c, err)
 		return
 	}
 
-	builder, err := builder.NewAurBuilderService(containerController)
-
+	builderService, err := builder.NewAurBuilderService(containerController, storageProvider)
 	if err != nil {
 		handleError(c, err)
 		return
 	}
 
-	containerId, err := builder.StartBuildAurPkgRoutine(&newPackage, app.PackagePath)
+	containerId, err := builderService.StartBuildAurPkgRoutine(&newPackage, app.PackagePath)
 
 	if err != nil {
 		handleError(c, err)

cmd/api/main.go

@@ -4,16 +4,17 @@ import (
 	"flag"
 	"fmt"
 
+	"github.com/gin-gonic/gin"
+	log "github.com/sirupsen/logrus"
+	"github.com/spf13/viper"
+	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
+
 	"github.com/drzombey/aur-package-builder-api/cmd/api/config"
 	"github.com/drzombey/aur-package-builder-api/cmd/api/handler"
 	"github.com/drzombey/aur-package-builder-api/cmd/api/tasks"
 	"github.com/drzombey/aur-package-builder-api/pkg/scheduler"
+	"github.com/drzombey/aur-package-builder-api/pkg/storage"
 	"github.com/drzombey/aur-package-builder-api/pkg/tracing"
-	log "github.com/sirupsen/logrus"
-	"go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin"
-
-	"github.com/gin-gonic/gin"
-	"github.com/spf13/viper"
 )
 
 var (
@@ -61,6 +62,7 @@ func loadConfig() {
 	viper.SetDefault("webserverMode", "production")
 	viper.SetDefault("jaegerURL", "http://localhost:14268/api/traces")
 	viper.SetDefault("packagePath", ".")
+	viper.SetDefault("storageProvider", "")
 	viper.SetDefault("database", map[string]interface{}{
 		"host":     "localhost",
 		"port":     27017,
@@ -87,7 +89,7 @@ func loadConfig() {
 }
 
 func registerHandlers(s *gin.Engine) {
-	handler.InitHandlers(&app)
+	handler.InitHandlers(&app, getStorageProvider())
 
 	version1 := "/api/v1"
 
@@ -96,8 +98,17 @@ func registerHandlers(s *gin.Engine) {
 	s.GET(fmt.Sprintf("%s/aurpackage", version1), handler.HandleGetAurPackageByName)
 }
 
+func getStorageProvider() storage.Provider {
+	switch app.StorageProvider {
+	case "object":
+		return storage.NewS3Provider(&app.Bucket)
+	default:
+		return storage.NewFilesystemProvider(app.PackagePath)
+	}
+}
+
 func initBackgroundTasks() {
 	taskScheduler = scheduler.NewTasksScheduler()
-	apiTask := tasks.NewApiTask(app)
-	taskScheduler.ScheduleTask(apiTask.UpdateAllPackages, 36000, "UpdatePackagesTask")
+	apiTask := tasks.NewApiTask(app, getStorageProvider())
+	taskScheduler.ScheduleTask(apiTask.UpdateAllPackages, 3600, "UpdatePackagesTask")
 }

cmd/api/tasks/types.go

@@ -1,13 +1,18 @@
 package tasks
 
-import "github.com/drzombey/aur-package-builder-api/cmd/api/config"
+import (
+	"github.com/drzombey/aur-package-builder-api/cmd/api/config"
+	"github.com/drzombey/aur-package-builder-api/pkg/storage"
+)
 
 type ApiTask struct {
-	app config.AppConfig
+	app             config.AppConfig
+	storageProvider storage.Provider
 }
 
-func NewApiTask(app config.AppConfig) *ApiTask {
+func NewApiTask(app config.AppConfig, provider storage.Provider) *ApiTask {
 	return &ApiTask{
-		app: app,
+		app:             app,
+		storageProvider: provider,
 	}
 }

cmd/api/tasks/update_packages.go

@@ -11,15 +11,13 @@ import (
 
 func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) {
 	repo, err := repository.NewPackageRepo(at.app.Database)
-
 	if err != nil {
 		message := fmt.Sprintf("Cannot establish database connection [Error: %s]", err)
 		at.logTaskError(taskId, taskName, message)
 		return
 	}
 
 	packages, err := repo.GetAlreadyBuildPackages()
-
 	if err != nil {
 		message := fmt.Sprintf("Cannot fetch packages [Error: %s]", err)
 		at.logTaskError(taskId, taskName, message)
@@ -34,17 +32,15 @@ func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) {
 	}
 
 	containerController, err := docker.NewContainerController(&registryData)
-
 	if err != nil {
 		message := fmt.Sprintf("Cannot initialize container controller [Error: %s]", err)
 		at.logTaskError(taskId, taskName, message)
 		return
 	}
 
-	builder, err := builder.NewAurBuilderService(containerController)
-
+	service, err := builder.NewAurBuilderService(containerController, at.storageProvider)
 	if err != nil {
-		message := fmt.Sprintf("Cannot initialize builder service [Error: %s]", err)
+		message := fmt.Sprintf("Cannot initialize service service [Error: %s]", err)
 		at.logTaskError(taskId, taskName, message)
 		return
 	}
@@ -68,9 +64,20 @@ func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) {
 			continue
 		}
 
-		repo.DeleteAurPackage(pkg)
-		repo.AddAurPackage(*aurPkg)
-		builder.StartBuildAurPkgRoutine(aurPkg, at.app.PackagePath)
+		err = repo.DeleteAurPackage(pkg)
+		if err != nil {
+			return
+		}
+
+		err = repo.AddAurPackage(*aurPkg)
+		if err != nil {
+			return
+		}
+
+		_, err = service.StartBuildAurPkgRoutine(aurPkg, at.app.PackagePath)
+		if err != nil {
+			return
+		}
 	}
 
 	at.logTaskCompleted(taskId, taskName)

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/docker/docker v20.10.24+incompatible
 	github.com/gin-gonic/gin v1.9.0
 	github.com/google/uuid v1.3.0
+	github.com/minio/minio-go/v7 v7.0.50
 	github.com/sirupsen/logrus v1.9.0
 	github.com/spf13/viper v1.12.0
 	go.mongodb.org/mongo-driver v1.10.1
@@ -22,6 +23,7 @@ require (
 	github.com/docker/distribution v2.8.1+incompatible // indirect
 	github.com/docker/go-connections v0.4.0 // indirect
 	github.com/docker/go-units v0.4.0 // indirect
+	github.com/dustin/go-humanize v1.0.1 // indirect
 	github.com/fsnotify/fsnotify v1.5.4 // indirect
 	github.com/gin-contrib/sse v0.1.0 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
@@ -34,11 +36,13 @@ require (
 	github.com/golang/snappy v0.0.4 // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
 	github.com/json-iterator/go v1.1.12 // indirect
-	github.com/klauspost/compress v1.15.9 // indirect
-	github.com/klauspost/cpuid/v2 v2.0.9 // indirect
+	github.com/klauspost/compress v1.16.0 // indirect
+	github.com/klauspost/cpuid/v2 v2.2.4 // indirect
 	github.com/leodido/go-urn v1.2.1 // indirect
 	github.com/magiconair/properties v1.8.6 // indirect
 	github.com/mattn/go-isatty v0.0.17 // indirect
+	github.com/minio/md5-simd v1.1.2 // indirect
+	github.com/minio/sha256-simd v1.0.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect
 	github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect
@@ -50,6 +54,7 @@ require (
 	github.com/pelletier/go-toml v1.9.5 // indirect
 	github.com/pelletier/go-toml/v2 v2.0.6 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
+	github.com/rs/xid v1.4.0 // indirect
 	github.com/spf13/afero v1.9.2 // indirect
 	github.com/spf13/cast v1.5.0 // indirect
 	github.com/spf13/jwalterweatherman v1.1.0 // indirect
@@ -63,7 +68,7 @@ require (
 	github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect
 	go.opentelemetry.io/otel/trace v1.11.1 // indirect
 	golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect
-	golang.org/x/crypto v0.5.0 // indirect
+	golang.org/x/crypto v0.6.0 // indirect
 	golang.org/x/net v0.7.0 // indirect
 	golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect
 	golang.org/x/sys v0.5.0 // indirect

go.sum

@@ -68,6 +68,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh
 github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec=
 github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw=
 github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk=
+github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY=
+github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto=
 github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4=
 github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98=
@@ -176,10 +178,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8=
 github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck=
 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk=
-github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY=
-github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU=
-github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4=
+github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4=
+github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE=
+github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
 github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg=
+github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk=
+github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY=
 github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg=
 github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo=
 github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0=
@@ -192,6 +197,12 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh
 github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60=
 github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng=
 github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM=
+github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34=
+github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM=
+github.com/minio/minio-go/v7 v7.0.50 h1:4IL4V8m/kI90ZL6GupCARZVrBv8/XrcKcJhaJ3iz68k=
+github.com/minio/minio-go/v7 v7.0.50/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU=
+github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g=
+github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM=
 github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY=
 github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo=
 github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI=
@@ -223,6 +234,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN
 github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8=
+github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY=
+github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg=
 github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
 github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0=
 github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ=
@@ -302,8 +315,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4=
 golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4=
-golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE=
-golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU=
+golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc=
+golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -432,6 +445,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w
 golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU=