From 74a7ce09c8f6755f138990dc4bee8e8c6cbd60af Mon Sep 17 00:00:00 2001 From: Tim Lange Date: Wed, 28 Dec 2022 19:19:18 +0100 Subject: [PATCH] Add S3 bucket upload functionality --- cmd/api/config.sample.yml | 9 ++- cmd/api/config/app_config.go | 17 +++-- cmd/api/handler/package.go | 17 ++--- cmd/api/main.go | 27 +++++--- cmd/api/tasks/types.go | 13 ++-- cmd/api/tasks/update_packages.go | 25 ++++--- go.mod | 11 ++- go.sum | 24 +++++-- pkg/builder/builder_service.go | 50 ++++++++++++-- pkg/builder/types.go | 12 ++-- pkg/docker/container.go | 41 ++---------- pkg/storage/filesystem.go | 37 +++++++++++ pkg/storage/filesystem_test.go | 104 +++++++++++++++++++++++++++++ pkg/storage/helper.go | 45 +++++++++++++ pkg/storage/s3.go | 111 +++++++++++++++++++++++++++++++ pkg/storage/types.go | 13 ++++ 16 files changed, 466 insertions(+), 90 deletions(-) create mode 100644 pkg/storage/filesystem.go create mode 100644 pkg/storage/filesystem_test.go create mode 100644 pkg/storage/helper.go create mode 100644 pkg/storage/s3.go create mode 100644 pkg/storage/types.go diff --git a/cmd/api/config.sample.yml b/cmd/api/config.sample.yml index 675de79..75d5d76 100644 --- a/cmd/api/config.sample.yml +++ b/cmd/api/config.sample.yml @@ -13,4 +13,11 @@ docker: auth: true username: password: - registryUrl: \ No newline at end of file + registryUrl: +bucket: + name: "test" + accessKey: "" + secretKey: "" + endpoint: "" + region: "" + ssl: false \ No newline at end of file diff --git a/cmd/api/config/app_config.go b/cmd/api/config/app_config.go index c3eccce..7f67ea1 100644 --- a/cmd/api/config/app_config.go +++ b/cmd/api/config/app_config.go @@ -3,14 +3,17 @@ package config import ( "github.com/drzombey/aur-package-builder-api/pkg/docker" "github.com/drzombey/aur-package-builder-api/pkg/mongo" + "github.com/drzombey/aur-package-builder-api/pkg/storage" ) type AppConfig struct { - WebserverPort int - Debug bool - LogLevel string - JaegerURL string - PackagePath string - Database mongo.MongoDbConfig - Docker docker.DockerConfig + WebserverPort int + Debug bool + LogLevel string + JaegerURL string + PackagePath string + StorageProvider string + Database mongo.MongoDbConfig + Docker docker.DockerConfig + Bucket storage.S3Config } diff --git a/cmd/api/handler/package.go b/cmd/api/handler/package.go index b014307..0a9b9fa 100644 --- a/cmd/api/handler/package.go +++ b/cmd/api/handler/package.go @@ -1,6 +1,7 @@ package handler import ( + "github.com/drzombey/aur-package-builder-api/pkg/storage" "net/http" "github.com/drzombey/aur-package-builder-api/cmd/api/config" @@ -12,10 +13,14 @@ import ( "github.com/sirupsen/logrus" ) -var app *config.AppConfig +var ( + app *config.AppConfig + storageProvider storage.Provider +) -func InitHandlers(a *config.AppConfig) { +func InitHandlers(a *config.AppConfig, sp storage.Provider) { app = a + storageProvider = sp } func HandleGetAlreadyBuildPackages(c *gin.Context) { @@ -96,7 +101,6 @@ func HandleBuildPackage(c *gin.Context) { } result, err := repo.GetAlreadyBuildPackageByAurIdAndVersion(newPackage.ID, newPackage.Version) - if err != nil { handleError(c, err) return @@ -114,7 +118,6 @@ func HandleBuildPackage(c *gin.Context) { } err = repo.AddAurPackage(newPackage) - if err != nil { logrus.Errorf("Error occured [error: %s]", err) c.JSON(http.StatusInternalServerError, gin.H{ @@ -132,20 +135,18 @@ func HandleBuildPackage(c *gin.Context) { } containerController, err := docker.NewContainerController(®istryData) - if err != nil { handleError(c, err) return } - builder, err := builder.NewAurBuilderService(containerController) - + builderService, err := builder.NewAurBuilderService(containerController, storageProvider) if err != nil { handleError(c, err) return } - containerId, err := builder.StartBuildAurPkgRoutine(&newPackage, app.PackagePath) + containerId, err := builderService.StartBuildAurPkgRoutine(&newPackage, app.PackagePath) if err != nil { handleError(c, err) diff --git a/cmd/api/main.go b/cmd/api/main.go index b2577eb..4e58cea 100644 --- a/cmd/api/main.go +++ b/cmd/api/main.go @@ -4,16 +4,17 @@ import ( "flag" "fmt" + "github.com/gin-gonic/gin" + log "github.com/sirupsen/logrus" + "github.com/spf13/viper" + "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" + "github.com/drzombey/aur-package-builder-api/cmd/api/config" "github.com/drzombey/aur-package-builder-api/cmd/api/handler" "github.com/drzombey/aur-package-builder-api/cmd/api/tasks" "github.com/drzombey/aur-package-builder-api/pkg/scheduler" + "github.com/drzombey/aur-package-builder-api/pkg/storage" "github.com/drzombey/aur-package-builder-api/pkg/tracing" - log "github.com/sirupsen/logrus" - "go.opentelemetry.io/contrib/instrumentation/github.com/gin-gonic/gin/otelgin" - - "github.com/gin-gonic/gin" - "github.com/spf13/viper" ) var ( @@ -61,6 +62,7 @@ func loadConfig() { viper.SetDefault("webserverMode", "production") viper.SetDefault("jaegerURL", "http://localhost:14268/api/traces") viper.SetDefault("packagePath", ".") + viper.SetDefault("storageProvider", "") viper.SetDefault("database", map[string]interface{}{ "host": "localhost", "port": 27017, @@ -87,7 +89,7 @@ func loadConfig() { } func registerHandlers(s *gin.Engine) { - handler.InitHandlers(&app) + handler.InitHandlers(&app, getStorageProvider()) version1 := "/api/v1" @@ -96,8 +98,17 @@ func registerHandlers(s *gin.Engine) { s.GET(fmt.Sprintf("%s/aurpackage", version1), handler.HandleGetAurPackageByName) } +func getStorageProvider() storage.Provider { + switch app.StorageProvider { + case "object": + return storage.NewS3Provider(&app.Bucket) + default: + return storage.NewFilesystemProvider(app.PackagePath) + } +} + func initBackgroundTasks() { taskScheduler = scheduler.NewTasksScheduler() - apiTask := tasks.NewApiTask(app) - taskScheduler.ScheduleTask(apiTask.UpdateAllPackages, 36000, "UpdatePackagesTask") + apiTask := tasks.NewApiTask(app, getStorageProvider()) + taskScheduler.ScheduleTask(apiTask.UpdateAllPackages, 3600, "UpdatePackagesTask") } diff --git a/cmd/api/tasks/types.go b/cmd/api/tasks/types.go index 924c2e9..0255e54 100644 --- a/cmd/api/tasks/types.go +++ b/cmd/api/tasks/types.go @@ -1,13 +1,18 @@ package tasks -import "github.com/drzombey/aur-package-builder-api/cmd/api/config" +import ( + "github.com/drzombey/aur-package-builder-api/cmd/api/config" + "github.com/drzombey/aur-package-builder-api/pkg/storage" +) type ApiTask struct { - app config.AppConfig + app config.AppConfig + storageProvider storage.Provider } -func NewApiTask(app config.AppConfig) *ApiTask { +func NewApiTask(app config.AppConfig, provider storage.Provider) *ApiTask { return &ApiTask{ - app: app, + app: app, + storageProvider: provider, } } diff --git a/cmd/api/tasks/update_packages.go b/cmd/api/tasks/update_packages.go index 0efdcec..e78533a 100644 --- a/cmd/api/tasks/update_packages.go +++ b/cmd/api/tasks/update_packages.go @@ -11,7 +11,6 @@ import ( func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) { repo, err := repository.NewPackageRepo(at.app.Database) - if err != nil { message := fmt.Sprintf("Cannot establish database connection [Error: %s]", err) at.logTaskError(taskId, taskName, message) @@ -19,7 +18,6 @@ func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) { } packages, err := repo.GetAlreadyBuildPackages() - if err != nil { message := fmt.Sprintf("Cannot fetch packages [Error: %s]", err) at.logTaskError(taskId, taskName, message) @@ -34,17 +32,15 @@ func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) { } containerController, err := docker.NewContainerController(®istryData) - if err != nil { message := fmt.Sprintf("Cannot initialize container controller [Error: %s]", err) at.logTaskError(taskId, taskName, message) return } - builder, err := builder.NewAurBuilderService(containerController) - + service, err := builder.NewAurBuilderService(containerController, at.storageProvider) if err != nil { - message := fmt.Sprintf("Cannot initialize builder service [Error: %s]", err) + message := fmt.Sprintf("Cannot initialize service service [Error: %s]", err) at.logTaskError(taskId, taskName, message) return } @@ -68,9 +64,20 @@ func (at *ApiTask) UpdateAllPackages(taskId uuid.UUID, taskName string) { continue } - repo.DeleteAurPackage(pkg) - repo.AddAurPackage(*aurPkg) - builder.StartBuildAurPkgRoutine(aurPkg, at.app.PackagePath) + err = repo.DeleteAurPackage(pkg) + if err != nil { + return + } + + err = repo.AddAurPackage(*aurPkg) + if err != nil { + return + } + + _, err = service.StartBuildAurPkgRoutine(aurPkg, at.app.PackagePath) + if err != nil { + return + } } at.logTaskCompleted(taskId, taskName) diff --git a/go.mod b/go.mod index 2d9e168..47ad2b6 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/docker/docker v20.10.24+incompatible github.com/gin-gonic/gin v1.9.0 github.com/google/uuid v1.3.0 + github.com/minio/minio-go/v7 v7.0.50 github.com/sirupsen/logrus v1.9.0 github.com/spf13/viper v1.12.0 go.mongodb.org/mongo-driver v1.10.1 @@ -22,6 +23,7 @@ require ( github.com/docker/distribution v2.8.1+incompatible // indirect github.com/docker/go-connections v0.4.0 // indirect github.com/docker/go-units v0.4.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect github.com/fsnotify/fsnotify v1.5.4 // indirect github.com/gin-contrib/sse v0.1.0 // indirect github.com/go-logr/logr v1.2.3 // indirect @@ -34,11 +36,13 @@ require ( github.com/golang/snappy v0.0.4 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/klauspost/compress v1.15.9 // indirect - github.com/klauspost/cpuid/v2 v2.0.9 // indirect + github.com/klauspost/compress v1.16.0 // indirect + github.com/klauspost/cpuid/v2 v2.2.4 // indirect github.com/leodido/go-urn v1.2.1 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mattn/go-isatty v0.0.17 // indirect + github.com/minio/md5-simd v1.1.2 // indirect + github.com/minio/sha256-simd v1.0.0 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae // indirect github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd // indirect @@ -50,6 +54,7 @@ require ( github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.6 // indirect github.com/pkg/errors v0.9.1 // indirect + github.com/rs/xid v1.4.0 // indirect github.com/spf13/afero v1.9.2 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect @@ -63,7 +68,7 @@ require ( github.com/youmark/pkcs8 v0.0.0-20201027041543-1326539a0a0a // indirect go.opentelemetry.io/otel/trace v1.11.1 // indirect golang.org/x/arch v0.0.0-20210923205945-b76863e36670 // indirect - golang.org/x/crypto v0.5.0 // indirect + golang.org/x/crypto v0.6.0 // indirect golang.org/x/net v0.7.0 // indirect golang.org/x/sync v0.0.0-20220819030929-7fc1605a5dde // indirect golang.org/x/sys v0.5.0 // indirect diff --git a/go.sum b/go.sum index 8df6065..84a4a80 100644 --- a/go.sum +++ b/go.sum @@ -68,6 +68,8 @@ github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKoh github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0 h1:3uh0PgVws3nIA0Q+MwDC8yjEPf9zjRfZZWXZYDct3Tw= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -176,10 +178,13 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X github.com/kisielk/errcheck v1.5.0/go.mod h1:pFxgyoBC7bSaBwPgfKdkLd5X25qrDl4LWUI2bnpBCr8= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.9 h1:wKRjX6JRtDdrE9qwa4b/Cip7ACOshUI4smpCQanqjSY= -github.com/klauspost/compress v1.15.9/go.mod h1:PhcZ0MbTNciWF3rruxRgKxI5NkcHHrHUDtV4Yw2GlzU= -github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= +github.com/klauspost/compress v1.16.0 h1:iULayQNOReoYUe+1qtKOqw9CwJv3aNQu8ivo7lw1HU4= +github.com/klauspost/compress v1.16.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/cpuid/v2 v2.0.1/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= +github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= +github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= @@ -192,6 +197,12 @@ github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamh github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/minio/md5-simd v1.1.2 h1:Gdi1DZK69+ZVMoNHRXJyNcxrMA4dSxoYHZSQbirFg34= +github.com/minio/md5-simd v1.1.2/go.mod h1:MzdKDxYpY2BT9XQFocsiZf/NKVtR7nkE4RoEpN+20RM= +github.com/minio/minio-go/v7 v7.0.50 h1:4IL4V8m/kI90ZL6GupCARZVrBv8/XrcKcJhaJ3iz68k= +github.com/minio/minio-go/v7 v7.0.50/go.mod h1:IbbodHyjUAguneyucUaahv+VMNs/EOTV9du7A7/Z3HU= +github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= +github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/moby/term v0.0.0-20220808134915-39b0c02b01ae h1:O4SWKdcHVCvYqyDV+9CJA1fcDN2L11Bule0iFy3YlAI= @@ -223,6 +234,8 @@ github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZN github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= +github.com/rs/xid v1.4.0 h1:qd7wPTDkN6KQx2VmMBLrpHkiyQwgFXRnkOLacUiaSNY= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= @@ -302,8 +315,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220622213112-05595931fe9d/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.5.0 h1:U/0M97KRkSFvyD/3FSmdP5W5swImpNgle/EHFhOsQPE= -golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -432,6 +445,7 @@ golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210616094352-59db8d763f22/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= diff --git a/pkg/builder/builder_service.go b/pkg/builder/builder_service.go index 2c56ad0..12b2189 100644 --- a/pkg/builder/builder_service.go +++ b/pkg/builder/builder_service.go @@ -1,11 +1,14 @@ package builder import ( + "archive/tar" + "context" "fmt" - "github.com/docker/docker/api/types/container" "github.com/drzombey/aur-package-builder-api/pkg/aur" "github.com/sirupsen/logrus" + "io" + "os" ) func (s *AurBuilderService) StartBuildAurPkgRoutine(aurPkg *aur.Package, dest string) (containerId string, err error) { @@ -16,7 +19,7 @@ func (s *AurBuilderService) StartBuildAurPkgRoutine(aurPkg *aur.Package, dest st } go func() { - _, err = s.copyPackageToDestination(containerId, dest, aurPkg) + err = s.savePackage(containerId, aurPkg) if err != nil { logrus.Errorf("Copying package from container %s not possible [error: %s]", containerId, err) @@ -59,21 +62,58 @@ func (s *AurBuilderService) buildAurPackage(aurPkg *aur.Package) (containerId st return containerId, nil } -func (s *AurBuilderService) copyPackageToDestination(containerId, dest string, aurPkg *aur.Package) (pkgPath string, err error) { +func (s *AurBuilderService) savePackage(containerId string, aurPkg *aur.Package) (err error) { pkgName := fmt.Sprintf("%s-%s%s", aurPkg.PackageBase, aurPkg.Version, packageSuffix) _, err = s.controller.WaitForContainer(containerId, container.WaitConditionNextExit) + if err != nil { + return err + } + + stream, err := s.controller.CopyFromContainer(containerId, packagePath+pkgName) + + if err != nil { + logrus.Errorf("Got error: %s", err) + return err + } + + err = s.storageProvider.AddFile(context.Background(), pkgName, stream) + if err != nil { + logrus.Errorf("Got error: %s", err) + return err + } + + return nil +} + +func (s *AurBuilderService) createPackageFileFromStream(stream io.ReadCloser, dest, pkgName string) (filenameWithPath string, err error) { + reader := tar.NewReader(stream) + + if _, err := reader.Next(); err != nil { + return "", err + } + + if _, err := os.Stat(dest); os.IsNotExist(err) { + os.MkdirAll(dest, os.ModePerm) + } + + filenameWithPath = dest + "/" + pkgName + + file, err := os.Create(filenameWithPath) + if err != nil { return "", err } - pkgPath, err = s.controller.CopyFromContainer(containerId, packagePath+pkgName, dest, pkgName) + defer file.Close() + + _, err = io.Copy(file, reader) if err != nil { return "", err } - return pkgPath, nil + return filenameWithPath, nil } func (s *AurBuilderService) cleanUpBuildEnv(containerId string) (err error) { diff --git a/pkg/builder/types.go b/pkg/builder/types.go index 7120b6d..b72352f 100644 --- a/pkg/builder/types.go +++ b/pkg/builder/types.go @@ -2,10 +2,12 @@ package builder import ( "github.com/drzombey/aur-package-builder-api/pkg/docker" + "github.com/drzombey/aur-package-builder-api/pkg/storage" ) type AurBuilderService struct { - controller *docker.ContainerController + controller *docker.ContainerController + storageProvider storage.Provider } const ( @@ -13,7 +15,9 @@ const ( packagePath = "/pkg/" ) -func NewAurBuilderService(dc *docker.ContainerController) (*AurBuilderService, error) { - - return &AurBuilderService{controller: dc}, nil +func NewAurBuilderService(dc *docker.ContainerController, sp storage.Provider) (*AurBuilderService, error) { + return &AurBuilderService{ + controller: dc, + storageProvider: sp, + }, nil } diff --git a/pkg/docker/container.go b/pkg/docker/container.go index a6c0473..05f9019 100644 --- a/pkg/docker/container.go +++ b/pkg/docker/container.go @@ -1,14 +1,11 @@ package docker import ( - "archive/tar" "context" - "io" - "os" - "github.com/docker/docker/api/types" "github.com/docker/docker/api/types/container" "github.com/docker/docker/api/types/filters" + "io" ) func (c *ContainerController) WaitForContainer(containerId string, condition container.WaitCondition) (state int64, err error) { @@ -54,40 +51,12 @@ func (c *ContainerController) ContainerById(containerId string) (container *type return container, nil } -func (c *ContainerController) CopyFromContainer(containerId string, src, dest, pkgName string) (filePathWithName string, err error) { - stream, _, err := c.cli.CopyFromContainer(context.Background(), containerId, src) - - if err != nil { - return "", err - } - - defer stream.Close() - - reader := tar.NewReader(stream) - - if _, err := reader.Next(); err != nil { - return "", err - } - - if _, err := os.Stat(dest); os.IsNotExist(err) { - os.MkdirAll(dest, os.ModePerm) - } - - filePathWithName = dest + "/" + pkgName - - file, err := os.Create(filePathWithName) - - if err != nil { - return "", err - } - - defer file.Close() - - _, err = io.Copy(file, reader) +func (c *ContainerController) CopyFromContainer(containerId string, src string) (stream io.ReadCloser, err error) { + stream, _, err = c.cli.CopyFromContainer(context.Background(), containerId, src) if err != nil { - return "", err + return nil, err } - return filePathWithName, nil + return stream, nil } diff --git a/pkg/storage/filesystem.go b/pkg/storage/filesystem.go new file mode 100644 index 0000000..79aac9d --- /dev/null +++ b/pkg/storage/filesystem.go @@ -0,0 +1,37 @@ +package storage + +import ( + "context" + "io" + "io/fs" + "os" +) + +type FileSystemProvider struct { + dest string +} + +func NewFilesystemProvider(dest string) Provider { + return &FileSystemProvider{dest: dest} +} + +func (f FileSystemProvider) AddFile(ctx context.Context, fileName string, stream io.Reader) error { + _, err := io.ReadAll(stream) + if err != nil { + return err + } + + _, err = createTarFileFromStream(fileName, f.dest, stream) + if err != nil { + return err + } + return nil +} + +func (f FileSystemProvider) GetFile(ctx context.Context, filename string) (*os.File, error) { + return os.OpenFile(f.dest+"/"+filename, os.O_RDONLY, fs.ModePerm) +} + +func (f FileSystemProvider) DeleteFile(ctx context.Context, filename string) error { + return os.Remove(f.dest + "/" + filename) +} diff --git a/pkg/storage/filesystem_test.go b/pkg/storage/filesystem_test.go new file mode 100644 index 0000000..b12d5b1 --- /dev/null +++ b/pkg/storage/filesystem_test.go @@ -0,0 +1,104 @@ +package storage + +import ( + "archive/tar" + "bytes" + "compress/gzip" + "context" + "io/ioutil" + "os" + "testing" +) + +var testFolder = os.TempDir() + "/test_files" + +func SetupTest() { + err := os.MkdirAll(testFolder, os.ModePerm) + if err != nil { + return + } +} + +func createTestTarGzFile(filename string, content []byte) error { + file, err := os.Create(filename) + if err != nil { + return err + } + defer file.Close() + + gw := gzip.NewWriter(file) + defer gw.Close() + + tw := tar.NewWriter(gw) + defer tw.Close() + + hdr := &tar.Header{ + Name: "test.tar", + Mode: 0644, + Size: int64(len(content)), + } + + if err := tw.WriteHeader(hdr); err != nil { + return err + } + + _, err = tw.Write(content) + if err != nil { + return err + } + + return nil +} + +func TestFileSystemProvider_GetFile(t *testing.T) { + SetupTest() + provider := NewFilesystemProvider(testFolder) + defer os.RemoveAll(testFolder) + + ctx := context.Background() + fileName := "test.txt" + content := []byte("This is a test file") + err := os.WriteFile(testFolder+"/"+fileName, content, 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + file, err := provider.GetFile(ctx, fileName) + if err != nil { + t.Fatalf("Failed to get file: %v", err) + } + defer file.Close() + + fileData, err := ioutil.ReadAll(file) + if err != nil { + t.Fatalf("Failed to read file: %v", err) + } + + if !bytes.Equal(fileData, content) { + t.Errorf("File content mismatch. Expected: %s, Got: %s", content, fileData) + } +} + +func TestFileSystemProvider_DeleteFile(t *testing.T) { + SetupTest() + provider := NewFilesystemProvider(testFolder) + defer os.RemoveAll(testFolder) + + ctx := context.Background() + fileName := "test.txt" + content := []byte("This is a test file") + err := os.WriteFile(testFolder+"/"+fileName, content, 0644) + if err != nil { + t.Fatalf("Failed to create test file: %v", err) + } + + err = provider.DeleteFile(ctx, fileName) + if err != nil { + t.Fatalf("Failed to delete file: %v", err) + } + + _, err = os.Stat(testFolder + "/" + fileName) + if !os.IsNotExist(err) { + t.Errorf("File was not deleted: %v", err) + } +} diff --git a/pkg/storage/helper.go b/pkg/storage/helper.go new file mode 100644 index 0000000..0bdbd5e --- /dev/null +++ b/pkg/storage/helper.go @@ -0,0 +1,45 @@ +package storage + +import ( + "archive/tar" + "bytes" + "io" + "os" +) + +func createTarFileFromStream(filename, dest string, stream io.Reader) (*os.File, error) { + reader := tar.NewReader(stream) + + if _, err := os.Stat(dest); os.IsNotExist(err) { + err := os.MkdirAll(dest, os.ModePerm) + if err != nil { + return nil, err + } + } + + filePathWithName := dest + "/" + filename + + file, err := os.Create(filePathWithName) + if err != nil { + return nil, err + } + defer file.Close() + + buf := new(bytes.Buffer) + _, err = io.Copy(buf, reader) + if err != nil { + return nil, err + } + + _, err = file.Write(buf.Bytes()) + if err != nil { + return nil, err + } + + _, err = file.Seek(0, io.SeekStart) + if err != nil { + return nil, err + } + + return file, nil +} diff --git a/pkg/storage/s3.go b/pkg/storage/s3.go new file mode 100644 index 0000000..15547b7 --- /dev/null +++ b/pkg/storage/s3.go @@ -0,0 +1,111 @@ +package storage + +import ( + "context" + "github.com/minio/minio-go/v7" + "github.com/minio/minio-go/v7/pkg/credentials" + log "github.com/sirupsen/logrus" + "io" + "os" +) + +type S3Provider struct { + client *minio.Client + cfg *S3Config +} + +type S3Config struct { + BucketName string + AccessKey string + SecretKey string + Endpoint string + Region string + Ssl bool +} + +func NewS3Provider(cfg *S3Config) Provider { + ctx := context.Background() + + minioClient, err := minio.New(cfg.Endpoint, &minio.Options{ + Creds: credentials.NewStaticV4(cfg.AccessKey, cfg.SecretKey, ""), + Secure: cfg.Ssl, + }) + + if err != nil { + log.Fatalln(err) + } + + err = minioClient.MakeBucket(ctx, cfg.BucketName, minio.MakeBucketOptions{Region: cfg.Region}) + if err != nil { + exists, err := minioClient.BucketExists(ctx, cfg.BucketName) + if err == nil && exists { + log.Printf("We already own %s\n", cfg.BucketName) + } else { + log.Fatalln(err) + } + } else { + log.Printf("Successfully created %s\n", cfg.BucketName) + } + + return &S3Provider{ + cfg: cfg, + client: minioClient, + } +} + +func (p S3Provider) AddFile(ctx context.Context, fileName string, stream io.Reader) error { + _, err := createTarFileFromStream(fileName, os.TempDir(), stream) + + if err != nil { + return err + } + _, err = p.client.FPutObject(ctx, p.cfg.BucketName, fileName, os.TempDir()+"/"+fileName, minio.PutObjectOptions{ContentType: "application/zst"}) + + if err != nil { + log.Errorf("Cannot upload following file %s to bucket\n", fileName) + return err + } + + return nil +} + +func (p S3Provider) GetFile(ctx context.Context, filename string) (*os.File, error) { + reader, err := p.client.GetObject(ctx, p.cfg.BucketName, filename, minio.GetObjectOptions{}) + if err != nil { + log.Error(err) + return nil, err + } + defer reader.Close() + + localFile, err := os.Create(filename) + if err != nil { + log.Error(err) + return nil, err + } + + stat, err := reader.Stat() + if err != nil { + log.Error(err) + return nil, err + } + + if _, err := io.CopyN(localFile, reader, stat.Size); err != nil { + log.Error(err) + return nil, err + } + + return localFile, err +} + +func (p S3Provider) DeleteFile(ctx context.Context, filename string) error { + opts := minio.RemoveObjectOptions{ + GovernanceBypass: true, + } + + if err := p.client.RemoveObject(ctx, p.cfg.BucketName, filename, opts); err != nil { + log.Error(err) + return err + } + + return nil +} diff --git a/pkg/storage/types.go b/pkg/storage/types.go new file mode 100644 index 0000000..8a1ef8a --- /dev/null +++ b/pkg/storage/types.go @@ -0,0 +1,13 @@ +package storage + +import ( + "context" + "io" + "os" +) + +type Provider interface { + AddFile(ctx context.Context, filename string, stream io.Reader) error + GetFile(ctx context.Context, filename string) (*os.File, error) + DeleteFile(ctx context.Context, filename string) error +}