From e144b3422822428b818f0e2b3e44529a81e0b642 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Mon, 8 Sep 2025 14:23:34 +0200 Subject: [PATCH 1/2] Update to use minio/minio docker image --- .github/workflows/ci.yml | 22 ++++++++++++---------- archives/archives_test.go | 34 ++++++++++++++++++++++++++++++---- 2 files changed, 42 insertions(+), 14 deletions(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 8ea9bf3..59b88bf 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -18,16 +18,6 @@ jobs: - 5432:5432 options: --health-cmd pg_isready --health-interval 10s --health-timeout 5s --health-retries 5 - minio: - image: bitnami/minio:latest - env: - MINIO_ROOT_USER: root - MINIO_ROOT_PASSWORD: tembatemba - MINIO_DEFAULT_BUCKETS: temba-archives - ports: - - 9000:9000 - options: --health-cmd "mc ready local" --health-interval 10s --health-timeout 5s --health-retries 5 - steps: - name: Checkout code uses: actions/checkout@v4 @@ -37,6 +27,18 @@ jobs: with: go-version: ${{ env.go-version }} + - name: Run MinIO docker container + run: | + docker run -d --rm \ + -p 9000:9000 \ + -p 9001:9001 \ + --env MINIO_ROOT_USER=root \ + --env MINIO_ROOT_PASSWORD=tembatemba \ + --health-cmd "curl -f http://127.0.0.1:9000/minio/health/live" \ + --health-interval 10s \ + --health-timeout 5s \ + --health-retries 5 \ + minio/minio:RELEASE.2025-07-23T15-54-02Z minio server /data --console-address ":9001" - name: Run tests run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./... diff --git a/archives/archives_test.go b/archives/archives_test.go index 933f307..52a6e74 100644 --- a/archives/archives_test.go +++ b/archives/archives_test.go @@ -9,9 +9,12 @@ import ( "testing" "time" + "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/service/s3" "github.com/jmoiron/sqlx" _ "github.com/lib/pq" "github.com/nyaruka/gocommon/aws/cwatch" + "github.com/nyaruka/gocommon/aws/s3x" "github.com/nyaruka/rp-archiver/runtime" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -26,6 +29,7 @@ func setup(t *testing.T) (context.Context, *runtime.Runtime) { config.AWSAccessKeyID = "root" config.AWSSecretAccessKey = "tembatemba" config.S3Endpoint = "http://localhost:9000" + config.S3Bucket = "test-archives" config.S3Minio = true config.DeploymentID = "test" @@ -38,19 +42,34 @@ func setup(t *testing.T) (context.Context, *runtime.Runtime) { _, err = db.Exec(string(testDB)) require.NoError(t, err) - s3Client, err := NewS3Client(config) - require.NoError(t, err) + svc, err := s3x.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, config.S3Endpoint, config.S3Minio) + assert.NoError(t, err) + + err = svc.Test(ctx, config.S3Bucket) + assert.ErrorContains(t, err, "NotFound") + + _, err = svc.Client.CreateBucket(ctx, &s3.CreateBucketInput{Bucket: aws.String(config.S3Bucket)}) + assert.NoError(t, err) slog.SetDefault(slog.New(slog.NewTextHandler(os.Stdout, &slog.HandlerOptions{Level: slog.LevelDebug}))) CW, err := cwatch.NewService(config.AWSAccessKeyID, config.AWSSecretAccessKey, config.AWSRegion, config.CloudwatchNamespace, config.DeploymentID) require.NoError(t, err) - return ctx, &runtime.Runtime{Config: config, DB: db, S3: s3Client, CW: CW} + return ctx, &runtime.Runtime{Config: config, DB: db, S3: svc, CW: CW} +} + +func teardown(t *testing.T, ctx context.Context, rt *runtime.Runtime) { + err := rt.S3.EmptyBucket(ctx, rt.Config.S3Bucket) + assert.NoError(t, err) + _, err = rt.S3.Client.DeleteBucket(ctx, &s3.DeleteBucketInput{Bucket: aws.String(rt.Config.S3Bucket)}) + assert.NoError(t, err) + } func TestGetMissingDayArchives(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) orgs, err := GetActiveOrgs(ctx, rt) assert.NoError(t, err) @@ -97,6 +116,7 @@ func TestGetMissingDayArchives(t *testing.T) { func TestGetMissingMonthArchives(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) orgs, err := GetActiveOrgs(ctx, rt) assert.NoError(t, err) @@ -125,6 +145,7 @@ func TestGetMissingMonthArchives(t *testing.T) { func TestCreateMsgArchive(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) err := EnsureTempArchiveDirectory("/tmp") assert.NoError(t, err) @@ -200,6 +221,7 @@ func assertArchiveFile(t *testing.T, archive *Archive, truthName string) { func TestCreateRunArchive(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) err := EnsureTempArchiveDirectory("/tmp") assert.NoError(t, err) @@ -258,6 +280,7 @@ func TestCreateRunArchive(t *testing.T) { func TestWriteArchiveToDB(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) orgs, err := GetActiveOrgs(ctx, rt) assert.NoError(t, err) @@ -309,6 +332,7 @@ func getCountInRange(db *sqlx.DB, query string, orgID int, start time.Time, end func TestArchiveOrgMessages(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) deleteTransactionSize = 1 @@ -417,6 +441,7 @@ func assertArchive(t *testing.T, a *Archive, startDate time.Time, period Archive func TestArchiveOrgRuns(t *testing.T) { ctx, rt := setup(t) + defer teardown(t, ctx, rt) orgs, err := GetActiveOrgs(ctx, rt) assert.NoError(t, err) @@ -494,7 +519,8 @@ func TestArchiveOrgRuns(t *testing.T) { } func TestArchiveActiveOrgs(t *testing.T) { - _, rt := setup(t) + ctx, rt := setup(t) + defer teardown(t, ctx, rt) err := ArchiveActiveOrgs(rt) assert.NoError(t, err) From 3c63a0d021435b44116452c78d08560242810f44 Mon Sep 17 00:00:00 2001 From: Norbert Kwizera Date: Thu, 18 Sep 2025 11:47:15 +0200 Subject: [PATCH 2/2] Use latest minio image in CI --- .github/workflows/ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 59b88bf..c8598b7 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -38,7 +38,7 @@ jobs: --health-interval 10s \ --health-timeout 5s \ --health-retries 5 \ - minio/minio:RELEASE.2025-07-23T15-54-02Z minio server /data --console-address ":9001" + minio/minio:latest minio server /data --console-address ":9001" - name: Run tests run: go test -p=1 -coverprofile=coverage.text -covermode=atomic ./...