Skip to content

Commit

Permalink
test(providers): cleanup GCS tests (kopia#4138)
Browse files Browse the repository at this point in the history 
* use `getEnvVarOrSkip()` helper
* use creds from JSON env for test to actually run the test
* prefer require
* nit: relocate helpers
* fail tests if credentials cannot be decoded
  • Loading branch information
@julio-lopez
julio-lopez authored Sep 27, 2024
1 parent 37c7dae commit 8ce8245
Show file tree
Hide file tree
Showing 3 changed files with 75 additions and 93 deletions.
11 changes: 5 additions & 6 deletions repo/blob/gcs/gcs_immu_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,7 +4,6 @@ import (
"context"
"crypto/rand"
"fmt"
"os"
"testing"
"time"

Expand All @@ -26,7 +25,7 @@ func TestGoogleStorageImmutabilityProtection(t *testing.T) {
testutil.ProviderTest(t)

opts := bucketOpts{
projectID: os.Getenv(testBucketProjectID),
projectID: getEnvVarOrSkip(t, testBucketProjectID),
bucket: getImmutableBucketNameOrSkip(t),
credentialsJSON: getCredJSONFromEnv(t),
isLockedBucket: true,
Expand Down Expand Up @@ -67,11 +66,12 @@ func TestGoogleStorageImmutabilityProtection(t *testing.T) {
}
err = st.PutBlob(ctx, dummyBlob, gather.FromSlice([]byte("x")), putOpts)
require.NoError(t, err)
cli := getGoogleCLI(t, opts.credentialsJSON)

count := getBlobCount(ctx, t, st, dummyBlob[:1])
require.Equal(t, 1, count)

cli := getGoogleCLI(t, opts.credentialsJSON)

attrs, err := cli.Bucket(opts.bucket).Object(blobNameFullPath).Attrs(ctx)
require.NoError(t, err)

Expand Down Expand Up @@ -117,9 +117,8 @@ func getGoogleCLI(t *testing.T, credentialsJSON []byte) *gcsclient.Client {

ctx := context.Background()
cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(credentialsJSON))
if err != nil {
t.Fatalf("unable to create GCS client: %v", err)
}

require.NoError(t, err, "unable to create GCS client")

return cli
}
93 changes: 10 additions & 83 deletions repo/blob/gcs/gcs_storage_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,10 @@ import (
"encoding/base64"
"io"
"os"
"strings"
"testing"

gcsclient "cloud.google.com/go/storage"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"google.golang.org/api/option"

"github.com/kopia/kopia/internal/blobtesting"
"github.com/kopia/kopia/internal/providervalidation"
Expand All @@ -26,69 +23,10 @@ import (
const (
testBucketEnv = "KOPIA_GCS_TEST_BUCKET"
testBucketProjectID = "KOPIA_GCS_TEST_PROJECT_ID"
testBucketCredentialsFile = "KOPIA_GCS_CREDENTIALS_FILE"
testBucketCredentialsJSONGzip = "KOPIA_GCS_CREDENTIALS_JSON_GZIP"
testImmutableBucketEnv = "KOPIA_GCS_TEST_IMMUTABLE_BUCKET"
)

type bucketOpts struct {
bucket string
credentialsJSON []byte
projectID string
isLockedBucket bool
}

func createBucket(t *testing.T, opts bucketOpts) {
t.Helper()
ctx := context.Background()

cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(opts.credentialsJSON))
if err != nil {
t.Fatalf("unable to create GCS client: %v", err)
}

attrs := &gcsclient.BucketAttrs{}

bucketHandle := cli.Bucket(opts.bucket)
if opts.isLockedBucket {
attrs.VersioningEnabled = true
bucketHandle = bucketHandle.SetObjectRetention(true)
}

err = bucketHandle.Create(ctx, opts.projectID, attrs)
if err == nil {
return
}

if strings.Contains(err.Error(), "The requested bucket name is not available") {
return
}

if strings.Contains(err.Error(), "Your previous request to create the named bucket succeeded and you already own it") {
return
}

t.Fatalf("issue creating bucket: %v", err)
}

func validateBucket(t *testing.T, opts bucketOpts) {
t.Helper()
ctx := context.Background()

cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(opts.credentialsJSON))
if err != nil {
t.Fatalf("unable to create GCS client: %v", err)
}

attrs, err := cli.Bucket(opts.bucket).Attrs(ctx)
require.NoError(t, err)

if opts.isLockedBucket {
require.True(t, attrs.VersioningEnabled)
require.Equal(t, "Enabled", attrs.ObjectRetentionMode)
}
}

func TestCleanupOldData(t *testing.T) {
t.Parallel()
testutil.ProviderTest(t)
Expand Down Expand Up @@ -128,16 +66,15 @@ func TestGCSStorageInvalid(t *testing.T) {
t.Parallel()
testutil.ProviderTest(t)

bucket := os.Getenv(testBucketEnv)
bucket := getEnvVarOrSkip(t, testBucketEnv)

ctx := testlogging.Context(t)

if _, err := gcs.New(ctx, &gcs.Options{
BucketName: bucket + "-no-such-bucket",
ServiceAccountCredentialsFile: os.Getenv(testBucketCredentialsFile),
}, false); err == nil {
t.Fatalf("unexpected success connecting to GCS, wanted error")
}
_, err := gcs.New(ctx, &gcs.Options{
BucketName: bucket + "-no-such-bucket",
ServiceAccountCredentialJSON: getCredJSONFromEnv(t),
}, false)
require.Error(t, err, "unexpected success connecting to GCS, wanted error")
}

func gunzip(d []byte) ([]byte, error) {
Expand Down Expand Up @@ -165,31 +102,21 @@ func getEnvVarOrSkip(t *testing.T, envVarName string) string {
func getCredJSONFromEnv(t *testing.T) []byte {
t.Helper()

b64Data := os.Getenv(testBucketCredentialsJSONGzip)
if b64Data == "" {
t.Skip(testBucketCredentialsJSONGzip + "is not set")
}
b64Data := getEnvVarOrSkip(t, testBucketCredentialsJSONGzip)

credDataGZ, err := base64.StdEncoding.DecodeString(b64Data)
if err != nil {
t.Skip("skipping test because GCS credentials file can't be decoded")
}
require.NoError(t, err, "GCS credentials env value can't be decoded")

credJSON, err := gunzip(credDataGZ)
if err != nil {
t.Skip("skipping test because GCS credentials file can't be unzipped")
}
require.NoError(t, err, "GCS credentials env can't be unzipped")

return credJSON
}

func mustGetOptionsOrSkip(t *testing.T, prefix string) *gcs.Options {
t.Helper()

bucket := os.Getenv(testBucketEnv)
if bucket == "" {
t.Skip("KOPIA_GCS_TEST_BUCKET not provided")
}
bucket := getEnvVarOrSkip(t, testBucketEnv)

return &gcs.Options{
BucketName: bucket,
Expand Down
64 changes: 60 additions & 4 deletions repo/blob/gcs/gcs_versioned_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,12 +4,14 @@ import (
"context"
"crypto/rand"
"fmt"
"os"
"strings"
"testing"
"time"

gcsclient "cloud.google.com/go/storage"
"github.com/pkg/errors"
"github.com/stretchr/testify/require"
"google.golang.org/api/option"

"github.com/kopia/kopia/internal/clock"
"github.com/kopia/kopia/internal/gather"
Expand All @@ -19,12 +21,19 @@ import (
"github.com/kopia/kopia/repo/blob/gcs"
)

type bucketOpts struct {
bucket string
credentialsJSON []byte
projectID string
isLockedBucket bool
}

func TestGetBlobVersionsFailsWhenVersioningDisabled(t *testing.T) {
t.Parallel()
testutil.ProviderTest(t)

// must be with Versioning disabled.
bucket := os.Getenv(testBucketEnv)
bucket := getEnvVarOrSkip(t, testBucketEnv)

ctx := testlogging.Context(t)
data := make([]byte, 8)
Expand Down Expand Up @@ -58,7 +67,7 @@ func TestGetBlobVersions(t *testing.T) {

// must be with Versioning enabled.
bOpts := bucketOpts{
projectID: os.Getenv(testBucketProjectID),
projectID: getEnvVarOrSkip(t, testBucketProjectID),
bucket: getImmutableBucketNameOrSkip(t),
credentialsJSON: getCredJSONFromEnv(t),
isLockedBucket: true,
Expand Down Expand Up @@ -165,7 +174,7 @@ func TestGetBlobVersionsWithDeletion(t *testing.T) {

// must be with Versioning enabled.
bOpts := bucketOpts{
projectID: os.Getenv(testBucketProjectID),
projectID: getEnvVarOrSkip(t, testBucketProjectID),
bucket: getImmutableBucketNameOrSkip(t),
credentialsJSON: getCredJSONFromEnv(t),
isLockedBucket: true,
Expand Down Expand Up @@ -257,6 +266,53 @@ func putBlobs(ctx context.Context, cli blob.Storage, blobID blob.ID, blobs []str
return putTimes, nil
}

func createBucket(t *testing.T, opts bucketOpts) {
t.Helper()
ctx := context.Background()

cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(opts.credentialsJSON))
require.NoError(t, err, "unable to create GCS client")

attrs := &gcsclient.BucketAttrs{}

bucketHandle := cli.Bucket(opts.bucket)
if opts.isLockedBucket {
attrs.VersioningEnabled = true
bucketHandle = bucketHandle.SetObjectRetention(true)
}

err = bucketHandle.Create(ctx, opts.projectID, attrs)
if err == nil {
return
}

if strings.Contains(err.Error(), "The requested bucket name is not available") {
return
}

if strings.Contains(err.Error(), "Your previous request to create the named bucket succeeded and you already own it") {
return
}

t.Fatalf("issue creating bucket: %v", err)
}

func validateBucket(t *testing.T, opts bucketOpts) {
t.Helper()
ctx := context.Background()

cli, err := gcsclient.NewClient(ctx, option.WithCredentialsJSON(opts.credentialsJSON))
require.NoError(t, err, "unable to create GCS client")

attrs, err := cli.Bucket(opts.bucket).Attrs(ctx)
require.NoError(t, err)

if opts.isLockedBucket {
require.True(t, attrs.VersioningEnabled)
require.Equal(t, "Enabled", attrs.ObjectRetentionMode)
}
}

func getImmutableBucketNameOrSkip(t *testing.T) string {
t.Helper()

Expand Down

0 comments on commit 8ce8245

Please sign in to comment.