diff --git a/cmd/migrate.go b/cmd/migrate.go index 4ad0f08..2ca8dbb 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -40,6 +40,8 @@ var ( chunkSize int64 chunkNumber int batchSize int + source string + accessToken string // if source is google drive or dropbox ) // migrateCmd is the migrateFromS3 sub command to migrate whole objects from some buckets. @@ -71,7 +73,8 @@ func init() { migrateCmd.Flags().Int64Var(&chunkSize, "chunk-size", 50*1024*1024, "chunk size in bytes") migrateCmd.Flags().IntVar(&chunkNumber, "chunk-number", 250, "number of chunks to upload") migrateCmd.Flags().IntVar(&batchSize, "batch-size", 20, "number of files to upload in a batch") - + migrateCmd.Flags().StringVar(&source, "source", "s3", "s3 or google_drive or dropbox") + migrateCmd.Flags().StringVar(&accessToken, "access-token", "", "access token for google drive or dropbox") } var migrateCmd = &cobra.Command{ @@ -111,7 +114,11 @@ var migrateCmd = &cobra.Command{ } } - if accessKey == "" || secretKey == "" { + if source == "" { + source = "s3" + } + + if (accessKey == "" || secretKey == "") && source == "s3" { if accessKey, secretKey = util.GetAwsCredentialsFromEnv(); accessKey == "" || secretKey == "" { if awsCredPath == "" { return errors.New("aws credentials missing") @@ -122,7 +129,12 @@ var migrateCmd = &cobra.Command{ } } - if bucket == "" { + if accessToken == "" { + if accessToken = util.GetAccessToken(); accessToken == "" { + return errors.New("Missing Access Token") + } + } + if bucket == "" && source == "s3" { bucket, region, prefix, err = util.GetBucketRegionPrefixFromFile(awsCredPath) if err != nil { return err @@ -232,6 +244,9 @@ var migrateCmd = &cobra.Command{ ChunkSize: chunkSize, ChunkNumber: chunkNumber, BatchSize: batchSize, + + Source: source, + AccessToken: accessToken, } if err := migration.InitMigration(&mConfig); err != nil { diff --git a/dropbox/core.go b/dropbox/core.go new file mode 100644 index 0000000..86bd28c --- /dev/null +++ b/dropbox/core.go @@ -0,0 +1,184 @@ +package dropbox + +import ( + "context" + "fmt" + "io" + "mime" + "os" + "path" + "path/filepath" + + T "github.com/0chain/s3migration/types" + "github.com/pkg/errors" + + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" +) + +type DropboxClient struct { + token string + workDir string + dropboxConf *dropbox.Config + dropboxFiles files.Client +} + +func GetDropboxClient(token string, workDir string) (*DropboxClient, error) { + config := dropbox.Config{ + Token: token, + } + + client := files.New(config) + + // for invalid Access token + arg := files.NewListFolderArg("") + + _, err := client.ListFolder(arg) + + if err != nil { + return nil, errors.Wrap(err, "invalid Client token") + } + + return &DropboxClient{ + token: token, + dropboxConf: &config, + dropboxFiles: client, + workDir: workDir, + }, nil +} + +func (d *DropboxClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) { + objectChan := make(chan *T.ObjectMeta) + errChan := make(chan error) + + go func() { + defer func() { + close(objectChan) + close(errChan) + }() + + arg := files.NewListFolderArg("") // "" for Root + arg.Recursive = true + arg.Limit = 100 + arg.IncludeNonDownloadableFiles = false + + res, err := d.dropboxFiles.ListFolder(arg) + if err != nil { + errChan <- err + return + } + + for _, entry := range res.Entries { + if meta, ok := entry.(*files.FileMetadata); ok { + objectChan <- &T.ObjectMeta{ + Key: meta.PathDisplay, + Size: int64(meta.Size), + ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + Ext: filepath.Ext(meta.PathDisplay), + } + } + } + + cursor := res.Cursor + hasMore := res.HasMore + + for hasMore { + continueArg := files.NewListFolderContinueArg(cursor) + res, err := d.dropboxFiles.ListFolderContinue(continueArg) + if err != nil { + errChan <- err + return + } + + for _, entry := range res.Entries { + if meta, ok := entry.(*files.FileMetadata); ok { + objectChan <- &T.ObjectMeta{ + Key: meta.PathDisplay, + Size: int64(meta.Size), + ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + } + } + } + + cursor = res.Cursor + hasMore = res.HasMore + } + }() + + return objectChan, errChan +} + +func (d *DropboxClient) GetFileContent(ctx context.Context, filePath string) (*T.Object, error) { + arg := files.NewDownloadArg(filePath) + res, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return nil, err + } + + return &T.Object{ + Body: content, + ContentType: mime.TypeByExtension(filepath.Ext(filePath)), + ContentLength: int64(res.Size), + }, nil +} + +func (d *DropboxClient) DeleteFile(ctx context.Context, filePath string) error { + arg := files.NewDeleteArg(filePath) + _, err := d.dropboxFiles.DeleteV2(arg) + return err +} + +func (d *DropboxClient) DownloadToFile(ctx context.Context, filePath string) (string, error) { + arg := files.NewDownloadArg(filePath) + _, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return "", err + } + + fileName := filepath.Base(filePath) + downloadPath := path.Join(d.workDir, fileName) + file, err := os.Create(downloadPath) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.Copy(file, content) + if err != nil { + return "", err + } + + return downloadPath, nil +} + +func (d *DropboxClient) DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) { + limit := offset + chunkSize - 1 + if limit > objectSize { + limit = objectSize + } + + rng := fmt.Sprintf("bytes=%d-%d", offset, limit) + + arg := files.NewDownloadArg(objectKey) + + arg.ExtraHeaders = map[string]string{"Range": rng} + + _, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return nil, err + } + defer content.Close() + + data := make([]byte, chunkSize) + n, err := io.ReadFull(content, data) + + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + + if int64(n) < chunkSize && objectSize != chunkSize { + data = data[:n] + } + + return data, nil +} diff --git a/dropbox/test.go b/dropbox/test.go new file mode 100644 index 0000000..dfe200e --- /dev/null +++ b/dropbox/test.go @@ -0,0 +1,150 @@ +package dropbox + +import ( + "context" + "fmt" + "testing" + + zlogger "github.com/0chain/s3migration/logger" +) + +var ( + dropboxAccessToken = "" + testFilePath = "" +) + +const TestFileContent = ` by Manuel Gutiérrez Nájera + +I want to die as the day declines, +at high sea and facing the sky, +while agony seems like a dream +and my soul like a bird that can fly. + +To hear not, at this last moment, +once alone with sky and sea, +any more voices nor weeping prayers +than the majestic beating of the waves. + +To die when the sad light retires +its golden network from the green waves +to be like the sun that slowly expires; +something very luminous that fades. + +To die, and die young, before +fleeting time removes the gentle crown, +while life still says: "I'm yours" +though we know with our hearts that she lies. +` + +func TestDropboxClient_ListFiles(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) + return + } + + ctx := context.Background() + objectChan, errChan := client.ListFiles(ctx) + + go func() { + for err := range errChan { + zlogger.Logger.Error(fmt.Sprintf("Error while listing files: %v", err)) + } + }() + + for object := range objectChan { + zlogger.Logger.Info(fmt.Sprintf("Key: %s, Type: %s, Size: %d bytes", object.Key, object.ContentType, object.Size)) + } +} + +func TestDropboxClient_GetFileContent(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) + } + + ctx := context.Background() + filePath := testFilePath + obj, err := client.GetFileContent(ctx, filePath) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Error while getting file content: %v", err)) + return + } + defer obj.Body.Close() + + zlogger.Logger.Info(fmt.Sprintf("File content type: %s, Length: %d", obj.ContentType, obj.ContentLength)) + + if (obj.Body == nil) || (obj.ContentLength == 0) { + fmt.Println("Empty file content") + return + } + + buf := make([]byte, obj.ContentLength) + n, err := obj.Body.Read(buf) + + if err != nil && err.Error() != "EOF" { + zlogger.Logger.Error(fmt.Sprintf("Error while reading file content: %v", err)) + return + } + + zlogger.Logger.Info(fmt.Sprintf("File content: %s", string(buf[:n]))) +} + +func TestDropboxClient_DeleteFile(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) + return + } + + ctx := context.Background() + filePath := testFilePath + err = client.DeleteFile(ctx, filePath) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Error while deleting file: %v", err)) + return + } + zlogger.Logger.Info(fmt.Sprintf("File %s deleted successfully", filePath)) +} + +func TestDropboxClient_DownloadToFile(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) + return + } + + ctx := context.Background() + filePath := testFilePath + downloadedPath, err := client.DownloadToFile(ctx, filePath) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Error while downloading file: %v", err)) + return + } + zlogger.Logger.Info(fmt.Sprintf("Downloaded to: %s", downloadedPath)) +} + +func TestDropboxClient_DownloadToMemory(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) + return + } + + ctx := context.Background() + + filePath := testFilePath + offset := int64(0) + + // half chunk + chunkSize := int64(313) + objectSize := int64(626) + + data, err := client.DownloadToMemory(ctx, filePath, offset, chunkSize, objectSize) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Error while downloading file: %v", err)) + return + } + + zlogger.Logger.Info(fmt.Sprintf("Downloaded data: %s", data)) +} diff --git a/gdrive/core.go b/gdrive/core.go new file mode 100644 index 0000000..5c85d21 --- /dev/null +++ b/gdrive/core.go @@ -0,0 +1,194 @@ +package gdrive + +import ( + "context" + "fmt" + "io" + "os" + "path" + + zlogger "github.com/0chain/s3migration/logger" + T "github.com/0chain/s3migration/types" + "github.com/pkg/errors" + "golang.org/x/oauth2" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +type GoogleDriveClient struct { + service *drive.Service + workDir string +} + +func NewGoogleDriveClient(accessToken string, workDir string) (*GoogleDriveClient, error) { + ctx := context.Background() + + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: accessToken}) + + httpClient := oauth2.NewClient(ctx, tokenSource) + + service, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) + if err != nil { + return nil, err + } + _, err = service.Files.List().Do() + + if err != nil { + return nil, errors.Wrap(err, "invalid Google Drive access token") + } + + return &GoogleDriveClient{ + service: service, + workDir: workDir, + }, nil +} + +func (g *GoogleDriveClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) { + objectChan := make(chan *T.ObjectMeta) + errChan := make(chan error) + + go func() { + defer func() { + close(objectChan) + close(errChan) + }() + + filesReq := g.service.Files.List().Context(ctx) + + filesReq.Q("trashed=false") + + filesReq.Fields( + "files(id, mimeType, size,fileExtension)", + ) + + filesReq.Pages(ctx, func(page *drive.FileList) error { + return nil + }) + + filesReq.PageSize(100) + + files, err := filesReq.Do() + if err != nil { + errChan <- err + return + } + + for _, file := range files.Files { + objectChan <- &T.ObjectMeta{ + Key: file.Id, + Size: file.Size, + ContentType: file.MimeType, + Ext: file.FileExtension, + } + } + + nextPgToken := files.NextPageToken + + for nextPgToken != "" { + filesReq.PageToken(nextPgToken) + + files, err := filesReq.Do() + + if err != nil { + errChan <- err + return + } + + for _, file := range files.Files { + objectChan <- &T.ObjectMeta{ + Key: file.Id, + Size: file.Size, + ContentType: file.MimeType, + } + } + + nextPgToken = files.NextPageToken + } + }() + + return objectChan, errChan +} + +func (g *GoogleDriveClient) GetFileContent(ctx context.Context, fileID string) (*T.Object, error) { + resp, err := g.service.Files.Get(fileID).Download() + if err != nil { + return nil, err + } + + // if !keepOpen { + // defer resp.Body.Close() + // } + + obj := &T.Object{ + Body: resp.Body, + ContentType: resp.Header.Get("Content-Type"), + ContentLength: resp.ContentLength, + } + + return obj, nil +} + +func (g *GoogleDriveClient) DeleteFile(ctx context.Context, fileID string) error { + err := g.service.Files.Delete(fileID).Do() + if err != nil { + return err + } + return nil +} + +func (g *GoogleDriveClient) DownloadToFile(ctx context.Context, fileID string) (string, error) { + resp, err := g.service.Files.Get(fileID).Download() + if err != nil { + return "", err + } + defer resp.Body.Close() + + destinationPath := path.Join(g.workDir, fileID) + + out, err := os.Create(destinationPath) + if err != nil { + return "", err + } + + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + return "", err + } + + zlogger.Logger.Info(fmt.Sprintf("Downloaded file ID: %s to %s\n", fileID, destinationPath)) + return destinationPath, nil +} + +func (g *GoogleDriveClient) DownloadToMemory(ctx context.Context, fileID string, offset int64, chunkSize, fileSize int64) ([]byte, error) { + limit := offset + chunkSize - 1 + if limit > fileSize { + limit = fileSize + } + + rng := fmt.Sprintf("bytes=%d-%d", offset, limit) + + req := g.service.Files.Get(fileID) + + req.Header().Set("Range", rng) + + resp, err := req.Download() + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data := make([]byte, chunkSize) + n, err := io.ReadFull(resp.Body, data) + + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + + if int64(n) < chunkSize && fileSize != chunkSize { + data = data[:n] + } + + return data, nil +} diff --git a/gdrive/test.go b/gdrive/test.go new file mode 100644 index 0000000..d9de872 --- /dev/null +++ b/gdrive/test.go @@ -0,0 +1,160 @@ +package gdrive + +import ( + "context" + "fmt" + "testing" + + zlogger "github.com/0chain/s3migration/logger" +) + +var ( + driveAccessToken = "ya29.a0Ad52N384rCQyODIBhljd7yg4RmP8czIUmIbJGkFHXNn-TnuEt3RL3ykzp0lEekSEoU0GDCkMnow31XOWFt0Dlw2l89fnO5sS8aiZKW8y0dDhqE6jtsNqpnA38EN5tBlRkNo0ipyY0Ps-cgX4hNxRRQcWhTZEqZgnOyLuaCgYKAWESARESFQHGX2Mi6ejaNEiKSZP6_H3hoXkeNA0171" + testFileID = "" +) + +// using: https://developers.google.com/oauthplayground + +// For reference (626 bytes text file) +const TestFileContent = ` by Manuel Gutiérrez Nájera + +I want to die as the day declines, +at high sea and facing the sky, +while agony seems like a dream +and my soul like a bird that can fly. + +To hear not, at this last moment, +once alone with sky and sea, +any more voices nor weeping prayers +than the majestic beating of the waves. + +To die when the sad light retires +its golden network from the green waves +to be like the sun that slowly expires; +something very luminous that fades. + +To die, and die young, before +fleeting time removes the gentle crown, +while life still says: "I'm yours" +though we know with our hearts that she lies. +` + +func TestGoogleDriveClient_ListFiles(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) + return + } + + ctx := context.Background() + objectChan, errChan := client.ListFiles(ctx) + + go func() { + for err := range errChan { + zlogger.Logger.Error(fmt.Sprintf("err while list files: %v", err)) + } + }() + + for object := range objectChan { + zlogger.Logger.Info(fmt.Sprintf("file:%s, size: %d bytes, type: %s", object.Key, object.Size, object.ContentType)) + } +} + +func TestGoogleDriveClient_GetFileContent(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("Failed to creating Google Drive client: %v", err)) + return + } + + ctx := context.Background() + fileID := testFileID + obj, err := client.GetFileContent(ctx, fileID) + + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while getting file content: %v", err)) + return + } + + defer obj.Body.Close() + + zlogger.Logger.Info(fmt.Sprintf("file content type: %s, length: %d", obj.ContentType, obj.ContentLength)) + + if (obj.Body == nil) || (obj.ContentLength == 0) { + zlogger.Logger.Info("empty file content") + return + } + + buf := make([]byte, obj.ContentLength) + n, err := obj.Body.Read(buf) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while read file content: %v", err)) + return + } + zlogger.Logger.Info(fmt.Sprintf("read data: %s", buf[:n])) +} + +func TestGoogleDriveClient_DeleteFile(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) + return + } + + ctx := context.Background() + fileID := testFileID + err = client.DeleteFile(ctx, fileID) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while delete file: %v", err)) + return + } + zlogger.Logger.Error(fmt.Sprintf("file: %s deleted successfully", fileID)) +} + +func TestGoogleDriveClient_DownloadToFile(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) + } + + ctx := context.Background() + fileID := testFileID + destinationPath, err := client.DownloadToFile(ctx, fileID) + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while downloading file: %v", err)) + return + } + zlogger.Logger.Info(fmt.Sprintf("downloaded to: %s", destinationPath)) +} + +func TestGoogleDriveClient_DownloadToMemory(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken, "./") + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) + } + + ctx := context.Background() + + fileID := testFileID + + offset := int64(0) + + // download only half chunk for testing + chunkSize := int64(313) + + fileSize := int64(626) + + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while getting file size: %v", err)) + return + } + + data, err := client.DownloadToMemory(ctx, fileID, offset, chunkSize, fileSize) + + if err != nil { + zlogger.Logger.Error(fmt.Sprintf("err while downloading file: %v", err)) + return + } + + zlogger.Logger.Info(fmt.Sprintf("downloaded data: %s", data)) +} diff --git a/go.mod b/go.mod index 3e09ae3..9d3d792 100644 --- a/go.mod +++ b/go.mod @@ -2,6 +2,7 @@ module github.com/0chain/s3migration go 1.21 + toolchain go1.22.2 require ( @@ -14,6 +15,36 @@ require ( github.com/golang/mock v1.6.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 + golang.org/x/oauth2 v0.18.0 +) + +require ( + cloud.google.com/go/compute v1.23.4 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect + github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/net v0.22.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect ) require ( @@ -41,11 +72,12 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 github.com/ethereum/go-ethereum v1.10.26 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-stack/stack v1.8.1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb // indirect github.com/hack-pad/go-webworkers v0.1.0 // indirect diff --git a/go.sum b/go.sum index 16eb80f..6f76a6e 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,10 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= +cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -144,6 +148,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeC github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU= +github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -154,6 +160,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.10.26 h1:i/7d9RBBwiXCEuyduBQzJw/mKmnvzsN14jqBmytw72s= github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= @@ -167,6 +175,11 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -177,6 +190,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -200,6 +215,10 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -213,7 +232,9 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= @@ -231,12 +252,18 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= @@ -244,10 +271,6 @@ github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWm github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb h1:GlQyMv2C48qmfPItvAXFoyN341Swxp9JNVeUZxnmbJw= github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= -github.com/hack-pad/go-webworkers v0.1.0 h1:QHBJpkXJgW0QRi2iiUGcxwGnmy7lQJL0F8UfsgMXKhA= -github.com/hack-pad/go-webworkers v0.1.0/go.mod h1:/rmjjgnlw0CursmeqRtP0NGIqo8CR+0o6AtzFydUHJ4= -github.com/hack-pad/safejs v0.1.1 h1:d5qPO0iQ7h2oVtpzGnLExE+Wn9AtytxIfltcS2b9KD8= -github.com/hack-pad/safejs v0.1.1/go.mod h1:HdS+bKF1NrE72VoXZeWzxFOVQVUSqZJAG0xNCnb+Tio= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -415,6 +438,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= @@ -432,6 +456,16 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -452,6 +486,7 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= @@ -489,6 +524,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -521,10 +557,14 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -534,6 +574,8 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -591,12 +633,15 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -660,6 +705,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -683,6 +729,8 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= +google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -690,6 +738,8 @@ google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -726,6 +776,11 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -742,6 +797,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -752,6 +809,10 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/migration/migrate.go b/migration/migrate.go index ff256a6..b5de939 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -14,6 +14,10 @@ import ( "syscall" "time" + "github.com/0chain/s3migration/dropbox" + "github.com/0chain/s3migration/gdrive" + T "github.com/0chain/s3migration/types" + "github.com/0chain/gosdk/zboxcore/sdk" "github.com/0chain/gosdk/zboxcore/zboxutil" dStorage "github.com/0chain/s3migration/dstorage" @@ -58,9 +62,9 @@ func abandonAllOperations(err error) { } type Migration struct { - zStore dStorage.DStoreI - awsStore s3.AwsI - fs util.FileSystem + zStore dStorage.DStoreI + dataSourceStore T.CloudStorageI + fs util.FileSystem skip int retryCount int @@ -119,36 +123,55 @@ func InitMigration(mConfig *MigrationConfig) error { return err } mConfig.ChunkSize = int64(mConfig.ChunkNumber) * dStorageService.GetChunkWriteSize() - zlogger.Logger.Info("Getting aws storage service") - awsStorageService, err := s3.GetAwsClient( - mConfig.Bucket, - mConfig.Prefix, - mConfig.Region, - mConfig.DeleteSource, - mConfig.NewerThan, - mConfig.OlderThan, - mConfig.StartAfter, - mConfig.WorkDir, - ) + zlogger.Logger.Info(fmt.Sprintf("Getting %v storage service", mConfig.Source)) + + var dataSourceStore T.CloudStorageI + if mConfig.Source == "s3" { + dataSourceStore, err = s3.GetAwsClient( + mConfig.Bucket, + mConfig.Prefix, + mConfig.Region, + mConfig.DeleteSource, + mConfig.NewerThan, + mConfig.OlderThan, + mConfig.StartAfter, + mConfig.WorkDir, + ) + } else if mConfig.Source == "dropbox" { + dataSourceStore, err = dropbox.GetDropboxClient( + mConfig.AccessToken, + mConfig.WorkDir, + ) + } else if mConfig.Source == "google_drive" { + dataSourceStore, err = gdrive.NewGoogleDriveClient( + mConfig.AccessToken, + mConfig.WorkDir, + ) + } else { + zlogger.Logger.Error("invalid source: ", mConfig.Source) + return err + } + + zlogger.Logger.Info(dataSourceStore, "data source info") if err != nil { zlogger.Logger.Error(err) return err } migration = Migration{ - zStore: dStorageService, - awsStore: awsStorageService, - skip: mConfig.Skip, - concurrency: mConfig.Concurrency, - retryCount: mConfig.RetryCount, - stateFilePath: mConfig.StateFilePath, - migrateTo: mConfig.MigrateToPath, - deleteSource: mConfig.DeleteSource, - workDir: mConfig.WorkDir, - bucket: mConfig.Bucket, - fs: util.Fs, - chunkSize: mConfig.ChunkSize, - batchSize: mConfig.BatchSize, + zStore: dStorageService, + dataSourceStore: dataSourceStore, + skip: mConfig.Skip, + concurrency: mConfig.Concurrency, + retryCount: mConfig.RetryCount, + stateFilePath: mConfig.StateFilePath, + migrateTo: mConfig.MigrateToPath, + deleteSource: mConfig.DeleteSource, + workDir: mConfig.WorkDir, + bucket: mConfig.Bucket, + fs: util.Fs, + chunkSize: mConfig.ChunkSize, + batchSize: mConfig.BatchSize, } rootContext, rootContextCancel = context.WithCancel(context.Background()) @@ -247,7 +270,7 @@ func (m *Migration) DownloadWorker(ctx context.Context, migrator *MigrationWorke totalObjChan := make(chan struct{}, 100) defer close(totalObjChan) go updateTotalObjects(totalObjChan, m.workDir) - objCh, errCh := migration.awsStore.ListFilesInBucket(rootContext) + objCh, errCh := migration.dataSourceStore.ListFiles(rootContext) wg := &sync.WaitGroup{} ops := make([]MigrationOperation, 0, m.batchSize) var opLock sync.Mutex @@ -283,6 +306,10 @@ func (m *Migration) DownloadWorker(ctx context.Context, migrator *MigrationWorke } wg.Add(1) go func() { + defer func(start time.Time) { + zlogger.Logger.Info("downloadObjMeta key: ", downloadObjMeta.ObjectKey, time.Since(start)) + }(time.Now()) + defer wg.Done() err := checkIsFileExist(ctx, downloadObjMeta) if err != nil { @@ -445,6 +472,11 @@ func checkDownloadStatus(downloadObj *DownloadObjectMeta) error { } func processOperation(ctx context.Context, downloadObj *DownloadObjectMeta) (MigrationOperation, error) { + + defer func(start time.Time) { + zlogger.Logger.Info("uploading object key: ", downloadObj.ObjectKey, time.Since(start)) + }(time.Now()) + remotePath := getRemotePath(downloadObj.ObjectKey) var op MigrationOperation fileObj, err := migration.fs.Open(downloadObj.LocalPath) @@ -554,7 +586,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp defer func() { for _, op := range ops { if migration.deleteSource && err == nil { - if deleteErr := migration.awsStore.DeleteFile(ctx, op.uploadObj.ObjectKey); deleteErr != nil { + if deleteErr := migration.dataSourceStore.DeleteFile(ctx, op.uploadObj.ObjectKey); deleteErr != nil { zlogger.Logger.Error(deleteErr) dsFileHandler.Write([]byte(op.uploadObj.ObjectKey + "\n")) } @@ -610,7 +642,7 @@ func (m *Migration) processChunkDownload(ctx context.Context, sw *util.StreamWri return default: } - data, err := m.awsStore.DownloadToMemory(ctx, downloadObjMeta.ObjectKey, int64(offset), int64(chunkSize), downloadObjMeta.Size) + data, err := m.dataSourceStore.DownloadToMemory(ctx, downloadObjMeta.ObjectKey, int64(offset), int64(chunkSize), downloadObjMeta.Size) if err != nil { migrator.DownloadDone(downloadObjMeta, "", err) ctx.Err() diff --git a/migration/migrateConfig.go b/migration/migrateConfig.go index c7d6b50..4a59945 100644 --- a/migration/migrateConfig.go +++ b/migration/migrateConfig.go @@ -22,4 +22,7 @@ type MigrationConfig struct { ChunkSize int64 ChunkNumber int BatchSize int + + Source string // "s3" (default) or "google_drive" or "dropbox" + AccessToken string // if Source == "google_drive" or "dropbox" } diff --git a/migration/migration_test.go b/migration/migration_test.go index 287748a..b02661d 100644 --- a/migration/migration_test.go +++ b/migration/migration_test.go @@ -6,8 +6,9 @@ import ( "log" "testing" + T "github.com/0chain/s3migration/types" + mock_dstorage "github.com/0chain/s3migration/dstorage/mocks" - "github.com/0chain/s3migration/s3" mock_s3 "github.com/0chain/s3migration/s3/mocks" mock_util "github.com/0chain/s3migration/util/mocks" "github.com/golang/mock/gomock" @@ -21,12 +22,12 @@ func TestMigrate(t *testing.T) { dStorageService := mock_dstorage.NewMockDStoreI(ctrl) awsStorageService := mock_s3.NewMockAwsI(ctrl) fileSystem := mock_util.NewMockFileSystem(ctrl) - migration = Migration{ - zStore: dStorageService, - awsStore: awsStorageService, - skip: Skip, - fs: fileSystem, - } + // migration = Migration{ + // zStore: dStorageService, + // awsStore: awsStorageService, + // skip: Skip, + // fs: fileSystem, + // } tests := []struct { name string @@ -39,16 +40,16 @@ func TestMigrate(t *testing.T) { name: "success in uploading files", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) - fileListChan <- &s3.ObjectMeta{ + fileListChan := make(chan *T.ObjectMeta, 1000) + fileListChan <- &T.ObjectMeta{ Key: "file1", Size: 1200, } - fileListChan <- &s3.ObjectMeta{ + fileListChan <- &T.ObjectMeta{ Key: "file2", Size: 1400, } - fileListChan <- &s3.ObjectMeta{ + fileListChan <- &T.ObjectMeta{ Key: "file3", Size: 1500, } @@ -56,7 +57,7 @@ func TestMigrate(t *testing.T) { errChan := make(chan error, 1) close(errChan) - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) awsStorageService.EXPECT().DownloadToFile(gomock.Any(), "file1").Return("/aws/file1", nil) awsStorageService.EXPECT().DownloadToFile(gomock.Any(), "file2").Return("/aws/file2", nil) @@ -106,8 +107,8 @@ func TestMigrate(t *testing.T) { name: "download to file error", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) - fileListChan <- &s3.ObjectMeta{ + fileListChan := make(chan *T.ObjectMeta, 1000) + fileListChan <- &T.ObjectMeta{ Key: "file11", Size: 1200, } @@ -115,7 +116,7 @@ func TestMigrate(t *testing.T) { errChan := make(chan error, 1) close(errChan) - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) updateKeyFunc = func(statePath string) (func(stateKey string), func(), error) { return func(stateKey string) {}, func() {}, nil @@ -132,8 +133,8 @@ func TestMigrate(t *testing.T) { name: "dstorage upload error", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) - fileListChan <- &s3.ObjectMeta{ + fileListChan := make(chan *T.ObjectMeta, 1000) + fileListChan <- &T.ObjectMeta{ Key: "file10", Size: 1200, } @@ -141,7 +142,7 @@ func TestMigrate(t *testing.T) { errChan := make(chan error, 1) close(errChan) - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) updateKeyFunc = func(statePath string) (func(stateKey string), func(), error) { return func(stateKey string) {}, func() {}, nil @@ -171,14 +172,14 @@ func TestMigrate(t *testing.T) { name: "aws list object error", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) + fileListChan := make(chan *T.ObjectMeta, 1000) close(fileListChan) errChan := make(chan error, 1) errChan <- errors.New("some error") - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) updateKeyFunc = func(statePath string) (func(stateKey string), func(), error) { return func(stateKey string) {}, func() {}, nil diff --git a/migration/migration_worker.go b/migration/migration_worker.go index 3c6e66e..6a2a405 100644 --- a/migration/migration_worker.go +++ b/migration/migration_worker.go @@ -38,6 +38,7 @@ type DownloadObjectMeta struct { ErrChan chan error IsFileAlreadyExist bool mimeType string + Ext string } type UploadObjectMeta struct { diff --git a/s3/aws.go b/s3/aws.go index 95394b1..ef803c5 100644 --- a/s3/aws.go +++ b/s3/aws.go @@ -3,12 +3,13 @@ package s3 import ( "context" "fmt" - "io" "os" "path/filepath" "strings" "time" + T "github.com/0chain/s3migration/types" + "github.com/0chain/gosdk/core/encryption" zlogger "github.com/0chain/s3migration/logger" "github.com/aws/aws-sdk-go-v2/aws" @@ -17,28 +18,6 @@ import ( awsS3 "github.com/aws/aws-sdk-go-v2/service/s3" ) -//go:generate mockgen -destination mocks/mock_aws.go -package mock_s3 github.com/0chain/s3migration/s3 AwsI -type AwsI interface { - ListFilesInBucket(ctx context.Context) (<-chan *ObjectMeta, <-chan error) - GetFileContent(ctx context.Context, objectKey string) (*Object, error) - DeleteFile(ctx context.Context, objectKey string) error - DownloadToFile(ctx context.Context, objectKey string) (string, error) - DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) -} - -type Object struct { - Body io.Reader - ContentType string - ContentLength int64 -} - -// ObjectMeta key: object key, size: size of object in bytes -type ObjectMeta struct { - Key string - Size int64 - ContentType string -} - type AwsClient struct { bucket string prefix string @@ -136,8 +115,8 @@ func (a *AwsClient) getBucketRegion() (region string, err error) { return } -func (a *AwsClient) ListFilesInBucket(ctx context.Context) (<-chan *ObjectMeta, <-chan error) { - objectMetaChan := make(chan *ObjectMeta, 1000) +func (a *AwsClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) { + objectMetaChan := make(chan *T.ObjectMeta, 1000) errChan := make(chan error, 1) go func() { @@ -192,20 +171,20 @@ func (a *AwsClient) ListFilesInBucket(ctx context.Context) (<-chan *ObjectMeta, errChan <- err return } - objectMetaChan <- &ObjectMeta{Key: aws.ToString(obj.Key), Size: obj.Size, ContentType: contentType} + objectMetaChan <- &T.ObjectMeta{Key: aws.ToString(obj.Key), Size: obj.Size, ContentType: contentType} } } }() return objectMetaChan, errChan } -func (a *AwsClient) GetFileContent(ctx context.Context, objectKey string) (*Object, error) { +func (a *AwsClient) GetFileContent(ctx context.Context, objectKey string) (*T.Object, error) { out, err := a.client.GetObject(ctx, &awsS3.GetObjectInput{Bucket: aws.String(a.bucket), Key: aws.String(objectKey)}) if err != nil { return nil, err } - return &Object{ + return &T.Object{ Body: out.Body, ContentType: aws.ToString(out.ContentType), ContentLength: out.ContentLength, diff --git a/s3/aws_integration_test.go b/s3/aws_integration_test.go index 9ea7209..b046644 100644 --- a/s3/aws_integration_test.go +++ b/s3/aws_integration_test.go @@ -5,9 +5,11 @@ package s3 import ( "context" - "github.com/0chain/s3migration/util" "log" "testing" + "time" + + "github.com/0chain/s3migration/util" ) func TestService_ListAllBuckets(t *testing.T) { @@ -16,7 +18,7 @@ func TestService_ListAllBuckets(t *testing.T) { util.SetAwsEnvCredentials(awsAccessKey, awsSecretKey) s3Svc, _ := GetAwsClient("", "", "", false, nil, nil, "", "") - fileListChan, errChan := s3Svc.ListFilesInBucket(context.Background()) + fileListChan, errChan := s3Svc.ListFiles(context.Background()) for { objKey, ok := <-fileListChan if ok { diff --git a/s3/mocks/mock_aws.go b/s3/mocks/mock_aws.go index c2232db..c677339 100644 --- a/s3/mocks/mock_aws.go +++ b/s3/mocks/mock_aws.go @@ -8,7 +8,8 @@ import ( context "context" reflect "reflect" - s3 "github.com/0chain/s3migration/s3" + T "github.com/0chain/s3migration/types" + gomock "github.com/golang/mock/gomock" ) @@ -65,10 +66,10 @@ func (mr *MockAwsIMockRecorder) DownloadToFile(arg0, arg1 interface{}) *gomock.C } // GetFileContent mocks base method. -func (m *MockAwsI) GetFileContent(arg0 context.Context, arg1 string) (*s3.Object, error) { +func (m *MockAwsI) GetFileContent(arg0 context.Context, arg1 string) (*T.Object, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetFileContent", arg0, arg1) - ret0, _ := ret[0].(*s3.Object) + ret0, _ := ret[0].(*T.Object) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -79,17 +80,17 @@ func (mr *MockAwsIMockRecorder) GetFileContent(arg0, arg1 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFileContent", reflect.TypeOf((*MockAwsI)(nil).GetFileContent), arg0, arg1) } -// ListFilesInBucket mocks base method. -func (m *MockAwsI) ListFilesInBucket(arg0 context.Context) (<-chan *s3.ObjectMeta, <-chan error) { +// ListFiles mocks base method. +func (m *MockAwsI) ListFiles(arg0 context.Context) (<-chan *T.ObjectMeta, <-chan error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListFilesInBucket", arg0) - ret0, _ := ret[0].(<-chan *s3.ObjectMeta) + ret := m.ctrl.Call(m, "ListFiles", arg0) + ret0, _ := ret[0].(<-chan *T.ObjectMeta) ret1, _ := ret[1].(<-chan error) return ret0, ret1 } -// ListFilesInBucket indicates an expected call of ListFilesInBucket. -func (mr *MockAwsIMockRecorder) ListFilesInBucket(arg0 interface{}) *gomock.Call { +// ListFiles indicates an expected call of ListFiles. +func (mr *MockAwsIMockRecorder) ListFiles(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFilesInBucket", reflect.TypeOf((*MockAwsI)(nil).ListFilesInBucket), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFiles", reflect.TypeOf((*MockAwsI)(nil).ListFiles), arg0) } diff --git a/types/index.go b/types/index.go new file mode 100644 index 0000000..a8d886c --- /dev/null +++ b/types/index.go @@ -0,0 +1,33 @@ +package types + +import ( + "context" + "io" +) + +type Object struct { + Body io.ReadCloser + ContentType string + ContentLength int64 +} + +// ObjectMeta key: object key, size: size of object in bytes +type ObjectMeta struct { + Key string + Size int64 + ContentType string + Ext string +} + +type CloudStorageI interface { + ListFiles(ctx context.Context) (<-chan *ObjectMeta, <-chan error) + GetFileContent(ctx context.Context, objectKey string) (*Object, error) + DeleteFile(ctx context.Context, objectKey string) error + DownloadToFile(ctx context.Context, objectKey string) (string, error) + DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) +} + +type CloudStorageClient struct { + name string +} + diff --git a/util/util.go b/util/util.go index a1f360d..686cf87 100644 --- a/util/util.go +++ b/util/util.go @@ -102,6 +102,9 @@ func GetAllocationIDFromEnv() string { func GetAwsCredentialsFromEnv() (string, string) { return os.Getenv("AWS_ACCESS_KEY"), os.Getenv("AWS_SECRET_KEY") } +func GetAccessToken() string { + return os.Getenv("AccessToken") +} func ConvertGoSDKTimeToTime(in string) time.Time { t, err := time.Parse(ZGoSDKTimeFormat, in)