From 6cf1591b2a4e07857f43e7fbc220a4e909554cf1 Mon Sep 17 00:00:00 2001 From: pewssh Date: Sun, 14 Apr 2024 22:54:03 +0545 Subject: [PATCH 01/11] Add Core Fetch Login for dropbox and gdrive --- dropbox/core.go | 179 ++++++++++++++++++++++++++++++++++++++++++++++++ dropbox/test.go | 147 +++++++++++++++++++++++++++++++++++++++ gdrive/core.go | 155 +++++++++++++++++++++++++++++++++++++++++ gdrive/test.go | 157 ++++++++++++++++++++++++++++++++++++++++++ go.mod | 36 ++++++++-- go.sum | 86 +++++++++++++++++++++++ 6 files changed, 755 insertions(+), 5 deletions(-) create mode 100644 dropbox/core.go create mode 100644 dropbox/test.go create mode 100644 gdrive/core.go create mode 100644 gdrive/test.go diff --git a/dropbox/core.go b/dropbox/core.go new file mode 100644 index 0000000..70ba1a4 --- /dev/null +++ b/dropbox/core.go @@ -0,0 +1,179 @@ +package dropbox + +import ( + "context" + "io" + "mime" + "os" + "path/filepath" + + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" +) + +type DropboxI interface { + ListFiles(ctx context.Context) ([]*ObjectMeta, error) + ListFilesInFolder(ctx context.Context) ([]*ObjectMeta, error) + GetFileContent(ctx context.Context, filePath string) (*Object, error) + DeleteFile(ctx context.Context, filePath string) error + DownloadToFile(ctx context.Context, filePath string) (string, error) + DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) +} + +type Object struct { + Body io.ReadCloser + ContentType string + ContentLength uint64 +} + +type ObjectMeta struct { + Name string + Path string + Size uint64 + ContentType string +} + +type DropboxClient struct { + token string + dropboxConf *dropbox.Config + dropboxFiles files.Client +} + +func GetDropboxClient(token string) (*DropboxClient, error) { + config := dropbox.Config{ + Token: token, + } + + client := files.New(config) + + return &DropboxClient{ + token: token, + dropboxConf: &config, + dropboxFiles: client, + }, nil +} + +func (d *DropboxClient) ListFiles(ctx context.Context) ([]*ObjectMeta, error) { + var objects []*ObjectMeta + + arg := files.NewListFolderArg("") + arg.Recursive = true + + res, err := d.dropboxFiles.ListFolder(arg) + + if err != nil { + return nil, err + } + + for _, entry := range res.Entries { + + if meta, ok := entry.(*files.FileMetadata); ok { + + objects = append(objects, &ObjectMeta{ + Name: meta.Name, + Path: meta.PathDisplay, + Size: meta.Size, + ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + }) + + } + } + + return objects, nil +} + +func (d *DropboxClient) ListFilesInFolder(ctx context.Context, folderName string) ([]*ObjectMeta, error) { + var objects []*ObjectMeta + + arg := files.NewListFolderArg(folderName) + arg.Recursive = true + + res, err := d.dropboxFiles.ListFolder(arg) + + if err != nil { + return nil, err + } + + for _, entry := range res.Entries { + + if meta, ok := entry.(*files.FileMetadata); ok { + + objects = append(objects, &ObjectMeta{ + Name: meta.Name, + Path: meta.PathDisplay, + Size: meta.Size, + ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + }) + + } + } + + return objects, nil +} + +func (d *DropboxClient) GetFileContent(ctx context.Context, filePath string) (*Object, error) { + arg := files.NewDownloadArg(filePath) + res, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return nil, err + } + + return &Object{ + Body: content, + ContentType: mime.TypeByExtension(filepath.Ext(filePath)), + ContentLength: res.Size, + }, nil +} + +func (d *DropboxClient) DeleteFile(ctx context.Context, filePath string) error { + arg := files.NewDeleteArg(filePath) + _, err := d.dropboxFiles.DeleteV2(arg) + return err +} + +func (d *DropboxClient) DownloadToFile(ctx context.Context, filePath string) (string, error) { + arg := files.NewDownloadArg(filePath) + _, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return "", err + } + + fileName := filepath.Base(filePath) + downloadPath := filepath.Join(".", fileName) + file, err := os.Create(downloadPath) + if err != nil { + return "", err + } + defer file.Close() + + _, err = io.Copy(file, content) + if err != nil { + return "", err + } + + return downloadPath, nil +} + +func (d *DropboxClient) DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) { + arg := files.NewDownloadArg(objectKey) + _, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return nil, err + } + defer content.Close() + + if _, err := io.CopyN(io.Discard, content, offset); err != nil { + return nil, err + } + + data := make([]byte, chunkSize) + n, err := io.ReadFull(content, data) + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + if int64(n) < chunkSize && objectSize != chunkSize { + data = data[:n] + } + + return data, nil +} diff --git a/dropbox/test.go b/dropbox/test.go new file mode 100644 index 0000000..c6308ed --- /dev/null +++ b/dropbox/test.go @@ -0,0 +1,147 @@ +package dropbox + +import ( + "context" + "fmt" + "log" + "testing" +) + +var ( + dropboxAccessToken = "" + testFilePath = "" +) + +const TestFileContent = ` by Manuel Gutiérrez Nájera + +I want to die as the day declines, +at high sea and facing the sky, +while agony seems like a dream +and my soul like a bird that can fly. + +To hear not, at this last moment, +once alone with sky and sea, +any more voices nor weeping prayers +than the majestic beating of the waves. + +To die when the sad light retires +its golden network from the green waves +to be like the sun that slowly expires; +something very luminous that fades. + +To die, and die young, before +fleeting time removes the gentle crown, +while life still says: "I'm yours" +though we know with our hearts that she lies. +` + +func TestDropboxClient_ListFiles(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken) + if err != nil { + log.Printf("Failed to create Dropbox client: %v", err) + return + } + + ctx := context.Background() + files, err := client.ListFiles(ctx) + if err != nil { + log.Printf("Error while listing files: %v", err) + return + } + + for _, file := range files { + log.Printf("File: %s, Name: %s, Size: %d bytes", file.Path, file.ContentType, file.Size) + } +} + +func TestDropboxClient_GetFileContent(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken) + if err != nil { + log.Printf("Failed to create Dropbox client: %v", err) + } + + ctx := context.Background() + filePath := testFilePath + obj, err := client.GetFileContent(ctx, filePath) + if err != nil { + log.Printf("Error while getting file content: %v", err) + return + } + defer obj.Body.Close() + + log.Printf("File content type: %s, Length: %d", obj.ContentType, obj.ContentLength) + + if (obj.Body == nil) || (obj.ContentLength == 0) { + fmt.Println("Empty file content") + return + } + + buf := make([]byte, obj.ContentLength) + n, err := obj.Body.Read(buf) + + if err != nil && err.Error() != "EOF" { + log.Printf("Error while reading file content: %v", err) + return + } + + log.Printf("File content: %s", string(buf[:n])) +} + +func TestDropboxClient_DeleteFile(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken) + if err != nil { + log.Printf("Failed to create Dropbox client: %v", err) + return + } + + ctx := context.Background() + filePath := testFilePath + err = client.DeleteFile(ctx, filePath) + if err != nil { + log.Printf("Error while deleting file: %v", err) + return + } + log.Printf("File %s deleted successfully", filePath) +} + +func TestDropboxClient_DownloadToFile(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken) + if err != nil { + log.Printf("Failed to create Dropbox client: %v", err) + return + } + + ctx := context.Background() + filePath := testFilePath + downloadedPath, err := client.DownloadToFile(ctx, filePath) + if err != nil { + log.Printf("Error while downloading file: %v", err) + return + } + log.Printf("Downloaded to: %s", downloadedPath) +} + +func TestDropboxClient_DownloadToMemory(t *testing.T) { + client, err := GetDropboxClient(dropboxAccessToken) + if err != nil { + log.Printf("Failed to create Dropbox client: %v", err) + return + } + + ctx := context.Background() + + filePath := testFilePath + offset := int64(0) + + // half chunk + chunkSize := int64(313) + objectSize := int64(626) + + data, err := client.DownloadToMemory(ctx, filePath, offset, chunkSize, objectSize) + if err != nil { + log.Printf("Error while downloading file: %v", err) + return + } + + log.Printf("Downloaded data: %s", data) +} diff --git a/gdrive/core.go b/gdrive/core.go new file mode 100644 index 0000000..595f867 --- /dev/null +++ b/gdrive/core.go @@ -0,0 +1,155 @@ +package gdrive + +import ( + "context" + "fmt" + "io" + "os" + "strings" + + zlogger "github.com/0chain/s3migration/logger" + "golang.org/x/oauth2" + "google.golang.org/api/drive/v3" + "google.golang.org/api/option" +) + +type GoogleDriveI interface { + ListFiles(ctx context.Context) ([]*ObjectMeta, error) + GetFileContent(ctx context.Context, fileID string, keepOpen bool) (*Object, error) + DeleteFile(ctx context.Context, fileID string) error + DownloadToFile(ctx context.Context, fileID, destinationPath string) error + DownloadToMemory(ctx context.Context, fileID string, offset int64, chunkSize, fileSize int64) ([]byte, error) +} + +type Object struct { + Body io.ReadCloser + ContentType string + ContentLength int64 +} + +type ObjectMeta struct { + ID string + Name string + Size int64 + ContentType string + LastModified string +} + +type GoogleDriveClient struct { + service *drive.Service +} + +func NewGoogleDriveClient(accessToken string) (*GoogleDriveClient, error) { + ctx := context.Background() + + tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: accessToken}) + + httpClient := oauth2.NewClient(ctx, tokenSource) + + service, err := drive.NewService(ctx, option.WithHTTPClient(httpClient)) + if err != nil { + return nil, err + } + + return &GoogleDriveClient{ + service: service, + }, nil +} + +func (g *GoogleDriveClient) ListFiles(ctx context.Context) ([]*ObjectMeta, error) { + files, err := g.service.Files.List().Context(ctx).Do() + if err != nil { + return nil, err + } + + var objects []*ObjectMeta + for _, file := range files.Files { + if strings.HasSuffix(file.Name, "/") { // Skip dirs + continue + } + + objects = append(objects, &ObjectMeta{ + ID: file.Id, + Name: file.Name, + Size: file.Size, + ContentType: file.MimeType, + LastModified: file.ModifiedTime, + }) + } + + return objects, nil +} + +func (g *GoogleDriveClient) GetFileContent(ctx context.Context, fileID string, keepOpen bool) (*Object, error) { + resp, err := g.service.Files.Get(fileID).Download() + if err != nil { + return nil, err + } + + if !keepOpen { + defer resp.Body.Close() + } + + obj := &Object{ + Body: resp.Body, + ContentType: resp.Header.Get("Content-Type"), + ContentLength: resp.ContentLength, + } + + return obj, nil +} + +func (g *GoogleDriveClient) DeleteFile(ctx context.Context, fileID string) error { + err := g.service.Files.Delete(fileID).Do() + if err != nil { + return err + } + return nil +} + +func (g *GoogleDriveClient) DownloadToFile(ctx context.Context, fileID, destinationPath string) error { + resp, err := g.service.Files.Get(fileID).Download() + if err != nil { + return err + } + defer resp.Body.Close() + + out, err := os.Create(destinationPath) + if err != nil { + return err + } + defer out.Close() + + _, err = io.Copy(out, resp.Body) + if err != nil { + return err + } + zlogger.Logger.Info(fmt.Sprintf("Downloaded file ID: %s to %s\n", fileID, destinationPath)) + return nil +} + +func (g *GoogleDriveClient) DownloadToMemory(ctx context.Context, fileID string, offset int64, chunkSize, fileSize int64) ([]byte, error) { + resp, err := g.service.Files.Get(fileID).Download() + if err != nil { + return nil, err + } + defer resp.Body.Close() + + if offset < 0 || offset >= fileSize || chunkSize <= 0 { + return nil, fmt.Errorf("invalid offset or chunk size") + } + + endPos := offset + chunkSize - 1 + if endPos >= fileSize { + endPos = fileSize - 1 + } + + data := make([]byte, endPos-offset+1) + n, err := io.ReadFull(resp.Body, data) + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + data = data[:n] + + return data, nil +} diff --git a/gdrive/test.go b/gdrive/test.go new file mode 100644 index 0000000..e771784 --- /dev/null +++ b/gdrive/test.go @@ -0,0 +1,157 @@ +package gdrive + +import ( + "context" + "log" + "testing" +) + +var ( + driveAccessToken="" + testFileID="" +) + +// using: https://developers.google.com/oauthplayground + +// For reference (626 bytes text file) +const TestFileContent = ` by Manuel Gutiérrez Nájera + +I want to die as the day declines, +at high sea and facing the sky, +while agony seems like a dream +and my soul like a bird that can fly. + +To hear not, at this last moment, +once alone with sky and sea, +any more voices nor weeping prayers +than the majestic beating of the waves. + +To die when the sad light retires +its golden network from the green waves +to be like the sun that slowly expires; +something very luminous that fades. + +To die, and die young, before +fleeting time removes the gentle crown, +while life still says: "I'm yours" +though we know with our hearts that she lies. +` + +func TestGoogleDriveClient_ListFiles(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken) + if err != nil { + log.Printf("err while creating Google Drive client: %v", err) + return + } + + ctx := context.Background() + files, err := client.ListFiles(ctx) + if err != nil { + log.Printf("err while list files: %v", err) + return + } + + for _, file := range files { + log.Printf("file: %s, name: %s, size: %d bytes", file.ID, file.Name, file.Size) + } +} + +func TestGoogleDriveClient_GetFileContent(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken) + if err != nil { + log.Printf("Failed to creating Google Drive client: %v", err) + return + } + + ctx := context.Background() + fileID := testFileID + obj, err := client.GetFileContent(ctx, fileID, true) + + if err != nil { + log.Printf("err while getting file content: %v", err) + return + } + + defer obj.Body.Close() + + log.Printf("file content type: %s, length: %d", obj.ContentType, obj.ContentLength) + + if (obj.Body == nil) || (obj.ContentLength == 0) { + log.Printf("empty file content") + return + } + + buf := make([]byte, obj.ContentLength) + n, err := obj.Body.Read(buf) + if err != nil { + log.Printf("err while read file content: %v", err) + return + } + log.Printf("read data: %s", buf[:n]) +} + +func TestGoogleDriveClient_DeleteFile(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken) + if err != nil { + log.Printf("err while creating Google Drive client: %v", err) + return + } + + ctx := context.Background() + fileID := testFileID + err = client.DeleteFile(ctx, fileID) + if err != nil { + log.Printf("err while delete file: %v", err) + return + } + log.Printf("file: %s deleted successfully", fileID) +} + +func TestGoogleDriveClient_DownloadToFile(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken) + if err != nil { + log.Printf("err while creating Google Drive client: %v", err) + } + + ctx := context.Background() + fileID := testFileID + destinationPath := "./downloaded_file.txt" + err = client.DownloadToFile(ctx, fileID, destinationPath) + if err != nil { + log.Printf("err while downloading file: %v", err) + return + } + log.Printf("downloaded to: %s", destinationPath) +} + +func TestGoogleDriveClient_DownloadToMemory(t *testing.T) { + client, err := NewGoogleDriveClient(driveAccessToken) + if err != nil { + log.Printf("err while creating Google Drive client: %v", err) + } + + ctx := context.Background() + + fileID := testFileID + + offset := int64(0) + + // download only half chunk for testing + chunkSize := int64(313) + + fileSize := int64(626) + + if err != nil { + log.Printf("err while getting file size: %v", err) + return + } + + data, err := client.DownloadToMemory(ctx, fileID, offset, chunkSize, fileSize) + + if err != nil { + log.Printf("err while downloading file: %v", err) + return + } + + log.Printf("downloaded data: %s", data) +} diff --git a/go.mod b/go.mod index 6c839e5..d862315 100644 --- a/go.mod +++ b/go.mod @@ -14,6 +14,30 @@ require ( github.com/spf13/viper v1.15.0 ) +require ( + cloud.google.com/go/compute v1.23.4 // indirect + cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/felixge/httpsnoop v1.0.4 // indirect + github.com/go-logr/logr v1.4.1 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect + github.com/golang/protobuf v1.5.3 // indirect + github.com/google/s2a-go v0.1.7 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect + github.com/googleapis/gax-go/v2 v2.12.3 // indirect + go.opencensus.io v0.24.0 // indirect + go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect + go.opentelemetry.io/otel v1.24.0 // indirect + go.opentelemetry.io/otel/metric v1.24.0 // indirect + go.opentelemetry.io/otel/trace v1.24.0 // indirect + golang.org/x/net v0.22.0 // indirect + golang.org/x/oauth2 v0.18.0 // indirect + google.golang.org/appengine v1.6.8 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect + google.golang.org/grpc v1.62.1 // indirect + google.golang.org/protobuf v1.33.0 // indirect +) + require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 // indirect github.com/Luzifer/go-openssl/v3 v3.1.0 // indirect @@ -38,11 +62,12 @@ require ( github.com/btcsuite/btcd/chaincfg/chainhash v1.0.2 // indirect github.com/deckarep/golang-set v1.8.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 github.com/ethereum/go-ethereum v1.10.26 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-stack/stack v1.8.1 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/h2non/filetype v1.1.3 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect @@ -79,10 +104,11 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.9.0 // indirect go.uber.org/zap v1.24.0 // indirect - golang.org/x/crypto v0.6.0 // indirect - golang.org/x/sync v0.1.0 // indirect - golang.org/x/sys v0.7.0 // indirect - golang.org/x/text v0.8.0 // indirect + golang.org/x/crypto v0.21.0 // indirect + golang.org/x/sync v0.6.0 // indirect + golang.org/x/sys v0.18.0 // indirect + golang.org/x/text v0.14.0 // indirect + google.golang.org/api v0.172.0 gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/lumberjack.v2 v2.2.1 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect diff --git a/go.sum b/go.sum index a73de0f..45f3e16 100644 --- a/go.sum +++ b/go.sum @@ -23,6 +23,10 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg= cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc= cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ= +cloud.google.com/go/compute v1.23.4 h1:EBT9Nw4q3zyE7G45Wvv3MzolIrCJEuHys5muLY0wvAw= +cloud.google.com/go/compute v1.23.4/go.mod h1:/EJMj55asU6kAFnuZET8zqgwgJ9FvXWXOkkfQZa4ioI= +cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= +cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE= cloud.google.com/go/datastore v1.1.0/go.mod h1:umbIZjpQpHh4hmRpGhH4tLFup+FVzqBi1b3c64qFpCk= cloud.google.com/go/pubsub v1.0.1/go.mod h1:R0Gpsv3s54REJCy4fxDixWD93lHJMoZTyQ2kNxGRt3I= @@ -140,6 +144,8 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeC github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= +github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU= +github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= @@ -149,6 +155,8 @@ github.com/envoyproxy/go-control-plane v0.9.9-0.20201210154907-fd9021fe5dad/go.m github.com/envoyproxy/protoc-gen-validate v0.1.0/go.mod h1:iSmxcyjqTsJpI2R4NaDN7+kN2VEUnK/pcBlmesArF7c= github.com/ethereum/go-ethereum v1.10.26 h1:i/7d9RBBwiXCEuyduBQzJw/mKmnvzsN14jqBmytw72s= github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= +github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= +github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= @@ -159,6 +167,11 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.6 h1:/Fpf6oFPoeFik9ty7siob0G6Ke8QvQEuVcuChpwXzpY= github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiUsvbm0= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= @@ -168,6 +181,8 @@ github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfU github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20200121045136-8c9f03a8e57e/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da h1:oI5xCqsCo564l8iNU+DwB5epxmsaqB+rhGL0m5jtYqE= +github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/mock v1.1.1/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/mock v1.3.1/go.mod h1:sBzyDLLjw3U8JLTeZvSv8jJB+tU5PVekmnlKIyFUx0Y= @@ -191,6 +206,11 @@ github.com/golang/protobuf v1.4.0/go.mod h1:jodUvKwWbYaEsadDk5Fwe5c77LiNKVO9IDvq github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QDs8UjoX8= github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= +github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= +github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= +github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= @@ -203,9 +223,12 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -220,12 +243,20 @@ github.com/google/pprof v0.0.0-20201023163331-3e6fc7fc9c4c/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= +github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= +github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/cLCKqA= +github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -367,6 +398,7 @@ github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9de github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= +github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/yusufpapurcu/wmi v1.2.2 h1:KBNDSne4vP5mbSWnJbO+51IMOXJB67QiYCSBrubbPRg= github.com/yusufpapurcu/wmi v1.2.2/go.mod h1:SBZ9tNy3G9/m5Oi98Zks0QjeHVDvuK0qfxQmPyzfmi0= go.dedis.ch/fixbuf v1.0.3 h1:hGcV9Cd/znUxlusJ64eAlExS+5cJDIyTyEG+otu5wQs= @@ -384,6 +416,16 @@ go.opencensus.io v0.22.2/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= +go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= +go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 h1:jq9TW8u3so/bN+JPT166wjOI6/vQPF6Xe7nMNIltagk= +go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0/go.mod h1:p8pYQP+m5XfbZm9fxtSKAbM6oIllS7s2AfxrChvc7iw= +go.opentelemetry.io/otel v1.24.0 h1:0LAOdjNmQeSTzGBzduGe/rU4tZhMwL5rWgtp9Ku5Jfo= +go.opentelemetry.io/otel v1.24.0/go.mod h1:W7b9Ozg4nkF5tWI5zsXkaKKDjdVjpD4oAt9Qi/MArHo= +go.opentelemetry.io/otel/metric v1.24.0 h1:6EhoGWWK28x1fbpA4tYTOWBkPefTDQnb8WSGXlc88kI= +go.opentelemetry.io/otel/metric v1.24.0/go.mod h1:VYhLe1rFfxuTXLgj4CBiyz+9WYBA8pNGJgDcSFRKBco= +go.opentelemetry.io/otel/trace v1.24.0 h1:CsKnnL4dUAr/0llH9FKuc698G04IrpWV0MQA/Y1YELI= +go.opentelemetry.io/otel/trace v1.24.0/go.mod h1:HPc3Xr/cOApsBI154IU0OI0HJexz+aw5uPdbs3UCjNU= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= @@ -403,9 +445,12 @@ golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= +golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= +golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= +golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -440,6 +485,7 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -472,10 +518,16 @@ golang.org/x/net v0.0.0-20200813134508-3edf25e44fcc/go.mod h1:/O7V0waA8r7cgGh81R golang.org/x/net v0.0.0-20200822124328-c89045814202/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA= golang.org/x/net v0.0.0-20201021035429-f5854403a974/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201031054903-ff519b6c9102/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= +golang.org/x/net v0.0.0-20201110031124-69a78807bb2b/go.mod h1:sp8m0HH+o8qH0wwXwYZr8TS3Oi6o0r6Gce1SSxlDquU= golang.org/x/net v0.0.0-20201209123823-ac852fbbde11/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= +golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= +golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= +golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -485,6 +537,10 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= +golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= +golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= +golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -496,8 +552,11 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -542,24 +601,34 @@ golang.org/x/sys v0.0.0-20210330210617-4fbd30eecc44/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20210423185535-09eb48e85fd7/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20210510120138-977fb7262007/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210615035016-665e8c7367d1/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= +golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= +golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= +golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -610,6 +679,7 @@ golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -633,13 +703,18 @@ google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz513 google.golang.org/api v0.35.0/go.mod h1:/XrVsuzM0rZmrsbjJutiuftIzeuTQcEeaYcSk/mQ1dg= google.golang.org/api v0.36.0/go.mod h1:+z5ficQTmoYpPn8LCUNVpK5I7hwkpjbcgqA7I34qYtE= google.golang.org/api v0.40.0/go.mod h1:fYKFpnQN0DsDSKRVRcQSDQNtqWPfM9i+zNPxepjRCQ8= +google.golang.org/api v0.172.0 h1:/1OcMZGPmW1rX2LCu2CmGUD1KXK1+pfzxotxyRUCCdk= +google.golang.org/api v0.172.0/go.mod h1:+fJZq6QXWfa9pXhnIzsjx4yI22d4aI9ZpLb58gvXjis= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20190307195333-5fe7a883aa19/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= google.golang.org/genproto v0.0.0-20190418145605-e7d98fc518a7/go.mod h1:VzzqZJRnGkLBvHegQrXjBqPurQTc5/KpmUdxsrq26oE= @@ -676,6 +751,9 @@ google.golang.org/genproto v0.0.0-20201210142538-e3217bee35cc/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= +google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.20.1/go.mod h1:10oTOabMzJvdu6/UiuZezV6QK5dSlG84ov/aaiqXj38= google.golang.org/grpc v1.21.1/go.mod h1:oYelfM1adQP15Ek0mdvEgi9Df8B9CZIaU1084ijfRaM= @@ -692,6 +770,8 @@ google.golang.org/grpc v1.31.1/go.mod h1:N36X2cJ7JwdamYAgDz+s+rVMFjt3numwzf/HckM google.golang.org/grpc v1.33.2/go.mod h1:JMHMWHQWaTccqQQlmk3MJZS+GWXOdAesneDmEnv2fbc= google.golang.org/grpc v1.34.0/go.mod h1:WotjhfgOW/POjDeRt8vscBtXq+2VjORFy659qA51WJ8= google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= +google.golang.org/grpc v1.62.1 h1:B4n+nfKzOICUXMgyrNd19h/I9oH0L1pizfk1d4zSgTk= +google.golang.org/grpc v1.62.1/go.mod h1:IWTG0VlJLCh1SkC58F7np9ka9mx/WNkjl4PGJaiq+QE= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= @@ -702,6 +782,12 @@ google.golang.org/protobuf v1.23.0/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2 google.golang.org/protobuf v1.23.1-0.20200526195155-81db48ad09cc/go.mod h1:EGpADcykh3NcUnDUJcl1+ZksZNG86OlYog2l/sGQquU= google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGjtUeSXeh4= google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= +google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= +google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= +google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= +google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= +google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 410ba0416ed4e548f39639fefa7539695f675179 Mon Sep 17 00:00:00 2001 From: pewssh Date: Sun, 14 Apr 2024 23:14:22 +0545 Subject: [PATCH 02/11] update for missing interface --- dropbox/core.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dropbox/core.go b/dropbox/core.go index 70ba1a4..2d9bb50 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -12,8 +12,8 @@ import ( ) type DropboxI interface { - ListFiles(ctx context.Context) ([]*ObjectMeta, error) - ListFilesInFolder(ctx context.Context) ([]*ObjectMeta, error) + ListFiles(ctx context.Context) ([]*ObjectMeta, error) + ListFilesInFolder(ctx context.Context, folderName string) ([]*ObjectMeta, error) GetFileContent(ctx context.Context, filePath string) (*Object, error) DeleteFile(ctx context.Context, filePath string) error DownloadToFile(ctx context.Context, filePath string) (string, error) From 932034e76b4cf306e110cf714d62dd875da1c1e3 Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 15 Apr 2024 12:42:28 +0545 Subject: [PATCH 03/11] updated download to memory --- dropbox/core.go | 44 ++++++++++++++++++++++++++------------------ gdrive/core.go | 44 +++++++++++++++++++++++++------------------- 2 files changed, 51 insertions(+), 37 deletions(-) diff --git a/dropbox/core.go b/dropbox/core.go index 2d9bb50..0aad701 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -155,25 +155,33 @@ func (d *DropboxClient) DownloadToFile(ctx context.Context, filePath string) (st } func (d *DropboxClient) DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) { - arg := files.NewDownloadArg(objectKey) - _, content, err := d.dropboxFiles.Download(arg) - if err != nil { - return nil, err - } - defer content.Close() + limit := offset + chunkSize - 1 + if limit > objectSize { + limit = objectSize + } - if _, err := io.CopyN(io.Discard, content, offset); err != nil { - return nil, err - } + rng := fmt.Sprintf("bytes=%d-%d", offset, limit) - data := make([]byte, chunkSize) - n, err := io.ReadFull(content, data) - if err != nil && err != io.ErrUnexpectedEOF { - return nil, err - } - if int64(n) < chunkSize && objectSize != chunkSize { - data = data[:n] - } + arg := files.NewDownloadArg(objectKey) + + arg.ExtraHeaders = map[string]string{"Range": rng} + + _, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return nil, err + } + defer content.Close() + + data := make([]byte, chunkSize) + n, err := io.ReadFull(content, data) + + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + + if int64(n) < chunkSize && objectSize != chunkSize { + data = data[:n] + } - return data, nil + return data, nil } diff --git a/gdrive/core.go b/gdrive/core.go index 595f867..42038c2 100644 --- a/gdrive/core.go +++ b/gdrive/core.go @@ -129,27 +129,33 @@ func (g *GoogleDriveClient) DownloadToFile(ctx context.Context, fileID, destinat } func (g *GoogleDriveClient) DownloadToMemory(ctx context.Context, fileID string, offset int64, chunkSize, fileSize int64) ([]byte, error) { - resp, err := g.service.Files.Get(fileID).Download() - if err != nil { - return nil, err - } - defer resp.Body.Close() + limit := offset + chunkSize - 1 + if limit > fileSize { + limit = fileSize + } - if offset < 0 || offset >= fileSize || chunkSize <= 0 { - return nil, fmt.Errorf("invalid offset or chunk size") - } + rng := fmt.Sprintf("bytes=%d-%d", offset, limit) - endPos := offset + chunkSize - 1 - if endPos >= fileSize { - endPos = fileSize - 1 - } + req := g.service.Files.Get(fileID) - data := make([]byte, endPos-offset+1) - n, err := io.ReadFull(resp.Body, data) - if err != nil && err != io.ErrUnexpectedEOF { - return nil, err - } - data = data[:n] + req.Header().Set("Range", rng) + + resp, err := req.Download() + if err != nil { + return nil, err + } + defer resp.Body.Close() + + data := make([]byte, chunkSize) + n, err := io.ReadFull(resp.Body, data) + + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } + + if int64(n) < chunkSize && fileSize != chunkSize { + data = data[:n] + } - return data, nil + return data, nil } From a00681601081ada087634b5bc35a43578e7eb578 Mon Sep 17 00:00:00 2001 From: pewssh Date: Tue, 16 Apr 2024 16:51:10 +0545 Subject: [PATCH 04/11] updated for zlogger instead of log --- dropbox/core.go | 1 + dropbox/test.go | 37 +++++++++++++++++++------------------ gdrive/test.go | 42 ++++++++++++++++++++++-------------------- 3 files changed, 42 insertions(+), 38 deletions(-) diff --git a/dropbox/core.go b/dropbox/core.go index 0aad701..24c52e4 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -2,6 +2,7 @@ package dropbox import ( "context" + "fmt" "io" "mime" "os" diff --git a/dropbox/test.go b/dropbox/test.go index c6308ed..b17d4dd 100644 --- a/dropbox/test.go +++ b/dropbox/test.go @@ -3,8 +3,9 @@ package dropbox import ( "context" "fmt" - "log" "testing" + + zlogger "github.com/0chain/s3migration/logger" ) var ( @@ -38,38 +39,38 @@ though we know with our hearts that she lies. func TestDropboxClient_ListFiles(t *testing.T) { client, err := GetDropboxClient(dropboxAccessToken) if err != nil { - log.Printf("Failed to create Dropbox client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return } ctx := context.Background() files, err := client.ListFiles(ctx) if err != nil { - log.Printf("Error while listing files: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Error while listing files: %v", err)) return } for _, file := range files { - log.Printf("File: %s, Name: %s, Size: %d bytes", file.Path, file.ContentType, file.Size) + zlogger.Logger.Info(fmt.Sprintf("File: %s, Name: %s, Size: %d bytes", file.Path, file.ContentType, file.Size)) } } func TestDropboxClient_GetFileContent(t *testing.T) { client, err := GetDropboxClient(dropboxAccessToken) if err != nil { - log.Printf("Failed to create Dropbox client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) } ctx := context.Background() filePath := testFilePath obj, err := client.GetFileContent(ctx, filePath) if err != nil { - log.Printf("Error while getting file content: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Error while getting file content: %v", err)) return } defer obj.Body.Close() - log.Printf("File content type: %s, Length: %d", obj.ContentType, obj.ContentLength) + zlogger.Logger.Info(fmt.Sprintf("File content type: %s, Length: %d", obj.ContentType, obj.ContentLength)) if (obj.Body == nil) || (obj.ContentLength == 0) { fmt.Println("Empty file content") @@ -80,17 +81,17 @@ func TestDropboxClient_GetFileContent(t *testing.T) { n, err := obj.Body.Read(buf) if err != nil && err.Error() != "EOF" { - log.Printf("Error while reading file content: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Error while reading file content: %v", err)) return } - log.Printf("File content: %s", string(buf[:n])) + zlogger.Logger.Info(fmt.Sprintf("File content: %s", string(buf[:n]))) } func TestDropboxClient_DeleteFile(t *testing.T) { client, err := GetDropboxClient(dropboxAccessToken) if err != nil { - log.Printf("Failed to create Dropbox client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return } @@ -98,16 +99,16 @@ func TestDropboxClient_DeleteFile(t *testing.T) { filePath := testFilePath err = client.DeleteFile(ctx, filePath) if err != nil { - log.Printf("Error while deleting file: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Error while deleting file: %v", err)) return } - log.Printf("File %s deleted successfully", filePath) + zlogger.Logger.Info(fmt.Sprintf("File %s deleted successfully", filePath)) } func TestDropboxClient_DownloadToFile(t *testing.T) { client, err := GetDropboxClient(dropboxAccessToken) if err != nil { - log.Printf("Failed to create Dropbox client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return } @@ -115,16 +116,16 @@ func TestDropboxClient_DownloadToFile(t *testing.T) { filePath := testFilePath downloadedPath, err := client.DownloadToFile(ctx, filePath) if err != nil { - log.Printf("Error while downloading file: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Error while downloading file: %v", err)) return } - log.Printf("Downloaded to: %s", downloadedPath) + zlogger.Logger.Info(fmt.Sprintf("Downloaded to: %s", downloadedPath)) } func TestDropboxClient_DownloadToMemory(t *testing.T) { client, err := GetDropboxClient(dropboxAccessToken) if err != nil { - log.Printf("Failed to create Dropbox client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return } @@ -139,9 +140,9 @@ func TestDropboxClient_DownloadToMemory(t *testing.T) { data, err := client.DownloadToMemory(ctx, filePath, offset, chunkSize, objectSize) if err != nil { - log.Printf("Error while downloading file: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Error while downloading file: %v", err)) return } - log.Printf("Downloaded data: %s", data) + zlogger.Logger.Info(fmt.Sprintf("Downloaded data: %s", data)) } diff --git a/gdrive/test.go b/gdrive/test.go index e771784..4104289 100644 --- a/gdrive/test.go +++ b/gdrive/test.go @@ -2,8 +2,10 @@ package gdrive import ( "context" - "log" + "fmt" "testing" + + zlogger "github.com/0chain/s3migration/logger" ) var ( @@ -40,26 +42,26 @@ though we know with our hearts that she lies. func TestGoogleDriveClient_ListFiles(t *testing.T) { client, err := NewGoogleDriveClient(driveAccessToken) if err != nil { - log.Printf("err while creating Google Drive client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) return } ctx := context.Background() files, err := client.ListFiles(ctx) if err != nil { - log.Printf("err while list files: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while list files: %v", err)) return } for _, file := range files { - log.Printf("file: %s, name: %s, size: %d bytes", file.ID, file.Name, file.Size) + zlogger.Logger.Info(fmt.Sprintf("file: %s, name: %s, size: %d bytes", file.ID, file.Name, file.Size)) } } func TestGoogleDriveClient_GetFileContent(t *testing.T) { client, err := NewGoogleDriveClient(driveAccessToken) if err != nil { - log.Printf("Failed to creating Google Drive client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("Failed to creating Google Drive client: %v", err)) return } @@ -68,32 +70,32 @@ func TestGoogleDriveClient_GetFileContent(t *testing.T) { obj, err := client.GetFileContent(ctx, fileID, true) if err != nil { - log.Printf("err while getting file content: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while getting file content: %v", err)) return } defer obj.Body.Close() - log.Printf("file content type: %s, length: %d", obj.ContentType, obj.ContentLength) + zlogger.Logger.Info(fmt.Sprintf("file content type: %s, length: %d", obj.ContentType, obj.ContentLength)) if (obj.Body == nil) || (obj.ContentLength == 0) { - log.Printf("empty file content") + zlogger.Logger.Info("empty file content") return } buf := make([]byte, obj.ContentLength) n, err := obj.Body.Read(buf) if err != nil { - log.Printf("err while read file content: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while read file content: %v", err)) return } - log.Printf("read data: %s", buf[:n]) + zlogger.Logger.Info(fmt.Sprintf("read data: %s", buf[:n])) } func TestGoogleDriveClient_DeleteFile(t *testing.T) { client, err := NewGoogleDriveClient(driveAccessToken) if err != nil { - log.Printf("err while creating Google Drive client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) return } @@ -101,16 +103,16 @@ func TestGoogleDriveClient_DeleteFile(t *testing.T) { fileID := testFileID err = client.DeleteFile(ctx, fileID) if err != nil { - log.Printf("err while delete file: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while delete file: %v", err)) return } - log.Printf("file: %s deleted successfully", fileID) + zlogger.Logger.Error(fmt.Sprintf("file: %s deleted successfully", fileID)) } func TestGoogleDriveClient_DownloadToFile(t *testing.T) { client, err := NewGoogleDriveClient(driveAccessToken) if err != nil { - log.Printf("err while creating Google Drive client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) } ctx := context.Background() @@ -118,16 +120,16 @@ func TestGoogleDriveClient_DownloadToFile(t *testing.T) { destinationPath := "./downloaded_file.txt" err = client.DownloadToFile(ctx, fileID, destinationPath) if err != nil { - log.Printf("err while downloading file: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while downloading file: %v", err)) return } - log.Printf("downloaded to: %s", destinationPath) + zlogger.Logger.Info(fmt.Sprintf("downloaded to: %s", destinationPath)) } func TestGoogleDriveClient_DownloadToMemory(t *testing.T) { client, err := NewGoogleDriveClient(driveAccessToken) if err != nil { - log.Printf("err while creating Google Drive client: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) } ctx := context.Background() @@ -142,16 +144,16 @@ func TestGoogleDriveClient_DownloadToMemory(t *testing.T) { fileSize := int64(626) if err != nil { - log.Printf("err while getting file size: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while getting file size: %v", err)) return } data, err := client.DownloadToMemory(ctx, fileID, offset, chunkSize, fileSize) if err != nil { - log.Printf("err while downloading file: %v", err) + zlogger.Logger.Error(fmt.Sprintf("err while downloading file: %v", err)) return } - log.Printf("downloaded data: %s", data) + zlogger.Logger.Info(fmt.Sprintf("downloaded data: %s", data)) } From 14b26a3136f419bbfd2156cafa0cd97a4e6d1778 Mon Sep 17 00:00:00 2001 From: pewssh Date: Tue, 23 Apr 2024 03:28:58 +0545 Subject: [PATCH 05/11] Modified code to support other cloud storage platform --- cmd/dropboxMigrate.go | 62 +++++++++++++ cmd/loadConfig.go | 59 ++++++++++++ cmd/migrate.go | 12 ++- dropbox/core.go | 176 ++++++++++++++++-------------------- dropbox/test.go | 28 +++--- gdrive/core.go | 172 ++++++++++++++++++++--------------- gdrive/test.go | 35 +++---- migration/migrate.go | 80 ++++++++++------ migration/migrateConfig.go | 3 + migration/migration_test.go | 41 +++++---- s3/aws.go | 35 ++----- s3/aws_integration_test.go | 6 +- s3/mocks/mock_aws.go | 21 +++-- types/index.go | 32 +++++++ 14 files changed, 473 insertions(+), 289 deletions(-) create mode 100644 cmd/dropboxMigrate.go create mode 100644 cmd/loadConfig.go create mode 100644 types/index.go diff --git a/cmd/dropboxMigrate.go b/cmd/dropboxMigrate.go new file mode 100644 index 0000000..728f559 --- /dev/null +++ b/cmd/dropboxMigrate.go @@ -0,0 +1,62 @@ +/* +Copyright © 2024 NAME HERE +*/ +package cmd + +import ( + "fmt" + "log" + "path/filepath" + + "github.com/0chain/s3migration/util" + "github.com/spf13/cobra" +) + + var ( + cfgDropbox = "" + dropboxMigrateCmd = &cobra.Command{ + Use: "dropboxmigration", + Short: "Migrate data from Dropbox to dStorage", + Long: `dropboxMigrate is a command-line utility that facilitates the migration of files and data + from Dropbox to a decentralized storage allocation on the 0chain network. It leverages the + Dropbox SDK for Go to interact with Dropbox and the 0chain gosdk for communication with the + 0chain network.`, + Run: func(cmd *cobra.Command, args []string) { + // Implementation for the migration process from Dropbox to dStorage + fmt.Println("Migration from Dropbox to dStorage started...") + startMigration() + // TODO: Add the logic for Dropbox to dStorage migration + }, + } +) + + + +func init() { + cobra.OnInitialize(initDropboxConfig) + rootCmd.AddCommand(dropboxMigrateCmd) + dropboxMigrateCmd.PersistentFlags().String("directory", "", "DirectoryName") + dropboxMigrateCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") + dropboxMigrateCmd.Flags().StringVar(&configDir, "configDir", util.GetDefaultConfigDir(), "configuration directory") + dropboxMigrateCmd.Flags().StringVar(&cfgDropbox, "dropbox", "dropbox.yaml", "config file") +} + + + +func startMigration() { + fmt.Printf("Intializing...") + +} + + +func initDropboxConfig() { + fmt.Println("Dropbox Migration in line...") + fmt.Println(cfgDropbox) + _, err :=loadDropboxFile(filepath.Join(configDir, cfgDropbox)) + if err != nil { + // Handle the error appropriately + log.Fatalf("Failed to load Dropbox config: %v", err) + } + fmt.Printf("config directory + %v\n", cfgDropbox) + + } \ No newline at end of file diff --git a/cmd/loadConfig.go b/cmd/loadConfig.go new file mode 100644 index 0000000..d7c2ec1 --- /dev/null +++ b/cmd/loadConfig.go @@ -0,0 +1,59 @@ +package cmd + +import ( + "errors" + "fmt" + "os" + "strings" + + thrown "github.com/0chain/errors" + "github.com/0chain/gosdk/core/sys" + "github.com/spf13/viper" +) + +type DropboxConfig struct { + accessToken string +} +var (ErrMssingConfig = errors.New("[conf]missing config file") + ErrBadParsing = errors.New("parsing error") + ErrInvalidToken=errors.New("invalid access Token")) + +func loadDropboxFile(file string) (DropboxConfig, error) { + var cfg DropboxConfig + var err error + + _, err = sys.Files.Stat(file) + + if err != nil { + if errors.Is(err, os.ErrNotExist) { + return cfg, thrown.Throw(ErrMssingConfig, file) + } + return cfg, err + } + + v := viper.New() + + v.SetConfigFile(file) + + if err := v.ReadInConfig(); err != nil { + return cfg, thrown.Throw(ErrBadParsing, err.Error()) + } + + return LoadConfig(v) + +} + + +func LoadConfig(v *viper.Viper) (DropboxConfig, error) { + var cfg DropboxConfig + + accessToken := strings.TrimSpace(v.GetString("access_token")) + + if len(accessToken) == 0 { + fmt.Printf("accessToken + %v\n", accessToken) + return cfg, thrown.Throw(ErrInvalidToken, "token is empty") + } + + + return cfg, nil +} \ No newline at end of file diff --git a/cmd/migrate.go b/cmd/migrate.go index 4ad0f08..9af197b 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -40,6 +40,8 @@ var ( chunkSize int64 chunkNumber int batchSize int + source string + accessToken string // if source is google drive or dropbox ) // migrateCmd is the migrateFromS3 sub command to migrate whole objects from some buckets. @@ -71,7 +73,8 @@ func init() { migrateCmd.Flags().Int64Var(&chunkSize, "chunk-size", 50*1024*1024, "chunk size in bytes") migrateCmd.Flags().IntVar(&chunkNumber, "chunk-number", 250, "number of chunks to upload") migrateCmd.Flags().IntVar(&batchSize, "batch-size", 20, "number of files to upload in a batch") - + migrateCmd.Flags().StringVar(&source, "source", "s3", "s3 or google_drive or dropbox") + migrateCmd.Flags().StringVar(&accessToken, "access-token", "", "access token for google drive or dropbox") } var migrateCmd = &cobra.Command{ @@ -129,6 +132,10 @@ var migrateCmd = &cobra.Command{ } } + if source == "" { + source = "s3" + } + if skip < 0 || skip > 2 { return fmt.Errorf("skip value not in range 0-2. Provided value is %v", skip) } @@ -232,6 +239,9 @@ var migrateCmd = &cobra.Command{ ChunkSize: chunkSize, ChunkNumber: chunkNumber, BatchSize: batchSize, + + Source: source, + AccessToken: accessToken, } if err := migration.InitMigration(&mConfig); err != nil { diff --git a/dropbox/core.go b/dropbox/core.go index 24c52e4..b447454 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -6,41 +6,23 @@ import ( "io" "mime" "os" + "path" "path/filepath" + T "github.com/0chain/s3migration/types" + "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" ) -type DropboxI interface { - ListFiles(ctx context.Context) ([]*ObjectMeta, error) - ListFilesInFolder(ctx context.Context, folderName string) ([]*ObjectMeta, error) - GetFileContent(ctx context.Context, filePath string) (*Object, error) - DeleteFile(ctx context.Context, filePath string) error - DownloadToFile(ctx context.Context, filePath string) (string, error) - DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) -} - -type Object struct { - Body io.ReadCloser - ContentType string - ContentLength uint64 -} - -type ObjectMeta struct { - Name string - Path string - Size uint64 - ContentType string -} - type DropboxClient struct { token string + workDir string dropboxConf *dropbox.Config dropboxFiles files.Client } -func GetDropboxClient(token string) (*DropboxClient, error) { +func GetDropboxClient(token string, workDir string) (*DropboxClient, error) { config := dropbox.Config{ Token: token, } @@ -51,78 +33,80 @@ func GetDropboxClient(token string) (*DropboxClient, error) { token: token, dropboxConf: &config, dropboxFiles: client, + workDir: workDir, }, nil } -func (d *DropboxClient) ListFiles(ctx context.Context) ([]*ObjectMeta, error) { - var objects []*ObjectMeta - - arg := files.NewListFolderArg("") - arg.Recursive = true - - res, err := d.dropboxFiles.ListFolder(arg) +func (d *DropboxClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) { + objectChan := make(chan *T.ObjectMeta) + errChan := make(chan error) - if err != nil { - return nil, err - } - - for _, entry := range res.Entries { - - if meta, ok := entry.(*files.FileMetadata); ok { + go func() { + defer func() { + close(objectChan) + close(errChan) + }() - objects = append(objects, &ObjectMeta{ - Name: meta.Name, - Path: meta.PathDisplay, - Size: meta.Size, - ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), - }) + arg := files.NewListFolderArg("") // "" for Root + arg.Recursive = true + arg.Limit = 100 + res, err := d.dropboxFiles.ListFolder(arg) + if err != nil { + errChan <- err + return } - } - - return objects, nil -} - -func (d *DropboxClient) ListFilesInFolder(ctx context.Context, folderName string) ([]*ObjectMeta, error) { - var objects []*ObjectMeta - arg := files.NewListFolderArg(folderName) - arg.Recursive = true - - res, err := d.dropboxFiles.ListFolder(arg) - - if err != nil { - return nil, err - } - - for _, entry := range res.Entries { - - if meta, ok := entry.(*files.FileMetadata); ok { - - objects = append(objects, &ObjectMeta{ - Name: meta.Name, - Path: meta.PathDisplay, - Size: meta.Size, - ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), - }) + for _, entry := range res.Entries { + if meta, ok := entry.(*files.FileMetadata); ok { + objectChan <- &T.ObjectMeta{ + Key: meta.PathDisplay, + Size: int64(meta.Size), + ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + } + } + } + cursor := res.Cursor + hasMore := res.HasMore + + for hasMore { + continueArg := files.NewListFolderContinueArg(cursor) + res, err := d.dropboxFiles.ListFolderContinue(continueArg) + if err != nil { + errChan <- err + return + } + + for _, entry := range res.Entries { + if meta, ok := entry.(*files.FileMetadata); ok { + objectChan <- &T.ObjectMeta{ + Key: meta.PathDisplay, + Size: int64(meta.Size), + ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + } + } + } + + cursor = res.Cursor + hasMore = res.HasMore } - } + }() - return objects, nil + return objectChan, errChan } -func (d *DropboxClient) GetFileContent(ctx context.Context, filePath string) (*Object, error) { +func (d *DropboxClient) GetFileContent(ctx context.Context, filePath string) (*T.Object, error) { arg := files.NewDownloadArg(filePath) res, content, err := d.dropboxFiles.Download(arg) if err != nil { return nil, err } - return &Object{ + return &T.Object{ Body: content, ContentType: mime.TypeByExtension(filepath.Ext(filePath)), - ContentLength: res.Size, + ContentLength: int64(res.Size), }, nil } @@ -140,7 +124,7 @@ func (d *DropboxClient) DownloadToFile(ctx context.Context, filePath string) (st } fileName := filepath.Base(filePath) - downloadPath := filepath.Join(".", fileName) + downloadPath := path.Join(d.workDir, fileName) file, err := os.Create(downloadPath) if err != nil { return "", err @@ -156,33 +140,33 @@ func (d *DropboxClient) DownloadToFile(ctx context.Context, filePath string) (st } func (d *DropboxClient) DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) { - limit := offset + chunkSize - 1 - if limit > objectSize { - limit = objectSize - } + limit := offset + chunkSize - 1 + if limit > objectSize { + limit = objectSize + } - rng := fmt.Sprintf("bytes=%d-%d", offset, limit) + rng := fmt.Sprintf("bytes=%d-%d", offset, limit) - arg := files.NewDownloadArg(objectKey) + arg := files.NewDownloadArg(objectKey) - arg.ExtraHeaders = map[string]string{"Range": rng} + arg.ExtraHeaders = map[string]string{"Range": rng} - _, content, err := d.dropboxFiles.Download(arg) - if err != nil { - return nil, err - } - defer content.Close() + _, content, err := d.dropboxFiles.Download(arg) + if err != nil { + return nil, err + } + defer content.Close() - data := make([]byte, chunkSize) - n, err := io.ReadFull(content, data) + data := make([]byte, chunkSize) + n, err := io.ReadFull(content, data) - if err != nil && err != io.ErrUnexpectedEOF { - return nil, err - } + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } - if int64(n) < chunkSize && objectSize != chunkSize { - data = data[:n] - } + if int64(n) < chunkSize && objectSize != chunkSize { + data = data[:n] + } - return data, nil + return data, nil } diff --git a/dropbox/test.go b/dropbox/test.go index b17d4dd..dfe200e 100644 --- a/dropbox/test.go +++ b/dropbox/test.go @@ -10,7 +10,7 @@ import ( var ( dropboxAccessToken = "" - testFilePath = "" + testFilePath = "" ) const TestFileContent = ` by Manuel Gutiérrez Nájera @@ -37,26 +37,28 @@ though we know with our hearts that she lies. ` func TestDropboxClient_ListFiles(t *testing.T) { - client, err := GetDropboxClient(dropboxAccessToken) + client, err := GetDropboxClient(dropboxAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return } ctx := context.Background() - files, err := client.ListFiles(ctx) - if err != nil { - zlogger.Logger.Error(fmt.Sprintf("Error while listing files: %v", err)) - return - } + objectChan, errChan := client.ListFiles(ctx) + + go func() { + for err := range errChan { + zlogger.Logger.Error(fmt.Sprintf("Error while listing files: %v", err)) + } + }() - for _, file := range files { - zlogger.Logger.Info(fmt.Sprintf("File: %s, Name: %s, Size: %d bytes", file.Path, file.ContentType, file.Size)) + for object := range objectChan { + zlogger.Logger.Info(fmt.Sprintf("Key: %s, Type: %s, Size: %d bytes", object.Key, object.ContentType, object.Size)) } } func TestDropboxClient_GetFileContent(t *testing.T) { - client, err := GetDropboxClient(dropboxAccessToken) + client, err := GetDropboxClient(dropboxAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) } @@ -89,7 +91,7 @@ func TestDropboxClient_GetFileContent(t *testing.T) { } func TestDropboxClient_DeleteFile(t *testing.T) { - client, err := GetDropboxClient(dropboxAccessToken) + client, err := GetDropboxClient(dropboxAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return @@ -106,7 +108,7 @@ func TestDropboxClient_DeleteFile(t *testing.T) { } func TestDropboxClient_DownloadToFile(t *testing.T) { - client, err := GetDropboxClient(dropboxAccessToken) + client, err := GetDropboxClient(dropboxAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return @@ -123,7 +125,7 @@ func TestDropboxClient_DownloadToFile(t *testing.T) { } func TestDropboxClient_DownloadToMemory(t *testing.T) { - client, err := GetDropboxClient(dropboxAccessToken) + client, err := GetDropboxClient(dropboxAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err)) return diff --git a/gdrive/core.go b/gdrive/core.go index 42038c2..36f4ccb 100644 --- a/gdrive/core.go +++ b/gdrive/core.go @@ -5,41 +5,21 @@ import ( "fmt" "io" "os" - "strings" + "path" zlogger "github.com/0chain/s3migration/logger" + T "github.com/0chain/s3migration/types" "golang.org/x/oauth2" "google.golang.org/api/drive/v3" "google.golang.org/api/option" ) -type GoogleDriveI interface { - ListFiles(ctx context.Context) ([]*ObjectMeta, error) - GetFileContent(ctx context.Context, fileID string, keepOpen bool) (*Object, error) - DeleteFile(ctx context.Context, fileID string) error - DownloadToFile(ctx context.Context, fileID, destinationPath string) error - DownloadToMemory(ctx context.Context, fileID string, offset int64, chunkSize, fileSize int64) ([]byte, error) -} - -type Object struct { - Body io.ReadCloser - ContentType string - ContentLength int64 -} - -type ObjectMeta struct { - ID string - Name string - Size int64 - ContentType string - LastModified string -} - type GoogleDriveClient struct { service *drive.Service + workDir string } -func NewGoogleDriveClient(accessToken string) (*GoogleDriveClient, error) { +func NewGoogleDriveClient(accessToken string, workDir string) (*GoogleDriveClient, error) { ctx := context.Background() tokenSource := oauth2.StaticTokenSource(&oauth2.Token{AccessToken: accessToken}) @@ -53,44 +33,86 @@ func NewGoogleDriveClient(accessToken string) (*GoogleDriveClient, error) { return &GoogleDriveClient{ service: service, + workDir: workDir, }, nil } -func (g *GoogleDriveClient) ListFiles(ctx context.Context) ([]*ObjectMeta, error) { - files, err := g.service.Files.List().Context(ctx).Do() - if err != nil { - return nil, err - } +func (g *GoogleDriveClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) { + objectChan := make(chan *T.ObjectMeta) + errChan := make(chan error) - var objects []*ObjectMeta - for _, file := range files.Files { - if strings.HasSuffix(file.Name, "/") { // Skip dirs - continue - } + go func() { + defer func() { + close(objectChan) + close(errChan) + }() + + filesReq := g.service.Files.List().Context(ctx) - objects = append(objects, &ObjectMeta{ - ID: file.Id, - Name: file.Name, - Size: file.Size, - ContentType: file.MimeType, - LastModified: file.ModifiedTime, + filesReq.Q("trashed=false") + + filesReq.Fields( + "id, mimeType, size", + ) + + filesReq.Pages(ctx, func(page *drive.FileList) error { + return nil }) - } - return objects, nil + filesReq.PageSize(100) + + files, err := filesReq.Do() + if err != nil { + errChan <- err + return + } + + for _, file := range files.Files { + objectChan <- &T.ObjectMeta{ + Key: file.Id, + Size: file.Size, + ContentType: file.MimeType, + } + } + + nextPgToken := files.NextPageToken + + for nextPgToken != "" { + filesReq.PageToken(nextPgToken) + + files, err := filesReq.Do() + + if err != nil { + errChan <- err + return + } + + for _, file := range files.Files { + objectChan <- &T.ObjectMeta{ + Key: file.Id, + Size: file.Size, + ContentType: file.MimeType, + } + } + + nextPgToken = files.NextPageToken + } + }() + + return objectChan, errChan } -func (g *GoogleDriveClient) GetFileContent(ctx context.Context, fileID string, keepOpen bool) (*Object, error) { +func (g *GoogleDriveClient) GetFileContent(ctx context.Context, fileID string) (*T.Object, error) { resp, err := g.service.Files.Get(fileID).Download() if err != nil { return nil, err } - if !keepOpen { - defer resp.Body.Close() - } + // if !keepOpen { + // defer resp.Body.Close() + // } - obj := &Object{ + obj := &T.Object{ Body: resp.Body, ContentType: resp.Header.Get("Content-Type"), ContentLength: resp.ContentLength, @@ -107,55 +129,59 @@ func (g *GoogleDriveClient) DeleteFile(ctx context.Context, fileID string) error return nil } -func (g *GoogleDriveClient) DownloadToFile(ctx context.Context, fileID, destinationPath string) error { +func (g *GoogleDriveClient) DownloadToFile(ctx context.Context, fileID string) (string, error) { resp, err := g.service.Files.Get(fileID).Download() if err != nil { - return err + return "", err } defer resp.Body.Close() + destinationPath := path.Join(g.workDir, fileID) + out, err := os.Create(destinationPath) if err != nil { - return err + return "", err } + defer out.Close() _, err = io.Copy(out, resp.Body) if err != nil { - return err + return "", err } + zlogger.Logger.Info(fmt.Sprintf("Downloaded file ID: %s to %s\n", fileID, destinationPath)) - return nil + return destinationPath, nil } func (g *GoogleDriveClient) DownloadToMemory(ctx context.Context, fileID string, offset int64, chunkSize, fileSize int64) ([]byte, error) { - limit := offset + chunkSize - 1 - if limit > fileSize { - limit = fileSize - } + limit := offset + chunkSize - 1 + if limit > fileSize { + limit = fileSize + } - rng := fmt.Sprintf("bytes=%d-%d", offset, limit) + rng := fmt.Sprintf("bytes=%d-%d", offset, limit) - req := g.service.Files.Get(fileID) + req := g.service.Files.Get(fileID) - req.Header().Set("Range", rng) + req.Header().Set("Range", rng) - resp, err := req.Download() - if err != nil { - return nil, err - } - defer resp.Body.Close() + resp, err := req.Download() + if err != nil { + return nil, err + } + defer resp.Body.Close() - data := make([]byte, chunkSize) - n, err := io.ReadFull(resp.Body, data) + data := make([]byte, chunkSize) + n, err := io.ReadFull(resp.Body, data) - if err != nil && err != io.ErrUnexpectedEOF { - return nil, err - } + if err != nil && err != io.ErrUnexpectedEOF { + return nil, err + } - if int64(n) < chunkSize && fileSize != chunkSize { - data = data[:n] - } + if int64(n) < chunkSize && fileSize != chunkSize { + data = data[:n] + } - return data, nil + return data, nil } diff --git a/gdrive/test.go b/gdrive/test.go index 4104289..d9de872 100644 --- a/gdrive/test.go +++ b/gdrive/test.go @@ -9,8 +9,8 @@ import ( ) var ( - driveAccessToken="" - testFileID="" + driveAccessToken = "ya29.a0Ad52N384rCQyODIBhljd7yg4RmP8czIUmIbJGkFHXNn-TnuEt3RL3ykzp0lEekSEoU0GDCkMnow31XOWFt0Dlw2l89fnO5sS8aiZKW8y0dDhqE6jtsNqpnA38EN5tBlRkNo0ipyY0Ps-cgX4hNxRRQcWhTZEqZgnOyLuaCgYKAWESARESFQHGX2Mi6ejaNEiKSZP6_H3hoXkeNA0171" + testFileID = "" ) // using: https://developers.google.com/oauthplayground @@ -40,26 +40,28 @@ though we know with our hearts that she lies. ` func TestGoogleDriveClient_ListFiles(t *testing.T) { - client, err := NewGoogleDriveClient(driveAccessToken) + client, err := NewGoogleDriveClient(driveAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) return } ctx := context.Background() - files, err := client.ListFiles(ctx) - if err != nil { - zlogger.Logger.Error(fmt.Sprintf("err while list files: %v", err)) - return - } + objectChan, errChan := client.ListFiles(ctx) + + go func() { + for err := range errChan { + zlogger.Logger.Error(fmt.Sprintf("err while list files: %v", err)) + } + }() - for _, file := range files { - zlogger.Logger.Info(fmt.Sprintf("file: %s, name: %s, size: %d bytes", file.ID, file.Name, file.Size)) + for object := range objectChan { + zlogger.Logger.Info(fmt.Sprintf("file:%s, size: %d bytes, type: %s", object.Key, object.Size, object.ContentType)) } } func TestGoogleDriveClient_GetFileContent(t *testing.T) { - client, err := NewGoogleDriveClient(driveAccessToken) + client, err := NewGoogleDriveClient(driveAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("Failed to creating Google Drive client: %v", err)) return @@ -67,7 +69,7 @@ func TestGoogleDriveClient_GetFileContent(t *testing.T) { ctx := context.Background() fileID := testFileID - obj, err := client.GetFileContent(ctx, fileID, true) + obj, err := client.GetFileContent(ctx, fileID) if err != nil { zlogger.Logger.Error(fmt.Sprintf("err while getting file content: %v", err)) @@ -93,7 +95,7 @@ func TestGoogleDriveClient_GetFileContent(t *testing.T) { } func TestGoogleDriveClient_DeleteFile(t *testing.T) { - client, err := NewGoogleDriveClient(driveAccessToken) + client, err := NewGoogleDriveClient(driveAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) return @@ -110,15 +112,14 @@ func TestGoogleDriveClient_DeleteFile(t *testing.T) { } func TestGoogleDriveClient_DownloadToFile(t *testing.T) { - client, err := NewGoogleDriveClient(driveAccessToken) + client, err := NewGoogleDriveClient(driveAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) } ctx := context.Background() fileID := testFileID - destinationPath := "./downloaded_file.txt" - err = client.DownloadToFile(ctx, fileID, destinationPath) + destinationPath, err := client.DownloadToFile(ctx, fileID) if err != nil { zlogger.Logger.Error(fmt.Sprintf("err while downloading file: %v", err)) return @@ -127,7 +128,7 @@ func TestGoogleDriveClient_DownloadToFile(t *testing.T) { } func TestGoogleDriveClient_DownloadToMemory(t *testing.T) { - client, err := NewGoogleDriveClient(driveAccessToken) + client, err := NewGoogleDriveClient(driveAccessToken, "./") if err != nil { zlogger.Logger.Error(fmt.Sprintf("err while creating Google Drive client: %v", err)) } diff --git a/migration/migrate.go b/migration/migrate.go index f59610b..1d8bdc2 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -13,6 +13,10 @@ import ( "syscall" "time" + "github.com/0chain/s3migration/dropbox" + "github.com/0chain/s3migration/gdrive" + T "github.com/0chain/s3migration/types" + "github.com/0chain/gosdk/zboxcore/sdk" "github.com/0chain/gosdk/zboxcore/zboxutil" dStorage "github.com/0chain/s3migration/dstorage" @@ -57,9 +61,9 @@ func abandonAllOperations(err error) { } type Migration struct { - zStore dStorage.DStoreI - awsStore s3.AwsI - fs util.FileSystem + zStore dStorage.DStoreI + dataSourceStore T.CloudStorageI + fs util.FileSystem skip int retryCount int @@ -119,35 +123,53 @@ func InitMigration(mConfig *MigrationConfig) error { } mConfig.ChunkSize = int64(mConfig.ChunkNumber) * dStorageService.GetChunkWriteSize() zlogger.Logger.Info("Getting aws storage service") - awsStorageService, err := s3.GetAwsClient( - mConfig.Bucket, - mConfig.Prefix, - mConfig.Region, - mConfig.DeleteSource, - mConfig.NewerThan, - mConfig.OlderThan, - mConfig.StartAfter, - mConfig.WorkDir, - ) + + var dataSourceStore T.CloudStorageI + if mConfig.Source == "s3" { + dataSourceStore, err = s3.GetAwsClient( + mConfig.Bucket, + mConfig.Prefix, + mConfig.Region, + mConfig.DeleteSource, + mConfig.NewerThan, + mConfig.OlderThan, + mConfig.StartAfter, + mConfig.WorkDir, + ) + } else if mConfig.Source == "dropbox" { + dataSourceStore, err = dropbox.GetDropboxClient( + mConfig.AccessToken, + mConfig.WorkDir, + ) + } else if mConfig.Source == "google_drive" { + dataSourceStore, err = gdrive.NewGoogleDriveClient( + mConfig.AccessToken, + mConfig.WorkDir, + ) + } else { + zlogger.Logger.Error("invalid source: ", mConfig.Source) + return err + } + if err != nil { zlogger.Logger.Error(err) return err } migration = Migration{ - zStore: dStorageService, - awsStore: awsStorageService, - skip: mConfig.Skip, - concurrency: mConfig.Concurrency, - retryCount: mConfig.RetryCount, - stateFilePath: mConfig.StateFilePath, - migrateTo: mConfig.MigrateToPath, - deleteSource: mConfig.DeleteSource, - workDir: mConfig.WorkDir, - bucket: mConfig.Bucket, - fs: util.Fs, - chunkSize: mConfig.ChunkSize, - batchSize: mConfig.BatchSize, + zStore: dStorageService, + dataSourceStore: dataSourceStore, + skip: mConfig.Skip, + concurrency: mConfig.Concurrency, + retryCount: mConfig.RetryCount, + stateFilePath: mConfig.StateFilePath, + migrateTo: mConfig.MigrateToPath, + deleteSource: mConfig.DeleteSource, + workDir: mConfig.WorkDir, + bucket: mConfig.Bucket, + fs: util.Fs, + chunkSize: mConfig.ChunkSize, + batchSize: mConfig.BatchSize, } rootContext, rootContextCancel = context.WithCancel(context.Background()) @@ -246,7 +268,7 @@ func (m *Migration) DownloadWorker(ctx context.Context, migrator *MigrationWorke totalObjChan := make(chan struct{}, 100) defer close(totalObjChan) go updateTotalObjects(totalObjChan, m.workDir) - objCh, errCh := migration.awsStore.ListFilesInBucket(rootContext) + objCh, errCh := migration.dataSourceStore.ListFiles(rootContext) wg := &sync.WaitGroup{} ops := make([]MigrationOperation, 0, m.batchSize) var opLock sync.Mutex @@ -553,7 +575,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp defer func() { for _, op := range ops { if migration.deleteSource && err == nil { - if deleteErr := migration.awsStore.DeleteFile(ctx, op.uploadObj.ObjectKey); deleteErr != nil { + if deleteErr := migration.dataSourceStore.DeleteFile(ctx, op.uploadObj.ObjectKey); deleteErr != nil { zlogger.Logger.Error(deleteErr) dsFileHandler.Write([]byte(op.uploadObj.ObjectKey + "\n")) } @@ -609,7 +631,7 @@ func (m *Migration) processChunkDownload(ctx context.Context, sw *util.StreamWri return default: } - data, err := m.awsStore.DownloadToMemory(ctx, downloadObjMeta.ObjectKey, int64(offset), int64(chunkSize), downloadObjMeta.Size) + data, err := m.dataSourceStore.DownloadToMemory(ctx, downloadObjMeta.ObjectKey, int64(offset), int64(chunkSize), downloadObjMeta.Size) if err != nil { migrator.DownloadDone(downloadObjMeta, "", err) ctx.Err() diff --git a/migration/migrateConfig.go b/migration/migrateConfig.go index c7d6b50..4a59945 100644 --- a/migration/migrateConfig.go +++ b/migration/migrateConfig.go @@ -22,4 +22,7 @@ type MigrationConfig struct { ChunkSize int64 ChunkNumber int BatchSize int + + Source string // "s3" (default) or "google_drive" or "dropbox" + AccessToken string // if Source == "google_drive" or "dropbox" } diff --git a/migration/migration_test.go b/migration/migration_test.go index 287748a..b02661d 100644 --- a/migration/migration_test.go +++ b/migration/migration_test.go @@ -6,8 +6,9 @@ import ( "log" "testing" + T "github.com/0chain/s3migration/types" + mock_dstorage "github.com/0chain/s3migration/dstorage/mocks" - "github.com/0chain/s3migration/s3" mock_s3 "github.com/0chain/s3migration/s3/mocks" mock_util "github.com/0chain/s3migration/util/mocks" "github.com/golang/mock/gomock" @@ -21,12 +22,12 @@ func TestMigrate(t *testing.T) { dStorageService := mock_dstorage.NewMockDStoreI(ctrl) awsStorageService := mock_s3.NewMockAwsI(ctrl) fileSystem := mock_util.NewMockFileSystem(ctrl) - migration = Migration{ - zStore: dStorageService, - awsStore: awsStorageService, - skip: Skip, - fs: fileSystem, - } + // migration = Migration{ + // zStore: dStorageService, + // awsStore: awsStorageService, + // skip: Skip, + // fs: fileSystem, + // } tests := []struct { name string @@ -39,16 +40,16 @@ func TestMigrate(t *testing.T) { name: "success in uploading files", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) - fileListChan <- &s3.ObjectMeta{ + fileListChan := make(chan *T.ObjectMeta, 1000) + fileListChan <- &T.ObjectMeta{ Key: "file1", Size: 1200, } - fileListChan <- &s3.ObjectMeta{ + fileListChan <- &T.ObjectMeta{ Key: "file2", Size: 1400, } - fileListChan <- &s3.ObjectMeta{ + fileListChan <- &T.ObjectMeta{ Key: "file3", Size: 1500, } @@ -56,7 +57,7 @@ func TestMigrate(t *testing.T) { errChan := make(chan error, 1) close(errChan) - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) awsStorageService.EXPECT().DownloadToFile(gomock.Any(), "file1").Return("/aws/file1", nil) awsStorageService.EXPECT().DownloadToFile(gomock.Any(), "file2").Return("/aws/file2", nil) @@ -106,8 +107,8 @@ func TestMigrate(t *testing.T) { name: "download to file error", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) - fileListChan <- &s3.ObjectMeta{ + fileListChan := make(chan *T.ObjectMeta, 1000) + fileListChan <- &T.ObjectMeta{ Key: "file11", Size: 1200, } @@ -115,7 +116,7 @@ func TestMigrate(t *testing.T) { errChan := make(chan error, 1) close(errChan) - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) updateKeyFunc = func(statePath string) (func(stateKey string), func(), error) { return func(stateKey string) {}, func() {}, nil @@ -132,8 +133,8 @@ func TestMigrate(t *testing.T) { name: "dstorage upload error", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) - fileListChan <- &s3.ObjectMeta{ + fileListChan := make(chan *T.ObjectMeta, 1000) + fileListChan <- &T.ObjectMeta{ Key: "file10", Size: 1200, } @@ -141,7 +142,7 @@ func TestMigrate(t *testing.T) { errChan := make(chan error, 1) close(errChan) - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) updateKeyFunc = func(statePath string) (func(stateKey string), func(), error) { return func(stateKey string) {}, func() {}, nil @@ -171,14 +172,14 @@ func TestMigrate(t *testing.T) { name: "aws list object error", setUpMock: func() { rootContext, rootContextCancel = context.WithCancel(context.Background()) - fileListChan := make(chan *s3.ObjectMeta, 1000) + fileListChan := make(chan *T.ObjectMeta, 1000) close(fileListChan) errChan := make(chan error, 1) errChan <- errors.New("some error") - awsStorageService.EXPECT().ListFilesInBucket(gomock.Any()).Return(fileListChan, errChan) + awsStorageService.EXPECT().ListFiles(gomock.Any()).Return(fileListChan, errChan) updateKeyFunc = func(statePath string) (func(stateKey string), func(), error) { return func(stateKey string) {}, func() {}, nil diff --git a/s3/aws.go b/s3/aws.go index 95394b1..ef803c5 100644 --- a/s3/aws.go +++ b/s3/aws.go @@ -3,12 +3,13 @@ package s3 import ( "context" "fmt" - "io" "os" "path/filepath" "strings" "time" + T "github.com/0chain/s3migration/types" + "github.com/0chain/gosdk/core/encryption" zlogger "github.com/0chain/s3migration/logger" "github.com/aws/aws-sdk-go-v2/aws" @@ -17,28 +18,6 @@ import ( awsS3 "github.com/aws/aws-sdk-go-v2/service/s3" ) -//go:generate mockgen -destination mocks/mock_aws.go -package mock_s3 github.com/0chain/s3migration/s3 AwsI -type AwsI interface { - ListFilesInBucket(ctx context.Context) (<-chan *ObjectMeta, <-chan error) - GetFileContent(ctx context.Context, objectKey string) (*Object, error) - DeleteFile(ctx context.Context, objectKey string) error - DownloadToFile(ctx context.Context, objectKey string) (string, error) - DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) -} - -type Object struct { - Body io.Reader - ContentType string - ContentLength int64 -} - -// ObjectMeta key: object key, size: size of object in bytes -type ObjectMeta struct { - Key string - Size int64 - ContentType string -} - type AwsClient struct { bucket string prefix string @@ -136,8 +115,8 @@ func (a *AwsClient) getBucketRegion() (region string, err error) { return } -func (a *AwsClient) ListFilesInBucket(ctx context.Context) (<-chan *ObjectMeta, <-chan error) { - objectMetaChan := make(chan *ObjectMeta, 1000) +func (a *AwsClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) { + objectMetaChan := make(chan *T.ObjectMeta, 1000) errChan := make(chan error, 1) go func() { @@ -192,20 +171,20 @@ func (a *AwsClient) ListFilesInBucket(ctx context.Context) (<-chan *ObjectMeta, errChan <- err return } - objectMetaChan <- &ObjectMeta{Key: aws.ToString(obj.Key), Size: obj.Size, ContentType: contentType} + objectMetaChan <- &T.ObjectMeta{Key: aws.ToString(obj.Key), Size: obj.Size, ContentType: contentType} } } }() return objectMetaChan, errChan } -func (a *AwsClient) GetFileContent(ctx context.Context, objectKey string) (*Object, error) { +func (a *AwsClient) GetFileContent(ctx context.Context, objectKey string) (*T.Object, error) { out, err := a.client.GetObject(ctx, &awsS3.GetObjectInput{Bucket: aws.String(a.bucket), Key: aws.String(objectKey)}) if err != nil { return nil, err } - return &Object{ + return &T.Object{ Body: out.Body, ContentType: aws.ToString(out.ContentType), ContentLength: out.ContentLength, diff --git a/s3/aws_integration_test.go b/s3/aws_integration_test.go index 9ea7209..b046644 100644 --- a/s3/aws_integration_test.go +++ b/s3/aws_integration_test.go @@ -5,9 +5,11 @@ package s3 import ( "context" - "github.com/0chain/s3migration/util" "log" "testing" + "time" + + "github.com/0chain/s3migration/util" ) func TestService_ListAllBuckets(t *testing.T) { @@ -16,7 +18,7 @@ func TestService_ListAllBuckets(t *testing.T) { util.SetAwsEnvCredentials(awsAccessKey, awsSecretKey) s3Svc, _ := GetAwsClient("", "", "", false, nil, nil, "", "") - fileListChan, errChan := s3Svc.ListFilesInBucket(context.Background()) + fileListChan, errChan := s3Svc.ListFiles(context.Background()) for { objKey, ok := <-fileListChan if ok { diff --git a/s3/mocks/mock_aws.go b/s3/mocks/mock_aws.go index c2232db..c677339 100644 --- a/s3/mocks/mock_aws.go +++ b/s3/mocks/mock_aws.go @@ -8,7 +8,8 @@ import ( context "context" reflect "reflect" - s3 "github.com/0chain/s3migration/s3" + T "github.com/0chain/s3migration/types" + gomock "github.com/golang/mock/gomock" ) @@ -65,10 +66,10 @@ func (mr *MockAwsIMockRecorder) DownloadToFile(arg0, arg1 interface{}) *gomock.C } // GetFileContent mocks base method. -func (m *MockAwsI) GetFileContent(arg0 context.Context, arg1 string) (*s3.Object, error) { +func (m *MockAwsI) GetFileContent(arg0 context.Context, arg1 string) (*T.Object, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetFileContent", arg0, arg1) - ret0, _ := ret[0].(*s3.Object) + ret0, _ := ret[0].(*T.Object) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -79,17 +80,17 @@ func (mr *MockAwsIMockRecorder) GetFileContent(arg0, arg1 interface{}) *gomock.C return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetFileContent", reflect.TypeOf((*MockAwsI)(nil).GetFileContent), arg0, arg1) } -// ListFilesInBucket mocks base method. -func (m *MockAwsI) ListFilesInBucket(arg0 context.Context) (<-chan *s3.ObjectMeta, <-chan error) { +// ListFiles mocks base method. +func (m *MockAwsI) ListFiles(arg0 context.Context) (<-chan *T.ObjectMeta, <-chan error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ListFilesInBucket", arg0) - ret0, _ := ret[0].(<-chan *s3.ObjectMeta) + ret := m.ctrl.Call(m, "ListFiles", arg0) + ret0, _ := ret[0].(<-chan *T.ObjectMeta) ret1, _ := ret[1].(<-chan error) return ret0, ret1 } -// ListFilesInBucket indicates an expected call of ListFilesInBucket. -func (mr *MockAwsIMockRecorder) ListFilesInBucket(arg0 interface{}) *gomock.Call { +// ListFiles indicates an expected call of ListFiles. +func (mr *MockAwsIMockRecorder) ListFiles(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFilesInBucket", reflect.TypeOf((*MockAwsI)(nil).ListFilesInBucket), arg0) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListFiles", reflect.TypeOf((*MockAwsI)(nil).ListFiles), arg0) } diff --git a/types/index.go b/types/index.go new file mode 100644 index 0000000..a2aa28f --- /dev/null +++ b/types/index.go @@ -0,0 +1,32 @@ +package types + +import ( + "context" + "io" +) + +type Object struct { + Body io.ReadCloser + ContentType string + ContentLength int64 +} + +// ObjectMeta key: object key, size: size of object in bytes +type ObjectMeta struct { + Key string + Size int64 + ContentType string +} + +type CloudStorageI interface { + ListFiles(ctx context.Context) (<-chan *ObjectMeta, <-chan error) + GetFileContent(ctx context.Context, objectKey string) (*Object, error) + DeleteFile(ctx context.Context, objectKey string) error + DownloadToFile(ctx context.Context, objectKey string) (string, error) + DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) +} + +type CloudStorageClient struct { + name string +} + From c0847fca2aa9a405381c61f992d6a3e0db6c37ae Mon Sep 17 00:00:00 2001 From: pewssh Date: Thu, 25 Apr 2024 11:07:54 +0545 Subject: [PATCH 06/11] fix --- cmd/dropboxMigrate.go | 62 ------------------------------------------- cmd/loadConfig.go | 59 ---------------------------------------- cmd/migrate.go | 17 +++++++----- util/util.go | 3 +++ 4 files changed, 14 insertions(+), 127 deletions(-) delete mode 100644 cmd/dropboxMigrate.go delete mode 100644 cmd/loadConfig.go diff --git a/cmd/dropboxMigrate.go b/cmd/dropboxMigrate.go deleted file mode 100644 index 728f559..0000000 --- a/cmd/dropboxMigrate.go +++ /dev/null @@ -1,62 +0,0 @@ -/* -Copyright © 2024 NAME HERE -*/ -package cmd - -import ( - "fmt" - "log" - "path/filepath" - - "github.com/0chain/s3migration/util" - "github.com/spf13/cobra" -) - - var ( - cfgDropbox = "" - dropboxMigrateCmd = &cobra.Command{ - Use: "dropboxmigration", - Short: "Migrate data from Dropbox to dStorage", - Long: `dropboxMigrate is a command-line utility that facilitates the migration of files and data - from Dropbox to a decentralized storage allocation on the 0chain network. It leverages the - Dropbox SDK for Go to interact with Dropbox and the 0chain gosdk for communication with the - 0chain network.`, - Run: func(cmd *cobra.Command, args []string) { - // Implementation for the migration process from Dropbox to dStorage - fmt.Println("Migration from Dropbox to dStorage started...") - startMigration() - // TODO: Add the logic for Dropbox to dStorage migration - }, - } -) - - - -func init() { - cobra.OnInitialize(initDropboxConfig) - rootCmd.AddCommand(dropboxMigrateCmd) - dropboxMigrateCmd.PersistentFlags().String("directory", "", "DirectoryName") - dropboxMigrateCmd.Flags().BoolP("toggle", "t", false, "Help message for toggle") - dropboxMigrateCmd.Flags().StringVar(&configDir, "configDir", util.GetDefaultConfigDir(), "configuration directory") - dropboxMigrateCmd.Flags().StringVar(&cfgDropbox, "dropbox", "dropbox.yaml", "config file") -} - - - -func startMigration() { - fmt.Printf("Intializing...") - -} - - -func initDropboxConfig() { - fmt.Println("Dropbox Migration in line...") - fmt.Println(cfgDropbox) - _, err :=loadDropboxFile(filepath.Join(configDir, cfgDropbox)) - if err != nil { - // Handle the error appropriately - log.Fatalf("Failed to load Dropbox config: %v", err) - } - fmt.Printf("config directory + %v\n", cfgDropbox) - - } \ No newline at end of file diff --git a/cmd/loadConfig.go b/cmd/loadConfig.go deleted file mode 100644 index d7c2ec1..0000000 --- a/cmd/loadConfig.go +++ /dev/null @@ -1,59 +0,0 @@ -package cmd - -import ( - "errors" - "fmt" - "os" - "strings" - - thrown "github.com/0chain/errors" - "github.com/0chain/gosdk/core/sys" - "github.com/spf13/viper" -) - -type DropboxConfig struct { - accessToken string -} -var (ErrMssingConfig = errors.New("[conf]missing config file") - ErrBadParsing = errors.New("parsing error") - ErrInvalidToken=errors.New("invalid access Token")) - -func loadDropboxFile(file string) (DropboxConfig, error) { - var cfg DropboxConfig - var err error - - _, err = sys.Files.Stat(file) - - if err != nil { - if errors.Is(err, os.ErrNotExist) { - return cfg, thrown.Throw(ErrMssingConfig, file) - } - return cfg, err - } - - v := viper.New() - - v.SetConfigFile(file) - - if err := v.ReadInConfig(); err != nil { - return cfg, thrown.Throw(ErrBadParsing, err.Error()) - } - - return LoadConfig(v) - -} - - -func LoadConfig(v *viper.Viper) (DropboxConfig, error) { - var cfg DropboxConfig - - accessToken := strings.TrimSpace(v.GetString("access_token")) - - if len(accessToken) == 0 { - fmt.Printf("accessToken + %v\n", accessToken) - return cfg, thrown.Throw(ErrInvalidToken, "token is empty") - } - - - return cfg, nil -} \ No newline at end of file diff --git a/cmd/migrate.go b/cmd/migrate.go index 9af197b..2ca8dbb 100644 --- a/cmd/migrate.go +++ b/cmd/migrate.go @@ -114,7 +114,11 @@ var migrateCmd = &cobra.Command{ } } - if accessKey == "" || secretKey == "" { + if source == "" { + source = "s3" + } + + if (accessKey == "" || secretKey == "") && source == "s3" { if accessKey, secretKey = util.GetAwsCredentialsFromEnv(); accessKey == "" || secretKey == "" { if awsCredPath == "" { return errors.New("aws credentials missing") @@ -125,17 +129,18 @@ var migrateCmd = &cobra.Command{ } } - if bucket == "" { + if accessToken == "" { + if accessToken = util.GetAccessToken(); accessToken == "" { + return errors.New("Missing Access Token") + } + } + if bucket == "" && source == "s3" { bucket, region, prefix, err = util.GetBucketRegionPrefixFromFile(awsCredPath) if err != nil { return err } } - if source == "" { - source = "s3" - } - if skip < 0 || skip > 2 { return fmt.Errorf("skip value not in range 0-2. Provided value is %v", skip) } diff --git a/util/util.go b/util/util.go index a1f360d..686cf87 100644 --- a/util/util.go +++ b/util/util.go @@ -102,6 +102,9 @@ func GetAllocationIDFromEnv() string { func GetAwsCredentialsFromEnv() (string, string) { return os.Getenv("AWS_ACCESS_KEY"), os.Getenv("AWS_SECRET_KEY") } +func GetAccessToken() string { + return os.Getenv("AccessToken") +} func ConvertGoSDKTimeToTime(in string) time.Time { t, err := time.Parse(ZGoSDKTimeFormat, in) From f6d4fc2b1b4418cea2a1486c07e20c75738b49bc Mon Sep 17 00:00:00 2001 From: pewssh Date: Fri, 26 Apr 2024 21:10:23 +0545 Subject: [PATCH 07/11] updated for native file --- dropbox/core.go | 1 + gdrive/core.go | 2 +- migration/migrate.go | 3 ++- 3 files changed, 4 insertions(+), 2 deletions(-) diff --git a/dropbox/core.go b/dropbox/core.go index b447454..3b0b4f4 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -50,6 +50,7 @@ func (d *DropboxClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <- arg := files.NewListFolderArg("") // "" for Root arg.Recursive = true arg.Limit = 100 + arg.IncludeNonDownloadableFiles=false res, err := d.dropboxFiles.ListFolder(arg) if err != nil { diff --git a/gdrive/core.go b/gdrive/core.go index 36f4ccb..55beb2c 100644 --- a/gdrive/core.go +++ b/gdrive/core.go @@ -52,7 +52,7 @@ func (g *GoogleDriveClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta filesReq.Q("trashed=false") filesReq.Fields( - "id, mimeType, size", + "files(id, mimeType, size)", ) filesReq.Pages(ctx, func(page *drive.FileList) error { diff --git a/migration/migrate.go b/migration/migrate.go index 1d8bdc2..fa757fc 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -122,7 +122,7 @@ func InitMigration(mConfig *MigrationConfig) error { return err } mConfig.ChunkSize = int64(mConfig.ChunkNumber) * dStorageService.GetChunkWriteSize() - zlogger.Logger.Info("Getting aws storage service") + zlogger.Logger.Info(fmt.Sprintf("Getting %v storage service", mConfig.Source)) var dataSourceStore T.CloudStorageI if mConfig.Source == "s3" { @@ -151,6 +151,7 @@ func InitMigration(mConfig *MigrationConfig) error { return err } + zlogger.Logger.Info(dataSourceStore, "data source info") if err != nil { zlogger.Logger.Error(err) return err From 6b3a7f70969b49a28c1bcb611bad124afe7714de Mon Sep 17 00:00:00 2001 From: pewssh Date: Sun, 5 May 2024 15:15:23 +0545 Subject: [PATCH 08/11] Fix as per the release sprint 1.14 --- dropbox/core.go | 1 + gdrive/core.go | 3 +- go.mod | 16 +++++-- go.sum | 80 ++++++++++++++++++++++++----------- migration/migrate.go | 2 +- migration/migration_worker.go | 1 + types/index.go | 1 + 7 files changed, 73 insertions(+), 31 deletions(-) diff --git a/dropbox/core.go b/dropbox/core.go index 3b0b4f4..93b8444 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -64,6 +64,7 @@ func (d *DropboxClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <- Key: meta.PathDisplay, Size: int64(meta.Size), ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)), + Ext: filepath.Ext(meta.PathDisplay), } } } diff --git a/gdrive/core.go b/gdrive/core.go index 55beb2c..dc57bb8 100644 --- a/gdrive/core.go +++ b/gdrive/core.go @@ -52,7 +52,7 @@ func (g *GoogleDriveClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta filesReq.Q("trashed=false") filesReq.Fields( - "files(id, mimeType, size)", + "files(id, mimeType, size,fileExtension)", ) filesReq.Pages(ctx, func(page *drive.FileList) error { @@ -72,6 +72,7 @@ func (g *GoogleDriveClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta Key: file.Id, Size: file.Size, ContentType: file.MimeType, + Ext: file.FileExtension, } } diff --git a/go.mod b/go.mod index d862315..bbe1941 100644 --- a/go.mod +++ b/go.mod @@ -1,10 +1,12 @@ module github.com/0chain/s3migration -go 1.20 +go 1.21 + +toolchain go1.22.1 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.11.0 + github.com/0chain/gosdk v1.14.0-RC7.0.20240503202847-67339fd170d5 github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.10 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37 @@ -12,11 +14,13 @@ require ( github.com/golang/mock v1.6.0 github.com/spf13/cobra v1.6.1 github.com/spf13/viper v1.15.0 + golang.org/x/oauth2 v0.18.0 ) require ( cloud.google.com/go/compute v1.23.4 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect + github.com/andybalholm/brotli v1.0.5 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -25,13 +29,17 @@ require ( github.com/google/s2a-go v0.1.7 // indirect github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.3 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect + github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17 // indirect + github.com/klauspost/compress v1.17.0 // indirect + github.com/valyala/bytebufferpool v1.0.0 // indirect + github.com/valyala/fasthttp v1.51.0 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.49.0 // indirect go.opentelemetry.io/otel v1.24.0 // indirect go.opentelemetry.io/otel/metric v1.24.0 // indirect go.opentelemetry.io/otel/trace v1.24.0 // indirect golang.org/x/net v0.22.0 // indirect - golang.org/x/oauth2 v0.18.0 // indirect google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 // indirect google.golang.org/grpc v1.62.1 // indirect @@ -69,7 +77,7 @@ require ( github.com/go-stack/stack v1.8.1 // indirect github.com/google/uuid v1.6.0 // indirect github.com/gorilla/websocket v1.5.0 // indirect - github.com/h2non/filetype v1.1.3 // indirect + github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/herumi/bls-go-binary v1.31.0 // indirect diff --git a/go.sum b/go.sum index 45f3e16..de4ede2 100644 --- a/go.sum +++ b/go.sum @@ -44,14 +44,17 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.11.0 h1:PSD4ohQaaSOsH/sHvfnCbq35Bs5fCtL1g9S4vyvxQOY= -github.com/0chain/gosdk v1.11.0/go.mod h1:DAg/de6vodjEa7CM1/LjElOwntRtNV5lb9rMRaR7fzU= +github.com/0chain/gosdk v1.14.0-RC7.0.20240503202847-67339fd170d5 h1:ekgGBWKuUNUdLjgayszT8FxWYtu21QlJF5Ji2YUV9h0= +github.com/0chain/gosdk v1.14.0-RC7.0.20240503202847-67339fd170d5/go.mod h1:tgAiVAuIy+Vs1tGfKCPEuuWWARwNQBEw32y950LrqrU= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw= github.com/Luzifer/go-openssl/v3 v3.1.0/go.mod h1:liy3FXuuS8hfDlYh1T+l78AwQ/NjZflJz0NDvjKhwDs= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= +github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= +github.com/andybalholm/brotli v1.0.5 h1:8uQZIdzKmjc/iuPu7O2ioW48L81FgatrcpfFmiq/cCs= +github.com/andybalholm/brotli v1.0.5/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= github.com/aws/aws-sdk-go-v2 v1.17.1 h1:02c72fDJr87N8RAC2s3Qu0YuvMRZKNZJ9F+lAehCazk= github.com/aws/aws-sdk-go-v2 v1.17.1/go.mod h1:JLnGeGONAyi2lWXI1p0PCIOIy333JMVK1U7Hf0aRFLw= github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.4.9 h1:RKci2D7tMwpvGpDNZnGQw9wk6v7o/xSwFcUAuNPoB8k= @@ -123,6 +126,7 @@ github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46f github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= +github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= @@ -147,6 +151,7 @@ github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0 github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5 h1:FT+t0UEDykcor4y3dMVKXIiWJETBpRgERYTGlmMd7HU= github.com/dropbox/dropbox-sdk-go-unofficial/v6 v6.0.5/go.mod h1:rSS3kM9XMzSQ6pw91Qgd6yB5jdt70N4OdtrAf74As5M= github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= +github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/envoyproxy/go-control-plane v0.9.0/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.1-0.20191026205805-5f8ba28d4473/go.mod h1:YTl/9mNaCwkRvm6d1a2C3ymFceY/DCBVvsKhRF0iEA4= github.com/envoyproxy/go-control-plane v0.9.4/go.mod h1:6rpuAdCZL397s3pYoYcLgu1mIlRU8Am5FuJP05cCM98= @@ -158,12 +163,15 @@ github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbL github.com/felixge/httpsnoop v1.0.4 h1:NFTV2Zj1bL4mc9sqWACXbQFVBBg2W3GPvqp8/ESS2Wg= github.com/felixge/httpsnoop v1.0.4/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= +github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqGNY4FhTFhk+o9oFHGINQ/+vhlm8HFzi6znCI= +github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -177,6 +185,7 @@ github.com/go-ole/go-ole v1.2.6/go.mod h1:pprOEPIfldk/42T2oK7lQ4v4JSDwmV0As9GaiU github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/golang-jwt/jwt/v4 v4.3.0 h1:kHL1vqdqWNfATmA0FNMdmZNMyZI1U6O31X4rlIPoBog= +github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzwAxVc6locg= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191227052852-215e87163ea7/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -207,12 +216,12 @@ github.com/golang/protobuf v1.4.1/go.mod h1:U8fpvMrcmy5pZrNK1lt4xCsGvpyWQ/VVv6QD github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= @@ -227,8 +236,8 @@ github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -247,8 +256,6 @@ github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= -github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.6.0 h1:NIvaJDMOsjHA8n1jAhLSgzrAzy1Hgr+hNrb57e+94F0= github.com/google/uuid v1.6.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= @@ -259,26 +266,36 @@ github.com/googleapis/gax-go/v2 v2.12.3 h1:5/zPPDvw8Q1SuXjrqrZslrqT7dL/uJT2CQii/ github.com/googleapis/gax-go/v2 v2.12.3/go.mod h1:AKloxT6GtNbaLm8QTNSidHUVsHYcBHwWRvkNFJUQcS4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= +github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= -github.com/h2non/filetype v1.1.3 h1:FKkx9QbD7HR/zjK1Ia5XiBsq9zdLi5Kf3zGyFTAFkGg= -github.com/h2non/filetype v1.1.3/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= +github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb h1:GlQyMv2C48qmfPItvAXFoyN341Swxp9JNVeUZxnmbJw= +github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb/go.mod h1:319b3zT68BvV+WRj7cwy856M2ehB3HqNOt6sy1HndBY= github.com/hashicorp/go-bexpr v0.1.10 h1:9kuI5PFotCboP3dkDYFr/wi0gg0QVbSNz5oFRpxn4uE= +github.com/hashicorp/go-bexpr v0.1.10/go.mod h1:oxlubA2vC/gFVfX1A6JGp7ls7uCDlfJn732ehYYg+g0= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= +github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= +github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17 h1:FbyIK0BfvXVZTOxKOe2dlxJqSPSF2ZXOv2Mc7dvS7sc= +github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= +github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= +github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= +github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= +github.com/jackpal/go-nat-pmp v1.0.2/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= @@ -290,6 +307,8 @@ github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1 github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+oQHNcck= github.com/kkdai/bstream v0.0.0-20161212061736-f391b8402d23/go.mod h1:J+Gs4SYgM6CZQHDETBtE9HaSEkGmuNXF86RwHhHUvq4= +github.com/klauspost/compress v1.17.0 h1:Rnbp4K9EjcDuVuHtd0dgA4qNuv9yKDYKK1ulpJwgrqM= +github.com/klauspost/compress v1.17.0/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid/v2 v2.2.4 h1:acbojRNwl3o09bUq+yDCtZFc1aiwaAAxtcn8YkZXnvk= github.com/klauspost/cpuid/v2 v2.2.4/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= @@ -297,9 +316,11 @@ github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/lithammer/shortuuid/v3 v3.0.7 h1:trX0KTHy4Pbwo/6ia8fscyHoGA+mf1jWbPJVuvyJQQ8= github.com/lithammer/shortuuid/v3 v3.0.7/go.mod h1:vMk8ke37EmiewwolSO1NLW8vP4ZaKlRuDIi8tWWmAts= github.com/machinebox/graphql v0.2.2 h1:dWKpJligYKhYKO5A2gvNhkJdQMNZeChZYyBbrZkBZfo= @@ -307,9 +328,13 @@ github.com/machinebox/graphql v0.2.2/go.mod h1:F+kbVMHuwrQ5tYgU9JXlnskM8nOaFxCAE github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= +github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-runewidth v0.0.14 h1:+xnbZSEeDbOIg5/mE6JF0w6n9duR1l3/WmbinWVwUuU= +github.com/mattn/go-runewidth v0.0.14/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -317,8 +342,10 @@ github.com/mitchellh/go-homedir v1.1.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrk github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyuac5Z2hdY= github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= +github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/nxadm/tail v1.4.4/go.mod h1:kenIhsEOeOJmVchQTgglprH7qJGnHDVpk1VPCcaMI8A= github.com/olekukonko/tablewriter v0.0.5 h1:P2Ga83D34wi1o9J6Wh1mRuqd4mF/x/lgBS7N7AbDhec= +github.com/olekukonko/tablewriter v0.0.5/go.mod h1:hPp6KlRPjbx+hW8ykQs1w3UBbZlj6HuIJcUGPhkA7kY= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.7.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.12.1/go.mod h1:zj2OWP4+oCPe1qIXoGWkgMRwljMUYCdkwsT2108oapk= @@ -340,13 +367,18 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/remeh/sizedwaitgroup v1.0.0 h1:VNGGFwNo/R5+MJBf6yrsr110p0m4/OX4S3DCy7Kyl5E= github.com/remeh/sizedwaitgroup v1.0.0/go.mod h1:3j2R4OIe/SeS6YDhICBy22RWjJC5eNCJ1V+9+NVNYlo= github.com/rivo/uniseg v0.4.3 h1:utMvzDsuh3suAEnhH0RdHmoPbU648o6CvXxTx4SBMOw= +github.com/rivo/uniseg v0.4.3/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= +github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rs/cors v1.7.0 h1:+88SsELBHx5r+hZ8TCkggzSstaWNbDvThkVK8H6f9ik= +github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/shirou/gopsutil v3.21.11+incompatible h1:+1+c1VGhc88SSonWP6foOcLhvnKlUeu/erjjvaPEYiI= @@ -366,6 +398,7 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSSt89Yw= github.com/stretchr/objx v0.5.0 h1:1zr/of2m5FGMsad5YfcqgdqdWrIhu+EBEJRhR1U7z/c= @@ -377,8 +410,9 @@ github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5 github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= @@ -392,7 +426,13 @@ github.com/tklauser/numcpus v0.6.0/go.mod h1:FEZLMke0lhOUG6w2JadTzp0a+Nl8PF/GFkQ github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= +github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= +github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/valyala/fasthttp v1.51.0 h1:8b30A5JlZ6C7AS81RsWjYMQmrZG6feChmgAolCl1SqA= +github.com/valyala/fasthttp v1.51.0/go.mod h1:oI2XroL+lI7vdXyYoQk03bXBThfFl2cVdIA3Xl7cH8g= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -431,6 +471,7 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI= +go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.9.0 h1:7fIwc/ZtS0q++VgcfqFDxSBZVv/Xo49/SYnDFupUwlI= go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= @@ -447,8 +488,6 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.6.0 h1:qfktjS5LUO+fFKeJXZ+ikTRijMmljikvG68fpMMruSc= -golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.21.0 h1:X31++rzVUdKhX5sWmSOFZxx8UW/ldWx55cbf08iNAMA= golang.org/x/crypto v0.21.0/go.mod h1:0BP7YvVV9gBbVKyeTG0Gyn+gZm94bibOW5BjDEYAOMs= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -524,8 +563,6 @@ golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v golang.org/x/net v0.0.0-20210226172049-e18ecbb05110/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg= golang.org/x/net v0.0.0-20210405180319-a5a99cb37ef4/go.mod h1:p54w0d4576C0XHj96bSt6lcn1PtDYWL6XObtHCRCNQM= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= golang.org/x/net v0.22.0 h1:9sGLhx7iRIHEiX0oAJ3MRZMUCElJgy7Br1nO+AMN3Tc= golang.org/x/net v0.22.0/go.mod h1:JKghWKKOSdJwpW2GEx0Ja7fmaKnMsbu+MWVZTokSYmg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -537,8 +574,6 @@ golang.org/x/oauth2 v0.0.0-20200902213428-5d25da1a8d43/go.mod h1:KelEdhl1UZF7XfJ golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20201208152858-08078c50e5b5/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= golang.org/x/oauth2 v0.0.0-20210218202405-ba52d332ba99/go.mod h1:KelEdhl1UZF7XfJ4dDtk6s++YSgaE7mD/BuKKDLBl4A= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783 h1:nt+Q6cXKz4MosCSpnbMtqiQ8Oz0pxTef2B4Vca2lvfk= -golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.18.0 h1:09qnuIAgzdx1XplqJvW6CQqMCtGZykZWcXzPMPUusvI= golang.org/x/oauth2 v0.18.0/go.mod h1:Wf7knwG0MPoWIMMBgFlEaSUDaKskp0dCfrlJRJXbBi8= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -553,8 +588,6 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -606,8 +639,6 @@ golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.7.0 h1:3jlCCIQZPdOYu1h8BkNvLz8Kgwtae2cagcG/VamtZRU= -golang.org/x/sys v0.7.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.18.0 h1:DBdB3niSjOA/O0blCZBqDefyWNYveAYMNF1Wum0DYQ4= golang.org/x/sys v0.18.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= @@ -620,15 +651,13 @@ golang.org/x/text v0.3.3/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= golang.org/x/time v0.5.0 h1:o7cqy6amK/52YcAKIPlM3a+Fpj35zvRj2TP+e1xFSfk= +golang.org/x/time v0.5.0/go.mod h1:3BpzKBy/shNhVucY/MWOyx10tF3SFh9QdLuxbVysPQM= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190114222345-bf090417da8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190226205152-f727befe758c/go.mod h1:9Yl7xja0Znq3iFh3HoIrodX9oNMXvdceNzlUR8zjMvY= @@ -711,7 +740,6 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= @@ -752,6 +780,8 @@ google.golang.org/genproto v0.0.0-20201214200347-8c77b98c765d/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210108203827-ffc7fda8c3d7/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210226172003-ab064af71705/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20240123012728-ef4313101c80 h1:KAeGQVN3M9nD0/bQXnr/ClcEMJ968gUXJQ9pwfSynuQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2 h1:rIo7ocm2roD9DcFIX67Ym8icoGCKSARAiPljFhh5suQ= +google.golang.org/genproto/googleapis/api v0.0.0-20240311132316-a219d84964c2/go.mod h1:O1cOfN1Cy6QEYr7VxtjOyP5AdAuR0aJ/MYZaaof623Y= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237 h1:NnYq6UN9ReLM9/Y01KWNOWyI5xQ9kbIms5GGJVwS/Yc= google.golang.org/genproto/googleapis/rpc v0.0.0-20240318140521-94a12d6c2237/go.mod h1:WtryC6hu0hhx87FDGxWCDptyssuo68sk10vYjF+T9fY= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= @@ -784,14 +814,13 @@ google.golang.org/protobuf v1.24.0/go.mod h1:r/3tXBNzIEhYS9I1OUVjXDlt8tc493IdKGj google.golang.org/protobuf v1.25.0/go.mod h1:9JNX74DMeImyA3h4bdi1ymwjUzf21/xIlbajtzgsN7c= google.golang.org/protobuf v1.26.0-rc.1/go.mod h1:jlhhOSvTdKEhbULTjvd4ARK9grFBp09yW+WbY/TyQbw= google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= -google.golang.org/protobuf v1.28.1 h1:d0NfwRgPtno5B1Wa6L2DAG+KivqkdutMf1UhdNx175w= -google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.33.0 h1:uNO2rsAINq/JlFpSdYEKIZ0uKD/R9cpdv0T+yoGwGmI= google.golang.org/protobuf v1.33.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= +gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= @@ -807,6 +836,7 @@ gopkg.in/yaml.v2 v2.2.4/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.8/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= +gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= diff --git a/migration/migrate.go b/migration/migrate.go index fa757fc..114427a 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -479,7 +479,7 @@ func processOperation(ctx context.Context, downloadObj *DownloadObjectMeta) (Mig zlogger.Logger.Error(err) return op, err } - mimeType, err := zboxutil.GetFileContentType(fileObj) + mimeType, err := zboxutil.GetFileContentType(downloadObj.Ext, fileObj) if err != nil { zlogger.Logger.Error("content type error: ", err, " file: ", fileInfo.Name(), " objKey:", downloadObj.ObjectKey) return op, err diff --git a/migration/migration_worker.go b/migration/migration_worker.go index 3c6e66e..6a2a405 100644 --- a/migration/migration_worker.go +++ b/migration/migration_worker.go @@ -38,6 +38,7 @@ type DownloadObjectMeta struct { ErrChan chan error IsFileAlreadyExist bool mimeType string + Ext string } type UploadObjectMeta struct { diff --git a/types/index.go b/types/index.go index a2aa28f..a8d886c 100644 --- a/types/index.go +++ b/types/index.go @@ -16,6 +16,7 @@ type ObjectMeta struct { Key string Size int64 ContentType string + Ext string } type CloudStorageI interface { From fdefd56ce7fb7cf0a366763c7742e40a4bcff406 Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 6 May 2024 00:46:04 +0545 Subject: [PATCH 09/11] added invalid access token error control --- dropbox/core.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/dropbox/core.go b/dropbox/core.go index 93b8444..e7684a3 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -10,6 +10,7 @@ import ( "path/filepath" T "github.com/0chain/s3migration/types" + "github.com/pkg/errors" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox" "github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files" @@ -29,6 +30,15 @@ func GetDropboxClient(token string, workDir string) (*DropboxClient, error) { client := files.New(config) + // for invalid Access token + arg := files.NewListFolderArg("") + + _, err := client.ListFolder(arg) + + if err != nil { + return nil, errors.Wrap(err, "invalid Dropbox token") + } + return &DropboxClient{ token: token, dropboxConf: &config, From d14b49379a0118d49ab85996c7eaf1cbfa9596f2 Mon Sep 17 00:00:00 2001 From: pewssh Date: Mon, 6 May 2024 02:09:31 +0545 Subject: [PATCH 10/11] updated comment --- dropbox/core.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/dropbox/core.go b/dropbox/core.go index e7684a3..86bd28c 100644 --- a/dropbox/core.go +++ b/dropbox/core.go @@ -36,7 +36,7 @@ func GetDropboxClient(token string, workDir string) (*DropboxClient, error) { _, err := client.ListFolder(arg) if err != nil { - return nil, errors.Wrap(err, "invalid Dropbox token") + return nil, errors.Wrap(err, "invalid Client token") } return &DropboxClient{ @@ -60,7 +60,7 @@ func (d *DropboxClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <- arg := files.NewListFolderArg("") // "" for Root arg.Recursive = true arg.Limit = 100 - arg.IncludeNonDownloadableFiles=false + arg.IncludeNonDownloadableFiles = false res, err := d.dropboxFiles.ListFolder(arg) if err != nil { From be8924dc6c9c41fcb58e09f3ceedde6c652f0074 Mon Sep 17 00:00:00 2001 From: pewssh Date: Wed, 15 May 2024 00:10:46 +0545 Subject: [PATCH 11/11] updated code for log --- gdrive/core.go | 6 ++++++ migration/migrate.go | 9 +++++++++ 2 files changed, 15 insertions(+) diff --git a/gdrive/core.go b/gdrive/core.go index dc57bb8..5c85d21 100644 --- a/gdrive/core.go +++ b/gdrive/core.go @@ -9,6 +9,7 @@ import ( zlogger "github.com/0chain/s3migration/logger" T "github.com/0chain/s3migration/types" + "github.com/pkg/errors" "golang.org/x/oauth2" "google.golang.org/api/drive/v3" "google.golang.org/api/option" @@ -30,6 +31,11 @@ func NewGoogleDriveClient(accessToken string, workDir string) (*GoogleDriveClien if err != nil { return nil, err } + _, err = service.Files.List().Do() + + if err != nil { + return nil, errors.Wrap(err, "invalid Google Drive access token") + } return &GoogleDriveClient{ service: service, diff --git a/migration/migrate.go b/migration/migrate.go index 114427a..db03ac8 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -305,6 +305,10 @@ func (m *Migration) DownloadWorker(ctx context.Context, migrator *MigrationWorke } wg.Add(1) go func() { + defer func(start time.Time) { + zlogger.Logger.Info("downloadObjMeta key: ", downloadObjMeta.ObjectKey, time.Since(start)) + }(time.Now()) + defer wg.Done() err := checkIsFileExist(ctx, downloadObjMeta) if err != nil { @@ -467,6 +471,11 @@ func checkDownloadStatus(downloadObj *DownloadObjectMeta) error { } func processOperation(ctx context.Context, downloadObj *DownloadObjectMeta) (MigrationOperation, error) { + + defer func(start time.Time) { + zlogger.Logger.Info("uploading object key: ", downloadObj.ObjectKey, time.Since(start)) + }(time.Now()) + remotePath := getRemotePath(downloadObj.ObjectKey) var op MigrationOperation fileObj, err := migration.fs.Open(downloadObj.LocalPath)