Merge pull request #124 from pewssh/dropbox-drive/migration/pewssh

Dropbox drive/migration/pewssh

cmd/migrate.go

@@ -40,6 +40,8 @@ var (
 	chunkSize                  int64
 	chunkNumber                int
 	batchSize                  int
+	source                     string
+	accessToken                string // if source is google drive or dropbox
 )
 
 // migrateCmd is the migrateFromS3 sub command to migrate whole objects from some buckets.
@@ -71,7 +73,8 @@ func init() {
 	migrateCmd.Flags().Int64Var(&chunkSize, "chunk-size", 50*1024*1024, "chunk size in bytes")
 	migrateCmd.Flags().IntVar(&chunkNumber, "chunk-number", 250, "number of chunks to upload")
 	migrateCmd.Flags().IntVar(&batchSize, "batch-size", 20, "number of files to upload in a batch")
-
+	migrateCmd.Flags().StringVar(&source, "source", "s3", "s3 or google_drive or dropbox")
+	migrateCmd.Flags().StringVar(&accessToken, "access-token", "", "access token for google drive or dropbox")
 }
 
 var migrateCmd = &cobra.Command{
@@ -111,7 +114,11 @@ var migrateCmd = &cobra.Command{
 			}
 		}
 
-		if accessKey == "" || secretKey == "" {
+		if source == "" {
+			source = "s3"
+		}
+
+		if (accessKey == "" || secretKey == "") && source == "s3" {
 			if accessKey, secretKey = util.GetAwsCredentialsFromEnv(); accessKey == "" || secretKey == "" {
 				if awsCredPath == "" {
 					return errors.New("aws credentials missing")
@@ -122,7 +129,12 @@ var migrateCmd = &cobra.Command{
 			}
 		}
 
-		if bucket == "" {
+		if accessToken == "" {
+			if accessToken = util.GetAccessToken(); accessToken == "" {
+				return errors.New("Missing Access Token")
+			}
+		}
+		if bucket == "" && source == "s3" {
 			bucket, region, prefix, err = util.GetBucketRegionPrefixFromFile(awsCredPath)
 			if err != nil {
 				return err
@@ -232,6 +244,9 @@ var migrateCmd = &cobra.Command{
 			ChunkSize:       chunkSize,
 			ChunkNumber:     chunkNumber,
 			BatchSize:       batchSize,
+
+			Source:      source,
+			AccessToken: accessToken,
 		}
 
 		if err := migration.InitMigration(&mConfig); err != nil {

dropbox/core.go

@@ -0,0 +1,184 @@
+package dropbox
+
+import (
+	"context"
+	"fmt"
+	"io"
+	"mime"
+	"os"
+	"path"
+	"path/filepath"
+
+	T "github.com/0chain/s3migration/types"
+	"github.com/pkg/errors"
+
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox"
+	"github.com/dropbox/dropbox-sdk-go-unofficial/v6/dropbox/files"
+)
+
+type DropboxClient struct {
+	token        string
+	workDir      string
+	dropboxConf  *dropbox.Config
+	dropboxFiles files.Client
+}
+
+func GetDropboxClient(token string, workDir string) (*DropboxClient, error) {
+	config := dropbox.Config{
+		Token: token,
+	}
+
+	client := files.New(config)
+
+	// for invalid Access token
+	arg := files.NewListFolderArg("")
+
+	_, err := client.ListFolder(arg)
+
+	if err != nil {
+		return nil, errors.Wrap(err, "invalid Client token")
+	}
+
+	return &DropboxClient{
+		token:        token,
+		dropboxConf:  &config,
+		dropboxFiles: client,
+		workDir:      workDir,
+	}, nil
+}
+
+func (d *DropboxClient) ListFiles(ctx context.Context) (<-chan *T.ObjectMeta, <-chan error) {
+	objectChan := make(chan *T.ObjectMeta)
+	errChan := make(chan error)
+
+	go func() {
+		defer func() {
+			close(objectChan)
+			close(errChan)
+		}()
+
+		arg := files.NewListFolderArg("") // "" for Root
+		arg.Recursive = true
+		arg.Limit = 100
+		arg.IncludeNonDownloadableFiles = false
+
+		res, err := d.dropboxFiles.ListFolder(arg)
+		if err != nil {
+			errChan <- err
+			return
+		}
+
+		for _, entry := range res.Entries {
+			if meta, ok := entry.(*files.FileMetadata); ok {
+				objectChan <- &T.ObjectMeta{
+					Key:         meta.PathDisplay,
+					Size:        int64(meta.Size),
+					ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)),
+					Ext:         filepath.Ext(meta.PathDisplay),
+				}
+			}
+		}
+
+		cursor := res.Cursor
+		hasMore := res.HasMore
+
+		for hasMore {
+			continueArg := files.NewListFolderContinueArg(cursor)
+			res, err := d.dropboxFiles.ListFolderContinue(continueArg)
+			if err != nil {
+				errChan <- err
+				return
+			}
+
+			for _, entry := range res.Entries {
+				if meta, ok := entry.(*files.FileMetadata); ok {
+					objectChan <- &T.ObjectMeta{
+						Key:         meta.PathDisplay,
+						Size:        int64(meta.Size),
+						ContentType: mime.TypeByExtension(filepath.Ext(meta.PathDisplay)),
+					}
+				}
+			}
+
+			cursor = res.Cursor
+			hasMore = res.HasMore
+		}
+	}()
+
+	return objectChan, errChan
+}
+
+func (d *DropboxClient) GetFileContent(ctx context.Context, filePath string) (*T.Object, error) {
+	arg := files.NewDownloadArg(filePath)
+	res, content, err := d.dropboxFiles.Download(arg)
+	if err != nil {
+		return nil, err
+	}
+
+	return &T.Object{
+		Body:          content,
+		ContentType:   mime.TypeByExtension(filepath.Ext(filePath)),
+		ContentLength: int64(res.Size),
+	}, nil
+}
+
+func (d *DropboxClient) DeleteFile(ctx context.Context, filePath string) error {
+	arg := files.NewDeleteArg(filePath)
+	_, err := d.dropboxFiles.DeleteV2(arg)
+	return err
+}
+
+func (d *DropboxClient) DownloadToFile(ctx context.Context, filePath string) (string, error) {
+	arg := files.NewDownloadArg(filePath)
+	_, content, err := d.dropboxFiles.Download(arg)
+	if err != nil {
+		return "", err
+	}
+
+	fileName := filepath.Base(filePath)
+	downloadPath := path.Join(d.workDir, fileName)
+	file, err := os.Create(downloadPath)
+	if err != nil {
+		return "", err
+	}
+	defer file.Close()
+
+	_, err = io.Copy(file, content)
+	if err != nil {
+		return "", err
+	}
+
+	return downloadPath, nil
+}
+
+func (d *DropboxClient) DownloadToMemory(ctx context.Context, objectKey string, offset int64, chunkSize, objectSize int64) ([]byte, error) {
+	limit := offset + chunkSize - 1
+	if limit > objectSize {
+		limit = objectSize
+	}
+
+	rng := fmt.Sprintf("bytes=%d-%d", offset, limit)
+
+	arg := files.NewDownloadArg(objectKey)
+
+	arg.ExtraHeaders = map[string]string{"Range": rng}
+
+	_, content, err := d.dropboxFiles.Download(arg)
+	if err != nil {
+		return nil, err
+	}
+	defer content.Close()
+
+	data := make([]byte, chunkSize)
+	n, err := io.ReadFull(content, data)
+
+	if err != nil && err != io.ErrUnexpectedEOF {
+		return nil, err
+	}
+
+	if int64(n) < chunkSize && objectSize != chunkSize {
+		data = data[:n]
+	}
+
+	return data, nil
+}

dropbox/test.go

@@ -0,0 +1,150 @@
+package dropbox
+
+import (
+	"context"
+	"fmt"
+	"testing"
+
+	zlogger "github.com/0chain/s3migration/logger"
+)
+
+var (
+	dropboxAccessToken = ""
+	testFilePath       = ""
+)
+
+const TestFileContent = ` by Manuel Gutiérrez Nájera
+
+I want to die as the day declines, 
+at high sea and facing the sky, 
+while agony seems like a dream 
+and my soul like a bird that can fly. 
+
+To hear not, at this last moment, 
+once alone with sky and sea, 
+any more voices nor weeping prayers 
+than the majestic beating of the waves. 
+
+To die when the sad light retires 
+its golden network from the green waves 
+to be like the sun that slowly expires; 
+something very luminous that fades. 
+
+To die, and die young, before 
+fleeting time removes the gentle crown, 
+while life still says: "I'm yours" 
+though we know with our hearts that she lies. 
+`
+
+func TestDropboxClient_ListFiles(t *testing.T) {
+	client, err := GetDropboxClient(dropboxAccessToken, "./")
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err))
+		return
+	}
+
+	ctx := context.Background()
+	objectChan, errChan := client.ListFiles(ctx)
+
+	go func() {
+		for err := range errChan {
+			zlogger.Logger.Error(fmt.Sprintf("Error while listing files: %v", err))
+		}
+	}()
+
+	for object := range objectChan {
+		zlogger.Logger.Info(fmt.Sprintf("Key: %s, Type: %s, Size: %d bytes", object.Key, object.ContentType, object.Size))
+	}
+}
+
+func TestDropboxClient_GetFileContent(t *testing.T) {
+	client, err := GetDropboxClient(dropboxAccessToken, "./")
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err))
+	}
+
+	ctx := context.Background()
+	filePath := testFilePath
+	obj, err := client.GetFileContent(ctx, filePath)
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Error while getting file content: %v", err))
+		return
+	}
+	defer obj.Body.Close()
+
+	zlogger.Logger.Info(fmt.Sprintf("File content type: %s, Length: %d", obj.ContentType, obj.ContentLength))
+
+	if (obj.Body == nil) || (obj.ContentLength == 0) {
+		fmt.Println("Empty file content")
+		return
+	}
+
+	buf := make([]byte, obj.ContentLength)
+	n, err := obj.Body.Read(buf)
+
+	if err != nil && err.Error() != "EOF" {
+		zlogger.Logger.Error(fmt.Sprintf("Error while reading file content: %v", err))
+		return
+	}
+
+	zlogger.Logger.Info(fmt.Sprintf("File content: %s", string(buf[:n])))
+}
+
+func TestDropboxClient_DeleteFile(t *testing.T) {
+	client, err := GetDropboxClient(dropboxAccessToken, "./")
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err))
+		return
+	}
+
+	ctx := context.Background()
+	filePath := testFilePath
+	err = client.DeleteFile(ctx, filePath)
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Error while deleting file: %v", err))
+		return
+	}
+	zlogger.Logger.Info(fmt.Sprintf("File %s deleted successfully", filePath))
+}
+
+func TestDropboxClient_DownloadToFile(t *testing.T) {
+	client, err := GetDropboxClient(dropboxAccessToken, "./")
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err))
+		return
+	}
+
+	ctx := context.Background()
+	filePath := testFilePath
+	downloadedPath, err := client.DownloadToFile(ctx, filePath)
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Error while downloading file: %v", err))
+		return
+	}
+	zlogger.Logger.Info(fmt.Sprintf("Downloaded to: %s", downloadedPath))
+}
+
+func TestDropboxClient_DownloadToMemory(t *testing.T) {
+	client, err := GetDropboxClient(dropboxAccessToken, "./")
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Failed to create Dropbox client: %v", err))
+		return
+	}
+
+	ctx := context.Background()
+
+	filePath := testFilePath
+	offset := int64(0)
+
+	// half chunk
+	chunkSize := int64(313)
+	objectSize := int64(626)
+
+	data, err := client.DownloadToMemory(ctx, filePath, offset, chunkSize, objectSize)
+	if err != nil {
+		zlogger.Logger.Error(fmt.Sprintf("Error while downloading file: %v", err))
+		return
+	}
+
+	zlogger.Logger.Info(fmt.Sprintf("Downloaded data: %s", data))
+}