Merge pull request #97 from 0chain/sprint-1.10

Sprint 1.10

.github/workflows/build-s3-migration.yaml

@@ -42,7 +42,7 @@ jobs:
           elif  [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then
             echo "TAG=v0.0.0-staging" >> $GITHUB_ENV
           else
-            echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV
+            echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV
           fi
 
       - name: add dependencies
@@ -54,17 +54,17 @@ jobs:
         run: |
           docker run --rm -v $PWD:/s3mgrt --workdir=/s3mgrt golang:1.20.4 make build
 
-      - name: Zip release
-        run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt
+      # - name: Zip release
+      #   run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt
 
-      - name: Upload binaries to release
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: s3mgrt-linux.tar.gz
-          tag: ${{ env.TAG }}
-          overwrite: true
-          file_glob: true
+      # - name: Upload binaries to release
+      #   uses: svenstaro/upload-release-action@v2
+      #   with:
+      #     repo_token: ${{ secrets.GITHUB_TOKEN }}
+      #     file: s3mgrt-linux.tar.gz
+      #     tag: ${{ env.TAG }}
+      #     overwrite: true
+      #     file_glob: true
 
       - name: Install AWS
         if: github.ref == 'refs/heads/staging'
@@ -114,24 +114,24 @@ jobs:
           } ElseIf ( "${{github.base_ref}}" -eq "staging" -OR "${{github.ref}}" -eq "refs/heads/staging" ){
             echo "TAG=v0.0.0-staging" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
           } ELSE {
-            echo "TAG=${{github.ref}}" | %{$_ -replace('refs/tags/', '')} | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+            echo "TAG=v0.0.0-${{github.base_ref}}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
           }
 
       - name: Install
         run: make build
 
-      - name: Zip Release
-        run: |
-          copy s3mgrt s3mgrt.exe
-          7z a s3mgrt-windows.zip s3mgrt.exe
-      - name: Upload binaries to release
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: s3mgrt-windows.zip
-          tag: ${{ env.TAG }}
-          overwrite: true
-          file_glob: true
+      # - name: Zip Release
+      #   run: |
+      #     copy s3mgrt s3mgrt.exe
+      #     7z a s3mgrt-windows.zip s3mgrt.exe
+      # - name: Upload binaries to release
+      #   uses: svenstaro/upload-release-action@v2
+      #   with:
+      #     repo_token: ${{ secrets.GITHUB_TOKEN }}
+      #     file: s3mgrt-windows.zip
+      #     tag: ${{ env.TAG }}
+      #     overwrite: true
+      #     file_glob: true
 
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v2
@@ -142,7 +142,7 @@ jobs:
 
   build-macos:
     name: Build-macos
-    runs-on: macos-latest
+    runs-on: macos-runner-binary
     steps:
       - name: Setup go 1.18
         uses: actions/setup-go@v2
@@ -161,22 +161,22 @@ jobs:
           elif  [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then
             echo "TAG=v0.0.0-staging" >> $GITHUB_ENV
           else
-            echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV
+            echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV
           fi
       - name: Install
         run: make build
 
-      - name: Zip release
-        run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt
-
-      - name: Upload binaries to release
-        uses: svenstaro/upload-release-action@v2
-        with:
-          repo_token: ${{ secrets.GITHUB_TOKEN }}
-          file: s3mgrt-macos.tar.gz
-          tag: ${{ env.TAG }}
-          overwrite: true
-          file_glob: true
+      # - name: Zip release
+      #   run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt
+
+      # - name: Upload binaries to release
+      #   uses: svenstaro/upload-release-action@v2
+      #   with:
+      #     repo_token: ${{ secrets.GITHUB_TOKEN }}
+      #     file: s3mgrt-macos.tar.gz
+      #     tag: ${{ env.TAG }}
+      #     overwrite: true
+      #     file_glob: true
 
       - name: 'Upload Artifact'
         uses: actions/upload-artifact@v2

go.mod

@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/0chain/errors v1.0.3
-	github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114
+	github.com/0chain/gosdk v1.8.18-0.20230826144722-16bc7138c41b
 	github.com/aws/aws-sdk-go-v2 v1.17.1
 	github.com/aws/aws-sdk-go-v2/config v1.17.10
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37

go.sum

@@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
 github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
 github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
 github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
-github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 h1:fgaUQSUpAqhjhD3ONmiY+3yWn56qHADEd0TCoRcDSZ0=
-github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
+github.com/0chain/gosdk v1.8.18-0.20230826144722-16bc7138c41b h1:WdriWJyVNK5WlhA6PvTRm+1IBIMFRJEQ5TLRw1d+Awo=
+github.com/0chain/gosdk v1.8.18-0.20230826144722-16bc7138c41b/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw=
@@ -728,4 +728,4 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9
 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k=
 rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8=
 rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0=
-rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=
+rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA=

migration/migrate.go

@@ -2,6 +2,8 @@ package migration
 
 import (
 	"context"
+	"crypto/sha1"
+	"encoding/hex"
 	"fmt"
 	"io"
 	"os"
@@ -22,7 +24,7 @@ import (
 
 const Batch = 10
 const (
-	Replace   = iota //Will replace existing file
+	Replace   = iota // Will replace existing file
 	Skip             // Will skip migration if file already exists
 	Duplicate        // Will add _copy prefix and uploads the file
 )
@@ -62,10 +64,10 @@ type Migration struct {
 	skip       int
 	retryCount int
 
-	//Number of goroutines to run. So at most concurrency * Batch goroutines will run. i.e. for bucket level and object level
+	// Number of goroutines to run. So at most concurrency * Batch goroutines will run. i.e. for bucket level and object level
 	concurrency int
 
-	szCtMu               sync.Mutex //size and count mutex; used to update migratedSize and totalMigratedObjects
+	szCtMu               sync.Mutex // size and count mutex; used to update migratedSize and totalMigratedObjects
 	migratedSize         uint64
 	totalMigratedObjects uint64
 
@@ -235,8 +237,16 @@ func StartMigration() error {
 	}
 
 	migrationWorker := NewMigrationWorker(migration.workDir)
-	go migration.DownloadWorker(rootContext, migrationWorker)
+
+	wg := sync.WaitGroup{}
+	wg.Add(1)
+	go func() {
+		defer wg.Done()
+		migration.DownloadWorker(rootContext, migrationWorker)
+	}()
 	// go migration.UploadWorker(rootContext, migrationWorker)
+	wg.Wait()
+
 	migration.UpdateStateFile(migrationWorker)
 	err := migrationWorker.GetMigrationError()
 	if err != nil {
@@ -387,8 +397,31 @@ func (m *Migration) UploadWorker(ctx context.Context, migrator *MigrationWorker)
 	wg.Wait()
 }
 
+func getUniqueShortObjKey(objectKey string) string {
+	// Max length to which objectKey would be trimmed to.
+	// Keeping this less than 100 chars to prevent longer name in case of uploading duplicate
+	// files with `_copy` suffixes.
+	const maxLength = 90
+
+	if len(objectKey) > maxLength {
+		// Generate a SHA-1 hash of the object key
+		hash := sha1.New()
+		hash.Write([]byte(objectKey))
+		hashSum := hash.Sum(nil)
+
+		// Convert the hash to a hexadecimal string
+		hashString := hex.EncodeToString(hashSum)
+
+		// Combine the first 10 characters of the hash with a truncated object key
+		shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:])
+		return shortKey
+	}
+
+	return objectKey
+}
+
 func getRemotePath(objectKey string) string {
-	return filepath.Join(migration.migrateTo, migration.bucket, objectKey)
+	return filepath.Join(migration.migrateTo, migration.bucket, getUniqueShortObjKey(objectKey))
 }
 
 func checkIsFileExist(ctx context.Context, downloadObj *DownloadObjectMeta) error {
@@ -466,7 +499,7 @@ func processOperationForMemory(ctx context.Context, downloadObj *DownloadObjectM
 			zlogger.Logger.Info("Replacing object" + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10))
 			fileOperation = migration.zStore.Replace(ctx, remotePath, r, downloadObj.Size, mimeType)
 		case Duplicate:
-			zlogger.Logger.Info("Duplicating object" + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10))
+			zlogger.Logger.Info("Duplicating object " + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10))
 			fileOperation = migration.zStore.Duplicate(ctx, remotePath, r, downloadObj.Size, mimeType)
 		}
 	} else {
@@ -547,7 +580,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp
 	fileOps := make([]sdk.OperationRequest, 0, len(ops))
 	for _, op := range ops {
 		migrator.UploadStart(op.uploadObj)
-		zlogger.Logger.Info("upload start: ", op.uploadObj.ObjectKey, "size: ", op.uploadObj.Size)
+		zlogger.Logger.Info("upload start: ", op.uploadObj.ObjectKey, " size: ", op.uploadObj.Size)
 		fileOps = append(fileOps, op.Operation)
 	}
 	err = util.Retry(3, time.Second*5, func() error {
@@ -563,7 +596,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp
 	})
 	for _, op := range ops {
 		migrator.UploadDone(op.uploadObj, err)
-		zlogger.Logger.Info("upload done: ", op.uploadObj.ObjectKey, "size ", op.uploadObj.Size, err)
+		zlogger.Logger.Info("upload done: ", op.uploadObj.ObjectKey, " size ", op.uploadObj.Size, err)
 	}
 	migrator.SetMigrationError(err)
 	return err