diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 92f26cc..ac77473 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -42,7 +42,7 @@ jobs: elif [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then echo "TAG=v0.0.0-staging" >> $GITHUB_ENV else - echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV fi - name: add dependencies @@ -54,17 +54,17 @@ jobs: run: | docker run --rm -v $PWD:/s3mgrt --workdir=/s3mgrt golang:1.20.4 make build - - name: Zip release - run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt + # - name: Zip release + # run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: s3mgrt-linux.tar.gz - tag: ${{ env.TAG }} - overwrite: true - file_glob: true + # - name: Upload binaries to release + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: s3mgrt-linux.tar.gz + # tag: ${{ env.TAG }} + # overwrite: true + # file_glob: true - name: Install AWS if: github.ref == 'refs/heads/staging' @@ -114,24 +114,24 @@ jobs: } ElseIf ( "${{github.base_ref}}" -eq "staging" -OR "${{github.ref}}" -eq "refs/heads/staging" ){ echo "TAG=v0.0.0-staging" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } ELSE { - echo "TAG=${{github.ref}}" | %{$_ -replace('refs/tags/', '')} | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + echo "TAG=v0.0.0-${{github.base_ref}}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } - name: Install run: make build - - name: Zip Release - run: | - copy s3mgrt s3mgrt.exe - 7z a s3mgrt-windows.zip s3mgrt.exe - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: s3mgrt-windows.zip - tag: ${{ env.TAG }} - overwrite: true - file_glob: true + # - name: Zip Release + # run: | + # copy s3mgrt s3mgrt.exe + # 7z a s3mgrt-windows.zip s3mgrt.exe + # - name: Upload binaries to release + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: s3mgrt-windows.zip + # tag: ${{ env.TAG }} + # overwrite: true + # file_glob: true - name: 'Upload Artifact' uses: actions/upload-artifact@v2 @@ -142,7 +142,7 @@ jobs: build-macos: name: Build-macos - runs-on: macos-latest + runs-on: macos-runner-binary steps: - name: Setup go 1.18 uses: actions/setup-go@v2 @@ -161,22 +161,22 @@ jobs: elif [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then echo "TAG=v0.0.0-staging" >> $GITHUB_ENV else - echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV fi - name: Install run: make build - - name: Zip release - run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt - - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: s3mgrt-macos.tar.gz - tag: ${{ env.TAG }} - overwrite: true - file_glob: true + # - name: Zip release + # run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt + + # - name: Upload binaries to release + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: s3mgrt-macos.tar.gz + # tag: ${{ env.TAG }} + # overwrite: true + # file_glob: true - name: 'Upload Artifact' uses: actions/upload-artifact@v2 diff --git a/go.mod b/go.mod index 031158f..d0df3ae 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 + github.com/0chain/gosdk v1.8.18-0.20230826144722-16bc7138c41b github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.10 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37 diff --git a/go.sum b/go.sum index 59422d4..1c92e4d 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 h1:fgaUQSUpAqhjhD3ONmiY+3yWn56qHADEd0TCoRcDSZ0= -github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= +github.com/0chain/gosdk v1.8.18-0.20230826144722-16bc7138c41b h1:WdriWJyVNK5WlhA6PvTRm+1IBIMFRJEQ5TLRw1d+Awo= +github.com/0chain/gosdk v1.8.18-0.20230826144722-16bc7138c41b/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw= @@ -728,4 +728,4 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= \ No newline at end of file +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= diff --git a/migration/migrate.go b/migration/migrate.go index 3c73cf1..38f0730 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -2,6 +2,8 @@ package migration import ( "context" + "crypto/sha1" + "encoding/hex" "fmt" "io" "os" @@ -22,7 +24,7 @@ import ( const Batch = 10 const ( - Replace = iota //Will replace existing file + Replace = iota // Will replace existing file Skip // Will skip migration if file already exists Duplicate // Will add _copy prefix and uploads the file ) @@ -62,10 +64,10 @@ type Migration struct { skip int retryCount int - //Number of goroutines to run. So at most concurrency * Batch goroutines will run. i.e. for bucket level and object level + // Number of goroutines to run. So at most concurrency * Batch goroutines will run. i.e. for bucket level and object level concurrency int - szCtMu sync.Mutex //size and count mutex; used to update migratedSize and totalMigratedObjects + szCtMu sync.Mutex // size and count mutex; used to update migratedSize and totalMigratedObjects migratedSize uint64 totalMigratedObjects uint64 @@ -235,8 +237,16 @@ func StartMigration() error { } migrationWorker := NewMigrationWorker(migration.workDir) - go migration.DownloadWorker(rootContext, migrationWorker) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + migration.DownloadWorker(rootContext, migrationWorker) + }() // go migration.UploadWorker(rootContext, migrationWorker) + wg.Wait() + migration.UpdateStateFile(migrationWorker) err := migrationWorker.GetMigrationError() if err != nil { @@ -387,8 +397,31 @@ func (m *Migration) UploadWorker(ctx context.Context, migrator *MigrationWorker) wg.Wait() } +func getUniqueShortObjKey(objectKey string) string { + // Max length to which objectKey would be trimmed to. + // Keeping this less than 100 chars to prevent longer name in case of uploading duplicate + // files with `_copy` suffixes. + const maxLength = 90 + + if len(objectKey) > maxLength { + // Generate a SHA-1 hash of the object key + hash := sha1.New() + hash.Write([]byte(objectKey)) + hashSum := hash.Sum(nil) + + // Convert the hash to a hexadecimal string + hashString := hex.EncodeToString(hashSum) + + // Combine the first 10 characters of the hash with a truncated object key + shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:]) + return shortKey + } + + return objectKey +} + func getRemotePath(objectKey string) string { - return filepath.Join(migration.migrateTo, migration.bucket, objectKey) + return filepath.Join(migration.migrateTo, migration.bucket, getUniqueShortObjKey(objectKey)) } func checkIsFileExist(ctx context.Context, downloadObj *DownloadObjectMeta) error { @@ -466,7 +499,7 @@ func processOperationForMemory(ctx context.Context, downloadObj *DownloadObjectM zlogger.Logger.Info("Replacing object" + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10)) fileOperation = migration.zStore.Replace(ctx, remotePath, r, downloadObj.Size, mimeType) case Duplicate: - zlogger.Logger.Info("Duplicating object" + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10)) + zlogger.Logger.Info("Duplicating object " + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10)) fileOperation = migration.zStore.Duplicate(ctx, remotePath, r, downloadObj.Size, mimeType) } } else { @@ -547,7 +580,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp fileOps := make([]sdk.OperationRequest, 0, len(ops)) for _, op := range ops { migrator.UploadStart(op.uploadObj) - zlogger.Logger.Info("upload start: ", op.uploadObj.ObjectKey, "size: ", op.uploadObj.Size) + zlogger.Logger.Info("upload start: ", op.uploadObj.ObjectKey, " size: ", op.uploadObj.Size) fileOps = append(fileOps, op.Operation) } err = util.Retry(3, time.Second*5, func() error { @@ -563,7 +596,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp }) for _, op := range ops { migrator.UploadDone(op.uploadObj, err) - zlogger.Logger.Info("upload done: ", op.uploadObj.ObjectKey, "size ", op.uploadObj.Size, err) + zlogger.Logger.Info("upload done: ", op.uploadObj.ObjectKey, " size ", op.uploadObj.Size, err) } migrator.SetMigrationError(err) return err