From cb52e1452fccb9c92e48c9e2080cb0d3d9754d08 Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Thu, 17 Aug 2023 20:44:01 +0530 Subject: [PATCH 01/15] Fix long files causing upload failures for the entire upload batch during migration. --- migration/migrate.go | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/migration/migrate.go b/migration/migrate.go index b5f27ac..55fba1a 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -359,8 +359,23 @@ func (m *Migration) UploadWorker(ctx context.Context, migrator *MigrationWorker) wg.Wait() } +func getShortObjectKey(objectKey string) string { + //max length to which objectKey would be trimmed to + //reserving 10 chars for duplicate suffixes. + const maxLength = 90 + + if len(objectKey) > maxLength { + //get the last 100 characters of the object key. + shortKey := objectKey[len(objectKey)-maxLength:] + return shortKey + } + + return objectKey + +} + func getRemotePath(objectKey string) string { - return filepath.Join(migration.migrateTo, migration.bucket, objectKey) + return filepath.Join(migration.migrateTo, migration.bucket, getShortObjectKey(objectKey)) } func checkIsFileExist(ctx context.Context, downloadObj *DownloadObjectMeta) error { @@ -490,7 +505,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp fileOps := make([]sdk.OperationRequest, 0, len(ops)) for _, op := range ops { migrator.UploadStart(op.uploadObj) - zlogger.Logger.Info("upload start: ", op.uploadObj.ObjectKey, "size: ", op.uploadObj.Size) + zlogger.Logger.Info("upload start: ", op.uploadObj.ObjectKey, " size: ", op.uploadObj.Size) fileOps = append(fileOps, op.Operation) } err = util.Retry(3, time.Second*5, func() error { From 3a42cdd58f12b45eb63e8ab5c8c9d35659c117ec Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Thu, 17 Aug 2023 21:21:44 +0530 Subject: [PATCH 02/15] minor comment refactor --- migration/migrate.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/migration/migrate.go b/migration/migrate.go index a761e83..8cc3900 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -393,7 +393,7 @@ func getShortObjectKey(objectKey string) string { const maxLength = 90 if len(objectKey) > maxLength { - //get the last 100 characters of the object key. + //get the last maxLength characters of the object key. shortKey := objectKey[len(objectKey)-maxLength:] return shortKey } From 2e16d29f1fa385ce1ba9f6e1454c3272685b695e Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Fri, 18 Aug 2023 18:57:47 +0530 Subject: [PATCH 03/15] reverting changes in yaml --- .github/workflows/build-s3-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 92f26cc..fcfd436 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -142,7 +142,7 @@ jobs: build-macos: name: Build-macos - runs-on: macos-latest + runs-on: macos-runner-binary steps: - name: Setup go 1.18 uses: actions/setup-go@v2 From 0ca18af62d7603f74a30414f1767f660733fa685 Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Fri, 18 Aug 2023 20:23:53 +0530 Subject: [PATCH 04/15] use hashing to avoid duplicates in the remote path --- migration/migrate.go | 24 ++++++++++++++++-------- 1 file changed, 16 insertions(+), 8 deletions(-) diff --git a/migration/migrate.go b/migration/migrate.go index 8cc3900..64a345f 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -2,6 +2,8 @@ package migration import ( "context" + "crypto/sha1" + "encoding/hex" "fmt" "io" "os" @@ -387,23 +389,29 @@ func (m *Migration) UploadWorker(ctx context.Context, migrator *MigrationWorker) wg.Wait() } -func getShortObjectKey(objectKey string) string { - //max length to which objectKey would be trimmed to - //reserving 10 chars for duplicate suffixes. - const maxLength = 90 +func getUniqueShortObjKey(objectKey string) string { + //Max length to which objectKey would be trimmed to. + const maxLength = 100 if len(objectKey) > maxLength { - //get the last maxLength characters of the object key. - shortKey := objectKey[len(objectKey)-maxLength:] + // Generate a SHA-1 hash of the object key + hash := sha1.New() + hash.Write([]byte(objectKey)) + hashSum := hash.Sum(nil) + + // Convert the hash to a hexadecimal string + hashString := hex.EncodeToString(hashSum) + + // Combine the first 10 characters of the hash with a truncated object key + shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:]) return shortKey } return objectKey - } func getRemotePath(objectKey string) string { - return filepath.Join(migration.migrateTo, migration.bucket, getShortObjectKey(objectKey)) + return filepath.Join(migration.migrateTo, migration.bucket, getUniqueShortObjKey(objectKey)) } func checkIsFileExist(ctx context.Context, downloadObj *DownloadObjectMeta) error { From fe6d72cb7e9769d8f1b7a37e24569762f24f9ec2 Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Fri, 18 Aug 2023 20:39:22 +0530 Subject: [PATCH 05/15] reverting the changes in build file for macos --- .github/workflows/build-s3-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index fcfd436..92f26cc 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -142,7 +142,7 @@ jobs: build-macos: name: Build-macos - runs-on: macos-runner-binary + runs-on: macos-latest steps: - name: Setup go 1.18 uses: actions/setup-go@v2 From d032ede3a0dedf0fdf0acfb5cb959b4267d73cb0 Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Fri, 18 Aug 2023 20:42:12 +0530 Subject: [PATCH 06/15] revert the macos build rule changes --- .github/workflows/build-s3-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 92f26cc..fcfd436 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -142,7 +142,7 @@ jobs: build-macos: name: Build-macos - runs-on: macos-latest + runs-on: macos-runner-binary steps: - name: Setup go 1.18 uses: actions/setup-go@v2 From bf68a699e59a1d41139ee44f682aa68691939f33 Mon Sep 17 00:00:00 2001 From: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:27:00 +0530 Subject: [PATCH 07/15] Update build-s3-migration.yaml --- .github/workflows/build-s3-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index fcfd436..0a6fdb7 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -42,7 +42,7 @@ jobs: elif [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then echo "TAG=v0.0.0-staging" >> $GITHUB_ENV else - echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV fi - name: add dependencies From 9d32254f39a65bd35050a6e4af6d0682b352f52e Mon Sep 17 00:00:00 2001 From: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:30:41 +0530 Subject: [PATCH 08/15] Update tag for macos --- .github/workflows/build-s3-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 0a6fdb7..68098b8 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -161,7 +161,7 @@ jobs: elif [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then echo "TAG=v0.0.0-staging" >> $GITHUB_ENV else - echo "TAG=$(echo ${GITHUB_REF#refs/tags/})" >> $GITHUB_ENV + echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV fi - name: Install run: make build From 6f368ce7510261b2ed14cd9444b5ddcdf466ee15 Mon Sep 17 00:00:00 2001 From: Kishan Dhakan <42718091+Kishan-Dhakan@users.noreply.github.com> Date: Fri, 18 Aug 2023 21:38:49 +0530 Subject: [PATCH 09/15] Update build-s3-migration.yaml --- .github/workflows/build-s3-migration.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 68098b8..0a099c7 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -114,7 +114,7 @@ jobs: } ElseIf ( "${{github.base_ref}}" -eq "staging" -OR "${{github.ref}}" -eq "refs/heads/staging" ){ echo "TAG=v0.0.0-staging" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } ELSE { - echo "TAG=${{github.ref}}" | %{$_ -replace('refs/tags/', '')} | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append + echo "TAG=v0.0.0-${{github.base_ref}}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append } - name: Install From a21c6fe405647b6c050b318a50c2b23ee9c4f08e Mon Sep 17 00:00:00 2001 From: Kishan Dhakan Date: Wed, 23 Aug 2023 00:33:03 +0530 Subject: [PATCH 10/15] use github ref for tag --- .github/workflows/build-s3-migration.yaml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 0a099c7..06ef570 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -62,7 +62,7 @@ jobs: with: repo_token: ${{ secrets.GITHUB_TOKEN }} file: s3mgrt-linux.tar.gz - tag: ${{ env.TAG }} + tag: ${{ github.ref }} overwrite: true file_glob: true @@ -129,7 +129,7 @@ jobs: with: repo_token: ${{ secrets.GITHUB_TOKEN }} file: s3mgrt-windows.zip - tag: ${{ env.TAG }} + tag: ${{ github.ref }} overwrite: true file_glob: true @@ -174,7 +174,7 @@ jobs: with: repo_token: ${{ secrets.GITHUB_TOKEN }} file: s3mgrt-macos.tar.gz - tag: ${{ env.TAG }} + tag: ${{ github.ref }} overwrite: true file_glob: true From 2bd86bf2df6f474a8e55141191d413ef91ba4238 Mon Sep 17 00:00:00 2001 From: shahnawaz-creator <117025384+shahnawaz-creator@users.noreply.github.com> Date: Wed, 23 Aug 2023 00:33:55 +0530 Subject: [PATCH 11/15] disabled auto release feature --- .github/workflows/build-s3-migration.yaml | 68 +++++++++++------------ 1 file changed, 34 insertions(+), 34 deletions(-) diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml index 92f26cc..a635ca6 100644 --- a/.github/workflows/build-s3-migration.yaml +++ b/.github/workflows/build-s3-migration.yaml @@ -54,17 +54,17 @@ jobs: run: | docker run --rm -v $PWD:/s3mgrt --workdir=/s3mgrt golang:1.20.4 make build - - name: Zip release - run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt - - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: s3mgrt-linux.tar.gz - tag: ${{ env.TAG }} - overwrite: true - file_glob: true + # - name: Zip release + # run: tar -czvf s3mgrt-linux.tar.gz ./s3mgrt + + # - name: Upload binaries to release + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: s3mgrt-linux.tar.gz + # tag: ${{ env.TAG }} + # overwrite: true + # file_glob: true - name: Install AWS if: github.ref == 'refs/heads/staging' @@ -120,18 +120,18 @@ jobs: - name: Install run: make build - - name: Zip Release - run: | - copy s3mgrt s3mgrt.exe - 7z a s3mgrt-windows.zip s3mgrt.exe - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: s3mgrt-windows.zip - tag: ${{ env.TAG }} - overwrite: true - file_glob: true + # - name: Zip Release + # run: | + # copy s3mgrt s3mgrt.exe + # 7z a s3mgrt-windows.zip s3mgrt.exe + # - name: Upload binaries to release + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: s3mgrt-windows.zip + # tag: ${{ env.TAG }} + # overwrite: true + # file_glob: true - name: 'Upload Artifact' uses: actions/upload-artifact@v2 @@ -166,17 +166,17 @@ jobs: - name: Install run: make build - - name: Zip release - run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt - - - name: Upload binaries to release - uses: svenstaro/upload-release-action@v2 - with: - repo_token: ${{ secrets.GITHUB_TOKEN }} - file: s3mgrt-macos.tar.gz - tag: ${{ env.TAG }} - overwrite: true - file_glob: true + # - name: Zip release + # run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt + + # - name: Upload binaries to release + # uses: svenstaro/upload-release-action@v2 + # with: + # repo_token: ${{ secrets.GITHUB_TOKEN }} + # file: s3mgrt-macos.tar.gz + # tag: ${{ env.TAG }} + # overwrite: true + # file_glob: true - name: 'Upload Artifact' uses: actions/upload-artifact@v2 From e834282a691757295e9be31c1b2b0c5547342d42 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 23 Aug 2023 17:20:18 +0530 Subject: [PATCH 12/15] update gosdk --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 031158f..89791fe 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 + github.com/0chain/gosdk v1.8.18-0.20230823095022-2ba5110a8e9a github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.10 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37 diff --git a/go.sum b/go.sum index 59422d4..edc9562 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 h1:fgaUQSUpAqhjhD3ONmiY+3yWn56qHADEd0TCoRcDSZ0= -github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= +github.com/0chain/gosdk v1.8.18-0.20230823095022-2ba5110a8e9a h1:haBisjN/T3Yk34K0R1p/Eiy9/tkAKykndh/nnXxZ3oE= +github.com/0chain/gosdk v1.8.18-0.20230823095022-2ba5110a8e9a/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw= @@ -728,4 +728,4 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= \ No newline at end of file +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 0fe39abc7d9010d94c3e4ea95c333fe5f2e28258 Mon Sep 17 00:00:00 2001 From: Kishan Dhakan Date: Wed, 23 Aug 2023 20:58:38 +0530 Subject: [PATCH 13/15] update gosdk --- go.mod | 2 +- go.sum | 6 +++--- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 031158f..70d150b 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.20 require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 + github.com/0chain/gosdk v1.8.18-0.20230823135357-61db9486f4c0 github.com/aws/aws-sdk-go-v2 v1.17.1 github.com/aws/aws-sdk-go-v2/config v1.17.10 github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37 diff --git a/go.sum b/go.sum index 59422d4..70732c6 100644 --- a/go.sum +++ b/go.sum @@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E= github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= -github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 h1:fgaUQSUpAqhjhD3ONmiY+3yWn56qHADEd0TCoRcDSZ0= -github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= +github.com/0chain/gosdk v1.8.18-0.20230823135357-61db9486f4c0 h1:XhLlPJKQxohZAS1+H2MHxy/e5I9Pw+Iyi+KaC5NHPl8= +github.com/0chain/gosdk v1.8.18-0.20230823135357-61db9486f4c0/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw= @@ -728,4 +728,4 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= -rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= \ No newline at end of file +rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= From 6127f1205e96b9458ee2af5db8b002bc2e1983dc Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Sat, 26 Aug 2023 21:28:39 +0530 Subject: [PATCH 14/15] fixing misbehaving error worker --- migration/migrate.go | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/migration/migrate.go b/migration/migrate.go index 3c73cf1..5852730 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -22,7 +22,7 @@ import ( const Batch = 10 const ( - Replace = iota //Will replace existing file + Replace = iota // Will replace existing file Skip // Will skip migration if file already exists Duplicate // Will add _copy prefix and uploads the file ) @@ -62,10 +62,10 @@ type Migration struct { skip int retryCount int - //Number of goroutines to run. So at most concurrency * Batch goroutines will run. i.e. for bucket level and object level + // Number of goroutines to run. So at most concurrency * Batch goroutines will run. i.e. for bucket level and object level concurrency int - szCtMu sync.Mutex //size and count mutex; used to update migratedSize and totalMigratedObjects + szCtMu sync.Mutex // size and count mutex; used to update migratedSize and totalMigratedObjects migratedSize uint64 totalMigratedObjects uint64 @@ -235,8 +235,16 @@ func StartMigration() error { } migrationWorker := NewMigrationWorker(migration.workDir) - go migration.DownloadWorker(rootContext, migrationWorker) + + wg := sync.WaitGroup{} + wg.Add(1) + go func() { + defer wg.Done() + migration.DownloadWorker(rootContext, migrationWorker) + }() // go migration.UploadWorker(rootContext, migrationWorker) + wg.Wait() + migration.UpdateStateFile(migrationWorker) err := migrationWorker.GetMigrationError() if err != nil { @@ -563,7 +571,7 @@ func (m *Migration) processMultiOperation(ctx context.Context, ops []MigrationOp }) for _, op := range ops { migrator.UploadDone(op.uploadObj, err) - zlogger.Logger.Info("upload done: ", op.uploadObj.ObjectKey, "size ", op.uploadObj.Size, err) + zlogger.Logger.Info("upload done: ", op.uploadObj.ObjectKey, " size ", op.uploadObj.Size, err) } migrator.SetMigrationError(err) return err From 53c5fef1c14e109481594427cbd4a3843a7c8a89 Mon Sep 17 00:00:00 2001 From: Sunil Kumar Date: Sun, 27 Aug 2023 21:09:50 +0530 Subject: [PATCH 15/15] fix long file names in case of duplicates --- migration/migrate.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/migration/migrate.go b/migration/migrate.go index 330aec3..38f0730 100644 --- a/migration/migrate.go +++ b/migration/migrate.go @@ -398,8 +398,10 @@ func (m *Migration) UploadWorker(ctx context.Context, migrator *MigrationWorker) } func getUniqueShortObjKey(objectKey string) string { - //Max length to which objectKey would be trimmed to. - const maxLength = 100 + // Max length to which objectKey would be trimmed to. + // Keeping this less than 100 chars to prevent longer name in case of uploading duplicate + // files with `_copy` suffixes. + const maxLength = 90 if len(objectKey) > maxLength { // Generate a SHA-1 hash of the object key @@ -497,7 +499,7 @@ func processOperationForMemory(ctx context.Context, downloadObj *DownloadObjectM zlogger.Logger.Info("Replacing object" + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10)) fileOperation = migration.zStore.Replace(ctx, remotePath, r, downloadObj.Size, mimeType) case Duplicate: - zlogger.Logger.Info("Duplicating object" + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10)) + zlogger.Logger.Info("Duplicating object " + downloadObj.ObjectKey + " size " + strconv.FormatInt(downloadObj.Size, 10)) fileOperation = migration.zStore.Duplicate(ctx, remotePath, r, downloadObj.Size, mimeType) } } else {