Merge pull request #113 from 0chain/fix/chunk-size (#114)

* README.md changes --added table of contents -- zus overview -- added sample migration command syntax -- added build instructions -- corrected typos * update macos runner * kishan dhakan suggestions * Update gosdk.yml: update create-pr workflow * remove ./build_mocks.sh * limit range * comment out build-windows and build-macos * disable macos windows builds * upgrade GoSDK to sprint-1.11 * fix chunk size --------- Co-authored-by: Harshit Mehndiratta <[email protected]> Co-authored-by: Manohar Reddy <[email protected]> Co-authored-by: Kishan Dhakan <[email protected]> Co-authored-by: Yury <[email protected]> Co-authored-by: Hitenjain14 <[email protected]> Co-authored-by: service-0chain <[email protected]> Co-authored-by: Hitenjain14 <[email protected]>
0chain · Dec 16, 2023 · 0919caa · 0919caa
1 parent 5b37aba
commit 0919caa
Show file tree

Hide file tree

Showing 7 changed files with 172 additions and 125 deletions.
diff --git a/.github/workflows/build-s3-migration.yaml b/.github/workflows/build-s3-migration.yaml
@@ -93,97 +93,97 @@ jobs:
           path: s3mgrt
           retention-days: 5
 
-  build-windows:
-    name: Build-windows
-    runs-on: windows-latest
-    steps:
-      - name: Setup go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.18' # The Go version to download (if necessary) and use.
-
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set GITHUB_ENV
-        run: |
-          IF ( "${{github.base_ref}}" -eq "master" -OR "${{github.ref}}" -eq "refs/heads/master" ){
-            echo "TAG=v0.0.0-master" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          } ElseIf ( "${{github.base_ref}}" -eq "dev" -OR "${{github.ref}}" -eq "refs/heads/dev" ){
-            echo "TAG=v0.0.0-dev" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          } ElseIf ( "${{github.base_ref}}" -eq "staging" -OR "${{github.ref}}" -eq "refs/heads/staging" ){
-            echo "TAG=v0.0.0-staging" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          } ELSE {
-            echo "TAG=v0.0.0-${{github.base_ref}}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
-          }
-
-      - name: Install
-        run: make build
-
-      # - name: Zip Release
-      #   run: |
-      #     copy s3mgrt s3mgrt.exe
-      #     7z a s3mgrt-windows.zip s3mgrt.exe
-      # - name: Upload binaries to release
-      #   uses: svenstaro/upload-release-action@v2
-      #   with:
-      #     repo_token: ${{ secrets.GITHUB_TOKEN }}
-      #     file: s3mgrt-windows.zip
-      #     tag: ${{ env.TAG }}
-      #     overwrite: true
-      #     file_glob: true
-
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v2
-        with:
-          name: s3mgrt-windows
-          path: s3mgrt
-          retention-days: 5
-
-  build-macos:
-    name: Build-macos
-    runs-on: macos-runner-binary
-    steps:
-      - name: Setup go 1.18
-        uses: actions/setup-go@v2
-        with:
-          go-version: '1.18' # The Go version to download (if necessary) and use.
-      - name: Checkout
-        uses: actions/checkout@v2
-
-      - name: Set GITHUB_ENV
-        run: |
-          echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV
-          if [[ "${{github.base_ref}}" == "master" || "${{github.ref}}" == "refs/heads/master" ]]; then
-            echo "TAG=v0.0.0-master" >> $GITHUB_ENV
-          elif  [[ "${{github.base_ref}}" == "dev" || "${{github.ref}}" == "refs/heads/dev" ]]; then
-            echo "TAG=v0.0.0-dev" >> $GITHUB_ENV
-          elif  [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then
-            echo "TAG=v0.0.0-staging" >> $GITHUB_ENV
-          else
-            echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV
-          fi
-      - name: Install
-        run: make build
-
-      # - name: Zip release
-      #   run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt
-
-      # - name: Upload binaries to release
-      #   uses: svenstaro/upload-release-action@v2
-      #   with:
-      #     repo_token: ${{ secrets.GITHUB_TOKEN }}
-      #     file: s3mgrt-macos.tar.gz
-      #     tag: ${{ env.TAG }}
-      #     overwrite: true
-      #     file_glob: true
-
-      - name: 'Upload Artifact'
-        uses: actions/upload-artifact@v2
-        with:
-          name: s3mgrt-macos
-          path: s3mgrt
-          retention-days: 5
+  # build-windows:
+  #   name: Build-windows
+  #   runs-on: windows-latest
+  #   steps:
+  #     - name: Setup go 1.18
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: '1.18' # The Go version to download (if necessary) and use.
+
+  #     - name: Checkout
+  #       uses: actions/checkout@v2
+
+  #     - name: Set GITHUB_ENV
+  #       run: |
+  #         IF ( "${{github.base_ref}}" -eq "master" -OR "${{github.ref}}" -eq "refs/heads/master" ){
+  #           echo "TAG=v0.0.0-master" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+  #         } ElseIf ( "${{github.base_ref}}" -eq "dev" -OR "${{github.ref}}" -eq "refs/heads/dev" ){
+  #           echo "TAG=v0.0.0-dev" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+  #         } ElseIf ( "${{github.base_ref}}" -eq "staging" -OR "${{github.ref}}" -eq "refs/heads/staging" ){
+  #           echo "TAG=v0.0.0-staging" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+  #         } ELSE {
+  #           echo "TAG=v0.0.0-${{github.base_ref}}" | Out-File -FilePath $Env:GITHUB_ENV -Encoding utf-8 -Append
+  #         }
+
+  #     - name: Install
+  #       run: make build
+
+  #     # - name: Zip Release
+  #     #   run: |
+  #     #     copy s3mgrt s3mgrt.exe
+  #     #     7z a s3mgrt-windows.zip s3mgrt.exe
+  #     # - name: Upload binaries to release
+  #     #   uses: svenstaro/upload-release-action@v2
+  #     #   with:
+  #     #     repo_token: ${{ secrets.GITHUB_TOKEN }}
+  #     #     file: s3mgrt-windows.zip
+  #     #     tag: ${{ env.TAG }}
+  #     #     overwrite: true
+  #     #     file_glob: true
+
+  #     - name: 'Upload Artifact'
+  #       uses: actions/upload-artifact@v2
+  #       with:
+  #         name: s3mgrt-windows
+  #         path: s3mgrt
+  #         retention-days: 5
+
+  # build-macos:
+  #   name: Build-macos
+  #   runs-on: macos-runner
+  #   steps:
+  #     - name: Setup go 1.18
+  #       uses: actions/setup-go@v2
+  #       with:
+  #         go-version: '1.18' # The Go version to download (if necessary) and use.
+  #     - name: Checkout
+  #       uses: actions/checkout@v2
+
+  #     - name: Set GITHUB_ENV
+  #       run: |
+  #         echo "BRANCH=$(echo ${GITHUB_REF#refs/heads/})" >> $GITHUB_ENV
+  #         if [[ "${{github.base_ref}}" == "master" || "${{github.ref}}" == "refs/heads/master" ]]; then
+  #           echo "TAG=v0.0.0-master" >> $GITHUB_ENV
+  #         elif  [[ "${{github.base_ref}}" == "dev" || "${{github.ref}}" == "refs/heads/dev" ]]; then
+  #           echo "TAG=v0.0.0-dev" >> $GITHUB_ENV
+  #         elif  [[ "${{github.base_ref}}" == "staging" || "${{github.ref}}" == "refs/heads/staging" ]]; then
+  #           echo "TAG=v0.0.0-staging" >> $GITHUB_ENV
+  #         else
+  #           echo "TAG=v0.0.0-${{github.base_ref}}" >> $GITHUB_ENV
+  #         fi
+  #     - name: Install
+  #       run: make build
+
+  #     # - name: Zip release
+  #     #   run: tar -czvf s3mgrt-macos.tar.gz ./s3mgrt
+
+  #     # - name: Upload binaries to release
+  #     #   uses: svenstaro/upload-release-action@v2
+  #     #   with:
+  #     #     repo_token: ${{ secrets.GITHUB_TOKEN }}
+  #     #     file: s3mgrt-macos.tar.gz
+  #     #     tag: ${{ env.TAG }}
+  #     #     overwrite: true
+  #     #     file_glob: true
+
+  #     - name: 'Upload Artifact'
+  #       uses: actions/upload-artifact@v2
+  #       with:
+  #         name: s3mgrt-macos
+  #         path: s3mgrt
+  #         retention-days: 5
 
   system-tests:
     if: github.event_name != 'workflow_dispatch'

diff --git a/.github/workflows/gosdk.yml b/.github/workflows/gosdk.yml
@@ -8,32 +8,37 @@ on:
   workflow_dispatch:
     inputs:
       gosdk:
-        description: 'The version of 0Chain GoSDK'
+        description: 'The version/branch of 0Chain GoSDK'
+        required: true
+      base:
+        description: the base branch to which PR needs to be raised
         required: true
 
 jobs:
   create-pr:
-    runs-on: [self-hosted, build]
+    runs-on: [arc-runner]
     steps:
-      - name: Setup go 1.18
+      - name: Setup go 1.20
         uses: actions/setup-go@v2
         with:
-          go-version: '1.18' 
+          go-version: '1.20'
 
       - name: Checkout
-        uses: actions/checkout@v1
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.inputs.base }}
 
       - name: Upgrade 0Chain GoSDK
         run: |
           echo "BRANCH=$(echo ${{github.event.inputs.gosdk}} | sed 's/\//-/g')" >> $GITHUB_ENV
-
+          
           go get github.com/0chain/gosdk@${{github.event.inputs.gosdk}}
           go mod tidy
 
       - name: Create Pull Request
         uses: peter-evans/create-pull-request@v3
         with:
-          base: staging
+          base: ${{ github.event.inputs.base }}
           token: ${{ secrets.GOSDK }}
           commit-message: upgrade GoSDK to ${{ github.event.inputs.gosdk }}
           branch: gosdk-upgrade-${{ env.BRANCH }}
@@ -43,5 +48,5 @@ jobs:
             0Chain GoSDK `${{ github.event.inputs.gosdk }}` is released.
             see full changelog on https://github.com/0chain/gosdk/releases/tag/${{ github.event.inputs.gosdk }}
           draft: false
-          reviewers: cnlangzi
+          reviewers: guruhubb
           labels: GoSDK
diff --git a/README.md b/README.md
@@ -1,38 +1,66 @@
-# s3-migration - a CLI for migrating AWS s3 bucket to 0Chain dStorage 
-This program helps to migrate files in s3 bucket to 0chain dStorage.
+# s3-migration - a CLI tool for migrating AWS S3 bucket to Züs dStorage 
 
+
+s3-migration is a command line interface (CLI) tool that helps migrate files from S3 bucket to Züs dStorage.
+
+- [Züs Overview](#züs-overview)
+- [Building s3 migration](#building-s3-migration)
+- [Running s3 migration](#running-s3-migration)
+- [Global Flags](#global-flags)
+- [Migration Commands](#migrate-command)
+- [Running Test Cases](#running-test-cases)
+
+## Züs Overview
+[Züs](https://zus.network/) is a high-performance cloud on a fast blockchain offering privacy and configurable uptime. It is an alternative to traditional cloud S3 and has shown better performance on a test network due to its parallel data architecture. The technology uses erasure code to distribute the data between data and parity servers. Züs storage is configurable to provide flexibility for IT managers to design for desired security and uptime, and can design a hybrid or a multi-cloud architecture with a few clicks using [Blimp's](https://blimp.software/) workflow, and can change redundancy and providers on the fly.
+
+For instance, the user can start with 10 data and 5 parity providers and select where they are located globally, and later decide to add a provider on-the-fly to increase resilience, performance, or switch to a lower cost provider.
+
+Users can also add their own servers to the network to operate in a hybrid cloud architecture. Such flexibility allows the user to improve their regulatory, content distribution, and security requirements with a true multi-cloud architecture. Users can also construct a private cloud with all of their own servers rented across the globe to have a better content distribution, highly available network, higher performance, and lower cost.
+
+[The QoS protocol](https://medium.com/0chain/qos-protocol-weekly-debrief-april-12-2023-44524924381f) is time-based where the blockchain challenges a provider on a file that the provider must respond within a certain time based on its size to pass. This forces the provider to have a good server and data center performance to earn rewards and income.
+
+The [privacy protocol](https://zus.network/build) from Züs is unique where a user can easily share their encrypted data with their business partners, friends, and family through a proxy key sharing protocol, where the key is given to the providers, and they re-encrypt the data using the proxy key so that only the recipient can decrypt it with their private key.
+
+Züs has ecosystem apps to encourage traditional storage consumption such as [Blimp](https://blimp.software/), a S3 server and cloud migration platform, and [Vult](https://vult.network/), a personal cloud app to store encrypted data and share privately with friends and family, and [Chalk](https://chalk.software/), a zero upfront cost permanent storage solution for NFT artists.
+
+Other apps are [Bolt](https://bolt.holdings/), a wallet that is very secure with air-gapped 2FA split-key protocol to prevent hacks from compromising your digital assets, and it enables you to stake and earn from the storage providers; [Atlus](https://atlus.cloud/), a blockchain explorer and [Chimney](https://demo.chimney.software/), which allows anyone to join the network and earn using their server or by just renting one, with no prior knowledge required.
+
+## Building s3-migration
+
+Prerequisites: [Go](https://go.dev/doc/install) 
+```
+git clone https://github.com/0chain/s3-migration.git
+cd s3-migration
+go build .
+```
 ## Running s3-migration
 
-When you run the `s3migration` command in terminal with no arguments, it will list all the available commands and the global flags.
+When you run the `./s3migration` command in terminal with no arguments inside the s3-migration directory, it will list all the available commands and the global flags.
 
 **Command**|**Description**
 :-----:|:-----:
 [help](#help)|Help about any command
 [migrate](#migrate)|Migrate user data from S3 bucket to dStorage
 
-## Running Test cases
-
-`go test ./...`
-
 ### Global Flags
 
-Global Flags are parameters in s3mgrt that can be used with any command to override the default configuration supports the following global parameters.
+Global Flags are parameters in s3migration that can be used with any command to override the default configuration supports the following global parameters.
 
 | Flags                      | Description                                                  | Usage                                             |
 | -------------------------- | ------------------------------------------------------------ | ------------------------------------------------- |
-| --config string            | Specify configuration file (default is [$HOME/.zcn/config.yaml](#zcnconfigyaml)) | s3mgrt migrate --config config1.yaml              |
-| --configDir string         | Specify a configuration directory (default is $HOME/.zcn) | s3migration migrate --configDir /$HOME/.zcn2           |
-| -h, --help                 | Gives more information about a particular command.           | s3migration migrate --help                             |
-| --network string           | Specify a network file to overwrite the network details(default is [$HOME/.zcn/network.yaml](#zcnnetworkyaml)) | s3migration migrate --network network1.yaml            |
-| --wallet string            | Specify a wallet file or 2nd wallet (default is $HOME/.zcn/wallet.json) | s3migration migrate --wallet wallet2.json              |
-| --wallet_client_id string  | Specify client id. If wallet key is also provided then wallet_private_key is required | s3migration migrate --wallet_client_id "wallet client id" |
-| --wallet_client_key string | Specify a wallet client_key (By default client_key specified in $HOME/.zcn/wallet.json is used) | s3migration migrate --wallet_client_key  "client key" |
-| --wallet_private_key string | Specify wallet private key| s3migration migrate --wallet_private_key "wallet private key"|
+| --config string            | Specify configuration file (default is [$HOME/.zcn/config.yaml](https://github.com/0chain/zboxcli/blob/staging/network/config.yaml)) | ./s3migration migrate --config config1.yaml              |
+| --configDir string         | Specify a configuration directory (default is $HOME/.zcn) | ./s3migration migrate --configDir /$HOME/.zcn2           |
+| -h, --help                 | Gives more information about a particular command.           | ./s3migration migrate --help                             |
+| --network string           | Specify a network file to overwrite the network details(default is [$HOME/.zcn/network.yaml](https://github.com/0chain/zwalletcli#override-network)) | s3migration migrate --network network1.yaml            |
+| --wallet string            | Specify a wallet file or 2nd wallet (default is $HOME/.zcn/wallet.json) | ./s3migration migrate --wallet wallet2.json              |
+| --wallet_client_id string  | Specify client id. If wallet key is also provided then wallet_private_key is required | ./s3migration migrate --wallet_client_id "wallet client id" |
+| --wallet_client_key string | Specify a wallet client_key (By default client_key specified in $HOME/.zcn/wallet.json is used) | ./s3migration migrate --wallet_client_key  "client key" |
+| --wallet_private_key string | Specify wallet private key| ./s3migration migrate --wallet_private_key "wallet private key"|
 
 ## Migrate Command
 
-`migrate` command is used to migrate files from s3 buckets to some remote directory(default is /) by using aws-s3-sdk and 0chain gosdk. All the objects from bucket will be migrated.
-However user can specify some prefix to migrate only the files with those prefix. Also if there is name conflict within dStorage file and bucket file use can
+`migrate` command is used to migrate files from s3 buckets to some remote directory(default is /) by using aws-s3-sdk and 0chain gosdk. All the objects from the bucket will be migrated.
+However users can specify some prefix to migrate only the files with those prefix. Also if there is name conflict within dStorage file and bucket file users can
 specify whether to skip, replace or duplicate them. Migration state is maintained in some file so user can also resume migration operation if some error had
 occurred in previous migration session. User can also specify whether to delete migrated file. Note the defaults.
 
@@ -60,6 +88,16 @@ occurred in previous migration session. User can also specify whether to delete
 | skip |  | 0 --> Replace existing files; 1 --> Skip migration; 2 --> Duplicate               | 1 | int    |
 | wd |  | Working directory               | $HOME/.s3migration | string    |
 
+Sample Command:
+```
+./s3migration migrate --allocation $ALLOC --access-key $AWS_ACCESS_KEY --secret-key $AWS_SECRET_KEY --bucket s3migfiles
+```
+Note: The bucket will be listed as directory in dStorage.
+
+## Running Test cases
+
+`go test ./...`
+
 ## BenchMark
 
 **Data Shard**|**Parity**|**Upload Size**|**File count**|**Time Taken**|**Network Speed**

diff --git a/go.mod b/go.mod
@@ -4,7 +4,7 @@ go 1.20
 
 require (
 	github.com/0chain/errors v1.0.3
-	github.com/0chain/gosdk v1.10.1-0.20231206175109-60d8a76df1e6
+	github.com/0chain/gosdk v1.10.1-0.20231210164918-9b9b8e0d5c89
 	github.com/aws/aws-sdk-go-v2 v1.17.1
 	github.com/aws/aws-sdk-go-v2/config v1.17.10
 	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.11.37

diff --git a/go.sum b/go.sum
@@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
 github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
 github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
 github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
-github.com/0chain/gosdk v1.10.1-0.20231206175109-60d8a76df1e6 h1:BpybQJUZmDAeuhn86XQW0arkntgcsOAf/R+PQwvSRPk=
-github.com/0chain/gosdk v1.10.1-0.20231206175109-60d8a76df1e6/go.mod h1:DAg/de6vodjEa7CM1/LjElOwntRtNV5lb9rMRaR7fzU=
+github.com/0chain/gosdk v1.10.1-0.20231210164918-9b9b8e0d5c89 h1:iEIDv2Sn2NISPGEusDRx9s5Ofk/Y8J0YdCtDN7ChNsw=
+github.com/0chain/gosdk v1.10.1-0.20231210164918-9b9b8e0d5c89/go.mod h1:DAg/de6vodjEa7CM1/LjElOwntRtNV5lb9rMRaR7fzU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw=

diff --git a/migration/migrate.go b/migration/migrate.go
@@ -129,7 +129,7 @@ func InitMigration(mConfig *MigrationConfig) error {
 		zlogger.Logger.Error(err)
 		return err
 	}
-	mConfig.ChunkSize = dStorageService.GetChunkWriteSize()
+	mConfig.ChunkSize = int64(mConfig.ChunkNumber) * dStorageService.GetChunkWriteSize()
 	zlogger.Logger.Info("Getting aws storage service")
 	awsStorageService, err := s3.GetAwsClient(
 		mConfig.Bucket,
@@ -616,7 +616,7 @@ func (m *Migration) processChunkDownload(ctx context.Context, sw *util.StreamWri
 			return
 		default:
 		}
-		data, err := m.awsStore.DownloadToMemory(ctx, downloadObjMeta.ObjectKey, int64(offset), int64(chunkSize))
+		data, err := m.awsStore.DownloadToMemory(ctx, downloadObjMeta.ObjectKey, int64(offset), int64(chunkSize), downloadObjMeta.Size)
 		if err != nil {
 			migrator.DownloadDone(downloadObjMeta, "", err)
 			ctx.Err()