Storage V2 (#1105)

* increase alloc size for v2

* skip move rollback test

* Revert wallet changes

* revert wallet changes

* skip rename rollback test

* increase alloc size

* revert mod changes

* increase size

* fix alloc size for download and recent refs

* increase alloc size

* fix alloc size

* increase alloc size

* increase alloc size for expired test

* unskip challenge tests

* Use Storage version v2

---------

Co-authored-by: Jayash <[email protected]>

go.mod

@@ -12,7 +12,7 @@ require (
 	github.com/shopspring/decimal v1.3.1
 	github.com/sirupsen/logrus v1.9.3
 	github.com/spf13/viper v1.15.0
-	github.com/stretchr/testify v1.8.4
+	github.com/stretchr/testify v1.9.0
 	github.com/tyler-smith/go-bip39 v1.1.0
 	github.com/ybbus/jsonrpc/v3 v3.1.5 // nolint
 	golang.org/x/crypto v0.24.0
@@ -23,6 +23,7 @@ require (
 
 require (
 	github.com/andybalholm/brotli v1.0.5 // indirect
+	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
 	github.com/google/go-cmp v0.6.0 // indirect
 	github.com/hack-pad/go-webworkers v0.1.0 // indirect
 	github.com/hack-pad/safejs v0.1.1 // indirect
@@ -35,6 +36,7 @@ require (
 	github.com/remeh/sizedwaitgroup v1.0.0 // indirect
 	github.com/valyala/bytebufferpool v1.0.0 // indirect
 	github.com/valyala/fasthttp v1.51.0 // indirect
+	github.com/x448/float16 v0.8.4 // indirect
 	go.uber.org/atomic v1.10.0 // indirect
 	go.uber.org/multierr v1.9.0 // indirect
 	go.uber.org/zap v1.24.0 // indirect
@@ -47,7 +49,7 @@ require (
 )
 
 require (
-	github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565
+	github.com/0chain/common v1.13.1-0.20240924052546-7261e0567e47
 	github.com/Luzifer/go-openssl/v3 v3.1.0 // indirect
 	github.com/PuerkitoBio/goquery v1.8.1 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
@@ -68,7 +70,7 @@ require (
 	github.com/gobwas/glob v0.2.3 // indirect
 	github.com/gocolly/colly v1.2.0
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
-	github.com/golang/protobuf v1.5.2 // indirect
+	github.com/golang/protobuf v1.5.3 // indirect
 	github.com/gorilla/websocket v1.5.0 // indirect
 	github.com/h2non/filetype v1.1.4-0.20231228185113-6469358c2bcb // indirect
 	github.com/hashicorp/hcl v1.0.0 // indirect
@@ -86,7 +88,6 @@ require (
 	github.com/philhofer/fwd v1.1.2-0.20210722190033-5c56ac6d0bb9 // indirect
 	github.com/pkg/errors v0.9.1 // indirect
 	github.com/pmezard/go-difflib v1.0.0 // indirect
-	github.com/rogpeppe/go-internal v1.8.0 // indirect
 	github.com/saintfish/chardet v0.0.0-20230101081208-5e3ef4b5456d // indirect
 	github.com/shirou/gopsutil v3.21.11+incompatible // indirect
 	github.com/spf13/afero v1.9.3 // indirect
@@ -106,7 +107,7 @@ require (
 	golang.org/x/sys v0.21.0 // indirect
 	golang.org/x/text v0.16.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/protobuf v1.28.1 // indirect
+	google.golang.org/protobuf v1.33.0 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect
 )

internal/api/model/api.go

@@ -490,6 +490,7 @@ type SCRestGetAllocationBlobbersResponse struct {
 	Blobbers           *[]string `json:"blobbers"`
 	BlobberAuthTickets []string  `json:"blobber_auth_tickets"`
 	BlobberRequirements
+	StorageVersion int `json:"storage_version"`
 }
 
 type SCRestGetAllocationRequest struct {

internal/api/model/helper.go

@@ -4,7 +4,7 @@ import "time"
 
 func DefaultBlobberRequirements(id, publicKey string) BlobberRequirements {
 	return BlobberRequirements{
-		Size:           60000,
+		Size:           64 * 1024 * 4 * 50,
 		DataShards:     3,
 		ParityShards:   1,
 		ExpirationDate: time.Now().Add(721 * time.Hour).Unix(),

internal/api/util/client/api_client.go

@@ -689,6 +689,7 @@ func (c *APIClient) V1SCRestGetAllocation(t *test.SystemTest, scRestGetAllocatio
 
 func (c *APIClient) V1SCRestGetAllocationBlobbers(t *test.SystemTest, scRestGetAllocationBlobbersRequest *model.SCRestGetAllocationBlobbersRequest, requiredStatusCode int) (*model.SCRestGetAllocationBlobbersResponse, *resty.Response, error) { //nolint
 	scRestGetAllocationBlobbersResponse := new(model.SCRestGetAllocationBlobbersResponse)
+	scRestGetAllocationBlobbersResponse.StorageVersion = 1
 
 	data, err := json.Marshal(scRestGetAllocationBlobbersRequest.BlobberRequirements)
 	if err != nil {
@@ -1233,6 +1234,8 @@ func (c *APIClient) GetAllocationBlobbers(t *test.SystemTest, wallet *model.Wall
 			BlobberRequirements: *blobberRequirements,
 		}, requiredStatusCode)
 
+	scRestGetAllocationBlobbersResponse.StorageVersion = 1
+
 	if requiredStatusCode == http.StatusOK {
 		require.Nil(t, err)
 		require.NotNil(t, resp)

internal/api/util/client/sdk.go

@@ -316,7 +316,7 @@ func (c *SDKClient) AddRenameOperation(t *test.SystemTest, allocationID, remoteP
 	return sdk.OperationRequest{
 		OperationType: constants.FileOperationRename,
 		RemotePath:    remotePath,
-		DestName:      "/" + filepath.Join("", newName),
+		DestName:      newName,
 	}
 }
 

tests/api_tests/config/api_tests_config.yaml

@@ -1,11 +1,11 @@
-block_worker:  https://dev.zus.network/dns
+block_worker:  https://dev-3.devnet-0chain.net/dns
 0box_url: http://0box.dev.devnet-0chain.net
 0box_phone_number: +917696229925
 zvault_url: http://zvault.dev.devnet-0chain.net
 zauth_url: http://zauth.dev.devnet-0chain.net
 default_test_case_timeout: 45s
 zs3_server_url: https://dev.0chain.net/zs3server/
-chimney_test_network: https://dev.zus.network/dns
+chimney_test_network: https://dev-3.devnet-0chain.net/dns
 blobber_owner_wallet_mnemonics: "economy day fan flower between rebuild valid bid catch bargain vivid hybrid room permit check manage mean twelve damage summer close churn boat either"
 owner_wallet_mnemonics: "cactus panther essence ability copper fox wise actual need cousin boat uncover ride diamond group jacket anchor current float rely tragic omit child payment"
 ethereum_address: 0xD8c9156e782C68EE671C09b6b92de76C97948432

tests/api_tests/config/wallets.json

@@ -1,4 +1,20 @@
 [
+  {
+    "client_id": "306517a2de8b1d994d0951ba45e276cedc1d1dca32c0e9b731066791eaf8dedc",
+    "client_key": "2875211706e5d0262913418e344baaf05794d6b23ce6dc81f71bdddf9096e21684df0141f2b3f03ff79e2a3dc5cf376b628a5bd6c9f5f93806c408f6c603a083",
+    "keys": [
+      {
+        "public_key": "2875211706e5d0262913418e344baaf05794d6b23ce6dc81f71bdddf9096e21684df0141f2b3f03ff79e2a3dc5cf376b628a5bd6c9f5f93806c408f6c603a083",
+        "private_key": "39293962c1064ab6de775e8da8095966f4fed36c4b9d1937ab537c6da2a69414"
+      }
+    ],
+    "mnemonics": "situate myself awkward material blame crucial head trick distance term system labor maximum pond trade disease prevent coach lion planet bounce afford gauge leisure",
+    "version": "1.0",
+    "date_created": "2023-11-26T01:23:58Z",
+    "nonce": 1,
+    "ChainID": "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe",
+    "SignatureScheme": null
+  },
   {
     "client_id": "306517a2de8b1d994d0951ba45e276cedc1d1dca32c0e9b731066791eaf8dedc",
     "client_key": "2875211706e5d0262913418e344baaf05794d6b23ce6dc81f71bdddf9096e21684df0141f2b3f03ff79e2a3dc5cf376b628a5bd6c9f5f93806c408f6c603a083",
@@ -39999,4 +40015,4 @@
     "ChainID": "0afc093ffb509f059c55478bc1a60351cef7b4e9c008a53a6cc8241ca8617dfe",
     "SignatureScheme": null
   }
-]
+]

tests/api_tests/multiop_rollback_test.go

@@ -136,16 +136,13 @@ func TestMultiOperationRollback(testSetup *testing.T) {
 		newOps := make([]sdk.OperationRequest, 0, 10)
 
 		for i := 0; i < 10; i++ {
-			switch i % 3 {
+			switch i % 2 {
 			case 0:
 				op := sdkClient.AddDeleteOperation(t, allocationID, ops[i].FileMeta.RemotePath)
 				newOps = append(newOps, op)
 			case 1:
 				op := sdkClient.AddUpdateOperation(t, ops[i].FileMeta.RemotePath, ops[i].FileMeta.RemoteName, ops[i].FileMeta.ActualSize)
 				newOps = append(newOps, op)
-			case 2:
-				op := sdkClient.AddRenameOperation(t, allocationID, ops[i].FileMeta.RemotePath, randName())
-				newOps = append(newOps, op)
 			}
 		}
 

tests/api_tests/repair_allocation_test.go

@@ -32,7 +32,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
 		blobberRequirements.DataShards = 2
 		blobberRequirements.ParityShards = 2
-		blobberRequirements.Size = 2056
+		blobberRequirements.Size = 64 * 1024 * 3
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
 		allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus)
 
@@ -76,7 +76,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 
 	t.RunSequentially("Repair allocation after update should work", func(t *test.SystemTest) {
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
-		blobberRequirements.Size = 6096
+		blobberRequirements.Size = 64 * 1024 * 10 * 2
 		blobberRequirements.DataShards = 2
 		blobberRequirements.ParityShards = 2
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
@@ -106,7 +106,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 
 	t.RunSequentially("Repair allocation after delete should work", func(t *test.SystemTest) {
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
-		blobberRequirements.Size = 2056
+		blobberRequirements.Size = 64 * 1024 * 3
 		blobberRequirements.DataShards = 2
 		blobberRequirements.ParityShards = 2
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
@@ -130,7 +130,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 
 	t.RunSequentially("Repair allocation after move should work", func(t *test.SystemTest) {
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
-		blobberRequirements.Size = 6096
+		blobberRequirements.Size = 64 * 1024 * 10 * 2
 		blobberRequirements.DataShards = 2
 		blobberRequirements.ParityShards = 2
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
@@ -155,7 +155,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 
 	t.RunSequentially("Repair allocation after copy should work", func(t *test.SystemTest) {
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
-		blobberRequirements.Size = 8096
+		blobberRequirements.Size = 64 * 1024 * 20 * 2
 		blobberRequirements.DataShards = 2
 		blobberRequirements.ParityShards = 2
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
@@ -182,7 +182,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 
 	t.RunSequentially("Repair allocation after rename should work", func(t *test.SystemTest) {
 		blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey)
-		blobberRequirements.Size = 6096
+		blobberRequirements.Size = 64 * 1024 * 10 * 2
 		blobberRequirements.DataShards = 2
 		blobberRequirements.ParityShards = 2
 		allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus)
@@ -350,6 +350,7 @@ func TestRepairAllocation(testSetup *testing.T) {
 
 func TestRepairSize(testSetup *testing.T) {
 	t := test.NewSystemTest(testSetup)
+	t.Skip("Skipping as repair size is not implemented for V2")
 	wallet := createWallet(t)
 	sdkClient.SetWallet(t, wallet)
 	apiClient.CreateReadPool(t, wallet, 0.5, client.TxSuccessfulStatus)

tests/cli_tests/0_expired_allocation_test.go

@@ -71,7 +71,7 @@ func TestExpiredAllocation(testSetup *testing.T) {
 	})
 
 	t.Run("Download File using Expired Allocation Should Fail", func(t *test.SystemTest) {
-		allocSize := int64(2048)
+		allocSize := int64(64 * KB * 2)
 		filesize := int64(256)
 		remotepath := "/"
 

tests/cli_tests/0_s3mgrt_migrate_test.go

@@ -59,7 +59,7 @@ func Test0S3Migration(testSetup *testing.T) {
 	})
 
 	t.RunSequentially("Should fail when bucket too large for allocation", func(t *test.SystemTest) {
-		allocSize := int64(5 * KB)
+		allocSize := int64(64 * KB)
 		allocationID := setupAllocation(t, configPath, map[string]interface{}{
 			"size": allocSize,
 		})

tests/cli_tests/zboxcli_file_copy_test.go

@@ -27,7 +27,7 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	t.Parallel()
 	t.Run("copy directory to another directry should work", func(t *test.SystemTest) {
 		allocSize := int64(2048000)
-		fileSize := int64(256)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -178,7 +178,7 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 		require.Equal(t, destpath+" directory created", output[0])
 
 		// Upload file
-		fileSize := int64(256)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err = createFileWithSize(file, fileSize)
@@ -290,8 +290,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.Run("copy file to existing directory", func(t *test.SystemTest) {
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 2)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -363,8 +363,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.RunWithTimeout("Copy file concurrently to existing directory, should work", 6*time.Minute, func(t *test.SystemTest) { // todo: way too slow
-		const allocSize int64 = 2048
-		const fileSize int64 = 256
+		const allocSize int64 = 64 * KB * 2 * 4
+		const fileSize int64 = 64 * KB
 
 		allocationID := setupAllocation(t, configPath, map[string]interface{}{
 			"size": allocSize,
@@ -446,8 +446,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.Run("copy file to non-existing directory should work", func(t *test.SystemTest) {
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 2)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -518,8 +518,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.Run("copy file to same directory should fail", func(t *test.SystemTest) {
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 2)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -581,8 +581,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.Run("copy file to dir with existing children should work", func(t *test.SystemTest) {
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 2)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -644,8 +644,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.Run("copy file to another directory with existing file with same name should fail", func(t *test.SystemTest) {
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 4)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -757,8 +757,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 
 		createWalletForName(nonAllocOwnerWallet)
 
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 2)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)
@@ -864,8 +864,8 @@ func TestFileCopy(testSetup *testing.T) { // nolint:gocyclo // team preference i
 	})
 
 	t.Run("copy file with allocation copy file options forbidden should fail", func(t *test.SystemTest) {
-		allocSize := int64(2048)
-		fileSize := int64(256)
+		allocSize := int64(64 * KB * 2 * 2)
+		fileSize := int64(64 * KB)
 
 		file := generateRandomTestFileName(t)
 		err := createFileWithSize(file, fileSize)

tests/cli_tests/zboxcli_file_delete_test.go

@@ -410,7 +410,7 @@ func TestFileDelete(testSetup *testing.T) {
 	})
 
 	t.RunWithTimeout("Delete file concurrently in existing directory, should work", 5*time.Minute, func(t *test.SystemTest) { // TODO: slow
-		const allocSize int64 = 2048
+		const allocSize int64 = 64 * KB * 2 * 2
 		const fileSize int64 = 256
 
 		allocationID := setupAllocation(t, configPath, map[string]interface{}{