Merge branch 'sprint-1.10' into tokenomics_suite_merge

.github/workflows/nightly-staging.yml

@@ -12,7 +12,6 @@ jobs:
   system-tests:
     name: "System Tests"
     runs-on: [ tests-suite ]
-    timeout-minutes: 120
     steps:
       - name: "Config: Deploy new 0Chain network then run tests against it"
         run: |
@@ -93,4 +92,4 @@ jobs:
               }
           ]
           }'
-          curl -X POST -H 'Content-type: application/json' --data "${payload}" ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }}
+          curl -X POST -H 'Content-type: application/json' --data "${payload}" ${{ secrets.DEVOPS_CHANNEL_WEBHOOK_URL }}

go.mod

@@ -4,7 +4,7 @@ go 1.18
 
 require (
 	github.com/0chain/errors v1.0.3
-	github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114
+	github.com/0chain/gosdk v1.8.18-0.20230823135357-61db9486f4c0
 	github.com/go-resty/resty/v2 v2.7.0
 	github.com/herumi/bls-go-binary v1.31.0
 	github.com/shopspring/decimal v1.3.1

go.sum

@@ -40,8 +40,8 @@ github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 h1:z+DtCR8mBsjPnEs
 github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565/go.mod h1:UyDC8Qyl5z9lGkCnf9RHJPMektnFX8XtCJZHXCCVj8E=
 github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM=
 github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc=
-github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114 h1:fgaUQSUpAqhjhD3ONmiY+3yWn56qHADEd0TCoRcDSZ0=
-github.com/0chain/gosdk v1.8.17-0.20230809212922-e71a28baf114/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
+github.com/0chain/gosdk v1.8.18-0.20230823135357-61db9486f4c0 h1:XhLlPJKQxohZAS1+H2MHxy/e5I9Pw+Iyi+KaC5NHPl8=
+github.com/0chain/gosdk v1.8.18-0.20230823135357-61db9486f4c0/go.mod h1:3NKNYzmnMIYqZwwwOgZwMmTW1DT1ZUAmKyVPmYQOiT4=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw=

internal/api/model/api.go

@@ -495,7 +495,6 @@ type UpdateAllocationRequest struct {
 	Size                 int64  `json:"size"`
 	Expiration           int64  `json:"expiration_date"`
 	SetImmutable         bool   `json:"set_immutable"`
-	UpdateTerms          bool   `json:"update_terms"`
 	AddBlobberId         string `json:"add_blobber_id"`
 	RemoveBlobberId      string `json:"remove_blobber_id"`
 	ThirdPartyExtendable bool   `json:"third_party_extendable"`

internal/cli/model/model.go

@@ -756,6 +756,7 @@ var StorageFloatSettings = []string{
 var StorageCurrencySettigs = []string{
 	"max_mint",
 	"min_stake",
+	"min_stake_per_delegate",
 	"max_stake",
 	"readpool.min_lock",
 	"writepool.min_lock",

tests/api_tests/0box_aggregate_endpoints_test.go

@@ -189,26 +189,6 @@ func Test0boxGraphAndTotalEndpoints(testSetup *testing.T) {
 			return cond
 		})
 
-		// Reduce allocation size
-		apiClient.UpdateAllocation(t, sdkWallet, allocationID, &model.UpdateAllocationRequest{
-			Size: -1024,
-		}, client.TxSuccessfulStatus)
-
-		// Check decreased
-		wait.PoolImmediately(t, 2*time.Minute, func() bool {
-			data, resp, err := zboxClient.GetGraphAllocatedStorage(t, &model.ZboxGraphRequest{DataPoints: "1"})
-			require.NoError(t, err)
-			require.Equal(t, 200, resp.StatusCode())
-			require.Equal(t, 1, len([]int64(*data)))
-			allocatedStorageAfter := (*data)[0]
-			cond := allocatedStorageAfter < allocatedStorage
-			if cond {
-				allocatedStorage = allocatedStorageAfter
-			}
-
-			return cond
-		})
-
 		// Add blobber to the allocation
 		allocation := apiClient.GetAllocation(t, allocationID, client.HttpOkStatus)
 		newBlobberID := getNotUsedStorageNodeID(allocationBlobbers.Blobbers, allocation.Blobbers)

tests/cli_tests/0_s3mgrt_migrate_alternate2_test.go

@@ -2,6 +2,8 @@ package cli_tests
 
 import (
 	"bytes"
+	"crypto/sha1" // #nosec
+	"encoding/hex"
 	"fmt"
 	"os"
 	"path"
@@ -350,7 +352,7 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) {
 		require.Equal(t, true, uploadStats, "The file migrated doesnot match with with required file")
 	})
 
-	t.RunSequentially("Should not pass when filename size is more than 100 character", func(t *test.SystemTest) {
+	t.RunSequentially("Should pass when filename size is more than 100 character with renamed file names", func(t *test.SystemTest) {
 		// Cleanup before test
 		err := cleanupBucket(S3Client, s3BucketNameAlternate)
 		if err != nil {
@@ -362,6 +364,8 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) {
 			"size": allocSize,
 		})
 
+		// As per the current logic in s3-migration even the longer file names would be migrated with file names
+		// trimmed to 100 chars.
 		fileKeyNew := "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaabbb.txt"
 		fileContents := []byte("Hello, World!")
 		_, err = S3Client.PutObject(&s3.PutObjectInput{
@@ -379,8 +383,38 @@ func Test0S3MigrationAlternatePart2(testSetup *testing.T) {
 			"allocation": allocationID,
 		}))
 		// mssg can be changed
-		require.NotNil(t, err, "Expected an error", strings.Join(output, "\n"))
-		require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n"))
-		require.Contains(t, output[0], "invalid_parameter: filename is longer than 100 characters", "Output was not as expected", strings.Join(output, "\n"))
+		require.Nil(t, err, "Unexpected error", strings.Join(output, "\n"))
+		require.Contains(t, "Migration completed successfully", output[0], "Output was not as expected", strings.Join(output, "\n"))
+		require.Equal(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n"))
+
+		remotepath := "/"
+		remoteFilePath := path.Join(remotepath, s3BucketNameAlternate)
+		shortFileKey := getUniqueShortObjKey(fileKeyNew)
+		remoteFilePath = path.Join(remoteFilePath, shortFileKey)
+		uploadStats := checkStats(t, remoteFilePath, shortFileKey, allocationID, false)
+		require.Equal(t, true, uploadStats, "The file migrated does not match with expected file")
 	})
 }
+
+// This func is a copy of new file name generation logic in s3-migration from migration/migrate.go
+// This func needs to be in sync with the original func.
+func getUniqueShortObjKey(objectKey string) string {
+	// Max length to which objectKey would be trimmed to.
+	const maxLength = 100
+
+	if len(objectKey) > maxLength {
+		// Generate a SHA-1 hash of the object key
+		hash := sha1.New() // #nosec
+		_, _ = hash.Write([]byte(objectKey))
+		hashSum := hash.Sum(nil)
+
+		// Convert the hash to a hexadecimal string
+		hashString := hex.EncodeToString(hashSum)
+
+		// Combine the first 10 characters of the hash with a truncated object key
+		shortKey := fmt.Sprintf("%s_%s", hashString[:10], objectKey[11+len(objectKey)-maxLength:])
+		return shortKey
+	}
+
+	return objectKey
+}

tests/cli_tests/zboxcli_file_update_test.go

@@ -38,10 +38,15 @@ func TestFileUpdate(testSetup *testing.T) {
 		filesize := int64(0.5 * MB)
 		remotepath := "/"
 		localFilePath := generateFileAndUpload(t, allocationID, remotepath, filesize)
+		os.Remove(localFilePath) //nolint: errcheck
+
+		newPath := generateRandomTestFileName(t)
+		err := createFileWithSize(newPath, filesize+1024)
+		require.Nil(t, err)
 
-		thumbnailFile, thumbnailSize := updateFileWithThumbnail(t, allocationID, "/"+filepath.Base(localFilePath), localFilePath, int64(filesize))
+		thumbnailFile, thumbnailSize := updateFileWithThumbnail(t, allocationID, "/"+filepath.Base(localFilePath), newPath, int64(filesize+1024))
 		os.Remove(thumbnailFile) //nolint: errcheck
-		os.Remove(localFilePath) //nolint: errcheck
+		os.Remove(newPath)       //nolint: errcheck
 
 		remotepath += filepath.Base(localFilePath)
 
@@ -145,10 +150,14 @@ func TestFileUpdate(testSetup *testing.T) {
 		require.Len(t, output, 2)
 
 		// Update with new thumbnail
-		newThumbnail, newThumbnailSize := updateFileWithThumbnail(t, allocationID, "/"+filepath.Base(localFilePath), localFilePath, int64(filesize))
-
-		os.Remove(newThumbnail)  //nolint: errcheck
 		os.Remove(localFilePath) //nolint: errcheck
+		newPath := generateRandomTestFileName(t)
+		err = createFileWithSize(newPath, filesize+1024)
+		require.Nil(t, err)
+		newThumbnail, newThumbnailSize := updateFileWithThumbnail(t, allocationID, "/"+filepath.Base(localFilePath), newPath, int64(filesize+1024))
+
+		os.Remove(newThumbnail) //nolint: errcheck
+		os.Remove(newPath)      //nolint: errcheck
 
 		downloadNewThumbnailDir := newThumbnail + "down"
 		defer os.RemoveAll(downloadNewThumbnailDir) //nolint: errcheck
@@ -531,6 +540,6 @@ func updateFileWithThumbnail(t *test.SystemTest, allocationID, remotePath, local
 	}, true)
 	require.Nil(t, err, strings.Join(output, "\n"))
 	require.Len(t, output, 2)
-	require.True(t, strings.HasPrefix(output[1], "Status completed callback.") && strings.HasSuffix(output[1], "Name = "+filepath.Base(localpath)))
+	require.True(t, strings.HasPrefix(output[1], "Status completed callback.") && strings.HasSuffix(output[1], "Name = "+filepath.Base(remotePath)))
 	return thumbnail, thumbnailSize
 }

tests/cli_tests/zboxcli_update_allocation_test.go

@@ -82,10 +82,9 @@ func TestUpdateAllocation(testSetup *testing.T) {
 		size := int64(2048)
 
 		params := createParams(map[string]interface{}{
-			"allocation":   allocationID,
-			"extend":       true,
-			"size":         size,
-			"update_terms": true,
+			"allocation": allocationID,
+			"extend":     true,
+			"size":       size,
 		})
 		output, err := updateAllocation(t, configPath, params, true)
 
@@ -100,7 +99,7 @@ func TestUpdateAllocation(testSetup *testing.T) {
 		require.Equal(t, allocationBeforeUpdate.Size+size, ac.Size)
 	})
 
-	t.Run("Update Negative Size Should Work", func(t *test.SystemTest) {
+	t.Run("Update Negative Size Should Fail", func(t *test.SystemTest) {
 		allocationID, allocationBeforeUpdate := setupAndParseAllocation(t, configPath)
 		size := int64(-256)
 
@@ -110,72 +109,14 @@ func TestUpdateAllocation(testSetup *testing.T) {
 		})
 		output, err := updateAllocation(t, configPath, params, true)
 
-		require.Nil(t, err, "Could not update allocation due to error", strings.Join(output, "\n"))
+		require.Error(t, err, "expected error updating allocation", strings.Join(output, "\n"))
 		require.Len(t, output, 1)
-		assertOutputMatchesAllocationRegex(t, updateAllocationRegex, output[0])
-
-		allocations := parseListAllocations(t, configPath)
-		ac, ok := allocations[allocationID]
-		require.True(t, ok, "current allocation not found", allocationID, allocations)
-		require.Equal(t, allocationBeforeUpdate.Size+size, ac.Size,
-			fmt.Sprint("Size doesn't match: Before:", allocationBeforeUpdate.Size, " After:", ac.Size),
-		)
-	})
-
-	t.Run("Update All Negative Parameters Should Work", func(t *test.SystemTest) {
-		allocationID, allocationBeforeUpdate := setupAndParseAllocation(t, configPath)
-		size := int64(-512)
-
-		params := createParams(map[string]interface{}{
-			"allocation": allocationID,
-			"size":       size,
-		})
-		output, err := updateAllocation(t, configPath, params, true)
-
-		require.Nil(t, err, "Could not update allocation due to error", strings.Join(output, "\n"))
-		require.Len(t, output, 1)
-		assertOutputMatchesAllocationRegex(t, updateAllocationRegex, output[0])
-
-		allocations := parseListAllocations(t, configPath)
-		ac, ok := allocations[allocationID]
-		require.True(t, ok, "current allocation not found", allocationID, allocations)
-		require.Equal(t, allocationBeforeUpdate.Size+size, ac.Size,
-			fmt.Sprint("Size doesn't match: Before:", allocationBeforeUpdate.Size, " After:", ac.Size),
-		)
-	})
-
-	t.Run("Update Size to less than occupied size should fail", func(t *test.SystemTest) {
-		allocationID, allocationBeforeUpdate := setupAndParseAllocation(t, configPath) // alloc size is 10000
-
-		filename := generateRandomTestFileName(t)
-		err := createFileWithSize(filename, 2048) // uploading a file of size 2048
-		require.Nil(t, err)
+		require.Equal(t, "Error updating allocation:allocation_updating_failed: allocation can't be reduced", output[0])
 
-		output, err := uploadFile(t, configPath, map[string]interface{}{
-			"allocation": allocationID,
-			"remotepath": "/dir/",
-			"localpath":  filename,
-		}, true)
-		require.Nil(t, err, strings.Join(output, "\n"))
-		require.Len(t, output, 2)
-
-		size := int64(-9000) // reducing it by 9000 should fail since 2048 is being used
-		params := createParams(map[string]interface{}{
-			"allocation": allocationID,
-			"size":       size,
-		})
-		output, err = updateAllocation(t, configPath, params, false)
-
-		require.NotNil(t, err, strings.Join(output, "\n"))
-		require.Len(t, output, 1)
-		require.Equal(t, output[0], "Error updating allocation:allocation_updating_failed: new allocation size is too small: 1000 < 1024")
+		alloc := getAllocation(t, allocationID)
 
-		allocations := parseListAllocations(t, configPath)
-		ac, ok := allocations[allocationID]
-		require.True(t, ok, "current allocation not found", allocationID, allocations)
-		require.Equal(t, allocationBeforeUpdate.Size, ac.Size,
-			fmt.Sprint("Size doesn't match: Before:", allocationBeforeUpdate.Size, " After:", ac.Size),
-		) // size should be unaffected
+		require.Equal(t, allocationBeforeUpdate.Size, alloc.Size)
+		require.Equal(t, allocationBeforeUpdate.ExpirationDate, alloc.ExpirationDate)
 	})
 
 	// FIXME extend or size should be required params - should not bother sharders with an empty update
@@ -211,23 +152,6 @@ func TestUpdateAllocation(testSetup *testing.T) {
 		require.Equal(t, "Error updating allocation:couldnt_find_allocation: Couldn't find the allocation required for update", output[0])
 	})
 
-	t.Run("Update Size To Less Than 1024 Should Fail", func(t *test.SystemTest) {
-		allocationID, allocationBeforeUpdate := setupAndParseAllocation(t, configPath)
-		size := -allocationBeforeUpdate.Size + 1023
-
-		params := createParams(map[string]interface{}{
-			"allocation": allocationID,
-			"size":       fmt.Sprintf("\"%d\"", size),
-		})
-		output, err := updateAllocation(t, configPath, params, false)
-
-		require.NotNil(t, err, "expected error updating "+
-			"allocation", strings.Join(output, "\n"))
-		require.True(t, len(output) > 0, "expected output "+
-			"length be at least 1", strings.Join(output, "\n"))
-		require.Equal(t, "Error updating allocation:allocation_updating_failed: new allocation size is too small: 1023 < 1024", output[0])
-	})
-
 	t.RunWithTimeout("Update Other's Allocation Should Fail", 5*time.Minute, func(t *test.SystemTest) { // todo: too slow
 		_, err := executeFaucetWithTokens(t, configPath, 10)
 		require.NoError(t, err, "faucet execution failed")
@@ -725,15 +649,6 @@ func TestUpdateAllocation(testSetup *testing.T) {
 		require.NotNil(t, err, "no error updating allocation by third party", strings.Join(output, "\n"))
 		require.Contains(t, strings.Join(output, "\n"), "only owner can update the allocation")
 
-		// set update_term should fail
-		params = createParams(map[string]interface{}{
-			"allocation":   allocationID,
-			"update_terms": false,
-		})
-		output, err = updateAllocationWithWallet(t, nonAllocOwnerWallet, configPath, params, false)
-		require.NotNil(t, err, "no error updating allocation by third party", strings.Join(output, "\n"))
-		require.Contains(t, strings.Join(output, "\n"), "only owner can update the allocation")
-
 		// set lock should fail
 		params = createParams(map[string]interface{}{
 			"allocation": allocationID,