diff --git a/.github/workflows/ci.yml b/.github/workflows/ci.yml index 231a353295..5cf539cb58 100644 --- a/.github/workflows/ci.yml +++ b/.github/workflows/ci.yml @@ -166,6 +166,8 @@ jobs: run_smoke_tests: ${{ env.RUN_SMOKE_TESTS }} S3_ACCESS_KEY: ${{ secrets.S3_ACCESS_KEY }} S3_SECRET_KEY: ${{ secrets.S3_SECRET_KEY }} + DROPBOX_ACCESS_TOKEN: ${{ secrets.DROPBOX_ACCESS_TOKEN }} + GDRIVE_ACCESS_TOKEN: ${{ secrets.GDRIVE_ACCESS_TOKEN }} - name: "Remove Tenderly virtual testnet" if: always() diff --git a/go.mod b/go.mod index 6fbabdfc4b..e75b651b5c 100644 --- a/go.mod +++ b/go.mod @@ -2,9 +2,11 @@ module github.com/0chain/system_test go 1.21 +toolchain go1.22.1 + require ( github.com/0chain/errors v1.0.3 - github.com/0chain/gosdk v1.14.0-RC2 + github.com/0chain/gosdk v1.15.2-0.20240614082721-1b8ad4f48bc6 github.com/go-resty/resty/v2 v2.7.0 github.com/herumi/bls-go-binary v1.31.0 github.com/shopspring/decimal v1.3.1 @@ -13,7 +15,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tyler-smith/go-bip39 v1.1.0 github.com/ybbus/jsonrpc/v3 v3.1.5 // nolint - golang.org/x/crypto v0.17.0 + golang.org/x/crypto v0.24.0 gopkg.in/errgo.v2 v2.1.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/gorm v1.24.1 @@ -21,9 +23,10 @@ require ( require ( github.com/andybalholm/brotli v1.0.5 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d // indirect github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect - github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17 // indirect + github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9 // indirect github.com/klauspost/compress v1.17.0 // indirect github.com/lithammer/shortuuid/v3 v3.0.7 // indirect github.com/minio/sha256-simd v1.0.1 // indirect @@ -96,10 +99,10 @@ require ( github.com/yusufpapurcu/wmi v1.2.3 // indirect go.dedis.ch/fixbuf v1.0.3 // indirect go.dedis.ch/kyber/v3 v3.1.0 // indirect - golang.org/x/net v0.19.0 // indirect - golang.org/x/sync v0.5.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/net v0.26.0 // indirect + golang.org/x/sync v0.7.0 // indirect + golang.org/x/sys v0.21.0 // indirect + golang.org/x/text v0.16.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/protobuf v1.28.1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect diff --git a/go.sum b/go.sum index 378c1268f0..0e466f738f 100644 --- a/go.sum +++ b/go.sum @@ -42,6 +42,8 @@ github.com/0chain/errors v1.0.3 h1:QQZPFxTfnMcRdt32DXbzRQIfGWmBsKoEdszKQDb0rRM= github.com/0chain/errors v1.0.3/go.mod h1:xymD6nVgrbgttWwkpSCfLLEJbFO6iHGQwk/yeSuYkIc= github.com/0chain/gosdk v1.14.0-RC2 h1:OIpjj7mDKxVqJVlxJKm8/fPzdJRN5roXO7/plBCtmTg= github.com/0chain/gosdk v1.14.0-RC2/go.mod h1:tgAiVAuIy+Vs1tGfKCPEuuWWARwNQBEw32y950LrqrU= +github.com/0chain/gosdk v1.15.2-0.20240614082721-1b8ad4f48bc6 h1:f8fN6PuvE/Iu1LqrJH5q20eWOFB/NjlDA8TIDMqZgAg= +github.com/0chain/gosdk v1.15.2-0.20240614082721-1b8ad4f48bc6/go.mod h1:Hzl56JJ66ZmoyNS7CbTJue7wUugBYvNx8/qJzTRWmkI= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/Luzifer/go-openssl/v3 v3.1.0 h1:QqKqo6kYXGGUsvtUoCpRZm8lHw+jDfhbzr36gVj+/gw= @@ -197,8 +199,8 @@ github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= @@ -240,6 +242,7 @@ github.com/herumi/bls-go-binary v1.31.0 h1:L1goQ2tMtGgpXCg5AwHAdJQpLs/pfnWWEc3Wo github.com/herumi/bls-go-binary v1.31.0/go.mod h1:O4Vp1AfR4raRGwFeQpr9X/PQtncEicMoOe6BQt1oX0Y= github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17 h1:FbyIK0BfvXVZTOxKOe2dlxJqSPSF2ZXOv2Mc7dvS7sc= github.com/hitenjain14/fasthttp v0.0.0-20240229173600-722723e15e17/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk= +github.com/hitenjain14/fasthttp v0.0.0-20240527123209-06019e79bff9/go.mod h1:RZMcXy7u4S+E97IXYTe7WHZ3+mCYOh4vys8PkIGZeXk= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= @@ -447,8 +450,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210421170649-83a5a9bb288b/go.mod h1:T9bdIzuCu7OtxOm1hfPfRQxPLYneinmdGuTeoZ9dtd4= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.24.0 h1:mnl8DM0o513X8fdIkmyFE/5hTYxbwYOjDS/+rK6qpRI= +golang.org/x/crypto v0.24.0/go.mod h1:Z1PMYSOR5nyMcyAVAIQSKCDwalqy85Aqn1x3Ws4L5DM= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -524,8 +527,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.26.0 h1:soB7SVo0PWrY4vPW/+ay0jKDNScG2X9wFeYlXIvJsOQ= +golang.org/x/net v0.26.0/go.mod h1:5YKkiSynbBIh3p6iOc/vibscux0x38BZDkn8sCUPxHE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -546,8 +549,8 @@ golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= +golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190124100055-b90733256f2e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -600,8 +603,8 @@ golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.21.0 h1:rF+pYz3DAGSQAxAu1CbC7catZg4ebC4UIeIhKxBZvws= +golang.org/x/sys v0.21.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -618,8 +621,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= -golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.16.0 h1:a94ExnEXNtEwYLGJSIUxnWoxoRz/ZcCsV63ROupILh4= +golang.org/x/text v0.16.0/go.mod h1:GhwF1Be+LQoKShO3cGOHzqOgRrGaYc9AvblQOmPVHnI= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/api/util/config/config.go b/internal/api/util/config/config.go index 59297a04b2..f18fdaa7eb 100644 --- a/internal/api/util/config/config.go +++ b/internal/api/util/config/config.go @@ -34,6 +34,8 @@ type Config struct { S3BucketNameAlternate string `yaml:"s3_bucket_name_alternate"` BlobberOwnerWalletMnemonics string `yaml:"blobber_owner_wallet_mnemonics"` OwnerWalletMnemonics string `yaml:"owner_wallet_mnemonics"` + DropboxAccessToken string `yaml:"dropboxAccessToken"` + GdriveAccessToken string `yaml:"gdriveAccessToken"` } func Parse(configPath string) *Config { diff --git a/internal/cli/model/model.go b/internal/cli/model/model.go index de279353f8..b43542add2 100644 --- a/internal/cli/model/model.go +++ b/internal/cli/model/model.go @@ -733,6 +733,14 @@ type ReadMarker struct { BlockNumber int64 `json:"block_number"` } +// holds result of repair size +type RepairSize struct { + // upload size in bytes + UploadSize uint64 `json:"upload_size"` + // download size in bytes + DownloadSize uint64 `json:"download_size"` +} + var StorageKeySettings = []string{ "owner_id", } diff --git a/tests/api_tests/repair_allocation_test.go b/tests/api_tests/repair_allocation_test.go index 8deecce837..3995adbb6b 100644 --- a/tests/api_tests/repair_allocation_test.go +++ b/tests/api_tests/repair_allocation_test.go @@ -347,3 +347,103 @@ func TestRepairAllocation(testSetup *testing.T) { } }) } + +func TestRepairSize(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + wallet := createWallet(t) + sdkClient.SetWallet(t, wallet) + apiClient.CreateReadPool(t, wallet, 0.5, client.TxSuccessfulStatus) + + t.RunSequentiallyWithTimeout("repair size in case of no blobber failure should be zero", 5 * time.Minute, func(t *test.SystemTest) { + // create allocation with default blobber requirements + blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey) + allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus) + allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus) + t.Logf("allocationID: %v", allocationID) + + // create and upload a file of 2KB to allocation. + op := sdkClient.AddUploadOperation(t, "", "", int64(1024 * 2)) + sdkClient.MultiOperation(t, allocationID, []sdk.OperationRequest{op}) + + // assert both upload and download size should be zero + alloc, err := sdk.GetAllocation(allocationID) + require.NoErrorf(t, err, "allocation ID %v is not found", allocationID) + rs, err := alloc.RepairSize("/") + require.Nil(t, err) + t.Logf("repair size: %v", rs) + require.Equal(t, uint64(0), rs.UploadSize, "upload size doesn't match") + require.Equal(t, uint64(0), rs.DownloadSize, "download size doesn't match") + }) + + t.RunSequentiallyWithTimeout("repair size on single blobber failure should match", 5 * time.Minute, func(t *test.SystemTest) { + // create allocation with default blobber requirements + blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey) + blobberRequirements.DataShards = 2 + blobberRequirements.ParityShards = 2 + blobberRequirements.Size = 2056 + allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus) + allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus) + t.Logf("allocationID: %v", allocationID) + + // create and upload a file of 2KB to allocation. + // one blobber url is set invalid to mimic failure. + alloc, err := sdk.GetAllocation(allocationID) + require.NoErrorf(t, err, "allocation ID %v is not found", allocationID) + alloc.Blobbers[0].Baseurl = "http://0zus.com/" + op := sdkClient.AddUploadOperation(t, "", "", int64(1024 * 2)) + sdkClient.MultiOperation(t, allocationID, []sdk.OperationRequest{op}, client.WithRepair(alloc.Blobbers)) + + // assert upload and download size should be 1KB and 2KB respectively + rs, err := alloc.RepairSize("/") + require.Nil(t, err) + t.Logf("repair size: %v", rs) + require.Equal(t, uint64(1024), rs.UploadSize, "upload size doesn't match") + require.Equal(t, uint64(1024 * 2), rs.DownloadSize, "download size doesn't match") + }) + + t.RunSequentiallyWithTimeout("repair size with nested directories and two blobber failure should match", 5 * time.Minute, func(t *test.SystemTest) { + // create allocation with default blobber requirements + blobberRequirements := model.DefaultBlobberRequirements(wallet.Id, wallet.PublicKey) + blobberRequirements.DataShards = 2 + blobberRequirements.ParityShards = 4 + allocationBlobbers := apiClient.GetAllocationBlobbers(t, wallet, &blobberRequirements, client.HttpOkStatus) + allocationID := apiClient.CreateAllocation(t, wallet, allocationBlobbers, client.TxSuccessfulStatus) + t.Logf("allocationID: %v", allocationID) + + // create and upload two files of 1KB each to / and /dir1. + // two blobber url is set invalid to mimic failure. + alloc, err := sdk.GetAllocation(allocationID) + require.NoErrorf(t, err, "allocation ID %v is not found", allocationID) + alloc.Blobbers[0].Baseurl = "http://0zus.com/" + alloc.Blobbers[1].Baseurl = "http://0zus.com/" + ops := []sdk.OperationRequest{ + sdkClient.AddUploadOperationWithPath(t, allocationID, "/dir1/"), + sdkClient.AddUploadOperationWithPath(t, allocationID, "/dir1/"), + sdkClient.AddUploadOperationWithPath(t, allocationID, "/"), + sdkClient.AddUploadOperationWithPath(t, allocationID, "/"), + } + sdkClient.MultiOperation(t, allocationID, ops, client.WithRepair(alloc.Blobbers)) + + // assert both upload and download size should be 2KB in /dir1 + rs, err := alloc.RepairSize("/dir1") + require.Nilf(t, err, "error getting repair size in /dir1: %v", err) + t.Logf("repair size: %v", rs) + require.Equal(t, uint64(1024 * 2), rs.UploadSize, "upload size in directory /dir1 doesn't match") + require.Equal(t, uint64(1024 * 2), rs.DownloadSize, "download size in directory dir1 doesn't match") + + // with trailing slash + // assert both upload and download size should be 2KB in /dir1/ + rs, err = alloc.RepairSize("/dir1/") + require.Nilf(t, err, "error getting repair size in /dir1/: %v", err) + t.Logf("repair size: %v", rs) + require.Equal(t, uint64(1024 * 2), rs.UploadSize, "upload size in directory /dir1/ doesn't match") + require.Equal(t, uint64(1024 * 2), rs.DownloadSize, "download size in directory /dir1/ doesn't match") + + // assert both upload and download size should be 4KB in root directory + rs, err = alloc.RepairSize("/") + require.Nilf(t, err, "error getting repair size in /: %v", err) + t.Logf("repair size: %v", rs) + require.Equal(t, uint64(1024 * 4), rs.UploadSize, "upload size in root directory doesn't match") + require.Equal(t, uint64(1024 * 4), rs.DownloadSize, "download size in root directory doesn't match") + }) +} \ No newline at end of file diff --git a/tests/cli_tests/0_dropboxmgrt_migrate_test.go b/tests/cli_tests/0_dropboxmgrt_migrate_test.go new file mode 100644 index 0000000000..2f357824b5 --- /dev/null +++ b/tests/cli_tests/0_dropboxmgrt_migrate_test.go @@ -0,0 +1,164 @@ +package cli_tests + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/0chain/system_test/internal/api/util/test" + cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/stretchr/testify/require" +) + +func Test0Dropbox(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if dropboxAccessToken == "" { + t.Skip("dropbox Access Token was missing") + } + + t.SetSmokeTests("Should migrate existing Dropbox folder and files successfully") + + t.RunSequentially("Should migrate existing Dropbox folder and files successfully", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationId := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, _ := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ + "access-token": dropboxAccessToken, + "wallet": escapedTestName(t) + "_wallet.json", + "allocation": allocationId, + "source": "dropbox", + "config": configPath, + "configDir": configDir, + "skip": 1, + })) + + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ + "access-token": dropboxAccessToken, + "wallet": escapedTestName(t) + "_wallet.json", + "allocation": allocationID, + "source": "dropbox", + "config": configPath, + "configDir": configDir, + "skip": 1, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + _ = setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, _ := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ + "access-token": dropboxAccessToken, + "wallet": escapedTestName(t) + "_wallet.json", + "source": "dropbox", + "config": configPath, + "configDir": configDir, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ + "access-token": "invalid", + "wallet": escapedTestName(t) + "_wallet.json", + "source": "dropbox", + "config": configPath, + "configDir": configDir, + "allocation": allocationID, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid Client token: invalid_access_token/", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ + "wallet": escapedTestName(t) + "_wallet.json", + "source": "dropbox", + "config": configPath, + "configDir": configDir, + "allocation": allocationID, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing Access Token", "Output was not as expected", strings.Join(output, "\n")) + }) + t.RunSequentially("Should fail when source is invalid", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromDropbox(t, configPath, createParams(map[string]interface{}{ + "wallet": escapedTestName(t) + "_wallet.json", + "source": "invalid", + "config": configPath, + "configDir": configDir, + "allocation": allocationID, + "access-token": dropboxAccessToken, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "invalid source", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder too large for allocation", func(t *test.SystemTest) { + allocSize := int64(5 * KB) + func() { + defer func() { + // recover from panic if one occurred + if r := recover(); r != nil { + fmt.Println("Panic occurred:", r) // Log the panic + t.Log("Test passed even though a panic occurred") + // Set the test status to passed + require.Equal(t, "", "") + } + }() + + // Set up allocation with wallet + _ = setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + }() + }) +} + +func migrateFromDropbox(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { + t.Logf("Migrating Dropbox to Zus...") + t.Logf(fmt.Sprintf("params %v", params)) + t.Logf(fmt.Sprintf("cli %v", cliConfigFilename)) + t.Logf(fmt.Sprintf("./s3migration migrate %s", params)) + return cliutils.RunCommand(t, fmt.Sprintf("./s3migration migrate %s", params), 1, time.Hour*2) +} diff --git a/tests/cli_tests/0_gdrivemgrt_migrate_test.go b/tests/cli_tests/0_gdrivemgrt_migrate_test.go new file mode 100644 index 0000000000..0a8d7fc4da --- /dev/null +++ b/tests/cli_tests/0_gdrivemgrt_migrate_test.go @@ -0,0 +1,144 @@ +package cli_tests + +import ( + "fmt" + "strings" + "testing" + "time" + + "github.com/0chain/system_test/internal/api/util/test" + cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/stretchr/testify/require" +) + +func Test0Gdrive(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + if gdriveAccessToken == "" { + t.Skip("Gdrive Access Token was missing") + } + + t.SetSmokeTests("Should migrate existing Gdrive folder and files successfully") + + t.RunSequentially("Should migrate existing Gdrive folder and files successfully", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, _ := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ + "access-token": gdriveAccessToken, + "allocation": allocationID, + "source": "google_drive", + "wallet": escapedTestName(t) + "_wallet.json", + "config": configPath, + "configDir": configDir, + })) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should migrate empty folder successfully", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ + "access-token": gdriveAccessToken, + "allocation": allocationID, + "source": "google_drive", + "wallet": escapedTestName(t) + "_wallet.json", + "config": configPath, + "configDir": configDir, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when folder does not exist", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ + "access-token": gdriveAccessToken, + "allocation": allocationID, + "source": "google_drive", + "wallet": escapedTestName(t) + "_wallet.json", + "config": configPath, + "configDir": configDir, + })) + + require.Nil(t, err, "Unexpected migration failure", strings.Join(output, "\n")) + require.GreaterOrEqual(t, len(output), 1, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Migration completed successfully", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when allocation flag missing", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + _ = setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, _ := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ + "access-token": dropboxAccessToken, + "wallet": escapedTestName(t) + "_wallet.json", + "source": "google_drive", + "config": configPath, + "configDir": configDir, + })) + + require.Contains(t, strings.Join(output, "\n"), "allocation id is missing", "Output was not as expected", strings.Join(output, "\n")) + }) + + t.RunSequentially("Should fail when access token invalid", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ + "access-token": "invalid", + "wallet": escapedTestName(t) + "_wallet.json", + "source": "google_drive", + "config": configPath, + "configDir": configDir, + "allocation": allocationID, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Invalid Credentials", "Output was not as expected", err) + }) + + t.RunSequentially("Should fail when access key missing", func(t *test.SystemTest) { + allocSize := int64(50 * MB) + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + }) + + output, err := migrateFromGdrive(t, configPath, createParams(map[string]interface{}{ + "wallet": escapedTestName(t) + "_wallet.json", + "source": "google_drive", + "config": configPath, + "configDir": configDir, + "allocation": allocationID, + })) + + require.NotNil(t, err, "Expected a migration failure but got no error", strings.Join(output, "\n")) + require.Greater(t, len(output), 0, "More/Less output was returned than expected", strings.Join(output, "\n")) + require.Contains(t, strings.Join(output, "\n"), "Missing Access Token", "Output was not as expected", strings.Join(output, "\n")) + }) +} + +func migrateFromGdrive(t *test.SystemTest, cliConfigFilename, params string) ([]string, error) { + t.Logf("Migrating Gdrive to Zus...") + t.Logf(fmt.Sprintf("params %v", params)) + t.Logf(fmt.Sprintf("cli %v", cliConfigFilename)) + t.Logf(fmt.Sprintf("./s3migration migrate %s", params)) + return cliutils.RunCommand(t, fmt.Sprintf("./s3migration migrate %s", params), 1, time.Hour*2) +} diff --git a/tests/cli_tests/config/cli_tests_config.yaml b/tests/cli_tests/config/cli_tests_config.yaml index 83f717be72..6867e623f2 100644 --- a/tests/cli_tests/config/cli_tests_config.yaml +++ b/tests/cli_tests/config/cli_tests_config.yaml @@ -1,5 +1,7 @@ default_test_case_timeout: 3m s3_access_key: s3_secret_key: -s3_bucket_name: system-tests-s3-migration-small +s3_bucket_name: s3_bucket_name_alternate: alternate-test-bucket +dropboxAccessToken: +gdriveAccessToken: diff --git a/tests/cli_tests/main_test.go b/tests/cli_tests/main_test.go index 742523e034..44d6ec10f6 100644 --- a/tests/cli_tests/main_test.go +++ b/tests/cli_tests/main_test.go @@ -58,6 +58,8 @@ func setupConfig() { s3SecretKey = parsedConfig.S3SecretKey s3bucketName = parsedConfig.S3BucketName s3BucketNameAlternate = parsedConfig.S3BucketNameAlternate + dropboxAccessToken = parsedConfig.DropboxAccessToken + gdriveAccessToken = parsedConfig.GdriveAccessToken if err != nil { log.Printf("Default test case timeout could not be parsed so has defaulted to [%v]", test.DefaultTestTimeout) @@ -107,6 +109,8 @@ var ( s3bucketName string s3BucketNameAlternate string S3Client *s3.S3 + dropboxAccessToken string + gdriveAccessToken string ) var ( @@ -134,7 +138,6 @@ func TestMain(m *testing.M) { } configDir, _ = filepath.Abs(configDir) - if !strings.EqualFold(strings.TrimSpace(os.Getenv("SKIP_CONFIG_CLEANUP")), "true") { if files, err := filepath.Glob("./config/*.json"); err == nil { for _, f := range files { @@ -177,13 +180,40 @@ func TestMain(m *testing.M) { Region: aws.String("us-east-2"), // Replace with your desired AWS region Credentials: credentials.NewStaticCredentials(s3AccessKey, s3SecretKey, ""), }) + if err != nil { log.Fatalln("Failed to create AWS session:", err) return } + // Create a session with Dropbox + sess_dp, err_dp := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials( + dropboxAccessToken, "", ""), + }) + + if err_dp != nil { + log.Fatalln("Failed to create Dropbox session:", err_dp) + } + + sess_gd, err_gd := session.NewSession(&aws.Config{ + Credentials: credentials.NewStaticCredentials( + gdriveAccessToken, "", ""), + }) + + if err_gd != nil { + log.Fatalln("Failed to create Gdrive session:", err_dp) + } // Create an S3 client - S3Client = s3.New(sess) + cloudService := os.Getenv("CLOUD_SERVICE") + + if cloudService == "dropbox" { + S3Client = s3.New(sess_dp) + } else if cloudService == "gdrive" { + S3Client = s3.New(sess_gd) + } else { + S3Client = s3.New(sess) + } walletMutex.Lock() // Read the content of the file diff --git a/tests/cli_tests/zboxcli_repair_test.go b/tests/cli_tests/zboxcli_repair_test.go new file mode 100644 index 0000000000..9fa1123324 --- /dev/null +++ b/tests/cli_tests/zboxcli_repair_test.go @@ -0,0 +1,93 @@ +package cli_tests + +import ( + "encoding/json" + "fmt" + "path/filepath" + "strings" + "testing" + "time" + + "github.com/0chain/system_test/internal/api/util/test" + "github.com/0chain/system_test/internal/cli/model" + cliutils "github.com/0chain/system_test/internal/cli/util" + "github.com/stretchr/testify/require" +) + +func TestRepairSize(testSetup *testing.T) { + t := test.NewSystemTest(testSetup) + + t.RunSequentiallyWithTimeout("repair size should work", 5 * time.Minute, func(t *test.SystemTest) { + allocSize := int64(1 * MB) + fileSize := int64(512 * KB) + + allocationID := setupAllocation(t, configPath, map[string]interface{}{ + "size": allocSize, + "parity": 1, + "data": 1, + }) + + filename := generateRandomTestFileName(t) + err := createFileWithSize(filename, fileSize) + require.Nil(t, err) + + output, err := uploadFile(t, configPath, map[string]interface{}{ + "allocation": allocationID, + "remotepath": "/", + "localpath": filename, + }, true) + require.Nil(t, err, strings.Join(output, "\n")) + require.Len(t, output, 2) + + expected := fmt.Sprintf( + "Status completed callback. Type = text/plain. Name = %s", + filepath.Base(filename), + ) + require.Equal(t, expected, output[1]) + + output, err = getRepairSize(t, configPath, map[string]interface{}{ + "allocation": allocationID, + "repairpath": "/", + }, true) + require.Nilf(t, err, "error getting repair size: %v", err) + var rs model.RepairSize + err = json.Unmarshal([]byte(output[0]), &rs) + require.Nilf(t, err, "error unmarshal repair size: %v", err) + require.Equal(t, uint64(0), rs.UploadSize, "upload size should be zero") + require.Equal(t, uint64(0), rs.DownloadSize, "download size should be zero") + + // optional repairpath + output, err = getRepairSize(t, configPath, map[string]interface{}{ + "allocation": allocationID, + }, true) + require.Nilf(t, err, "error getting repair size: %v", err) + var rs2 model.RepairSize + err = json.Unmarshal([]byte(output[0]), &rs2) + require.Nilf(t, err, "error unmarshal repair size: %v", err) + require.Equal(t, uint64(0), rs2.UploadSize, "upload size should be zero") + require.Equal(t, uint64(0), rs2.DownloadSize, "download size should be zero") + }) + +} + +func getRepairSize(t *test.SystemTest, cliConfigFilename string, param map[string]interface{}, retry bool) ([]string, error) { + return getRepairSizeForWallet(t, escapedTestName(t), cliConfigFilename, param, retry) +} + +func getRepairSizeForWallet(t *test.SystemTest, wallet, cliConfigFilename string, param map[string]interface{}, retry bool) ([]string, error) { + t.Logf("getting Repair size...") + + p := createParams(param) + cmd := fmt.Sprintf( + "./zbox repair-size %s --silent --wallet %s_wallet.json --configDir ./config --config %s", + p, + wallet, + cliConfigFilename, + ) + + if retry { + return cliutils.RunCommand(t, cmd, 3, time.Second*40) + } else { + return cliutils.RunCommandWithoutRetry(cmd) + } +} \ No newline at end of file