From f9eb6751cae3a29bf73cdaee396c7e1cc8d0fff9 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 03:17:50 +0530 Subject: [PATCH 01/45] add version marker and block hasher --- zboxcore/blockchain/entity.go | 20 ++- zboxcore/marker/versionmarker.go | 50 ++++++ zboxcore/sdk/allocation.go | 10 +- zboxcore/sdk/chunked_upload_blobber.go | 3 - zboxcore/sdk/chunked_upload_form_builder.go | 56 ++---- zboxcore/sdk/chunked_upload_hasher.go | 35 +++- zboxcore/sdk/chunked_upload_model.go | 21 +-- zboxcore/sdk/chunked_upload_process.go | 9 +- zboxcore/sdk/chunked_upload_process_js.go | 11 +- zboxcore/sdk/commitworker.go | 182 +++----------------- zboxcore/sdk/multi_operation_worker.go | 8 + zboxcore/sdk/rollback.go | 114 +++++------- 12 files changed, 199 insertions(+), 320 deletions(-) create mode 100644 zboxcore/marker/versionmarker.go diff --git a/zboxcore/blockchain/entity.go b/zboxcore/blockchain/entity.go index e5c85b5e0..8e562a8c7 100644 --- a/zboxcore/blockchain/entity.go +++ b/zboxcore/blockchain/entity.go @@ -2,11 +2,12 @@ package blockchain import ( "encoding/json" - "github.com/0chain/gosdk/core/util" "math" "sync" "sync/atomic" + "github.com/0chain/gosdk/core/util" + "github.com/0chain/gosdk/core/conf" "github.com/0chain/gosdk/core/node" ) @@ -51,16 +52,16 @@ type ChainConfig struct { // StakePoolSettings information. type StakePoolSettings struct { - DelegateWallet string `json:"delegate_wallet"` - NumDelegates int `json:"num_delegates"` - ServiceCharge float64 `json:"service_charge"` + DelegateWallet string `json:"delegate_wallet"` + NumDelegates int `json:"num_delegates"` + ServiceCharge float64 `json:"service_charge"` } // UpdateStakePoolSettings information. type UpdateStakePoolSettings struct { - DelegateWallet *string `json:"delegate_wallet,omitempty"` - NumDelegates *int `json:"num_delegates,omitempty"` - ServiceCharge *float64 `json:"service_charge,omitempty"` + DelegateWallet *string `json:"delegate_wallet,omitempty"` + NumDelegates *int `json:"num_delegates,omitempty"` + ServiceCharge *float64 `json:"service_charge,omitempty"` } type ValidationNode struct { @@ -76,8 +77,9 @@ type UpdateValidationNode struct { } type StorageNode struct { - ID string `json:"id"` - Baseurl string `json:"url"` + ID string `json:"id"` + Baseurl string `json:"url"` + AllocationVersion int64 `json:"-"` skip uint64 `json:"-"` // skip on error } diff --git a/zboxcore/marker/versionmarker.go b/zboxcore/marker/versionmarker.go new file mode 100644 index 000000000..b150a3446 --- /dev/null +++ b/zboxcore/marker/versionmarker.go @@ -0,0 +1,50 @@ +package marker + +import ( + "fmt" + + "github.com/0chain/errors" + "github.com/0chain/gosdk/core/encryption" + "github.com/0chain/gosdk/core/sys" + "github.com/0chain/gosdk/zboxcore/client" +) + +type VersionMarker struct { + ClientID string `json:"client_id"` + BlobberID string `json:"blobber_id"` + AllocationID string `json:"allocation_id"` + Version int64 `json:"version"` + Timestamp int64 `json:"timestamp"` + Signature string `json:"signature"` + IsRepair bool `json:"is_repair"` + RepairVersion int64 `json:"repair_version"` + RepairOffset string `json:"repair_offset"` +} + +func (vm *VersionMarker) GetHashData() string { + return fmt.Sprintf("%s:%s:%s:%d:%d", vm.AllocationID, vm.ClientID, vm.BlobberID, vm.Version, vm.Timestamp) +} + +func (vm *VersionMarker) GetHash() string { + sigData := vm.GetHashData() + return encryption.Hash(sigData) +} + +func (vm *VersionMarker) Sign() error { + var err error + vm.Signature, err = client.Sign(vm.GetHash()) + return err +} + +func (vm *VersionMarker) VerifySignature(clientPublicKey string) error { + hashData := vm.GetHashData() + signatureHash := encryption.Hash(hashData) + sigOK, err := sys.Verify(vm.Signature, signatureHash) + if err != nil { + return errors.New("write_marker_validation_failed", "Error during verifying signature. "+err.Error()) + } + if !sigOK { + return errors.New("write_marker_validation_failed", "Write marker signature is not valid") + } + return nil +} diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 0c53eacfd..2a44fe467 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -722,7 +722,7 @@ func (a *Allocation) GetCurrentVersion() (bool, error) { } else { markerChan <- &RollbackBlobber{ blobber: blobber, - lpm: wr, + lvm: wr, commitResult: &CommitResult{}, } } @@ -737,15 +737,15 @@ func (a *Allocation) GetCurrentVersion() (bool, error) { for rb := range markerChan { - if rb == nil || rb.lpm.LatestWM == nil { + if rb == nil || rb.lvm == nil { continue } - if _, ok := versionMap[rb.lpm.LatestWM.Timestamp]; !ok { - versionMap[rb.lpm.LatestWM.Timestamp] = make([]*RollbackBlobber, 0) + if _, ok := versionMap[rb.lvm.VersionMarker.Version]; !ok { + versionMap[rb.lvm.VersionMarker.Version] = make([]*RollbackBlobber, 0) } - versionMap[rb.lpm.LatestWM.Timestamp] = append(versionMap[rb.lpm.LatestWM.Timestamp], rb) + versionMap[rb.lvm.VersionMarker.Version] = append(versionMap[rb.lvm.VersionMarker.Version], rb) if len(versionMap) > 2 { return false, fmt.Errorf("more than 2 versions found") diff --git a/zboxcore/sdk/chunked_upload_blobber.go b/zboxcore/sdk/chunked_upload_blobber.go index 886464aff..430c1920a 100644 --- a/zboxcore/sdk/chunked_upload_blobber.go +++ b/zboxcore/sdk/chunked_upload_blobber.go @@ -154,9 +154,6 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest( // fixed fileRef in last chunk on stream if isFinal { - sb.fileRef.FixedMerkleRoot = formData.FixedMerkleRoot - sb.fileRef.ValidationRoot = formData.ValidationRoot - sb.fileRef.ChunkSize = su.chunkSize sb.fileRef.Size = su.shardUploadedSize sb.fileRef.Path = su.fileMeta.RemotePath diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index 1de228fa5..3db24225b 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -6,7 +6,6 @@ import ( "encoding/json" "io" "mime/multipart" - "sync" "github.com/0chain/gosdk/zboxcore/client" @@ -29,9 +28,8 @@ type ChunkedUploadFormMetadata struct { FileBytesLen int ThumbnailBytesLen int ContentType string - FixedMerkleRoot string - ValidationRoot string ThumbnailContentHash string + DataHash string } // CreateChunkedUploadFormBuilder create ChunkedUploadFormBuilder instance @@ -116,12 +114,7 @@ func (b *chunkedUploadFormBuilder) Build( return res, err } - err = hasher.WriteToFixedMT(chunkBytes) - if err != nil { - return res, err - } - - err = hasher.WriteToValidationMT(chunkBytes) + err = hasher.WriteToBlockHasher(chunkBytes) if err != nil { return res, err } @@ -130,50 +123,24 @@ func (b *chunkedUploadFormBuilder) Build( } if isFinal && i == numBodies-1 { - err = hasher.Finalize() + + actualHashSignature, err := client.Sign(fileMeta.ActualHash) if err != nil { return res, err } - var ( - wg sync.WaitGroup - errChan = make(chan error, 2) - ) - wg.Add(2) - go func() { - formData.FixedMerkleRoot, err = hasher.GetFixedMerkleRoot() - if err != nil { - errChan <- err - } - wg.Done() - }() - go func() { - formData.ValidationRoot, err = hasher.GetValidationRoot() - if err != nil { - errChan <- err - } - wg.Done() - }() - wg.Wait() - close(errChan) - for err := range errChan { - return res, err - } - actualHashSignature, err := client.Sign(fileMeta.ActualHash) + formData.ActualHash = fileMeta.ActualHash + formData.ActualFileHashSignature = actualHashSignature + formData.ActualSize = fileMeta.ActualSize + dataHash, err := hasher.GetBlockHash() if err != nil { return res, err } - - validationRootSignature, err := client.Sign(actualHashSignature + formData.ValidationRoot) + formData.DataHash = dataHash + formData.DataHashSignature, err = client.Sign(dataHash) if err != nil { return res, err } - - formData.ActualHash = fileMeta.ActualHash - formData.ActualFileHashSignature = actualHashSignature - formData.ValidationRootSignature = validationRootSignature - formData.ActualSize = fileMeta.ActualSize - } thumbnailSize := len(thumbnailChunkData) @@ -225,9 +192,8 @@ func (b *chunkedUploadFormBuilder) Build( contentSlice = append(contentSlice, formWriter.FormDataContentType()) dataBuffers = append(dataBuffers, body) } - metadata.FixedMerkleRoot = formData.FixedMerkleRoot - metadata.ValidationRoot = formData.ValidationRoot metadata.ThumbnailContentHash = formData.ThumbnailContentHash + metadata.DataHash = formData.DataHash res.dataBuffers = dataBuffers res.contentSlice = contentSlice res.formData = metadata diff --git a/zboxcore/sdk/chunked_upload_hasher.go b/zboxcore/sdk/chunked_upload_hasher.go index bd1935d3b..e58da0220 100644 --- a/zboxcore/sdk/chunked_upload_hasher.go +++ b/zboxcore/sdk/chunked_upload_hasher.go @@ -26,6 +26,10 @@ type Hasher interface { WriteToValidationMT(buf []byte) error // Finalize will let merkle tree know that tree is finalized with the content it has received Finalize() error + // WriteToBlockHasher write bytes to block hasher + WriteToBlockHasher(buf []byte) error + // GetBlockHash get block hash + GetBlockHash() (string, error) } // see more detail about hash on https://github.com/0chain/blobber/wiki/Protocols#file-hash @@ -33,14 +37,14 @@ type hasher struct { File hash.Hash `json:"-"` FixedMT *util.FixedMerkleTree `json:"fixed_merkle_tree"` ValidationMT *util.ValidationTree `json:"validation_merkle_tree"` + BlockHasher hash.Hash `json:"-"` } // CreateHasher creat Hasher instance func CreateHasher(dataSize int64) Hasher { return &hasher{ - File: md5.New(), - FixedMT: util.NewFixedMerkleTree(), - ValidationMT: util.NewValidationTree(dataSize), + File: md5.New(), + BlockHasher: md5.New(), } } @@ -123,6 +127,31 @@ func (h *hasher) WriteToValidationMT(buf []byte) error { return err } +func (h *hasher) WriteToBlockHasher(buf []byte) error { + if h == nil { + return errors.Throw(constants.ErrInvalidParameter, "h") + } + + if h.BlockHasher == nil { + return errors.Throw(constants.ErrInvalidParameter, "h.BlockHasher") + } + + _, err := h.BlockHasher.Write(buf) + return err +} + +func (h *hasher) GetBlockHash() (string, error) { + if h == nil { + return "", errors.Throw(constants.ErrInvalidParameter, "h") + } + + if h.BlockHasher == nil { + return "", errors.Throw(constants.ErrInvalidParameter, "h.BlockHasher") + } + + return hex.EncodeToString(h.BlockHasher.Sum(nil)), nil +} + func (h *hasher) Finalize() error { var ( wg sync.WaitGroup diff --git a/zboxcore/sdk/chunked_upload_model.go b/zboxcore/sdk/chunked_upload_model.go index 326fb7e73..ec6788353 100644 --- a/zboxcore/sdk/chunked_upload_model.go +++ b/zboxcore/sdk/chunked_upload_model.go @@ -140,15 +140,9 @@ type UploadFormData struct { // Path remote path Path string `json:"filepath,omitempty"` - // ValidationRoot is merkle root of sha256 of 64KB as leaf - ValidationRoot string `json:"validation_root,omitempty"` - ValidationRootSignature string `json:"validation_root_signature,omitempty"` // Hash hash of shard thumbnail (encoded,encrypted) ThumbnailContentHash string `json:"thumbnail_content_hash,omitempty"` - // ChallengeHash challenge hash of shard data (encoded, encrypted) - FixedMerkleRoot string `json:"fixed_merkle_root,omitempty"` - // ActualHash hash of original file (un-encoded, un-encrypted) ActualHash string `json:"actual_hash,omitempty"` ActualFileHashSignature string `json:"actual_file_hash_signature,omitempty"` @@ -164,13 +158,14 @@ type UploadFormData struct { EncryptedKey string `json:"encrypted_key,omitempty"` EncryptedKeyPoint string `json:"encrypted_key_point,omitempty"` - IsFinal bool `json:"is_final,omitempty"` // all of chunks are uploaded - ChunkStartIndex int `json:"chunk_start_index,omitempty"` // start index of chunks. - ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of streaming merkle hash - ChunkSize int64 `json:"chunk_size,omitempty"` // the size of a chunk. 64*1024 is default - UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to - Size int64 `json:"size"` // total size of shard - + IsFinal bool `json:"is_final,omitempty"` // all of chunks are uploaded + ChunkStartIndex int `json:"chunk_start_index,omitempty"` // start index of chunks. + ChunkEndIndex int `json:"chunk_end_index,omitempty"` // end index of chunks. all chunks MUST be uploaded one by one because of streaming merkle hash + ChunkSize int64 `json:"chunk_size,omitempty"` // the size of a chunk. 64*1024 is default + UploadOffset int64 `json:"upload_offset,omitempty"` // It is next position that new incoming chunk should be append to + Size int64 `json:"size"` // total size of shard + DataHash string `json:"data_hash,omitempty"` // hash of shard data (encoded,encrypted) + DataHashSignature string `json:"data_hash_signature,omitempty"` } // UploadProgress progress of upload diff --git a/zboxcore/sdk/chunked_upload_process.go b/zboxcore/sdk/chunked_upload_process.go index da58f19de..597217612 100644 --- a/zboxcore/sdk/chunked_upload_process.go +++ b/zboxcore/sdk/chunked_upload_process.go @@ -49,14 +49,7 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, for i, blobberShard := range fileShards { hasher := su.blobbers[i].progress.Hasher for _, chunkBytes := range blobberShard { - err := hasher.WriteToFixedMT(chunkBytes) - if err != nil { - if su.statusCallback != nil { - su.statusCallback.Error(su.allocationObj.ID, su.fileMeta.RemotePath, su.opCode, err) - } - return err - } - err = hasher.WriteToValidationMT(chunkBytes) + err := hasher.WriteToBlockHasher(chunkBytes) if err != nil { if su.statusCallback != nil { su.statusCallback.Error(su.allocationObj.ID, su.fileMeta.RemotePath, su.opCode, err) diff --git a/zboxcore/sdk/chunked_upload_process_js.go b/zboxcore/sdk/chunked_upload_process_js.go index b6a0c8d3b..303a3739c 100644 --- a/zboxcore/sdk/chunked_upload_process_js.go +++ b/zboxcore/sdk/chunked_upload_process_js.go @@ -212,8 +212,7 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, } type FinalWorkerResult struct { - FixedMerkleRoot string - ValidationRoot string + DataHash string ThumbnailContentHash string } @@ -335,8 +334,6 @@ func (su *ChunkedUpload) listen(allEventChan []chan worker.MessageEvent, respCha } return } - blobber.fileRef.FixedMerkleRoot = finalResultObj.FixedMerkleRoot - blobber.fileRef.ValidationRoot = finalResultObj.ValidationRoot blobber.fileRef.ThumbnailHash = finalResultObj.ThumbnailContentHash isFinal = true } @@ -408,8 +405,7 @@ func ProcessEventData(data safejs.Value) { if formInfo.OnlyHash { if formInfo.IsFinal { finalResult := &FinalWorkerResult{ - FixedMerkleRoot: uploadData.formData.FixedMerkleRoot, - ValidationRoot: uploadData.formData.ValidationRoot, + DataHash: uploadData.formData.DataHash, ThumbnailContentHash: uploadData.formData.ThumbnailContentHash, } selfPostMessage(true, true, "", remotePath, formInfo.ChunkEndIndex, finalResult) @@ -449,8 +445,7 @@ func ProcessEventData(data safejs.Value) { } if formInfo.IsFinal { finalResult := &FinalWorkerResult{ - FixedMerkleRoot: blobberData.formData.FixedMerkleRoot, - ValidationRoot: blobberData.formData.ValidationRoot, + DataHash: blobberData.formData.DataHash, ThumbnailContentHash: blobberData.formData.ThumbnailContentHash, } selfPostMessage(true, true, "", remotePath, formInfo.ChunkEndIndex, finalResult) diff --git a/zboxcore/sdk/commitworker.go b/zboxcore/sdk/commitworker.go index 39494064f..c353a1252 100644 --- a/zboxcore/sdk/commitworker.go +++ b/zboxcore/sdk/commitworker.go @@ -3,7 +3,6 @@ package sdk import ( "bytes" "context" - "encoding/hex" "encoding/json" "fmt" "io" @@ -25,7 +24,6 @@ import ( l "github.com/0chain/gosdk/zboxcore/logger" "github.com/0chain/gosdk/zboxcore/marker" "github.com/0chain/gosdk/zboxcore/zboxutil" - "github.com/minio/sha256-simd" ) type ReferencePathResult struct { @@ -52,15 +50,19 @@ func SuccessCommitResult() *CommitResult { const MARKER_VERSION = "v2" type CommitRequest struct { - changes []allocationchange.AllocationChange - blobber *blockchain.StorageNode - allocationID string - allocationTx string - connectionID string - wg *sync.WaitGroup - result *CommitResult - timestamp int64 - blobberInd uint64 + changes []allocationchange.AllocationChange + blobber *blockchain.StorageNode + allocationID string + allocationTx string + connectionID string + wg *sync.WaitGroup + result *CommitResult + timestamp int64 + blobberInd uint64 + version int64 + isRepair bool + repairVersion int64 + repairOffset string } var commitChan map[string]chan *CommitRequest @@ -100,115 +102,12 @@ func (commitreq *CommitRequest) processCommit() { defer commitreq.wg.Done() start := time.Now() l.Logger.Debug("received a commit request") - paths := make([]string, 0) - for _, change := range commitreq.changes { - paths = append(paths, change.GetAffectedPath()...) - } - if len(paths) == 0 { + if len(commitreq.changes) == 0 { l.Logger.Debug("Nothing to commit") commitreq.result = SuccessCommitResult() return } - var req *http.Request - var lR ReferencePathResult - req, err := zboxutil.NewReferencePathRequest(commitreq.blobber.Baseurl, commitreq.allocationID, commitreq.allocationTx, paths) - if err != nil { - l.Logger.Error("Creating ref path req", err) - return - } - ctx, cncl := context.WithTimeout(context.Background(), (time.Second * 30)) - err = zboxutil.HttpDo(ctx, cncl, req, func(resp *http.Response, err error) error { - if err != nil { - l.Logger.Error("Ref path error:", err) - return err - } - defer resp.Body.Close() - if resp.StatusCode != http.StatusOK { - l.Logger.Error("Ref path response : ", resp.StatusCode) - } - resp_body, err := ioutil.ReadAll(resp.Body) - if err != nil { - l.Logger.Error("Ref path: Resp", err) - return err - } - if resp.StatusCode != http.StatusOK { - return errors.New( - strconv.Itoa(resp.StatusCode), - fmt.Sprintf("Reference path error response: Status: %d - %s ", - resp.StatusCode, string(resp_body))) - } - err = json.Unmarshal(resp_body, &lR) - if err != nil { - l.Logger.Error("Reference path json decode error: ", err) - return err - } - return nil - }) - - if err != nil { - commitreq.result = ErrorCommitResult(err.Error()) - return - } - rootRef, err := lR.GetDirTree(commitreq.allocationID) - - if err != nil { - commitreq.result = ErrorCommitResult(err.Error()) - return - } - hasher := sha256.New() - if lR.LatestWM != nil { - err = lR.LatestWM.VerifySignature(client.GetClientPublicKey()) - if err != nil { - e := errors.New("signature_verification_failed", err.Error()) - commitreq.result = ErrorCommitResult(e.Error()) - return - } - if commitreq.timestamp <= lR.LatestWM.Timestamp { - commitreq.timestamp = lR.LatestWM.Timestamp + 1 - } - - rootRef.CalculateHash() - prevAllocationRoot := rootRef.Hash - if prevAllocationRoot != lR.LatestWM.AllocationRoot { - l.Logger.Error("Allocation root from latest writemarker mismatch. Expected: " + prevAllocationRoot + " got: " + lR.LatestWM.AllocationRoot) - errMsg := fmt.Sprintf( - "calculated allocation root mismatch from blobber %s. Expected: %s, Got: %s", - commitreq.blobber.Baseurl, prevAllocationRoot, lR.LatestWM.AllocationRoot) - commitreq.result = ErrorCommitResult(errMsg) - return - } - if lR.LatestWM.ChainHash != "" { - prevChainHash, err := hex.DecodeString(lR.LatestWM.ChainHash) - if err != nil { - commitreq.result = ErrorCommitResult(err.Error()) - return - } - hasher.Write(prevChainHash) //nolint:errcheck - } - } - - var size int64 - fileIDMeta := make(map[string]string) - - for _, change := range commitreq.changes { - err = change.ProcessChange(rootRef, fileIDMeta) - if err != nil { - if !errors.Is(err, allocationchange.ErrRefNotFound) { - commitreq.result = ErrorCommitResult(err.Error()) - return - } - } else { - size += change.GetSize() - } - } - rootRef.CalculateHash() - var chainHash string - if lR.Version == MARKER_VERSION { - decodedHash, _ := hex.DecodeString(rootRef.Hash) - hasher.Write(decodedHash) //nolint:errcheck - chainHash = hex.EncodeToString(hasher.Sum(nil)) - } - err = commitreq.commitBlobber(rootRef, chainHash, lR.LatestWM, size, fileIDMeta) + err := commitreq.commitBlobber() if err != nil { commitreq.result = ErrorCommitResult(err.Error()) return @@ -217,42 +116,20 @@ func (commitreq *CommitRequest) processCommit() { commitreq.result = SuccessCommitResult() } -func (req *CommitRequest) commitBlobber( - rootRef *fileref.Ref, chainHash string, latestWM *marker.WriteMarker, size int64, - fileIDMeta map[string]string) (err error) { - - fileIDMetaData, err := json.Marshal(fileIDMeta) - if err != nil { - l.Logger.Error("Marshalling inode metadata failed: ", err) - return err - } - - wm := &marker.WriteMarker{} - wm.AllocationRoot = rootRef.Hash - wm.ChainSize = size - if latestWM != nil { - wm.PreviousAllocationRoot = latestWM.AllocationRoot - wm.ChainSize += latestWM.ChainSize - } else { - wm.PreviousAllocationRoot = "" - } - if wm.AllocationRoot == wm.PreviousAllocationRoot { - l.Logger.Debug("Allocation root and previous allocation root are same") - return nil +func (req *CommitRequest) commitBlobber() (err error) { + vm := &marker.VersionMarker{ + Version: req.version, + Timestamp: req.timestamp, + ClientID: client.GetClientID(), + AllocationID: req.allocationID, + BlobberID: req.blobber.ID, } - wm.ChainHash = chainHash - wm.FileMetaRoot = rootRef.FileMetaHash - wm.AllocationID = req.allocationID - wm.Size = size - wm.BlobberID = req.blobber.ID - wm.Timestamp = req.timestamp - wm.ClientID = client.GetClientID() - err = wm.Sign() + err = vm.Sign() if err != nil { l.Logger.Error("Signing writemarker failed: ", err) return err } - wmData, err := json.Marshal(wm) + vmData, err := json.Marshal(vm) if err != nil { l.Logger.Error("Creating writemarker failed: ", err) return err @@ -266,7 +143,7 @@ func (req *CommitRequest) commitBlobber( for retries := 0; retries < 6; retries++ { err, shouldContinue = func() (err error, shouldContinue bool) { body := new(bytes.Buffer) - formWriter, err := getFormWritter(req.connectionID, wmData, fileIDMetaData, body) + formWriter, err := getFormWritter(req.connectionID, vmData, body) if err != nil { l.Logger.Error("Creating form writer failed: ", err) return @@ -379,19 +256,14 @@ func (commitreq *CommitRequest) calculateHashRequest(ctx context.Context, paths return err } -func getFormWritter(connectionID string, wmData, fileIDMetaData []byte, body *bytes.Buffer) (*multipart.Writer, error) { +func getFormWritter(connectionID string, vmData []byte, body *bytes.Buffer) (*multipart.Writer, error) { formWriter := multipart.NewWriter(body) err := formWriter.WriteField("connection_id", connectionID) if err != nil { return nil, err } - err = formWriter.WriteField("write_marker", string(wmData)) - if err != nil { - return nil, err - } - - err = formWriter.WriteField("file_id_meta", string(fileIDMetaData)) + err = formWriter.WriteField("version_marker", string(vmData)) if err != nil { return nil, err } diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index 0ab370207..2a5f5a316 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -292,6 +292,7 @@ func (mo *MultiOperation) Process() error { wg: wg, timestamp: timestamp, blobberInd: pos, + version: mo.allocationObj.Blobbers[pos].AllocationVersion + 1, } commitReq.changes = append(commitReq.changes, mo.changes[pos]...) @@ -335,6 +336,13 @@ func (mo *MultiOperation) Process() error { for _, op := range mo.operations { op.Completed(mo.allocationObj) } + if singleClientMode && !mo.isRepair { + for _, commitReq := range commitReqs { + if commitReq.result.Success { + mo.allocationObj.Blobbers[commitReq.blobberInd].AllocationVersion++ + } + } + } } return nil diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index e0e10dc2c..32788cae0 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -3,7 +3,6 @@ package sdk import ( "bytes" "context" - "encoding/hex" "encoding/json" "fmt" "io" @@ -24,7 +23,6 @@ import ( l "github.com/0chain/gosdk/zboxcore/logger" "github.com/0chain/gosdk/zboxcore/marker" "github.com/0chain/gosdk/zboxcore/zboxutil" - "github.com/minio/sha256-simd" "go.uber.org/zap" ) @@ -34,6 +32,10 @@ type LatestPrevWriteMarker struct { Version string `json:"version"` } +type LatestVersionMarker struct { + VersionMarker *marker.VersionMarker `json:"version_marker"` +} + type AllocStatus byte const ( @@ -51,7 +53,7 @@ var ( type RollbackBlobber struct { blobber *blockchain.StorageNode commitResult *CommitResult - lpm *LatestPrevWriteMarker + lvm *LatestVersionMarker blobIndex int } @@ -60,9 +62,9 @@ type BlobberStatus struct { Status string } -func GetWritemarker(allocID, allocTx, id, baseUrl string) (*LatestPrevWriteMarker, error) { +func GetWritemarker(allocID, allocTx, id, baseUrl string) (*LatestVersionMarker, error) { - var lpm LatestPrevWriteMarker + var lvm LatestVersionMarker req, err := zboxutil.NewWritemarkerRequest(baseUrl, allocID, allocTx) if err != nil { @@ -96,75 +98,49 @@ func GetWritemarker(allocID, allocTx, id, baseUrl string) (*LatestPrevWriteMarke if err != nil { return nil, err } - err = json.Unmarshal(body, &lpm) + err = json.Unmarshal(body, &lvm) if err != nil { return nil, err } - if lpm.LatestWM != nil { - err = lpm.LatestWM.VerifySignature(client.GetClientPublicKey()) + if lvm.VersionMarker != nil && lvm.VersionMarker.Version != 0 { + err = lvm.VersionMarker.VerifySignature(client.GetClientPublicKey()) if err != nil { return nil, fmt.Errorf("signature verification failed for latest writemarker: %s", err.Error()) } - if lpm.PrevWM != nil { - err = lpm.PrevWM.VerifySignature(client.GetClientPublicKey()) - if err != nil { - return nil, fmt.Errorf("signature verification failed for latest writemarker: %s", err.Error()) - } - } } - return &lpm, nil + return &lvm, nil } return nil, fmt.Errorf("writemarker error response %d", http.StatusTooManyRequests) } func (rb *RollbackBlobber) processRollback(ctx context.Context, tx string) error { - - wm := &marker.WriteMarker{} - wm.AllocationID = rb.lpm.LatestWM.AllocationID - wm.Timestamp = rb.lpm.LatestWM.Timestamp - wm.BlobberID = rb.lpm.LatestWM.BlobberID - wm.ClientID = client.GetClientID() - wm.Size = -rb.lpm.LatestWM.Size - wm.ChainSize = wm.Size + rb.lpm.LatestWM.ChainSize - - if rb.lpm.PrevWM != nil { - wm.AllocationRoot = rb.lpm.PrevWM.AllocationRoot - wm.PreviousAllocationRoot = rb.lpm.PrevWM.AllocationRoot - wm.FileMetaRoot = rb.lpm.PrevWM.FileMetaRoot - if wm.AllocationRoot == rb.lpm.LatestWM.AllocationRoot { - return nil - } - } - if rb.lpm.Version == MARKER_VERSION { - decodedHash, _ := hex.DecodeString(wm.AllocationRoot) - prevChainHash, _ := hex.DecodeString(rb.lpm.LatestWM.ChainHash) - hasher := sha256.New() - hasher.Write(prevChainHash) //nolint:errcheck - hasher.Write(decodedHash) //nolint:errcheck - wm.ChainHash = hex.EncodeToString(hasher.Sum(nil)) - } else if rb.lpm.Version == "" { - wm.Size = 0 + vm := &marker.VersionMarker{ + ClientID: client.GetClientID(), + BlobberID: rb.lvm.VersionMarker.BlobberID, + AllocationID: rb.lvm.VersionMarker.AllocationID, + Version: rb.lvm.VersionMarker.Version - 1, + Timestamp: rb.lvm.VersionMarker.Timestamp, } - err := wm.Sign() + err := vm.Sign() if err != nil { l.Logger.Error("Signing writemarker failed: ", err) return err } body := new(bytes.Buffer) formWriter := multipart.NewWriter(body) - wmData, err := json.Marshal(wm) + vmData, err := json.Marshal(vm) if err != nil { l.Logger.Error("Creating writemarker failed: ", err) return err } connID := zboxutil.NewConnectionId() - formWriter.WriteField("write_marker", string(wmData)) + formWriter.WriteField("version_marker", string(vmData)) formWriter.WriteField("connection_id", connID) formWriter.Close() - req, err := zboxutil.NewRollbackRequest(rb.blobber.Baseurl, wm.AllocationID, tx, body) + req, err := zboxutil.NewRollbackRequest(rb.blobber.Baseurl, vm.AllocationID, tx, body) if err != nil { l.Logger.Error("Creating rollback request failed: ", err) return err @@ -268,22 +244,23 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { ID: blobber.ID, Status: "available", } - wr, err := GetWritemarker(a.ID, a.Tx, blobber.ID, blobber.Baseurl) + lvm, err := GetWritemarker(a.ID, a.Tx, blobber.ID, blobber.Baseurl) if err != nil { atomic.AddInt32(&errCnt, 1) markerError = err l.Logger.Error("error during getWritemarker", zap.Error(err)) blobStatus.Status = "unavailable" } - if wr == nil { + if lvm == nil { markerChan <- nil } else { markerChan <- &RollbackBlobber{ blobber: blobber, - lpm: wr, + lvm: lvm, commitResult: &CommitResult{}, blobIndex: ind, } + blobber.AllocationVersion = lvm.VersionMarker.Version } blobberRes[ind] = blobStatus }(blobber, ind) @@ -295,30 +272,24 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { return Broken, blobberRes, common.NewError("check_alloc_status_failed", markerError.Error()) } - versionMap := make(map[string][]*RollbackBlobber) + versionMap := make(map[int64][]*RollbackBlobber) var ( - prevVersion string - latestVersion string - highestTS int64 + consensusReached bool + latestVersion int64 + prevVersion int64 ) for rb := range markerChan { - if rb == nil || rb.lpm.LatestWM == nil { + if rb == nil || rb.lvm == nil { continue } - version := rb.lpm.LatestWM.FileMetaRoot - - if highestTS < rb.lpm.LatestWM.Timestamp { - prevVersion = latestVersion - highestTS = rb.lpm.LatestWM.Timestamp + version := rb.lvm.VersionMarker.Version + if version > latestVersion { latestVersion = version - } - - if prevVersion == "" && version != latestVersion { - prevVersion = version + prevVersion = latestVersion } if _, ok := versionMap[version]; !ok { @@ -326,6 +297,9 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { } versionMap[version] = append(versionMap[version], rb) + if len(versionMap[version]) >= a.DataShards && version == latestVersion { + consensusReached = true + } } req := a.DataShards @@ -334,17 +308,15 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { return Commit, blobberRes, nil } - if len(versionMap[latestVersion]) > req || len(versionMap[prevVersion]) > req { + if consensusReached { return Commit, blobberRes, nil } - if len(versionMap[latestVersion]) >= req || len(versionMap[prevVersion]) >= req || len(versionMap) > 2 { - for _, rb := range versionMap[prevVersion] { - blobberRes[rb.blobIndex].Status = "repair" - } + if len(versionMap[latestVersion]) >= req { + // for _, rb := range versionMap[prevVersion] { + // blobberRes[rb.blobIndex].Status = "repair" + // } return Repair, blobberRes, nil - } else { - l.Logger.Info("versionMapLen", zap.Int("versionMapLen", len(versionMap)), zap.Int("latestLen", len(versionMap[latestVersion])), zap.Int("prevLen", len(versionMap[prevVersion]))) } // rollback to previous version @@ -401,7 +373,7 @@ func (a *Allocation) RollbackWithMask(mask zboxutil.Uint128) { } else { markerChan <- &RollbackBlobber{ blobber: blobber, - lpm: wr, + lvm: wr, commitResult: &CommitResult{}, } } @@ -412,7 +384,7 @@ func (a *Allocation) RollbackWithMask(mask zboxutil.Uint128) { close(markerChan) for rb := range markerChan { - if rb == nil || rb.lpm.LatestWM == nil { + if rb == nil || rb.lvm == nil { continue } wg.Add(1) From 02e203d91330793495685777093910c4b495636d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 22 Jul 2024 03:25:19 +0530 Subject: [PATCH 02/45] add repair info in commit worker --- zboxcore/sdk/commitworker.go | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/zboxcore/sdk/commitworker.go b/zboxcore/sdk/commitworker.go index c353a1252..61830cbf3 100644 --- a/zboxcore/sdk/commitworker.go +++ b/zboxcore/sdk/commitworker.go @@ -118,11 +118,14 @@ func (commitreq *CommitRequest) processCommit() { func (req *CommitRequest) commitBlobber() (err error) { vm := &marker.VersionMarker{ - Version: req.version, - Timestamp: req.timestamp, - ClientID: client.GetClientID(), - AllocationID: req.allocationID, - BlobberID: req.blobber.ID, + Version: req.version, + Timestamp: req.timestamp, + ClientID: client.GetClientID(), + AllocationID: req.allocationID, + BlobberID: req.blobber.ID, + IsRepair: req.isRepair, + RepairVersion: req.repairVersion, + RepairOffset: req.repairOffset, } err = vm.Sign() if err != nil { From c981f068ef753702ebf4475de4d742eba3648565 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 23 Jul 2024 23:26:34 +0530 Subject: [PATCH 03/45] add check for repair marker --- zboxcore/sdk/rollback.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index 32788cae0..02018e29f 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -115,6 +115,10 @@ func GetWritemarker(allocID, allocTx, id, baseUrl string) (*LatestVersionMarker, } func (rb *RollbackBlobber) processRollback(ctx context.Context, tx string) error { + // don't rollback if the blobber is already in repair mode otherwise it will lead to inconsistent state + if rb.lvm == nil || rb.lvm.VersionMarker.IsRepair { + return nil + } vm := &marker.VersionMarker{ ClientID: client.GetClientID(), BlobberID: rb.lvm.VersionMarker.BlobberID, From b5cbb792ef034f3c71200460d3b646da41670b90 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 25 Jul 2024 02:10:45 +0530 Subject: [PATCH 04/45] add allocation version and check for directory --- zboxcore/fileref/fileref.go | 1 + zboxcore/sdk/allocation.go | 2 ++ zboxcore/sdk/deleteworker.go | 3 +++ zboxcore/sdk/multi_operation_worker.go | 13 ++++++++++++- zboxcore/sdk/rollback.go | 7 ++++--- 5 files changed, 22 insertions(+), 4 deletions(-) diff --git a/zboxcore/fileref/fileref.go b/zboxcore/fileref/fileref.go index bb46a0e11..4cc2f1147 100644 --- a/zboxcore/fileref/fileref.go +++ b/zboxcore/fileref/fileref.go @@ -90,6 +90,7 @@ type Ref struct { ThumbnailSize int64 `json:"thumbnail_size" mapstructure:"thumbnail_size"` ActualThumbnailHash string `json:"actual_thumbnail_hash" mapstructure:"actual_thumbnail_hash"` ActualThumbnailSize int64 `json:"actual_thumbnail_size" mapstructure:"actual_thumbnail_size"` + IsEmpty bool `json:"is_empty" mapstructure:"is_empty"` HashToBeComputed bool ChildrenLoaded bool Children []RefEntity `json:"-" mapstructure:"-"` diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 2a44fe467..00e68aed1 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -201,6 +201,7 @@ type Allocation struct { MovedToValidators common.Balance `json:"moved_to_validators,omitempty"` FileOptions uint16 `json:"file_options"` ThirdPartyExtendable bool `json:"third_party_extendable"` + AllocationVersion int64 `json:"-"` numBlockDownloads int downloadChan chan *DownloadRequest @@ -338,6 +339,7 @@ func (a *Allocation) InitAllocation() { a.startWorker(a.ctx) InitCommitWorker(a.Blobbers) InitBlockDownloader(a.Blobbers, downloadWorkerCount) + a.CheckAllocStatus() //nolint:errcheck a.initialized = true } diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index 3ac0b6eb1..e173ea759 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -369,6 +369,9 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( l.Logger.Error(err.Error()) return } + if refEntity.Type == fileref.DIRECTORY && !refEntity.IsEmpty { + //TODO: Handle directory delete + } err = deleteReq.deleteBlobberFile(deleteReq.blobbers[blobberIdx], blobberIdx) if err != nil { blobberErrors[blobberIdx] = err diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index 2a5f5a316..cb2807d1f 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -275,11 +275,21 @@ func (mo *MultiOperation) Process() error { } logger.Logger.Debug("[checkAllocStatus]", time.Since(start).Milliseconds()) mo.Consensus.Reset() + var pos uint64 + for i := mo.operationMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + if mo.allocationObj.Blobbers[pos].AllocationVersion != mo.allocationObj.AllocationVersion { + mo.operationMask = mo.operationMask.And(zboxutil.NewUint128(1).Lsh(pos).Not()) + } + } activeBlobbers := mo.operationMask.CountOnes() + if activeBlobbers < mo.consensusThresh { + return errors.New("consensus_not_met", "Active blobbers less than consensus threshold") + } commitReqs := make([]*CommitRequest, activeBlobbers) start = time.Now() wg.Add(activeBlobbers) - var pos uint64 + var counter = 0 timestamp := int64(common.Now()) for i := mo.operationMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { @@ -342,6 +352,7 @@ func (mo *MultiOperation) Process() error { mo.allocationObj.Blobbers[commitReq.blobberInd].AllocationVersion++ } } + mo.allocationObj.AllocationVersion += 1 } } diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index 02018e29f..a288c5de2 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -313,13 +313,14 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { } if consensusReached { + a.AllocationVersion = latestVersion return Commit, blobberRes, nil } if len(versionMap[latestVersion]) >= req { - // for _, rb := range versionMap[prevVersion] { - // blobberRes[rb.blobIndex].Status = "repair" - // } + for _, rb := range versionMap[prevVersion] { + blobberRes[rb.blobIndex].Status = "repair" + } return Repair, blobberRes, nil } From 526a11784672a66f914b278f01e840f0bca10a1d Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 25 Jul 2024 18:53:19 +0530 Subject: [PATCH 05/45] prefix delete --- wasmsdk/common.go | 8 ++ zboxcore/fileref/fileref.go | 7 ++ zboxcore/sdk/allocation.go | 12 ++- zboxcore/sdk/deleteworker.go | 132 ++++++++++++++++++++++++++++++++- zboxcore/sdk/filerefsworker.go | 61 ++++++++++++++- 5 files changed, 212 insertions(+), 8 deletions(-) diff --git a/wasmsdk/common.go b/wasmsdk/common.go index ba2bfe408..7c9f4f8ac 100644 --- a/wasmsdk/common.go +++ b/wasmsdk/common.go @@ -89,3 +89,11 @@ func (h *hasher) WriteToValidationMT(_ []byte) error { func (h *hasher) Finalize() error { return nil } + +func (h *hasher) GetBlockHash() (string, error) { + return "", nil +} + +func (h *hasher) WriteToBlockHasher(buf []byte) error { + return nil +} diff --git a/zboxcore/fileref/fileref.go b/zboxcore/fileref/fileref.go index 4cc2f1147..c79f343ce 100644 --- a/zboxcore/fileref/fileref.go +++ b/zboxcore/fileref/fileref.go @@ -17,6 +17,7 @@ const CHUNK_SIZE = 64 * 1024 const ( FILE = "f" DIRECTORY = "d" + REGULAR = "regular" ) var fileCache, _ = lru.New[string, FileRef](100) @@ -70,6 +71,7 @@ type RefEntity interface { GetFileID() string GetCreatedAt() common.Timestamp GetUpdatedAt() common.Timestamp + GetAllocationVersion() int64 } type Ref struct { @@ -91,6 +93,7 @@ type Ref struct { ActualThumbnailHash string `json:"actual_thumbnail_hash" mapstructure:"actual_thumbnail_hash"` ActualThumbnailSize int64 `json:"actual_thumbnail_size" mapstructure:"actual_thumbnail_size"` IsEmpty bool `json:"is_empty" mapstructure:"is_empty"` + AllocationVersion int64 `json:"allocation_version" mapstructure:"allocation_version"` HashToBeComputed bool ChildrenLoaded bool Children []RefEntity `json:"-" mapstructure:"-"` @@ -233,6 +236,10 @@ func (r *Ref) RemoveChild(idx int) { r.Children = append(r.Children[:idx], r.Children[idx+1:]...) } +func (r *Ref) GetAllocationVersion() int64 { + return r.AllocationVersion +} + func (fr *FileRef) GetFileMetaHash() string { return fr.FileMetaHash } diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 00e68aed1..6154cb69d 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -64,6 +64,7 @@ const ( const ( emptyFileDataHash = "d41d8cd98f00b204e9800998ecf8427e" + getRefPageLimit = 1000 ) // Expected success rate is calculated (NumDataShards)*100/(NumDataShards+NumParityShards) @@ -1401,7 +1402,7 @@ func (a *Allocation) ListDir(path string, opts ...ListRequestOptions) (*ListResu return nil, errors.New("list_request_failed", "Failed to get list response from the blobbers") } -func (a *Allocation) getRefs(path, pathHash, authToken, offsetPath, updatedDate, offsetDate, fileType, refType string, level, pageLimit int) (*ObjectTreeResult, error) { +func (a *Allocation) getRefs(path, pathHash, authToken, offsetPath, updatedDate, offsetDate, fileType, refType string, level, pageLimit int, opts ...ObjectTreeRequestOption) (*ObjectTreeResult, error) { if !a.isInitialized() { return nil, notInitialized } @@ -1421,9 +1422,16 @@ func (a *Allocation) getRefs(path, pathHash, authToken, offsetPath, updatedDate, fileType: fileType, refType: refType, ctx: a.ctx, + reqMask: zboxutil.NewUint128(1).Lsh(uint64(len(a.Blobbers))).Sub64(1), } oTreeReq.fullconsensus = a.fullconsensus oTreeReq.consensusThresh = a.DataShards + for _, opt := range opts { + opt(oTreeReq) + } + if singleClientMode { + oTreeReq.singleBlobber = true + } return oTreeReq.GetRefs() } @@ -1516,7 +1524,7 @@ func (a *Allocation) GetRefsWithAuthTicket(authToken, offsetPath, updatedDate, o } // This function will retrieve paginated objectTree and will handle concensus; Required tree should be made in application side. -func (a *Allocation) GetRefs(path, offsetPath, updatedDate, offsetDate, fileType, refType string, level, pageLimit int) (*ObjectTreeResult, error) { +func (a *Allocation) GetRefs(path, offsetPath, updatedDate, offsetDate, fileType, refType string, level, pageLimit int, opts ...ObjectTreeRequestOption) (*ObjectTreeResult, error) { if len(path) == 0 || !zboxutil.IsRemoteAbs(path) { return nil, errors.New("invalid_path", fmt.Sprintf("Absolute path required. Path provided: %v", path)) } diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index e173ea759..83be0414e 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" "net/url" + "strings" "sync" "sync/atomic" "time" @@ -352,8 +353,11 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( numList := len(deleteReq.blobbers) objectTreeRefs := make([]fileref.RefEntity, numList) blobberErrors := make([]error, numList) - - var pos uint64 + versionMap := make(map[int64]int) + var ( + pos uint64 + consensusRef *fileref.FileRef + ) for i := deleteReq.deleteMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) @@ -372,7 +376,57 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( if refEntity.Type == fileref.DIRECTORY && !refEntity.IsEmpty { //TODO: Handle directory delete } - err = deleteReq.deleteBlobberFile(deleteReq.blobbers[blobberIdx], blobberIdx) + // err = deleteReq.deleteBlobberFile(deleteReq.blobbers[blobberIdx], blobberIdx) + // if err != nil { + // blobberErrors[blobberIdx] = err + // } + // if singleClientMode { + // lookuphash := fileref.GetReferenceLookup(deleteReq.allocationID, deleteReq.remotefilepath) + // cacheKey := fileref.GetCacheKey(lookuphash, deleteReq.blobbers[blobberIdx].ID) + // fileref.DeleteFileRef(cacheKey) + // } + objectTreeRefs[blobberIdx] = refEntity + deleteReq.maskMu.Lock() + versionMap[refEntity.AllocationVersion]++ + if versionMap[refEntity.AllocationVersion] > deleteReq.consensus.consensusThresh { + consensusRef = refEntity + } + deleteReq.maskMu.Unlock() + }(int(pos)) + } + deleteReq.wg.Wait() + + if !deleteReq.consensus.isConsensusOk() { + err := zboxutil.MajorError(blobberErrors) + if err != nil { + return nil, deleteReq.deleteMask, thrown.New("delete_failed", fmt.Sprintf("Delete failed. %s", err.Error())) + } + + return nil, deleteReq.deleteMask, thrown.New("consensus_not_met", + fmt.Sprintf("Delete failed. Required consensus %d, got %d", + deleteReq.consensus.consensusThresh, deleteReq.consensus.consensus)) + } + if consensusRef == nil { + return nil, deleteReq.deleteMask, thrown.New("delete_failed", "Delete failed. No consensus found") + } + if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { + for ind, refEntity := range objectTreeRefs { + if refEntity.GetAllocationVersion() != consensusRef.AllocationVersion { + deleteReq.deleteMask = deleteReq.deleteMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) + } + } + err := deleteReq.deleteSubDirectories() + if err != nil { + return nil, deleteReq.deleteMask, err + } + } + pos = 0 + for i := deleteReq.deleteMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + deleteReq.wg.Add(1) + go func(blobberIdx int) { + defer deleteReq.wg.Done() + err := deleteReq.deleteBlobberFile(deleteReq.blobbers[blobberIdx], blobberIdx) if err != nil { blobberErrors[blobberIdx] = err } @@ -381,7 +435,6 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( cacheKey := fileref.GetCacheKey(lookuphash, deleteReq.blobbers[blobberIdx].ID) fileref.DeleteFileRef(cacheKey) } - objectTreeRefs[blobberIdx] = refEntity }(int(pos)) } deleteReq.wg.Wait() @@ -396,6 +449,7 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( fmt.Sprintf("Delete failed. Required consensus %d, got %d", deleteReq.consensus.consensusThresh, deleteReq.consensus.consensus)) } + l.Logger.Debug("Delete Process Ended ") return objectTreeRefs, deleteReq.deleteMask, nil } @@ -453,3 +507,73 @@ func NewDeleteOperation(remotePath string, deleteMask zboxutil.Uint128, maskMu * dop.ctx, dop.ctxCncl = context.WithCancel(ctx) return dop } + +func (req *DeleteRequest) deleteSubDirectories() error { + // list all files + var ( + offsetPath string + pathLevel int + ) + for { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WitObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + break + } + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.deleteMask + op := OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: ref.Path, + Mask: &opMask, + } + ops = append(ops, op) + } + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err + } + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + pathLevel = oResult.Refs[len(oResult.Refs)-1].PathLevel + if len(oResult.Refs) < getRefPageLimit { + break + } + } + // reset offsetPath + offsetPath = "" + level := len(strings.Split(strings.TrimSuffix(req.remotefilepath, "/"), "/")) + // list all directories by descending order of path level + for pathLevel > level { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.DIRECTORY, fileref.REGULAR, pathLevel, getRefPageLimit, WithObjectContext(req.ctx), WitObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + pathLevel-- + } else { + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.deleteMask + op := OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: ref.Path, + Mask: &opMask, + } + ops = append(ops, op) + } + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err + } + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + if len(oResult.Refs) < getRefPageLimit { + pathLevel-- + } + } + } + + return nil +} diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index 93af36190..dccea9af2 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -7,6 +7,7 @@ import ( "fmt" "io/ioutil" "math" + "math/rand" "net/http" "sync" "time" @@ -43,7 +44,9 @@ type ObjectTreeRequest struct { offsetPath string updatedDate string // must have "2006-01-02T15:04:05.99999Z07:00" format offsetDate string // must have "2006-01-02T15:04:05.99999Z07:00" format + reqMask zboxutil.Uint128 ctx context.Context + singleBlobber bool Consensus } @@ -54,18 +57,72 @@ type oTreeResponse struct { idx int } +type ObjectTreeRequestOption func(*ObjectTreeRequest) + +func WithObjectContext(ctx context.Context) ObjectTreeRequestOption { + return func(o *ObjectTreeRequest) { + o.ctx = ctx + } +} + +func WitObjectMask(mask zboxutil.Uint128) ObjectTreeRequestOption { + return func(o *ObjectTreeRequest) { + o.reqMask = mask + } +} + +func WithObjectConsensusThresh(thresh int) ObjectTreeRequestOption { + return func(o *ObjectTreeRequest) { + o.consensusThresh = thresh + } +} + +func WithSingleBlobber(singleBlobber bool) ObjectTreeRequestOption { + return func(o *ObjectTreeRequest) { + o.singleBlobber = singleBlobber + } +} + // Paginated tree should not be collected as this will stall the client // It should rather be handled by application that uses gosdk func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { totalBlobbersCount := len(o.blobbers) oTreeResponses := make([]oTreeResponse, totalBlobbersCount) respChan := make(chan *oTreeResponse, totalBlobbersCount) - for i, blob := range o.blobbers { + activeCount := o.reqMask.CountOnes() + if o.singleBlobber { + var respErr error + for i := 0; i < activeCount; i++ { + var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) + num := rnd.Intn(activeCount) + blob := o.blobbers[num] + l.Logger.Debug(fmt.Sprintf("Getting file refs for path %v from blobber %v", o.remotefilepath, blob.Baseurl)) + idx := num + baseURL := blob.Baseurl + go o.getFileRefs(baseURL, respChan, idx) + select { + case <-o.ctx.Done(): + return nil, o.ctx.Err() + case oTreeResponse := <-respChan: + if oTreeResponse.err != nil { + respErr = oTreeResponse.err + } else { + return oTreeResponse.oTResult, nil + } + } + } + return nil, respErr + } + var pos uint64 + for i := o.reqMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + blob := o.blobbers[pos] l.Logger.Debug(fmt.Sprintf("Getting file refs for path %v from blobber %v", o.remotefilepath, blob.Baseurl)) - idx := i + idx := int(pos) baseURL := blob.Baseurl go o.getFileRefs(baseURL, respChan, idx) } + hashCount := make(map[string]int) hashRefsMap := make(map[string]*ObjectTreeResult) oTreeResponseErrors := make([]error, totalBlobbersCount) From 6486442dcc77436f3917ae36d88b024aa6ae883c Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 25 Jul 2024 23:40:28 +0530 Subject: [PATCH 06/45] fix consensus in delete --- zboxcore/sdk/deleteworker.go | 13 +------------ 1 file changed, 1 insertion(+), 12 deletions(-) diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index 83be0414e..dfb116a53 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -373,18 +373,7 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( l.Logger.Error(err.Error()) return } - if refEntity.Type == fileref.DIRECTORY && !refEntity.IsEmpty { - //TODO: Handle directory delete - } - // err = deleteReq.deleteBlobberFile(deleteReq.blobbers[blobberIdx], blobberIdx) - // if err != nil { - // blobberErrors[blobberIdx] = err - // } - // if singleClientMode { - // lookuphash := fileref.GetReferenceLookup(deleteReq.allocationID, deleteReq.remotefilepath) - // cacheKey := fileref.GetCacheKey(lookuphash, deleteReq.blobbers[blobberIdx].ID) - // fileref.DeleteFileRef(cacheKey) - // } + deleteReq.consensus.Done() objectTreeRefs[blobberIdx] = refEntity deleteReq.maskMu.Lock() versionMap[refEntity.AllocationVersion]++ From f9f82373ef8084338b08eb4b0f11925dfe29c4c1 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Fri, 26 Jul 2024 14:48:49 +0530 Subject: [PATCH 07/45] fix consensus thresh check --- zboxcore/sdk/deleteworker.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index dfb116a53..927abcd07 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -376,8 +376,8 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( deleteReq.consensus.Done() objectTreeRefs[blobberIdx] = refEntity deleteReq.maskMu.Lock() - versionMap[refEntity.AllocationVersion]++ - if versionMap[refEntity.AllocationVersion] > deleteReq.consensus.consensusThresh { + versionMap[refEntity.AllocationVersion] += 1 + if versionMap[refEntity.AllocationVersion] >= deleteReq.consensus.consensusThresh { consensusRef = refEntity } deleteReq.maskMu.Unlock() From e015027ef52aca37ba042399a963cf1819b323e1 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Fri, 26 Jul 2024 19:43:56 +0530 Subject: [PATCH 08/45] fix path level in delete --- zboxcore/sdk/chunked_upload.go | 3 +-- zboxcore/sdk/deleteworker.go | 11 +++++++++-- 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index f30f318ef..00f514340 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -181,9 +181,9 @@ func CreateChunkedUpload( // su.ctx, su.ctxCncl = context.WithCancel(allocationObj.ctx) su.ctx, su.ctxCncl = context.WithCancelCause(ctx) + su.httpMethod = http.MethodPost if isUpdate { - su.httpMethod = http.MethodPut su.buildChange = func(ref *fileref.FileRef, _ uuid.UUID, ts common.Timestamp) allocationchange.AllocationChange { change := &allocationchange.UpdateFileChange{} change.NewFile = ref @@ -193,7 +193,6 @@ func CreateChunkedUpload( return change } } else { - su.httpMethod = http.MethodPost su.buildChange = func(ref *fileref.FileRef, uid uuid.UUID, ts common.Timestamp) allocationchange.AllocationChange { change := &allocationchange.NewFileChange{} change.File = ref diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index 927abcd07..1db2e9a2c 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -384,7 +384,6 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( }(int(pos)) } deleteReq.wg.Wait() - if !deleteReq.consensus.isConsensusOk() { err := zboxutil.MajorError(blobberErrors) if err != nil { @@ -514,6 +513,12 @@ func (req *DeleteRequest) deleteSubDirectories() error { ops := make([]OperationRequest, 0, len(oResult.Refs)) for _, ref := range oResult.Refs { opMask := req.deleteMask + if ref.Type == fileref.DIRECTORY { + continue + } + if ref.PathLevel > pathLevel { + pathLevel = ref.PathLevel + } op := OperationRequest{ OperationType: constants.FileOperationDelete, RemotePath: ref.Path, @@ -526,7 +531,6 @@ func (req *DeleteRequest) deleteSubDirectories() error { return err } offsetPath = oResult.Refs[len(oResult.Refs)-1].Path - pathLevel = oResult.Refs[len(oResult.Refs)-1].PathLevel if len(oResult.Refs) < getRefPageLimit { break } @@ -534,6 +538,9 @@ func (req *DeleteRequest) deleteSubDirectories() error { // reset offsetPath offsetPath = "" level := len(strings.Split(strings.TrimSuffix(req.remotefilepath, "/"), "/")) + if pathLevel == 0 { + pathLevel = level + 1 + } // list all directories by descending order of path level for pathLevel > level { oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.DIRECTORY, fileref.REGULAR, pathLevel, getRefPageLimit, WithObjectContext(req.ctx), WitObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) From a8bf8bb9edeaac95691d7209d2a21c7f0481567a Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Fri, 26 Jul 2024 22:58:49 +0530 Subject: [PATCH 09/45] remove vt from downloadworker --- zboxcore/sdk/blockdownloadworker.go | 13 ------------- zboxcore/sdk/downloadworker.go | 11 ----------- 2 files changed, 24 deletions(-) diff --git a/zboxcore/sdk/blockdownloadworker.go b/zboxcore/sdk/blockdownloadworker.go index 603003903..7dc4d5375 100644 --- a/zboxcore/sdk/blockdownloadworker.go +++ b/zboxcore/sdk/blockdownloadworker.go @@ -11,7 +11,6 @@ import ( "github.com/0chain/errors" "github.com/0chain/gosdk/core/common" - "github.com/0chain/gosdk/core/util" "github.com/0chain/gosdk/zboxcore/blockchain" "github.com/0chain/gosdk/zboxcore/client" "github.com/0chain/gosdk/zboxcore/fileref" @@ -29,7 +28,6 @@ const ( type BlockDownloadRequest struct { blobber *blockchain.StorageNode - blobberFile *blobberFile allocationID string allocationTx string allocOwnerID string @@ -199,18 +197,7 @@ func (req *BlockDownloadRequest) downloadBlobberBlock(fastClient *fasthttp.Clien dR.Data = respBuf } if req.contentMode == DOWNLOAD_CONTENT_FULL && req.shouldVerify { - - vmp := util.MerklePathForMultiLeafVerification{ - Nodes: dR.Nodes, - Index: dR.Indexes, - RootHash: req.blobberFile.validationRoot, - DataSize: req.blobberFile.size, - } zlogger.Logger.Info("verifying multiple blocks") - err = vmp.VerifyMultipleBlocks(dR.Data) - if err != nil { - return errors.New("merkle_path_verification_error", err.Error()) - } } rspData.idx = req.blobberIdx diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 3bcd518ed..e9c56c131 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -80,7 +80,6 @@ type DownloadRequest struct { endBlock int64 chunkSize int numBlocks int64 - validationRootMap map[string]*blobberFile statusCallback StatusCallback ctx context.Context ctxCncl context.CancelFunc @@ -255,7 +254,6 @@ func (req *DownloadRequest) downloadBlock( remotefilepathhash: req.remotefilepathhash, numBlocks: totalBlock, encryptedKey: req.encryptedKey, - shouldVerify: req.shouldVerify, connectionID: req.connectionID, } @@ -268,8 +266,6 @@ func (req *DownloadRequest) downloadBlock( } if !skipDownload { - bf := req.validationRootMap[blockDownloadReq.blobber.ID] - blockDownloadReq.blobberFile = bf if req.shouldVerify { go AddBlockDownloadReq(req.ctx, blockDownloadReq, nil, req.effectiveBlockSize) } else { @@ -1181,7 +1177,6 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse) return nil, errors.New("consensus_not_met", "") } - req.validationRootMap = make(map[string]*blobberFile) blobberCount := 0 countThreshold := req.consensusThresh + 1 if countThreshold > req.fullconsensus { @@ -1215,12 +1210,6 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse) continue } - blobber := req.blobbers[fmr.blobberIdx] - vr, _ := hex.DecodeString(fmr.fileref.ValidationRoot) - req.validationRootMap[blobber.ID] = &blobberFile{ - size: fmr.fileref.Size, - validationRoot: vr, - } shift := zboxutil.NewUint128(1).Lsh(uint64(fmr.blobberIdx)) foundMask = foundMask.Or(shift) req.downloadQueue[fmr.blobberIdx] = downloadPriority{ From b91b592f21d0f9aa3f0d2cc50827f2feb71d4198 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 13:37:24 +0530 Subject: [PATCH 10/45] rm vt in reader --- zboxcore/sdk/downloadworker.go | 10 ---------- zboxcore/sdk/reader.go | 19 +++++++++---------- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index 3bcd518ed..f4ba045b9 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -80,7 +80,6 @@ type DownloadRequest struct { endBlock int64 chunkSize int numBlocks int64 - validationRootMap map[string]*blobberFile statusCallback StatusCallback ctx context.Context ctxCncl context.CancelFunc @@ -268,8 +267,6 @@ func (req *DownloadRequest) downloadBlock( } if !skipDownload { - bf := req.validationRootMap[blockDownloadReq.blobber.ID] - blockDownloadReq.blobberFile = bf if req.shouldVerify { go AddBlockDownloadReq(req.ctx, blockDownloadReq, nil, req.effectiveBlockSize) } else { @@ -1181,7 +1178,6 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse) return nil, errors.New("consensus_not_met", "") } - req.validationRootMap = make(map[string]*blobberFile) blobberCount := 0 countThreshold := req.consensusThresh + 1 if countThreshold > req.fullconsensus { @@ -1215,12 +1211,6 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse) continue } - blobber := req.blobbers[fmr.blobberIdx] - vr, _ := hex.DecodeString(fmr.fileref.ValidationRoot) - req.validationRootMap[blobber.ID] = &blobberFile{ - size: fmr.fileref.Size, - validationRoot: vr, - } shift := zboxutil.NewUint128(1).Lsh(uint64(fmr.blobberIdx)) foundMask = foundMask.Or(shift) req.downloadQueue[fmr.blobberIdx] = downloadPriority{ diff --git a/zboxcore/sdk/reader.go b/zboxcore/sdk/reader.go index ca380c00f..711bc9c15 100644 --- a/zboxcore/sdk/reader.go +++ b/zboxcore/sdk/reader.go @@ -168,16 +168,15 @@ func GetDStorageFileReader(alloc *Allocation, ref *ORef, sdo *StreamDownloadOpti sd := &StreamDownload{ DownloadRequest: &DownloadRequest{ - allocationID: alloc.ID, - allocationTx: alloc.Tx, - allocOwnerID: alloc.Owner, - allocOwnerPubKey: alloc.OwnerPublicKey, - datashards: alloc.DataShards, - parityshards: alloc.ParityShards, - remotefilepath: ref.Path, - numBlocks: int64(sdo.BlocksPerMarker), - validationRootMap: make(map[string]*blobberFile), - shouldVerify: sdo.VerifyDownload, + allocationID: alloc.ID, + allocationTx: alloc.Tx, + allocOwnerID: alloc.Owner, + allocOwnerPubKey: alloc.OwnerPublicKey, + datashards: alloc.DataShards, + parityshards: alloc.ParityShards, + remotefilepath: ref.Path, + numBlocks: int64(sdo.BlocksPerMarker), + shouldVerify: sdo.VerifyDownload, Consensus: Consensus{ RWMutex: &sync.RWMutex{}, fullconsensus: alloc.fullconsensus, From f5b51c041d86592ff8c7ac37e0075ac0acb2f786 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 14:28:48 +0530 Subject: [PATCH 11/45] remove single blobber for single client mode --- zboxcore/sdk/allocation.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 6154cb69d..117fb8941 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -1429,9 +1429,10 @@ func (a *Allocation) getRefs(path, pathHash, authToken, offsetPath, updatedDate, for _, opt := range opts { opt(oTreeReq) } - if singleClientMode { - oTreeReq.singleBlobber = true - } + //TODO: Have a mask on allocation object to track blobbers which are on latest version + // if singleClientMode { + // oTreeReq.singleBlobber = true + // } return oTreeReq.GetRefs() } From 4d33f272e936abd3365b2a905a0dbcf56f3cf84c Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 16:54:01 +0530 Subject: [PATCH 12/45] add check for nil ref --- zboxcore/sdk/deleteworker.go | 6 +++++- zboxcore/sdk/multi_operation_worker.go | 2 +- 2 files changed, 6 insertions(+), 2 deletions(-) diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index 1db2e9a2c..00c203ef3 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -395,7 +395,8 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( deleteReq.consensus.consensusThresh, deleteReq.consensus.consensus)) } if consensusRef == nil { - return nil, deleteReq.deleteMask, thrown.New("delete_failed", "Delete failed. No consensus found") + //Already deleted + return objectTreeRefs, dop.deleteMask, nil } if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { for ind, refEntity := range objectTreeRefs { @@ -409,6 +410,7 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( } } pos = 0 + deleteReq.consensus.Reset() for i := deleteReq.deleteMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) deleteReq.wg.Add(1) @@ -416,8 +418,10 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( defer deleteReq.wg.Done() err := deleteReq.deleteBlobberFile(deleteReq.blobbers[blobberIdx], blobberIdx) if err != nil { + logger.Logger.Error("error during deleteBlobberFile", err) blobberErrors[blobberIdx] = err } + deleteReq.consensus.Done() if singleClientMode { lookuphash := fileref.GetReferenceLookup(deleteReq.allocationID, deleteReq.remotefilepath) cacheKey := fileref.GetCacheKey(lookuphash, deleteReq.blobbers[blobberIdx].ID) diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index cb2807d1f..fcea1047d 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -189,7 +189,7 @@ func (mo *MultiOperation) Process() error { return } mo.maskMU.Lock() - mo.operationMask = mo.operationMask.Or(mask) + mo.operationMask = mo.operationMask.And(mask) mo.maskMU.Unlock() changes := op.buildChange(refs, uid) mo.changes[idx] = changes From 796de38ecaaf700a7525d52d639780f590310dd8 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 17:14:41 +0530 Subject: [PATCH 13/45] remove mask reset --- zboxcore/sdk/allocation.go | 1 + zboxcore/sdk/multi_operation_worker.go | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 117fb8941..c81a5de9c 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -977,6 +977,7 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...Mul if len(mo.operations) > 0 { err := mo.Process() if err != nil { + logger.Logger.Error("Error in multi operation", zap.Error(err)) return err } diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index fcea1047d..7a200b403 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -167,7 +167,6 @@ func (mo *MultiOperation) Process() error { defer ctxCncl(nil) swg := sizedwaitgroup.New(BatchSize) errsSlice := make([]error, len(mo.operations)) - mo.operationMask = zboxutil.NewUint128(0) for idx, op := range mo.operations { uid := util.GetNewUUID() swg.Add() @@ -312,7 +311,7 @@ func (mo *MultiOperation) Process() error { counter++ } wg.Wait() - logger.Logger.Debug("[commitRequests]", time.Since(start).Milliseconds()) + logger.Logger.Info("[commitRequests]", time.Since(start).Milliseconds()) rollbackMask := zboxutil.NewUint128(0) errSlice := make([]error, len(commitReqs)) for idx, commitReq := range commitReqs { @@ -353,6 +352,7 @@ func (mo *MultiOperation) Process() error { } } mo.allocationObj.AllocationVersion += 1 + logger.Logger.Info("Allocation version updated to ", mo.allocationObj.AllocationVersion) } } From da1aa5526abad6f130ae4fdb4e89f8fc00ab1995 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 17:28:16 +0530 Subject: [PATCH 14/45] check change count --- zboxcore/sdk/commitworker.go | 5 ----- zboxcore/sdk/deleteworker.go | 2 +- zboxcore/sdk/multi_operation_worker.go | 15 ++++++++------- 3 files changed, 9 insertions(+), 13 deletions(-) diff --git a/zboxcore/sdk/commitworker.go b/zboxcore/sdk/commitworker.go index 61830cbf3..ea847b456 100644 --- a/zboxcore/sdk/commitworker.go +++ b/zboxcore/sdk/commitworker.go @@ -102,11 +102,6 @@ func (commitreq *CommitRequest) processCommit() { defer commitreq.wg.Done() start := time.Now() l.Logger.Debug("received a commit request") - if len(commitreq.changes) == 0 { - l.Logger.Debug("Nothing to commit") - commitreq.result = SuccessCommitResult() - return - } err := commitreq.commitBlobber() if err != nil { commitreq.result = ErrorCommitResult(err.Error()) diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index 00c203ef3..9d5750b6d 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -396,7 +396,7 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( } if consensusRef == nil { //Already deleted - return objectTreeRefs, dop.deleteMask, nil + return nil, dop.deleteMask, nil } if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { for ind, refEntity := range objectTreeRefs { diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index 7a200b403..d6d9694c8 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -14,7 +14,6 @@ import ( "github.com/remeh/sizedwaitgroup" "github.com/0chain/gosdk/core/common" - "github.com/0chain/gosdk/core/util" "github.com/0chain/gosdk/zboxcore/allocationchange" "github.com/0chain/gosdk/zboxcore/client" "github.com/0chain/gosdk/zboxcore/fileref" @@ -167,8 +166,8 @@ func (mo *MultiOperation) Process() error { defer ctxCncl(nil) swg := sizedwaitgroup.New(BatchSize) errsSlice := make([]error, len(mo.operations)) + var changeCount int for idx, op := range mo.operations { - uid := util.GetNewUUID() swg.Add() go func(op Operationer, idx int) { defer swg.Done() @@ -189,9 +188,10 @@ func (mo *MultiOperation) Process() error { } mo.maskMU.Lock() mo.operationMask = mo.operationMask.And(mask) + if refs != nil { + changeCount += 1 + } mo.maskMU.Unlock() - changes := op.buildChange(refs, uid) - mo.changes[idx] = changes }(op, idx) } swg.Wait() @@ -212,12 +212,15 @@ func (mo *MultiOperation) Process() error { return nil } + if changeCount == 0 { + return nil + } + // Take transpose of mo.change because it will be easier to iterate mo if it contains blobber changes // in row instead of column. Currently mo.change[0] contains allocationChange for operation 1 and so on. // But we want mo.changes[0] to have allocationChange for blobber 1 and mo.changes[1] to have allocationChange for // blobber 2 and so on. start := time.Now() - mo.changes = zboxutil.Transpose(mo.changes) writeMarkerMutex, err := CreateWriteMarkerMutex(client.GetClient(), mo.allocationObj) if err != nil { @@ -303,8 +306,6 @@ func (mo *MultiOperation) Process() error { blobberInd: pos, version: mo.allocationObj.Blobbers[pos].AllocationVersion + 1, } - - commitReq.changes = append(commitReq.changes, mo.changes[pos]...) commitReqs[counter] = commitReq l.Logger.Debug("Commit request sending to blobber ", commitReq.blobber.Baseurl) go AddCommitRequest(commitReq) From a9814dfb788b56077fcf7cf019749d810ec42de9 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Jul 2024 17:59:07 +0530 Subject: [PATCH 15/45] chane ref check --- zboxcore/sdk/deleteworker.go | 4 +++- zboxcore/sdk/multi_operation_worker.go | 14 +++++++------- 2 files changed, 10 insertions(+), 8 deletions(-) diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index 9d5750b6d..b1a07026e 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -43,6 +43,8 @@ type DeleteRequest struct { timestamp int64 } +var ErrFileDeleted = errors.New("file_deleted", "file is already deleted") + func (req *DeleteRequest) deleteBlobberFile( blobber *blockchain.StorageNode, blobberIdx int) error { @@ -396,7 +398,7 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( } if consensusRef == nil { //Already deleted - return nil, dop.deleteMask, nil + return nil, dop.deleteMask, ErrFileDeleted } if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { for ind, refEntity := range objectTreeRefs { diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index d6d9694c8..fbb71aab6 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -179,18 +179,18 @@ func (mo *MultiOperation) Process() error { default: } - refs, mask, err := op.Process(mo.allocationObj, mo.connectionID) // Process with each blobber + _, mask, err := op.Process(mo.allocationObj, mo.connectionID) // Process with each blobber if err != nil { - l.Logger.Error(err) - errsSlice[idx] = errors.New("", err.Error()) - ctxCncl(err) + if err != ErrFileDeleted { + l.Logger.Error(err) + errsSlice[idx] = errors.New("", err.Error()) + ctxCncl(err) + } return } mo.maskMU.Lock() mo.operationMask = mo.operationMask.And(mask) - if refs != nil { - changeCount += 1 - } + changeCount += 1 mo.maskMU.Unlock() }(op, idx) } From 08de511abcd90db3697c2afb065c71c6242b260b Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Mon, 29 Jul 2024 11:22:44 +0530 Subject: [PATCH 16/45] remove validation root from Ref --- zboxcore/allocationchange/newfile.go | 11 ----------- zboxcore/allocationchange/updatefile.go | 11 ----------- zboxcore/fileref/fileref.go | 19 +++++++------------ zboxcore/sdk/allocation_test.go | 9 +-------- zboxcore/sdk/downloadworker.go | 14 -------------- 5 files changed, 8 insertions(+), 56 deletions(-) diff --git a/zboxcore/allocationchange/newfile.go b/zboxcore/allocationchange/newfile.go index 8e3fd91ea..cdc501e4f 100644 --- a/zboxcore/allocationchange/newfile.go +++ b/zboxcore/allocationchange/newfile.go @@ -35,23 +35,12 @@ func (ch *NewFileChange) ProcessChange(rootRef *fileref.Ref, fileIDMeta map[stri return } - if ch.File.ValidationRoot == "" { - err = errors.New("empty validation root field") - return - } - fileHashSign, err := client.Sign(ch.File.ActualFileHash) if err != nil { return } - validationRootSign, err := client.Sign(fileHashSign + ch.File.ValidationRoot) - if err != nil { - return - } - ch.File.ActualFileHashSignature = fileHashSign - ch.File.ValidationRootSignature = validationRootSign rootRef.HashToBeComputed = true dirRef := rootRef diff --git a/zboxcore/allocationchange/updatefile.go b/zboxcore/allocationchange/updatefile.go index f3b89ae8d..b95b9669f 100644 --- a/zboxcore/allocationchange/updatefile.go +++ b/zboxcore/allocationchange/updatefile.go @@ -23,23 +23,12 @@ func (ch *UpdateFileChange) ProcessChange(rootRef *fileref.Ref, _ map[string]str return } - if ch.NewFile.ValidationRoot == "" { - err = fmt.Errorf("empty validation root field") - return - } - fileHashSign, err := client.Sign(ch.NewFile.ActualFileHash) if err != nil { return } - validationRootSign, err := client.Sign(fileHashSign + ch.NewFile.ValidationRoot) - if err != nil { - return - } - ch.NewFile.ActualFileHashSignature = fileHashSign - ch.NewFile.ValidationRootSignature = validationRootSign fields, err := common.GetPathFields(pathutil.Dir(ch.NewFile.Path)) diff --git a/zboxcore/fileref/fileref.go b/zboxcore/fileref/fileref.go index c79f343ce..7dc024093 100644 --- a/zboxcore/fileref/fileref.go +++ b/zboxcore/fileref/fileref.go @@ -29,16 +29,13 @@ type Collaborator struct { } type FileRef struct { - Ref `mapstructure:",squash"` - CustomMeta string `json:"custom_meta" mapstructure:"custom_meta"` - ValidationRoot string `json:"validation_root" mapstructure:"validation_root"` + Ref `mapstructure:",squash"` + CustomMeta string `json:"custom_meta" mapstructure:"custom_meta"` // ValidationRootSignature is signature signed by client for hash_of(ActualFileHashSignature + ValidationRoot) - ValidationRootSignature string `json:"validation_root_signature" mapstructure:"validation_root_signature"` - FixedMerkleRoot string `json:"fixed_merkle_root" mapstructure:"fixed_merkle_root"` - ThumbnailSize int64 `json:"thumbnail_size" mapstructure:"thumbnail_size"` - ThumbnailHash string `json:"thumbnail_hash" mapstructure:"thumbnail_hash"` - ActualFileSize int64 `json:"actual_file_size" mapstructure:"actual_file_size"` - ActualFileHash string `json:"actual_file_hash" mapstructure:"actual_file_hash"` + ThumbnailSize int64 `json:"thumbnail_size" mapstructure:"thumbnail_size"` + ThumbnailHash string `json:"thumbnail_hash" mapstructure:"thumbnail_hash"` + ActualFileSize int64 `json:"actual_file_size" mapstructure:"actual_file_size"` + ActualFileHash string `json:"actual_file_hash" mapstructure:"actual_file_hash"` // ActualFileHashSignature is signature signed by client for ActualFileHash ActualFileHashSignature string `json:"actual_file_hash_signature" mapstructure:"actual_file_hash_signature"` ActualThumbnailSize int64 `json:"actual_thumbnail_size" mapstructure:"actual_thumbnail_size"` @@ -252,14 +249,12 @@ func (fr *FileRef) GetFileMetaHashData() string { func (fr *FileRef) GetHashData() string { return fmt.Sprintf( - "%s:%s:%s:%s:%d:%s:%s:%d:%s:%d:%s", + "%s:%s:%s:%s:%d:%d:%s:%d:%s", fr.AllocationID, fr.Type, // don't need to add it as well fr.Name, // don't see any utility as fr.Path below has name in it fr.Path, fr.Size, - fr.ValidationRoot, - fr.FixedMerkleRoot, fr.ActualFileSize, fr.ActualFileHash, fr.ChunkSize, diff --git a/zboxcore/sdk/allocation_test.go b/zboxcore/sdk/allocation_test.go index cf283343c..e98d2c23c 100644 --- a/zboxcore/sdk/allocation_test.go +++ b/zboxcore/sdk/allocation_test.go @@ -1171,7 +1171,6 @@ func TestAllocation_GetAuthTicketForShare(t *testing.T) { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: mockValidationRoot, }) require.NoError(t, err) return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) @@ -1226,7 +1225,6 @@ func TestAllocation_GetAuthTicket(t *testing.T) { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: "mock validation root", }) require.NoError(t, err) return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) @@ -1258,7 +1256,6 @@ func TestAllocation_GetAuthTicket(t *testing.T) { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: "mock validation root", }) require.NoError(t, err) return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) @@ -1310,7 +1307,6 @@ func TestAllocation_GetAuthTicket(t *testing.T) { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: "mock validation root", }) require.NoError(t, err) return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) @@ -1351,7 +1347,6 @@ func TestAllocation_GetAuthTicket(t *testing.T) { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: "mock validation root", }) require.NoError(t, err) return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) @@ -2384,7 +2379,6 @@ func setupMockGetFileInfoResponse(t *testing.T, mockClient *mocks.HttpClient) { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: "mock validation root", }) require.NoError(t, err) return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) @@ -2425,8 +2419,7 @@ func getMockAuthTicket(t *testing.T) string { Ref: fileref.Ref{ Name: mockFileRefName, }, - ValidationRoot: "mock validation root", - EncryptedKey: "encrypted key", + EncryptedKey: "encrypted key", }) require.NoError(t, err) diff --git a/zboxcore/sdk/downloadworker.go b/zboxcore/sdk/downloadworker.go index e9c56c131..93d174b91 100644 --- a/zboxcore/sdk/downloadworker.go +++ b/zboxcore/sdk/downloadworker.go @@ -1196,20 +1196,6 @@ func (req *DownloadRequest) getFileMetaConsensus(fMetaResp []*fileMetaResponse) continue } - isValid, err := sys.VerifyWith( - req.allocOwnerPubKey, - fRef.ValidationRootSignature, - fRef.ActualFileHashSignature+fRef.ValidationRoot, - ) - if err != nil { - l.Logger.Error(err, "allocOwnerPubKey: ", req.allocOwnerPubKey, " validationRootSignature: ", fRef.ValidationRootSignature, " actualFileHashSignature: ", fRef.ActualFileHashSignature, " validationRoot: ", fRef.ValidationRoot) - continue - } - if !isValid { - l.Logger.Error("invalid validation root signature") - continue - } - shift := zboxutil.NewUint128(1).Lsh(uint64(fmr.blobberIdx)) foundMask = foundMask.Or(shift) req.downloadQueue[fmr.blobberIdx] = downloadPriority{ From e7df719aa344bcb1094551be1c9b404a5732e3ab Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 31 Jul 2024 13:43:34 +0530 Subject: [PATCH 17/45] add list logs --- zboxcore/sdk/allocation.go | 2 +- zboxcore/sdk/filerefsworker.go | 12 +++++++----- zboxcore/sdk/multi_operation_worker.go | 6 +++--- zboxcore/sdk/repairworker.go | 3 +++ zboxcore/sdk/rollback.go | 2 +- 5 files changed, 15 insertions(+), 10 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index c81a5de9c..13c1ee086 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -202,7 +202,6 @@ type Allocation struct { MovedToValidators common.Balance `json:"moved_to_validators,omitempty"` FileOptions uint16 `json:"file_options"` ThirdPartyExtendable bool `json:"third_party_extendable"` - AllocationVersion int64 `json:"-"` numBlockDownloads int downloadChan chan *DownloadRequest @@ -220,6 +219,7 @@ type Allocation struct { // conseususes consensusThreshold int fullconsensus int + allocationVersion int64 } type OperationRequest struct { diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index dccea9af2..6b7a477ed 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -86,10 +86,9 @@ func WithSingleBlobber(singleBlobber bool) ObjectTreeRequestOption { // Paginated tree should not be collected as this will stall the client // It should rather be handled by application that uses gosdk func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { - totalBlobbersCount := len(o.blobbers) - oTreeResponses := make([]oTreeResponse, totalBlobbersCount) - respChan := make(chan *oTreeResponse, totalBlobbersCount) activeCount := o.reqMask.CountOnes() + oTreeResponses := make([]oTreeResponse, activeCount) + respChan := make(chan *oTreeResponse, activeCount) if o.singleBlobber { var respErr error for i := 0; i < activeCount; i++ { @@ -125,15 +124,16 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { hashCount := make(map[string]int) hashRefsMap := make(map[string]*ObjectTreeResult) - oTreeResponseErrors := make([]error, totalBlobbersCount) + oTreeResponseErrors := make([]error, activeCount) var successCount int - for i := 0; i < totalBlobbersCount; i++ { + for i := 0; i < activeCount; i++ { select { case <-o.ctx.Done(): return nil, o.ctx.Err() case oTreeResponse := <-respChan: oTreeResponseErrors[oTreeResponse.idx] = oTreeResponse.err if oTreeResponse.err != nil { + l.Logger.Error(oTreeResponse.err) if code, _ := zboxutil.GetErrorMessageCode(oTreeResponse.err.Error()); code != INVALID_PATH { l.Logger.Error("Error while getting file refs from blobber:", oTreeResponse.err) } @@ -141,6 +141,7 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { } successCount++ hash := oTreeResponse.hash + l.Logger.Info("getRefsHash", hash) if _, ok := hashCount[hash]; ok { hashCount[hash]++ } else { @@ -255,6 +256,7 @@ func (o *ObjectTreeRequest) getFileRefs(bUrl string, respChan chan *oTreeRespons similarFieldRefs = append(similarFieldRefs, decodeBytes...) } oTR.hash = zboxutil.GetRefsHash(similarFieldRefs) + l.Logger.Info("getFileRefs: ", " blobber: ", bUrl, " path: ", o.remotefilepath, " hash: ", oTR.hash) } // Blobber response will be different from each other so we should only consider similar fields diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index fbb71aab6..0c18cefdc 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -280,7 +280,7 @@ func (mo *MultiOperation) Process() error { var pos uint64 for i := mo.operationMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) - if mo.allocationObj.Blobbers[pos].AllocationVersion != mo.allocationObj.AllocationVersion { + if mo.allocationObj.Blobbers[pos].AllocationVersion != mo.allocationObj.allocationVersion { mo.operationMask = mo.operationMask.And(zboxutil.NewUint128(1).Lsh(pos).Not()) } } @@ -352,8 +352,8 @@ func (mo *MultiOperation) Process() error { mo.allocationObj.Blobbers[commitReq.blobberInd].AllocationVersion++ } } - mo.allocationObj.AllocationVersion += 1 - logger.Logger.Info("Allocation version updated to ", mo.allocationObj.AllocationVersion) + mo.allocationObj.allocationVersion += 1 + logger.Logger.Info("Allocation version updated to ", mo.allocationObj.allocationVersion, " activeBlobbers ", activeBlobbers) } } diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index 466e9ebab..84255e054 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -315,3 +315,6 @@ func (r *RepairRequest) checkForCancel(a *Allocation) bool { } return false } + +//repair for enterprise workflow +// list all files in the allocation recursively and repair them diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index a288c5de2..7295ed51c 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -313,7 +313,7 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { } if consensusReached { - a.AllocationVersion = latestVersion + a.allocationVersion = latestVersion return Commit, blobberRes, nil } From c2e8b89de390bee37d5c236ad5da6b90037f9411 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 31 Jul 2024 14:06:03 +0530 Subject: [PATCH 18/45] add response --- zboxcore/sdk/filerefsworker.go | 5 +++-- zboxcore/zboxutil/util.go | 3 ++- 2 files changed, 5 insertions(+), 3 deletions(-) diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index 6b7a477ed..ca5276d86 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -139,9 +139,10 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { } continue } + oTreeResponses[oTreeResponse.idx] = *oTreeResponse successCount++ hash := oTreeResponse.hash - l.Logger.Info("getRefsHash", hash) + l.Logger.Info("getRefsHash", hash, " blobber: ", o.blobbers[oTreeResponse.idx].Baseurl) if _, ok := hashCount[hash]; ok { hashCount[hash]++ } else { @@ -171,7 +172,7 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { selected = &ObjectTreeResult{} minPage := int64(math.MaxInt64) for _, oTreeResponse := range oTreeResponses { - if oTreeResponse.err != nil { + if oTreeResponse.err != nil || oTreeResponse.oTResult == nil { continue } if oTreeResponse.oTResult.TotalPages < minPage { diff --git a/zboxcore/zboxutil/util.go b/zboxcore/zboxutil/util.go index c216305d1..3c38d91f9 100644 --- a/zboxcore/zboxutil/util.go +++ b/zboxcore/zboxutil/util.go @@ -5,6 +5,7 @@ import ( "crypto/cipher" "crypto/rand" "encoding/base64" + "encoding/hex" "encoding/json" "fmt" "io" @@ -218,7 +219,7 @@ func GetRefsHash(r []byte) string { hash.Write(r) var buf []byte buf = hash.Sum(buf) - return string(buf) + return hex.EncodeToString(buf) } func GetActiveBlobbers(dirMask uint32, blobbers []*blockchain.StorageNode) []*blockchain.StorageNode { From 4496ab62f025baea21a3065709d54975d1a7bbad Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 31 Jul 2024 14:19:45 +0530 Subject: [PATCH 19/45] add debug logs --- zboxcore/sdk/allocation.go | 5 +++++ zboxcore/sdk/filerefsworker.go | 11 ++++++++++- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 13c1ee086..427bd1f47 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -1434,6 +1434,11 @@ func (a *Allocation) getRefs(path, pathHash, authToken, offsetPath, updatedDate, // if singleClientMode { // oTreeReq.singleBlobber = true // } + for _, blob := range a.Blobbers { + if blob.AllocationVersion != a.allocationVersion { + l.Logger.Error("blobber not on latest version: ", blob.Baseurl, " version: ", blob.AllocationVersion, " latest version: ", a.allocationVersion) + } + } return oTreeReq.GetRefs() } diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index ca5276d86..5c1e3f3aa 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -87,6 +87,7 @@ func WithSingleBlobber(singleBlobber bool) ObjectTreeRequestOption { // It should rather be handled by application that uses gosdk func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { activeCount := o.reqMask.CountOnes() + l.Logger.Info("GetRefs: ", " path: ", o.remotefilepath, " pageLimit: ", o.pageLimit, " offsetPath: ", o.offsetPath, " activeCount: ", activeCount) oTreeResponses := make([]oTreeResponse, activeCount) respChan := make(chan *oTreeResponse, activeCount) if o.singleBlobber { @@ -142,7 +143,6 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { oTreeResponses[oTreeResponse.idx] = *oTreeResponse successCount++ hash := oTreeResponse.hash - l.Logger.Info("getRefsHash", hash, " blobber: ", o.blobbers[oTreeResponse.idx].Baseurl) if _, ok := hashCount[hash]; ok { hashCount[hash]++ } else { @@ -154,6 +154,7 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { } } } + l.Logger.Info("no consensus found: ", o.remotefilepath, " pageLimit: ", o.pageLimit, " offsetPath: ", o.offsetPath) var selected *ObjectTreeResult if successCount < o.consensusThresh { majorError := zboxutil.MajorError(oTreeResponseErrors) @@ -190,6 +191,14 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { } } if len(selected.Refs) > 0 { + for _, oTreeResponse := range oTreeResponses { + if oTreeResponse.err != nil || oTreeResponse.oTResult == nil { + continue + } + if len(selected.Refs) != len(oTreeResponse.oTResult.Refs) { + l.Logger.Error("Consensus failed for refs: ", o.blobbers[oTreeResponse.idx].Baseurl) + } + } selected.OffsetPath = selected.Refs[len(selected.Refs)-1].Path return selected, nil } From daa353d079dc500c80e6c7703a70578a577ec1a3 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 31 Jul 2024 14:27:48 +0530 Subject: [PATCH 20/45] add log for selected file ref --- zboxcore/sdk/filerefsworker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index 5c1e3f3aa..cb8bc655b 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -181,6 +181,7 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { selected.TotalPages = minPage } for _, ref := range oTreeResponse.oTResult.Refs { + l.Logger.Info("selectFileRef: ", " blobber: ", o.blobbers[oTreeResponse.idx].Baseurl, " path: ", ref.Path, " hash: ", ref.FileMetaHash) if refHash[ref.FileMetaHash] == o.consensusThresh { continue } From 974959f729468524f32c4c5a99eb288dbb1eaf33 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 31 Jul 2024 22:41:25 +0530 Subject: [PATCH 21/45] add check for rate limit --- zboxcore/sdk/filerefsworker.go | 103 ++++++++++++++++++++------------- 1 file changed, 62 insertions(+), 41 deletions(-) diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index cb8bc655b..4ea47b928 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -15,6 +15,7 @@ import ( "github.com/0chain/errors" "github.com/0chain/gosdk/core/common" "github.com/0chain/gosdk/zboxcore/blockchain" + "github.com/0chain/gosdk/zboxcore/logger" l "github.com/0chain/gosdk/zboxcore/logger" "github.com/0chain/gosdk/zboxcore/marker" "github.com/0chain/gosdk/zboxcore/zboxutil" @@ -57,6 +58,8 @@ type oTreeResponse struct { idx int } +var errTooManyRequests = errors.New("too_many_requests", "Too many requests") + type ObjectTreeRequestOption func(*ObjectTreeRequest) func WithObjectContext(ctx context.Context) ObjectTreeRequestOption { @@ -154,7 +157,7 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { } } } - l.Logger.Info("no consensus found: ", o.remotefilepath, " pageLimit: ", o.pageLimit, " offsetPath: ", o.offsetPath) + l.Logger.Error("no consensus found: ", o.remotefilepath, " pageLimit: ", o.pageLimit, " offsetPath: ", o.offsetPath) var selected *ObjectTreeResult if successCount < o.consensusThresh { majorError := zboxutil.MajorError(oTreeResponseErrors) @@ -181,7 +184,6 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { selected.TotalPages = minPage } for _, ref := range oTreeResponse.oTResult.Refs { - l.Logger.Info("selectFileRef: ", " blobber: ", o.blobbers[oTreeResponse.idx].Baseurl, " path: ", ref.Path, " hash: ", ref.FileMetaHash) if refHash[ref.FileMetaHash] == o.consensusThresh { continue } @@ -213,52 +215,72 @@ func (o *ObjectTreeRequest) getFileRefs(bUrl string, respChan chan *oTreeRespons defer func() { respChan <- oTR }() - oReq, err := zboxutil.NewRefsRequest( - bUrl, - o.allocationID, - o.allocationTx, - o.remotefilepath, - o.pathHash, - o.authToken, - o.offsetPath, - o.updatedDate, - o.offsetDate, - o.fileType, - o.refType, - o.level, - o.pageLimit, - ) - if err != nil { - oTR.err = err - return - } + oResult := ObjectTreeResult{} - ctx, cncl := context.WithTimeout(o.ctx, 2*time.Minute) - err = zboxutil.HttpDo(ctx, cncl, oReq, func(resp *http.Response, err error) error { - if err != nil { - l.Logger.Error(err) - return err - } - defer resp.Body.Close() - respBody, err := ioutil.ReadAll(resp.Body) + for i := 0; i < 3; i++ { + oReq, err := zboxutil.NewRefsRequest( + bUrl, + o.allocationID, + o.allocationTx, + o.remotefilepath, + o.pathHash, + o.authToken, + o.offsetPath, + o.updatedDate, + o.offsetDate, + o.fileType, + o.refType, + o.level, + o.pageLimit, + ) if err != nil { - l.Logger.Error(err) - return err + oTR.err = err + return } - if resp.StatusCode == http.StatusOK { - err := json.Unmarshal(respBody, &oResult) + ctx, cncl := context.WithTimeout(o.ctx, 2*time.Minute) + defer cncl() + err = zboxutil.HttpDo(ctx, cncl, oReq, func(resp *http.Response, err error) error { if err != nil { l.Logger.Error(err) return err } - return nil - } else { - return errors.New("response_error", fmt.Sprintf("got status %d, err: %s", resp.StatusCode, respBody)) + defer resp.Body.Close() + respBody, err := ioutil.ReadAll(resp.Body) + if err != nil { + l.Logger.Error(err) + return err + } + if resp.StatusCode == http.StatusOK { + err := json.Unmarshal(respBody, &oResult) + if err != nil { + l.Logger.Error(err) + return err + } + return nil + } else { + if resp.StatusCode == http.StatusTooManyRequests { + l.Logger.Error("Too many requests") + r, err := zboxutil.GetRateLimitValue(resp) + if err != nil { + logger.Logger.Error(err) + return err + } + + time.Sleep(time.Duration(r) * time.Second) + return errTooManyRequests + } + + return errors.New("response_error", fmt.Sprintf("got status %d, err: %s", resp.StatusCode, respBody)) + } + }) + if err != nil { + if err == errTooManyRequests && i < 2 { + continue + } + oTR.err = err + return } - }) - if err != nil { - oTR.err = err - return + break } oTR.oTResult = &oResult similarFieldRefs := make([]byte, 0, 32*len(oResult.Refs)) @@ -267,7 +289,6 @@ func (o *ObjectTreeRequest) getFileRefs(bUrl string, respChan chan *oTreeRespons similarFieldRefs = append(similarFieldRefs, decodeBytes...) } oTR.hash = zboxutil.GetRefsHash(similarFieldRefs) - l.Logger.Info("getFileRefs: ", " blobber: ", bUrl, " path: ", o.remotefilepath, " hash: ", oTR.hash) } // Blobber response will be different from each other so we should only consider similar fields From ec1a470a3fe7bfd8340b29c223863d9788216a1f Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 1 Aug 2024 00:15:19 +0530 Subject: [PATCH 22/45] remove get ref log --- zboxcore/sdk/filerefsworker.go | 1 - 1 file changed, 1 deletion(-) diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index 4ea47b928..6b2f5714d 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -90,7 +90,6 @@ func WithSingleBlobber(singleBlobber bool) ObjectTreeRequestOption { // It should rather be handled by application that uses gosdk func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { activeCount := o.reqMask.CountOnes() - l.Logger.Info("GetRefs: ", " path: ", o.remotefilepath, " pageLimit: ", o.pageLimit, " offsetPath: ", o.offsetPath, " activeCount: ", activeCount) oTreeResponses := make([]oTreeResponse, activeCount) respChan := make(chan *oTreeResponse, activeCount) if o.singleBlobber { From facdb4f89c04d7d1f419fe83ba26b69cd3c14c2c Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Mon, 5 Aug 2024 15:02:13 +0530 Subject: [PATCH 23/45] add repair for enterprise blobbers --- zboxcore/sdk/allocation.go | 27 +- zboxcore/sdk/chunked_upload.go | 11 +- zboxcore/sdk/common.go | 96 +++++ zboxcore/sdk/copyworker.go | 169 ++++++++- zboxcore/sdk/copyworker_test.go | 2 +- zboxcore/sdk/deleteworker.go | 10 +- zboxcore/sdk/filerefsworker.go | 24 +- zboxcore/sdk/moveworker.go | 92 ++++- zboxcore/sdk/multi_operation_worker.go | 26 +- zboxcore/sdk/renameworker.go | 80 +++- zboxcore/sdk/renameworker_test.go | 2 +- zboxcore/sdk/repairworker.go | 506 +++++++++++++++++-------- 12 files changed, 794 insertions(+), 251 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 427bd1f47..9f06ed3b9 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -39,12 +39,13 @@ import ( ) var ( - noBLOBBERS = errors.New("", "No Blobbers set in this allocation") - notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object") - IsWasm = false - MultiOpBatchSize = 50 - RepairBatchSize = 50 - Workdir string + noBLOBBERS = errors.New("", "No Blobbers set in this allocation") + notInitialized = errors.New("sdk_not_initialized", "Please call InitStorageSDK Init and use GetAllocation to get the allocation object") + IsWasm = false + MultiOpBatchSize = 50 + RepairBatchSize = 50 + Workdir string + multiOpRepairBatchSize = 10 ) const ( @@ -409,12 +410,11 @@ func (a *Allocation) UploadFile(workdir, localpath string, remotepath string, return a.StartChunkedUpload(workdir, localpath, remotepath, status, false, false, "", false, false) } -func (a *Allocation) RepairFile(file sys.File, remotepath string, statusCallback StatusCallback, mask zboxutil.Uint128, ref *fileref.FileRef) *OperationRequest { +func (a *Allocation) RepairFile(file sys.File, remotepath string, statusCallback StatusCallback, mask zboxutil.Uint128, ref ORef) *OperationRequest { idr, _ := homedir.Dir() if Workdir != "" { idr = Workdir } - mask = mask.Not().And(zboxutil.NewUint128(1).Lsh(uint64(len(a.Blobbers))).Sub64(1)) fileMeta := FileMeta{ ActualSize: ref.ActualFileSize, MimeType: ref.MimeType, @@ -937,7 +937,7 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...Mul case constants.FileOperationDelete: if op.Mask != nil { - operation = NewDeleteOperation(op.RemotePath, *op.Mask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) + operation = NewDeleteOperation(op.RemotePath, *op.Mask, mo.maskMU, op.Mask.CountOnes(), mo.fullconsensus, mo.ctx) } else { operation = NewDeleteOperation(op.RemotePath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) } @@ -1434,11 +1434,6 @@ func (a *Allocation) getRefs(path, pathHash, authToken, offsetPath, updatedDate, // if singleClientMode { // oTreeReq.singleBlobber = true // } - for _, blob := range a.Blobbers { - if blob.AllocationVersion != a.allocationVersion { - l.Logger.Error("blobber not on latest version: ", blob.Baseurl, " version: ", blob.AllocationVersion, " latest version: ", a.allocationVersion) - } - } return oTreeReq.GetRefs() } @@ -1536,7 +1531,7 @@ func (a *Allocation) GetRefs(path, offsetPath, updatedDate, offsetDate, fileType return nil, errors.New("invalid_path", fmt.Sprintf("Absolute path required. Path provided: %v", path)) } - return a.getRefs(path, "", "", offsetPath, updatedDate, offsetDate, fileType, refType, level, pageLimit) + return a.getRefs(path, "", "", offsetPath, updatedDate, offsetDate, fileType, refType, level, pageLimit, opts...) } func (a *Allocation) GetRefsFromLookupHash(pathHash, offsetPath, updatedDate, offsetDate, fileType, refType string, level, pageLimit int) (*ObjectTreeResult, error) { @@ -2316,11 +2311,13 @@ func (a *Allocation) StartRepair(localRootPath, pathToRepair string, statusCB St if err != nil { return err } + a.CheckAllocStatus() //nolint:errcheck repairReq := &RepairRequest{ listDir: listDir, localRootPath: localRootPath, statusCB: statusCB, + repairPath: pathToRepair, } repairReq.completedCallback = func() { diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index 00f514340..2798b851b 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -154,11 +154,6 @@ func CreateChunkedUpload( } uploadMask := zboxutil.NewUint128(1).Lsh(uint64(len(allocationObj.Blobbers))).Sub64(1) - if isRepair { - opCode = OpUpdate - consensus.fullconsensus = uploadMask.CountOnes() - consensus.consensusThresh = 1 - } su := &ChunkedUpload{ allocationObj: allocationObj, @@ -216,6 +211,12 @@ func CreateChunkedUpload( opt(su) } + if isRepair { + opCode = OpUpdate + su.consensus.fullconsensus = su.uploadMask.CountOnes() + su.consensus.consensusThresh = su.uploadMask.CountOnes() + } + if su.progressStorer == nil && shouldSaveProgress { su.progressStorer = createFsChunkedUploadProgress(context.Background()) } diff --git a/zboxcore/sdk/common.go b/zboxcore/sdk/common.go index 211effe76..6f5a6fd8a 100644 --- a/zboxcore/sdk/common.go +++ b/zboxcore/sdk/common.go @@ -8,6 +8,7 @@ import ( "net/http" "path" "strconv" + "strings" "sync" "time" @@ -120,3 +121,98 @@ func ValidateRemoteFileName(remotePath string) error { return nil } + +type subDirRequest struct { + opType string + remotefilepath string + destPath string + allocationObj *Allocation + ctx context.Context + consensusThresh int + mask zboxutil.Uint128 +} + +func (req *subDirRequest) processSubDirectories() error { + var ( + offsetPath string + pathLevel int + ) + + for { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WithObjectConsensusThresh(req.consensusThresh), WithSingleBlobber(true), WithObjectMask(req.mask)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + break + } + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.mask + if ref.Type == fileref.DIRECTORY { + continue + } + if ref.PathLevel > pathLevel { + pathLevel = ref.PathLevel + } + destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + op := OperationRequest{ + OperationType: req.opType, + RemotePath: ref.Path, + DestPath: destPath, + Mask: &opMask, + } + ops = append(ops, op) + } + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err + } + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + if len(oResult.Refs) < getRefPageLimit { + break + } + } + + offsetPath = "" + level := len(strings.Split(strings.TrimSuffix(req.remotefilepath, "/"), "/")) + if pathLevel == 0 { + pathLevel = level + 1 + } + + for pathLevel > level { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.DIRECTORY, fileref.REGULAR, pathLevel, getRefPageLimit, WithObjectContext(req.ctx), WithObjectMask(req.mask), WithObjectConsensusThresh(req.consensusThresh), WithSingleBlobber(true)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + pathLevel-- + } else { + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.mask + if ref.Type == fileref.FILE { + continue + } + destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + op := OperationRequest{ + OperationType: req.opType, + RemotePath: ref.Path, + DestPath: destPath, + Mask: &opMask, + } + ops = append(ops, op) + } + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err + } + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + if len(oResult.Refs) < getRefPageLimit { + pathLevel-- + } + } + } + + return nil +} diff --git a/zboxcore/sdk/copyworker.go b/zboxcore/sdk/copyworker.go index cb47ee6a0..98678545d 100644 --- a/zboxcore/sdk/copyworker.go +++ b/zboxcore/sdk/copyworker.go @@ -47,8 +47,27 @@ func (req *CopyRequest) getObjectTreeFromBlobber(blobber *blockchain.StorageNode return getObjectTreeFromBlobber(req.ctx, req.allocationID, req.allocationTx, req.remotefilepath, blobber) } +func (req *CopyRequest) getFileMetaFromBlobber(pos int) (fileRef *fileref.FileRef, err error) { + listReq := &ListRequest{ + allocationID: req.allocationID, + allocationTx: req.allocationTx, + blobbers: req.blobbers, + remotefilepath: req.remotefilepath, + ctx: req.ctx, + } + respChan := make(chan *fileMetaResponse) + go listReq.getFileMetaInfoFromBlobber(req.blobbers[pos], int(pos), respChan) + refRes := <-respChan + if refRes.err != nil { + err = refRes.err + return + } + fileRef = refRes.fileref + return +} + func (req *CopyRequest) copyBlobberObject( - blobber *blockchain.StorageNode, blobberIdx int) (refEntity fileref.RefEntity, err error) { + blobber *blockchain.StorageNode, blobberIdx int) (err error) { defer func() { if err != nil { @@ -58,11 +77,6 @@ func (req *CopyRequest) copyBlobberObject( req.maskMU.Unlock() } }() - refEntity, err = req.getObjectTreeFromBlobber(req.blobbers[blobberIdx]) - if err != nil { - return nil, err - } - var resp *http.Response var shouldContinue bool var latestRespMsg string @@ -159,15 +173,19 @@ func (req *CopyRequest) copyBlobberObject( } return } - return nil, errors.New("unknown_issue", + return errors.New("unknown_issue", fmt.Sprintf("last status code: %d, last response message: %s", latestStatusCode, latestRespMsg)) } -func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, []error) { - var pos uint64 +func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { + var ( + pos uint64 + consensusRef *fileref.FileRef + ) numList := len(req.blobbers) objectTreeRefs := make([]fileref.RefEntity, numList) blobberErrors := make([]error, numList) + versionMap := make(map[int64]int) wg := &sync.WaitGroup{} for i := req.copyMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { @@ -175,17 +193,55 @@ func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, []error) { wg.Add(1) go func(blobberIdx int) { defer wg.Done() - refEntity, err := req.copyBlobberObject(req.blobbers[blobberIdx], blobberIdx) + // refEntity, err := req.copyBlobberObject(req.blobbers[blobberIdx], blobberIdx) + refEntity, err := req.getFileMetaFromBlobber(blobberIdx) if err != nil { blobberErrors[blobberIdx] = err l.Logger.Debug(err.Error()) return } objectTreeRefs[blobberIdx] = refEntity + req.maskMU.Lock() + versionMap[refEntity.AllocationVersion] += 1 + if versionMap[refEntity.AllocationVersion] >= req.consensusThresh { + consensusRef = refEntity + } + req.maskMU.Unlock() + }(int(pos)) + } + wg.Wait() + if consensusRef == nil { + return nil, zboxutil.MajorError(blobberErrors) + } + + if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { + for ind, refEntity := range objectTreeRefs { + if refEntity.GetAllocationVersion() != consensusRef.AllocationVersion { + req.copyMask = req.copyMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) + } + } + err := req.copySubDirectoriees() + if err != nil { + return nil, err + } + } + + for i := req.copyMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + wg.Add(1) + go func(blobberIdx int) { + defer wg.Done() + err := req.copyBlobberObject(req.blobbers[blobberIdx], blobberIdx) + if err != nil { + blobberErrors[blobberIdx] = err + l.Logger.Debug(err.Error()) + return + } }(int(pos)) } wg.Wait() - return objectTreeRefs, blobberErrors + + return objectTreeRefs, zboxutil.MajorError(blobberErrors) } func (req *CopyRequest) ProcessCopy() error { @@ -194,10 +250,9 @@ func (req *CopyRequest) ProcessCopy() error { wg := &sync.WaitGroup{} var pos uint64 - objectTreeRefs, blobberErrors := req.ProcessWithBlobbers() + objectTreeRefs, err := req.ProcessWithBlobbers() if !req.isConsensusOk() { - err := zboxutil.MajorError(blobberErrors) if err != nil { return errors.New("copy_failed", fmt.Sprintf("Copy failed. %s", err.Error())) } @@ -321,11 +376,10 @@ func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]f cR.consensusThresh = co.consensusThresh cR.fullconsensus = co.fullconsensus - objectTreeRefs, blobberErrors := cR.ProcessWithBlobbers() + objectTreeRefs, err := cR.ProcessWithBlobbers() if !cR.isConsensusOk() { l.Logger.Error("copy failed: ", cR.remotefilepath, cR.destPath) - err := zboxutil.MajorError(blobberErrors) if err != nil { return nil, cR.copyMask, errors.New("copy_failed", fmt.Sprintf("Copy failed. %s", err.Error())) } @@ -403,3 +457,88 @@ func NewCopyOperation(remotePath string, destPath string, copyMask zboxutil.Uint return co } + +func (req *CopyRequest) copySubDirectoriees() error { + var ( + offsetPath string + pathLevel int + ) + + for { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WithObjectConsensusThresh(req.consensusThresh), WithSingleBlobber(true)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + break + } + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.copyMask + if ref.Type == fileref.DIRECTORY { + continue + } + if ref.PathLevel > pathLevel { + pathLevel = ref.PathLevel + } + destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + op := OperationRequest{ + OperationType: constants.FileOperationCopy, + RemotePath: ref.Path, + DestPath: destPath, + Mask: &opMask, + } + ops = append(ops, op) + } + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err + } + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + if len(oResult.Refs) < getRefPageLimit { + break + } + } + + offsetPath = "" + level := len(strings.Split(strings.TrimSuffix(req.remotefilepath, "/"), "/")) + if pathLevel == 0 { + pathLevel = level + 1 + } + + for pathLevel > level { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.DIRECTORY, fileref.REGULAR, pathLevel, getRefPageLimit, WithObjectContext(req.ctx), WithObjectMask(req.copyMask), WithObjectConsensusThresh(req.consensusThresh), WithSingleBlobber(true)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + pathLevel-- + } else { + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.copyMask + if ref.Type == fileref.FILE { + continue + } + destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + op := OperationRequest{ + OperationType: constants.FileOperationCopy, + RemotePath: ref.Path, + DestPath: destPath, + Mask: &opMask, + } + ops = append(ops, op) + } + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err + } + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + if len(oResult.Refs) < getRefPageLimit { + pathLevel-- + } + } + } + + return nil +} diff --git a/zboxcore/sdk/copyworker_test.go b/zboxcore/sdk/copyworker_test.go index 007b4b837..374021805 100644 --- a/zboxcore/sdk/copyworker_test.go +++ b/zboxcore/sdk/copyworker_test.go @@ -216,7 +216,7 @@ func TestCopyRequest_copyBlobberObject(t *testing.T) { Baseurl: tt.name, }) req.copyMask = zboxutil.NewUint128(1).Lsh(uint64(len(req.blobbers))).Sub64(1) - _, err := req.copyBlobberObject(req.blobbers[0], 0) + err := req.copyBlobberObject(req.blobbers[0], 0) require.EqualValues(tt.wantErr, err != nil) if err != nil { require.Contains(errors.Top(err), tt.errMsg) diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index b1a07026e..db3922422 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -402,6 +402,9 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( } if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { for ind, refEntity := range objectTreeRefs { + if refEntity == nil { + continue + } if refEntity.GetAllocationVersion() != consensusRef.AllocationVersion { deleteReq.deleteMask = deleteReq.deleteMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) } @@ -411,6 +414,9 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( return nil, deleteReq.deleteMask, err } } + if dop.remotefilepath == "/" { + return objectTreeRefs, deleteReq.deleteMask, nil + } pos = 0 deleteReq.consensus.Reset() for i := deleteReq.deleteMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { @@ -509,7 +515,7 @@ func (req *DeleteRequest) deleteSubDirectories() error { pathLevel int ) for { - oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WitObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WithObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) if err != nil { return err } @@ -549,7 +555,7 @@ func (req *DeleteRequest) deleteSubDirectories() error { } // list all directories by descending order of path level for pathLevel > level { - oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.DIRECTORY, fileref.REGULAR, pathLevel, getRefPageLimit, WithObjectContext(req.ctx), WitObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.DIRECTORY, fileref.REGULAR, pathLevel, getRefPageLimit, WithObjectContext(req.ctx), WithObjectMask(req.deleteMask), WithObjectConsensusThresh(req.consensus.consensusThresh), WithSingleBlobber(true)) if err != nil { return err } diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index 6b2f5714d..10c9edad0 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -17,16 +17,14 @@ import ( "github.com/0chain/gosdk/zboxcore/blockchain" "github.com/0chain/gosdk/zboxcore/logger" l "github.com/0chain/gosdk/zboxcore/logger" - "github.com/0chain/gosdk/zboxcore/marker" "github.com/0chain/gosdk/zboxcore/zboxutil" ) type ObjectTreeResult struct { - TotalPages int64 `json:"total_pages"` - OffsetPath string `json:"offset_path"` - OffsetDate string `json:"offset_date"` - Refs []ORef `json:"refs"` - LatestWM *marker.WriteMarker `json:"latest_write_marker"` + TotalPages int64 `json:"total_pages"` + OffsetPath string `json:"offset_path"` + OffsetDate string `json:"offset_date"` + Refs []ORef `json:"refs"` } const INVALID_PATH = "invalid_path" @@ -68,7 +66,7 @@ func WithObjectContext(ctx context.Context) ObjectTreeRequestOption { } } -func WitObjectMask(mask zboxutil.Uint128) ObjectTreeRequestOption { +func WithObjectMask(mask zboxutil.Uint128) ObjectTreeRequestOption { return func(o *ObjectTreeRequest) { o.reqMask = mask } @@ -97,7 +95,16 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { for i := 0; i < activeCount; i++ { var rnd = rand.New(rand.NewSource(time.Now().UnixNano())) num := rnd.Intn(activeCount) - blob := o.blobbers[num] + var blob *blockchain.StorageNode + var pos uint64 + for i := o.reqMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + num-- + if num < 0 { + blob = o.blobbers[pos] + break + } + } l.Logger.Debug(fmt.Sprintf("Getting file refs for path %v from blobber %v", o.remotefilepath, blob.Baseurl)) idx := num baseURL := blob.Baseurl @@ -312,6 +319,7 @@ type SimilarField struct { PathLevel int `json:"level"` Size int64 `json:"size"` EncryptedKey string `json:"encrypted_key"` + EncryptedKeyPoint string `json:"encrypted_key_point"` ActualFileSize int64 `json:"actual_file_size"` ActualFileHash string `json:"actual_file_hash"` MimeType string `json:"mimetype"` diff --git a/zboxcore/sdk/moveworker.go b/zboxcore/sdk/moveworker.go index 9e9e08dc8..dc9891ce8 100644 --- a/zboxcore/sdk/moveworker.go +++ b/zboxcore/sdk/moveworker.go @@ -47,8 +47,27 @@ func (req *MoveRequest) getObjectTreeFromBlobber(blobber *blockchain.StorageNode return getObjectTreeFromBlobber(req.ctx, req.allocationID, req.allocationTx, req.remotefilepath, blobber) } +func (req *MoveRequest) getFileMetaFromBlobber(pos int) (fileRef *fileref.FileRef, err error) { + listReq := &ListRequest{ + allocationID: req.allocationID, + allocationTx: req.allocationTx, + blobbers: req.blobbers, + remotefilepath: req.remotefilepath, + ctx: req.ctx, + } + respChan := make(chan *fileMetaResponse) + go listReq.getFileMetaInfoFromBlobber(req.blobbers[pos], int(pos), respChan) + refRes := <-respChan + if refRes.err != nil { + err = refRes.err + return + } + fileRef = refRes.fileref + return +} + func (req *MoveRequest) moveBlobberObject( - blobber *blockchain.StorageNode, blobberIdx int) (refEntity fileref.RefEntity, err error) { + blobber *blockchain.StorageNode, blobberIdx int) (err error) { defer func() { if err != nil { @@ -58,10 +77,6 @@ func (req *MoveRequest) moveBlobberObject( req.maskMU.Unlock() } }() - refEntity, err = req.getObjectTreeFromBlobber(req.blobbers[blobberIdx]) - if err != nil { - return nil, err - } var resp *http.Response var shouldContinue bool @@ -156,32 +171,81 @@ func (req *MoveRequest) moveBlobberObject( } return } - return nil, errors.New("unknown_issue", + return errors.New("unknown_issue", fmt.Sprintf("last status code: %d, last response message: %s", latestStatusCode, latestRespMsg)) } -func (req *MoveRequest) ProcessWithBlobbers() ([]fileref.RefEntity, []error) { - var pos uint64 +func (req *MoveRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { + var ( + pos uint64 + consensusRef *fileref.FileRef + ) numList := len(req.blobbers) objectTreeRefs := make([]fileref.RefEntity, numList) blobberErrors := make([]error, numList) + versionMap := make(map[int64]int) wg := &sync.WaitGroup{} for i := req.moveMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) wg.Add(1) go func(blobberIdx int) { defer wg.Done() - refEntity, err := req.moveBlobberObject(req.blobbers[blobberIdx], blobberIdx) + refEntity, err := req.getFileMetaFromBlobber(blobberIdx) if err != nil { blobberErrors[blobberIdx] = err - l.Logger.Error(err.Error()) + l.Logger.Debug(err.Error()) return } objectTreeRefs[blobberIdx] = refEntity + req.maskMU.Lock() + versionMap[refEntity.AllocationVersion] += 1 + if versionMap[refEntity.AllocationVersion] >= req.consensusThresh { + consensusRef = refEntity + } + req.maskMU.Unlock() + }(int(pos)) + } + wg.Wait() + if consensusRef == nil { + return nil, zboxutil.MajorError(blobberErrors) + } + + if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { + for ind, refEntity := range objectTreeRefs { + if refEntity.GetAllocationVersion() != consensusRef.AllocationVersion { + req.moveMask = req.moveMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) + } + } + subRequest := &subDirRequest{ + allocationObj: req.allocationObj, + remotefilepath: req.remotefilepath, + destPath: req.destPath, + ctx: req.ctx, + consensusThresh: req.consensusThresh, + opType: constants.FileOperationMove, + mask: req.moveMask, + } + err := subRequest.processSubDirectories() + if err != nil { + return nil, err + } + } + + for i := req.moveMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + wg.Add(1) + go func(blobberIdx int) { + defer wg.Done() + err := req.moveBlobberObject(req.blobbers[blobberIdx], blobberIdx) + if err != nil { + blobberErrors[blobberIdx] = err + l.Logger.Debug(err.Error()) + return + } }(int(pos)) } wg.Wait() - return objectTreeRefs, blobberErrors + return objectTreeRefs, zboxutil.MajorError(blobberErrors) } func (req *MoveRequest) ProcessMove() error { @@ -190,10 +254,9 @@ func (req *MoveRequest) ProcessMove() error { wg := &sync.WaitGroup{} var pos uint64 - objectTreeRefs, blobberErrors := req.ProcessWithBlobbers() + objectTreeRefs, err := req.ProcessWithBlobbers() if !req.isConsensusOk() { - err := zboxutil.MajorError(blobberErrors) if err != nil { return errors.New("move_failed", fmt.Sprintf("Move failed. %s", err.Error())) } @@ -314,10 +377,9 @@ func (mo *MoveOperation) Process(allocObj *Allocation, connectionID string) ([]f mR.Consensus.fullconsensus = mo.consensus.fullconsensus mR.Consensus.consensusThresh = mo.consensus.consensusThresh - objectTreeRefs, blobberErrors := mR.ProcessWithBlobbers() + objectTreeRefs, err := mR.ProcessWithBlobbers() if !mR.Consensus.isConsensusOk() { - err := zboxutil.MajorError(blobberErrors) if err != nil { return nil, mR.moveMask, thrown.New("move_failed", fmt.Sprintf("Move failed. %s", err.Error())) } diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index 0c18cefdc..fdbac3959 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -32,10 +32,12 @@ var BatchSize = 6 type MultiOperationOption func(mo *MultiOperation) -func WithRepair() MultiOperationOption { +func WithRepair(latestVersion int64, repairOffsetPath string) MultiOperationOption { return func(mo *MultiOperation) { mo.Consensus.consensusThresh = 0 mo.isRepair = true + mo.repairVersion = latestVersion + mo.repairOffset = repairOffsetPath } } @@ -56,8 +58,10 @@ type MultiOperation struct { operationMask zboxutil.Uint128 maskMU *sync.Mutex Consensus - changes [][]allocationchange.AllocationChange - isRepair bool + changes [][]allocationchange.AllocationChange + isRepair bool + repairVersion int64 + repairOffset string } func (mo *MultiOperation) createConnectionObj(blobberIdx int) (err error) { @@ -278,10 +282,12 @@ func (mo *MultiOperation) Process() error { logger.Logger.Debug("[checkAllocStatus]", time.Since(start).Milliseconds()) mo.Consensus.Reset() var pos uint64 - for i := mo.operationMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { - pos = uint64(i.TrailingZeros()) - if mo.allocationObj.Blobbers[pos].AllocationVersion != mo.allocationObj.allocationVersion { - mo.operationMask = mo.operationMask.And(zboxutil.NewUint128(1).Lsh(pos).Not()) + if !mo.isRepair { + for i := mo.operationMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + if mo.allocationObj.Blobbers[pos].AllocationVersion != mo.allocationObj.allocationVersion { + mo.operationMask = mo.operationMask.And(zboxutil.NewUint128(1).Lsh(pos).Not()) + } } } activeBlobbers := mo.operationMask.CountOnes() @@ -306,6 +312,12 @@ func (mo *MultiOperation) Process() error { blobberInd: pos, version: mo.allocationObj.Blobbers[pos].AllocationVersion + 1, } + if mo.isRepair { + commitReq.isRepair = true + commitReq.version = mo.allocationObj.Blobbers[pos].AllocationVersion + commitReq.repairVersion = mo.repairVersion + commitReq.repairOffset = mo.repairOffset + } commitReqs[counter] = commitReq l.Logger.Debug("Commit request sending to blobber ", commitReq.blobber.Baseurl) go AddCommitRequest(commitReq) diff --git a/zboxcore/sdk/renameworker.go b/zboxcore/sdk/renameworker.go index 491c6f08d..de494b983 100644 --- a/zboxcore/sdk/renameworker.go +++ b/zboxcore/sdk/renameworker.go @@ -47,8 +47,27 @@ func (req *RenameRequest) getObjectTreeFromBlobber(blobber *blockchain.StorageNo return getObjectTreeFromBlobber(req.ctx, req.allocationID, req.allocationTx, req.remotefilepath, blobber) } +func (req *RenameRequest) getFileMetaFromBlobber(pos int) (fileRef *fileref.FileRef, err error) { + listReq := &ListRequest{ + allocationID: req.allocationID, + allocationTx: req.allocationTx, + blobbers: req.blobbers, + remotefilepath: req.remotefilepath, + ctx: req.ctx, + } + respChan := make(chan *fileMetaResponse) + go listReq.getFileMetaInfoFromBlobber(req.blobbers[pos], int(pos), respChan) + refRes := <-respChan + if refRes.err != nil { + err = refRes.err + return + } + fileRef = refRes.fileref + return +} + func (req *RenameRequest) renameBlobberObject( - blobber *blockchain.StorageNode, blobberIdx int) (refEntity fileref.RefEntity, err error) { + blobber *blockchain.StorageNode, blobberIdx int) (err error) { defer func() { if err != nil { @@ -58,11 +77,6 @@ func (req *RenameRequest) renameBlobberObject( } }() - refEntity, err = req.getObjectTreeFromBlobber(req.blobbers[blobberIdx]) - if err != nil { - return nil, err - } - var ( resp *http.Response shouldContinue bool @@ -159,37 +173,76 @@ func (req *RenameRequest) renameBlobberObject( return } -func (req *RenameRequest) ProcessWithBlobbers() ([]fileref.RefEntity, []error) { - var pos uint64 +func (req *RenameRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { + var ( + pos uint64 + consensusRef *fileref.FileRef + ) numList := len(req.blobbers) objectTreeRefs := make([]fileref.RefEntity, numList) blobberErrors := make([]error, numList) + versionMap := make(map[int64]int) req.wg = &sync.WaitGroup{} for i := req.renameMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { pos = uint64(i.TrailingZeros()) req.wg.Add(1) go func(blobberIdx int) { defer req.wg.Done() - refEntity, err := req.renameBlobberObject(req.blobbers[blobberIdx], blobberIdx) + refEntity, err := req.getFileMetaFromBlobber(blobberIdx) if err != nil { blobberErrors[blobberIdx] = err l.Logger.Error(err.Error()) return } objectTreeRefs[blobberIdx] = refEntity + req.maskMU.Lock() + versionMap[refEntity.AllocationVersion] += 1 + if versionMap[refEntity.AllocationVersion] >= req.consensus.consensusThresh { + consensusRef = refEntity + } + req.maskMU.Unlock() }(int(pos)) } req.wg.Wait() - return objectTreeRefs, blobberErrors + if consensusRef == nil { + return nil, zboxutil.MajorError(blobberErrors) + } + if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { + for ind, refEntity := range objectTreeRefs { + if refEntity.GetAllocationVersion() != consensusRef.AllocationVersion { + req.renameMask = req.renameMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) + } + } + // err := req.copySubDirectoriees() + // if err != nil { + // return nil, err + // } + } + + for i := req.renameMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + pos = uint64(i.TrailingZeros()) + req.wg.Add(1) + go func(blobberIdx int) { + defer req.wg.Done() + err := req.renameBlobberObject(req.blobbers[blobberIdx], blobberIdx) + if err != nil { + blobberErrors[blobberIdx] = err + l.Logger.Debug(err.Error()) + return + } + }(int(pos)) + } + req.wg.Wait() + + return objectTreeRefs, zboxutil.MajorError(blobberErrors) } func (req *RenameRequest) ProcessRename() error { defer req.ctxCncl() - objectTreeRefs, blobberErrors := req.ProcessWithBlobbers() + objectTreeRefs, err := req.ProcessWithBlobbers() if !req.consensus.isConsensusOk() { - err := zboxutil.MajorError(blobberErrors) if err != nil { return errors.New("rename_failed", fmt.Sprintf("Rename failed. %s", err.Error())) @@ -324,10 +377,9 @@ func (ro *RenameOperation) Process(allocObj *Allocation, connectionID string) ([ rR.consensus.fullconsensus = ro.consensus.fullconsensus rR.consensus.consensusThresh = ro.consensus.consensusThresh - objectTreeRefs, blobberErrors := rR.ProcessWithBlobbers() + objectTreeRefs, err := rR.ProcessWithBlobbers() if !rR.consensus.isConsensusOk() { - err := zboxutil.MajorError(blobberErrors) if err != nil { return nil, rR.renameMask, errors.New("rename_failed", fmt.Sprintf("Renamed failed. %s", err.Error())) } diff --git a/zboxcore/sdk/renameworker_test.go b/zboxcore/sdk/renameworker_test.go index 29aa6304f..e43a66d83 100644 --- a/zboxcore/sdk/renameworker_test.go +++ b/zboxcore/sdk/renameworker_test.go @@ -244,7 +244,7 @@ func TestRenameRequest_renameBlobberObject(t *testing.T) { req.blobbers = append(req.blobbers, &blockchain.StorageNode{ Baseurl: tt.name, }) - _, err := req.renameBlobberObject(req.blobbers[0], 0) + err := req.renameBlobberObject(req.blobbers[0], 0) require.EqualValues(tt.wantErr, err != nil, "Error: ", err) if err != nil { require.Contains(errors.Top(err), tt.errMsg) diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index 84255e054..38f6fd94f 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -2,9 +2,10 @@ package sdk import ( "context" - "fmt" + "errors" "io" "sync" + "time" "github.com/0chain/gosdk/constants" "github.com/0chain/gosdk/core/sys" @@ -23,6 +24,9 @@ type RepairRequest struct { filesRepaired int wg *sync.WaitGroup allocation *Allocation + repairPath string + versionMap map[int64]zboxutil.Uint128 + resMap map[int64]*getRes } type RepairStatusCB struct { @@ -32,6 +36,15 @@ type RepairStatusCB struct { statusCB StatusCallback } +type getRes struct { + oTR *ObjectTreeResult + err error + listCompleted bool + idx int + lastMatchedPath string + lastIndex int +} + func (cb *RepairStatusCB) Started(allocationId, filePath string, op int, totalBytes int) { cb.statusCB.Started(allocationId, filePath, op, totalBytes) } @@ -66,7 +79,11 @@ func (r *RepairRequest) processRepair(ctx context.Context, a *Allocation) { return } SetNumBlockDownloads(100) - r.iterateDir(a, r.listDir) + currentSize := MultiOpBatchSize + SetMultiOpBatchSize(multiOpRepairBatchSize) + defer SetMultiOpBatchSize(currentSize) + r.allocation = a + r.iterateDir(ctx) if r.statusCB != nil { r.statusCB.RepairCompleted(r.filesRepaired) @@ -85,197 +102,367 @@ type RepairSize struct { func (r *RepairRequest) Size(ctx context.Context, dir *ListResult) (RepairSize, error) { var rs RepairSize var err error - switch dir.Type { - case fileref.DIRECTORY: - if len(dir.Children) == 0 { - // fetch dir - dir, err = r.allocation.ListDir(dir.Path, WithListRequestForRepair(true), WithListRequestPageLimit(-1)) - if err != nil { - return rs, err - } + // switch dir.Type { + // case fileref.DIRECTORY: + // if len(dir.Children) == 0 { + // // fetch dir + // dir, err = r.allocation.ListDir(dir.Path, WithListRequestForRepair(true), WithListRequestPageLimit(-1)) + // if err != nil { + // return rs, err + // } + // } + // for _, subDir := range dir.Children { + // subDirSz, err := r.Size(ctx, subDir) + // if err != nil { + // return rs, err + // } + // rs.UploadSize += subDirSz.UploadSize + // rs.DownloadSize += subDirSz.DownloadSize + // } + // case fileref.FILE: + // // this returns repair operations required + // repairOps := r.repairFile(r.allocation, dir) + // if repairOps == nil { + // err = fmt.Errorf("fetch repairOps failed") + // return rs, err + // } + // for _, repairOp := range repairOps { + // if repairOp.OperationType == constants.FileOperationInsert { + // rs.UploadSize += uint64(repairOp.Mask.CountOnes()) * uint64(getShardSize(repairOp.FileMeta.ActualSize, r.allocation.DataShards, repairOp.EncryptedKey != "")) + // rs.DownloadSize += uint64(repairOp.FileMeta.ActualSize) + // } + // } + // } + return rs, err +} + +func (r *RepairRequest) iterateDir(ctx context.Context) { + r.versionMap = make(map[int64]zboxutil.Uint128) + latestVersion := r.allocation.allocationVersion + for idx, blobber := range r.allocation.Blobbers { + r.versionMap[blobber.AllocationVersion] = r.versionMap[blobber.AllocationVersion].Or(zboxutil.NewUint128(1).Lsh(uint64(idx))) + } + if r.versionMap[latestVersion].CountOnes() < r.allocation.DataShards { + l.Logger.Error("No consensus on latest allocation version: ", latestVersion) + if r.statusCB != nil { + r.statusCB.Error(r.allocation.ID, "/", OpRepair, errors.New("no consensus on latest allocation version")) } - for _, subDir := range dir.Children { - subDirSz, err := r.Size(ctx, subDir) - if err != nil { - return rs, err - } - rs.UploadSize += subDirSz.UploadSize - rs.DownloadSize += subDirSz.DownloadSize + return + } + if len(r.versionMap) == 1 { + return + } + + r.resMap = make(map[int64]*getRes) + err := r.repairObjects(ctx, latestVersion, fileref.FILE) + if err != nil { + return + } + l.Logger.Debug("repair file completed") + err = r.repairObjects(ctx, latestVersion, fileref.DIRECTORY) + if err != nil { + return + } + l.Logger.Debug("repair directory completed") + + r.allocation.CheckAllocStatus() //nolint:errcheck + if r.allocation.allocationVersion != latestVersion { + l.Logger.Error("Allocation version changed during repair operation") + if r.statusCB != nil { + r.statusCB.Error(r.allocation.ID, r.repairPath, OpRepair, errors.New("allocation version changed during repair operation")) } - case fileref.FILE: - // this returns repair operations required - repairOps := r.repairFile(r.allocation, dir) - if repairOps == nil { - err = fmt.Errorf("fetch repairOps failed") - return rs, err + return + } + wg := &sync.WaitGroup{} + commiReqs := make([]*CommitRequest, 0) + //send final commit to remaining blobbers + for version, mask := range r.versionMap { + if version == latestVersion { + continue } - for _, repairOp := range repairOps { - if repairOp.OperationType == constants.FileOperationInsert { - rs.UploadSize += uint64(repairOp.Mask.CountOnes()) * uint64(getShardSize(repairOp.FileMeta.ActualSize, r.allocation.DataShards, repairOp.EncryptedKey != "")) - rs.DownloadSize += uint64(repairOp.FileMeta.ActualSize) + currMask := mask + var pos uint64 + for i := currMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { + wg.Add(1) + pos = uint64(i.TrailingZeros()) + commitReq := &CommitRequest{ + allocationID: r.allocation.ID, + allocationTx: r.allocation.Tx, + blobber: r.allocation.Blobbers[pos], + connectionID: zboxutil.NewConnectionId(), + wg: wg, + timestamp: time.Now().Unix(), + blobberInd: pos, + version: latestVersion, } + go AddCommitRequest(commitReq) + commiReqs = append(commiReqs, commitReq) + } + } + wg.Wait() + for _, commitReq := range commiReqs { + if commitReq.result != nil && !commitReq.result.Success { + l.Logger.Error("Failed to commit repair operation: ", commitReq.result.ErrorMessage) + if r.statusCB != nil { + r.statusCB.Error(r.allocation.ID, r.repairPath, OpRepair, errors.New(commitReq.result.ErrorMessage)) + } + return } } - return rs, err } -func (r *RepairRequest) iterateDir(a *Allocation, dir *ListResult) []OperationRequest { - ops := make([]OperationRequest, 0) - switch dir.Type { - case fileref.DIRECTORY: - if len(dir.Children) == 0 { - var err error - dir, err = a.ListDir(dir.Path, WithListRequestForRepair(true), WithListRequestPageLimit(-1)) - if err != nil { - l.Logger.Error("Failed to get listDir for path ", zap.Any("path", dir.Path), zap.Error(err)) - return nil +func (r *RepairRequest) uploadFileOp(file ORef, opMask zboxutil.Uint128) OperationRequest { + var wg sync.WaitGroup + wg.Add(1) + statusCB := &RepairStatusCB{ + wg: &wg, + statusCB: r.statusCB, + } + memFile := &sys.MemChanFile{ + Buffer: make(chan []byte, 100), + ChunkWriteSize: int(r.allocation.GetChunkReadSize(file.EncryptedKey != "")), + } + op := r.allocation.RepairFile(memFile, file.Path, statusCB, opMask, file) + if op.FileMeta.ActualSize > 0 { + op.DownloadFile = true + } + return *op +} + +func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, fileType string) error { + clear(r.resMap) + for { + r.getRefsWithVersion(ctx, fileType) + if r.resMap[latestVersion].err != nil { + l.Logger.Error("Failed to get refs for repair path: ", r.repairPath, r.resMap[latestVersion].err) + if r.statusCB != nil { + r.statusCB.Error(r.allocation.ID, r.repairPath, OpRepair, r.resMap[latestVersion].err) } + return r.resMap[latestVersion].err } - if len(dir.Children) == 0 { - if dir.deleteMask.CountOnes() > 0 { - l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", dir.Path)) - consensus := dir.deleteMask.CountOnes() - if consensus < a.DataShards { + for version, res := range r.resMap { + if res.err != nil { + l.Logger.Error("Failed to get refs for repair path: ", r.repairPath, res.err) + delete(r.versionMap, version) + delete(r.resMap, version) + } + } + if len(r.versionMap) == 1 { + l.Logger.Error("all other versions failed to get refs for repair path: ", r.repairPath) + err := errors.New("all other versions failed to get refs for repair path") + if r.statusCB != nil { + r.statusCB.Error(r.allocation.ID, r.repairPath, OpRepair, err) + } + return err + } + ops := make([]OperationRequest, 0) + for idx, file := range r.resMap[latestVersion].oTR.Refs { + if r.checkForCancel(r.allocation) { + return errors.New("repair cancelled") + } - err := a.deleteFile(dir.Path, 0, consensus, dir.deleteMask) - if err != nil { - l.Logger.Error("repair_file_failed", zap.Error(err)) - if r.statusCB != nil { - r.statusCB.Error(a.ID, dir.Path, OpRepair, err) + for version, res := range r.resMap { + if version == latestVersion { + continue + } + + if file.Path <= res.lastMatchedPath { + continue + } + + if res.idx == len(res.oTR.Refs) { + if res.listCompleted { + // upload the file + opMask := r.versionMap[version] + if fileType == fileref.FILE { + op := r.uploadFileOp(file, opMask) + ops = append(ops, op) + } else { + op := OperationRequest{ + OperationType: constants.FileOperationCreateDir, + RemotePath: file.Path, + } + ops = append(ops, op) + } + res.lastMatchedPath = file.Path + res.lastIndex = idx + } + continue + } + + currentPath := res.oTR.Refs[res.idx].Path + if currentPath == file.Path { + if file.Type == fileref.FILE && file.ActualFileHash != res.oTR.Refs[res.idx].ActualFileHash { + // upload the file + opMask := r.versionMap[version] + op := r.uploadFileOp(file, opMask) + ops = append(ops, op) + } + res.lastMatchedPath = currentPath + res.idx++ + res.lastIndex = idx + } else if currentPath < file.Path { + for res.oTR.Refs[res.idx].Path < file.Path && res.idx < len(res.oTR.Refs) { + //delete the file + opMask := r.versionMap[version] + op := OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: res.oTR.Refs[res.idx].Path, + Mask: &opMask, } - return nil + res.idx++ + ops = append(ops, op) } - r.filesRepaired++ - } else if consensus < len(a.Blobbers) { - createMask := dir.deleteMask.Not().And(zboxutil.NewUint128(1).Lsh(uint64(len(a.Blobbers))).Sub64(1)) - err := a.createDir(dir.Path, 0, createMask.CountOnes(), createMask) - if err != nil { - l.Logger.Error("repair_file_failed", zap.Error(err)) - if r.statusCB != nil { - r.statusCB.Error(a.ID, dir.Path, OpRepair, err) + if res.idx == len(res.oTR.Refs) { + res.lastIndex = idx + } else if res.oTR.Refs[res.idx].Path > file.Path { + opMask := r.versionMap[version] + // upload the file + if fileType == fileref.FILE { + op := r.uploadFileOp(file, opMask) + ops = append(ops, op) + } else { + op := OperationRequest{ + OperationType: constants.FileOperationCreateDir, + RemotePath: file.Path, + } + ops = append(ops, op) } - return nil } - r.filesRepaired++ + res.lastMatchedPath = file.Path + res.idx++ + res.lastIndex = idx + } else { + //TODO: take a union of mask so we don't have duplicate upload operations + // upload the file + opMask := r.versionMap[version] + if fileType == fileref.FILE { + op := r.uploadFileOp(file, opMask) + ops = append(ops, op) + } else { + op := OperationRequest{ + OperationType: constants.FileOperationCreateDir, + RemotePath: file.Path, + } + ops = append(ops, op) + } + res.lastMatchedPath = file.Path + res.lastIndex = idx } } } - for _, childDir := range dir.Children { - if r.checkForCancel(a) { - return nil + + if r.resMap[latestVersion].listCompleted { + for version, res := range r.resMap { + if version == latestVersion { + continue + } + for res.idx < len(res.oTR.Refs) { + //delete the file + l.Logger.Debug("Deleting file: ", res.oTR.Refs[res.idx].Path) + opMask := r.versionMap[version] + op := OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: res.oTR.Refs[res.idx].Path, + Mask: &opMask, + } + res.idx++ + ops = append(ops, op) + } } - ops = append(ops, r.iterateDir(a, childDir)...) - if len(ops) >= RepairBatchSize/2 { - r.repairOperation(a, ops) - ops = nil + } else { + minLastIndex := len(r.resMap[latestVersion].oTR.Refs) + for version, res := range r.resMap { + if version == latestVersion { + continue + } + if !res.listCompleted { + if res.lastIndex < minLastIndex { + minLastIndex = res.lastIndex + } + if res.idx < len(res.oTR.Refs) { + res.oTR.Refs = res.oTR.Refs[res.idx:] + } else { + res.oTR.Refs = nil + } + } + } + if minLastIndex < len(r.resMap[latestVersion].oTR.Refs) { + r.resMap[latestVersion].oTR.Refs = r.resMap[latestVersion].oTR.Refs[minLastIndex:] } } - if len(ops) > 0 { - r.repairOperation(a, ops) - ops = nil + + err := r.repairOperation(ops, latestVersion, r.resMap[latestVersion].oTR.OffsetPath) + if err != nil { + l.Logger.Error("Failed to repair files: ", err) + return err } - case fileref.FILE: - // this returns op object and mask - repairOps := r.repairFile(a, dir) - if repairOps != nil { - ops = append(ops, repairOps...) + if r.resMap[latestVersion].listCompleted { + break } - - default: - l.Logger.Info("Invalid directory type", zap.Any("type", dir.Type)) } - return ops + + return nil } -func (r *RepairRequest) repairFile(a *Allocation, file *ListResult) []OperationRequest { - ops := make([]OperationRequest, 0) - if r.checkForCancel(a) { - return nil - } - l.Logger.Info("Checking file for the path :", zap.Any("path", file.Path)) - found, deleteMask, repairRequired, ref, err := a.RepairRequired(file.Path) - if err != nil { - l.Logger.Error("repair_required_failed", zap.Error(err)) - return nil - } - if repairRequired { - l.Logger.Info("Repair required for the path :", zap.Any("path", file.Path)) - if found.CountOnes() >= a.DataShards { - l.Logger.Info("Repair by upload", zap.Any("path", file.Path)) - var wg sync.WaitGroup - statusCB := &RepairStatusCB{ - wg: &wg, - statusCB: r.statusCB, +func (r *RepairRequest) getRefsWithVersion(ctx context.Context, fileType string) { + mpLock := sync.Mutex{} + resWg := sync.WaitGroup{} + for version, mask := range r.versionMap { + resWg.Add(1) + currVersion := version + currMask := mask + go func(version int64, mask zboxutil.Uint128) { + defer resWg.Done() + var offsetPath string + mpLock.Lock() + if r.resMap[version] != nil { + offsetPath = r.resMap[version].oTR.OffsetPath } - - if deleteMask.CountOnes() > 0 { - l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path)) - op := OperationRequest{ - OperationType: constants.FileOperationDelete, - RemotePath: file.Path, - Mask: &deleteMask, + mpLock.Unlock() + l.Logger.Debug("Getting refs for repair path: ", r.repairPath, " version: ", version, " mask: ", mask.CountOnes()) + res, err := r.allocation.GetRefs(r.repairPath, offsetPath, "", "", fileType, fileref.REGULAR, 0, getRefPageLimit, WithSingleBlobber(true), WithObjectMask(mask), WithObjectContext(ctx)) + if err != nil { + if code, _ := zboxutil.GetErrorMessageCode(err.Error()); code == INVALID_PATH { + err = nil + res = &ObjectTreeResult{Refs: make([]ORef, 0)} } - ops = append(ops, op) } - wg.Add(1) - localPath := r.getLocalPath(file) - var op *OperationRequest - if !checkFileExists(localPath) { - if r.checkForCancel(a) { - return nil - } - memFile := &sys.MemChanFile{ - Buffer: make(chan []byte, 100), - ChunkWriteSize: int(a.GetChunkReadSize(ref.EncryptedKey != "")), - } - op = a.RepairFile(memFile, file.Path, statusCB, found, ref) - if op.FileMeta.ActualSize > 0 { - op.DownloadFile = true + mpLock.Lock() + currRes := r.resMap[version] + if currRes != nil { + if res != nil { + currRes.oTR.Refs = append(currRes.oTR.Refs, res.Refs...) + if len(res.Refs) < getRefPageLimit { + currRes.listCompleted = true + } } + currRes.idx = 0 + currRes.lastIndex = 0 } else { - f, err := sys.Files.Open(localPath) - if err != nil { - l.Logger.Error("repair_file_failed", zap.Error(err)) - return nil + getResult := &getRes{oTR: res, err: err} + if res != nil && len(res.Refs) < getRefPageLimit { + getResult.listCompleted = true } - op = a.RepairFile(f, file.Path, statusCB, found, ref) - } - ops = append(ops, *op) - if r.checkForCancel(a) { - return nil - } - } else { - l.Logger.Info("Repair by delete", zap.Any("path", file.Path)) - op := OperationRequest{ - OperationType: constants.FileOperationDelete, - RemotePath: file.Path, - Mask: &found, + r.resMap[version] = getResult } - ops = append(ops, op) - } - } else if deleteMask.CountOnes() > 0 { - l.Logger.Info("Deleting minority shards for the path :", zap.Any("path", file.Path)) - op := OperationRequest{ - OperationType: constants.FileOperationDelete, - RemotePath: file.Path, - Mask: &deleteMask, - } - ops = append(ops, op) + mpLock.Unlock() + }(currVersion, currMask) } - return ops + resWg.Wait() } -func (r *RepairRequest) repairOperation(a *Allocation, ops []OperationRequest) { - err := a.DoMultiOperation(ops, WithRepair()) +func (r *RepairRequest) repairOperation(ops []OperationRequest, latestVersion int64, repairOffsetPath string) error { + err := r.allocation.DoMultiOperation(ops, WithRepair(latestVersion, repairOffsetPath)) if err != nil { l.Logger.Error("repair_file_failed", zap.Error(err)) status := r.statusCB != nil for _, op := range ops { if op.DownloadFile { - _ = a.CancelDownload(op.RemotePath) + _ = r.allocation.CancelDownload(op.RemotePath) } if status { - r.statusCB.Error(a.ID, op.RemotePath, OpRepair, err) + r.statusCB.Error(r.allocation.ID, op.RemotePath, OpRepair, err) } } } else { @@ -288,21 +475,7 @@ func (r *RepairRequest) repairOperation(a *Allocation, ops []OperationRequest) { } } } -} - -func (r *RepairRequest) getLocalPath(file *ListResult) string { - return r.localRootPath + file.Path -} - -func checkFileExists(localPath string) bool { - if IsWasm { - return false - } - info, err := sys.Files.Stat(localPath) - if err != nil { - return false - } - return !info.IsDir() + return err } func (r *RepairRequest) checkForCancel(a *Allocation) bool { @@ -315,6 +488,3 @@ func (r *RepairRequest) checkForCancel(a *Allocation) bool { } return false } - -//repair for enterprise workflow -// list all files in the allocation recursively and repair them From d15f3ce27ff33cc4fb5c9ad96bf9655fe43a7a99 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Wed, 7 Aug 2024 13:07:31 +0530 Subject: [PATCH 24/45] add delete for dir remotepath in move --- zboxcore/sdk/common.go | 2 ++ zboxcore/sdk/copyworker.go | 6 ++++++ zboxcore/sdk/deleteworker.go | 4 ++-- zboxcore/sdk/moveworker.go | 18 +++++++++++++++--- zboxcore/sdk/multi_operation_worker.go | 2 +- zboxcore/sdk/renameworker.go | 6 ++++++ 6 files changed, 32 insertions(+), 6 deletions(-) diff --git a/zboxcore/sdk/common.go b/zboxcore/sdk/common.go index 6f5a6fd8a..f866c58ee 100644 --- a/zboxcore/sdk/common.go +++ b/zboxcore/sdk/common.go @@ -20,6 +20,8 @@ import ( "github.com/0chain/gosdk/zboxcore/zboxutil" ) +const alreadyExists = "file already exists" + func getObjectTreeFromBlobber(ctx context.Context, allocationID, allocationTx string, remoteFilePath string, blobber *blockchain.StorageNode) (fileref.RefEntity, error) { httpreq, err := zboxutil.NewObjectTreeRequest(blobber.Baseurl, allocationID, allocationTx, remoteFilePath) if err != nil { diff --git a/zboxcore/sdk/copyworker.go b/zboxcore/sdk/copyworker.go index 98678545d..501b10818 100644 --- a/zboxcore/sdk/copyworker.go +++ b/zboxcore/sdk/copyworker.go @@ -147,6 +147,10 @@ func (req *CopyRequest) copyBlobberObject( latestRespMsg = string(respBody) latestStatusCode = resp.StatusCode + if strings.Contains(latestRespMsg, alreadyExists) { + req.Consensus.Done() + return + } if resp.StatusCode == http.StatusTooManyRequests { logger.Logger.Error("Got too many request error") @@ -224,6 +228,8 @@ func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { if err != nil { return nil, err } + req.consensus = req.copyMask.CountOnes() + return objectTreeRefs, nil } for i := req.copyMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { diff --git a/zboxcore/sdk/deleteworker.go b/zboxcore/sdk/deleteworker.go index db3922422..264c3576e 100644 --- a/zboxcore/sdk/deleteworker.go +++ b/zboxcore/sdk/deleteworker.go @@ -43,7 +43,7 @@ type DeleteRequest struct { timestamp int64 } -var ErrFileDeleted = errors.New("file_deleted", "file is already deleted") +var errFileDeleted = errors.New("file_deleted", "file is already deleted") func (req *DeleteRequest) deleteBlobberFile( blobber *blockchain.StorageNode, blobberIdx int) error { @@ -398,7 +398,7 @@ func (dop *DeleteOperation) Process(allocObj *Allocation, connectionID string) ( } if consensusRef == nil { //Already deleted - return nil, dop.deleteMask, ErrFileDeleted + return nil, dop.deleteMask, errFileDeleted } if consensusRef.Type == fileref.DIRECTORY && !consensusRef.IsEmpty { for ind, refEntity := range objectTreeRefs { diff --git a/zboxcore/sdk/moveworker.go b/zboxcore/sdk/moveworker.go index dc9891ce8..27f10fbc4 100644 --- a/zboxcore/sdk/moveworker.go +++ b/zboxcore/sdk/moveworker.go @@ -43,9 +43,7 @@ type MoveRequest struct { Consensus } -func (req *MoveRequest) getObjectTreeFromBlobber(blobber *blockchain.StorageNode) (fileref.RefEntity, error) { - return getObjectTreeFromBlobber(req.ctx, req.allocationID, req.allocationTx, req.remotefilepath, blobber) -} +var errNoChange = errors.New("no_change", "No change in the operation") func (req *MoveRequest) getFileMetaFromBlobber(pos int) (fileRef *fileref.FileRef, err error) { listReq := &ListRequest{ @@ -146,6 +144,11 @@ func (req *MoveRequest) moveBlobberObject( latestRespMsg = string(respBody) latestStatusCode = resp.StatusCode + if strings.Contains(latestRespMsg, alreadyExists) { + req.Consensus.Done() + return + } + if resp.StatusCode == http.StatusTooManyRequests { logger.Logger.Error("Got too many request error") var r int @@ -229,6 +232,15 @@ func (req *MoveRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { if err != nil { return nil, err } + op := OperationRequest{ + OperationType: constants.FileOperationDelete, + RemotePath: req.remotefilepath, + } + err = req.allocationObj.DoMultiOperation([]OperationRequest{op}) + if err != nil { + return nil, err + } + return nil, errNoChange } for i := req.moveMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index fdbac3959..ad057af47 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -185,7 +185,7 @@ func (mo *MultiOperation) Process() error { _, mask, err := op.Process(mo.allocationObj, mo.connectionID) // Process with each blobber if err != nil { - if err != ErrFileDeleted { + if err != errFileDeleted || err != errNoChange { l.Logger.Error(err) errsSlice[idx] = errors.New("", err.Error()) ctxCncl(err) diff --git a/zboxcore/sdk/renameworker.go b/zboxcore/sdk/renameworker.go index de494b983..235297476 100644 --- a/zboxcore/sdk/renameworker.go +++ b/zboxcore/sdk/renameworker.go @@ -8,6 +8,7 @@ import ( "mime/multipart" "net/http" "path/filepath" + "strings" "sync" "time" @@ -142,6 +143,11 @@ func (req *RenameRequest) renameBlobberObject( return } + if strings.Contains(latestRespMsg, alreadyExists) { + req.consensus.Done() + return + } + if resp.StatusCode == http.StatusTooManyRequests { logger.Logger.Error("Got too many request error") var r int From f2bb064791d0a3504fcb061cd9b4f35dbaf28daa Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Wed, 7 Aug 2024 13:09:30 +0530 Subject: [PATCH 25/45] fix no change check --- zboxcore/sdk/multi_operation_worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index ad057af47..950616124 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -185,7 +185,7 @@ func (mo *MultiOperation) Process() error { _, mask, err := op.Process(mo.allocationObj, mo.connectionID) // Process with each blobber if err != nil { - if err != errFileDeleted || err != errNoChange { + if err != errFileDeleted && err != errNoChange { l.Logger.Error(err) errsSlice[idx] = errors.New("", err.Error()) ctxCncl(err) From b0e385f040e841a8621188f813b896aced7eed48 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Wed, 7 Aug 2024 20:22:26 +0530 Subject: [PATCH 26/45] fix consensus and add check for no change err --- zboxcore/sdk/moveworker.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/zboxcore/sdk/moveworker.go b/zboxcore/sdk/moveworker.go index 27f10fbc4..2a4f5b381 100644 --- a/zboxcore/sdk/moveworker.go +++ b/zboxcore/sdk/moveworker.go @@ -240,6 +240,7 @@ func (req *MoveRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { if err != nil { return nil, err } + req.consensus = req.moveMask.CountOnes() return nil, errNoChange } @@ -393,6 +394,9 @@ func (mo *MoveOperation) Process(allocObj *Allocation, connectionID string) ([]f if !mR.Consensus.isConsensusOk() { if err != nil { + if err == errNoChange { + return nil, mR.moveMask, err + } return nil, mR.moveMask, thrown.New("move_failed", fmt.Sprintf("Move failed. %s", err.Error())) } From a2081f3159c1c1b36aaacf72d299685f6027bb00 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Wed, 7 Aug 2024 21:39:16 +0530 Subject: [PATCH 27/45] fix list completed repair --- zboxcore/sdk/allocation.go | 1 + zboxcore/sdk/repairworker.go | 48 +++++++++++++++++++++++++++--------- 2 files changed, 37 insertions(+), 12 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 9f06ed3b9..565c84d9b 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -420,6 +420,7 @@ func (a *Allocation) RepairFile(file sys.File, remotepath string, statusCallback MimeType: ref.MimeType, RemoteName: ref.Name, RemotePath: remotepath, + CustomMeta: ref.CustomMeta, } var opts []ChunkedUploadOption if ref.EncryptedKey != "" { diff --git a/zboxcore/sdk/repairworker.go b/zboxcore/sdk/repairworker.go index 38f6fd94f..b42d7cc20 100644 --- a/zboxcore/sdk/repairworker.go +++ b/zboxcore/sdk/repairworker.go @@ -302,7 +302,7 @@ func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, res.idx++ res.lastIndex = idx } else if currentPath < file.Path { - for res.oTR.Refs[res.idx].Path < file.Path && res.idx < len(res.oTR.Refs) { + for res.idx < len(res.oTR.Refs) && res.oTR.Refs[res.idx].Path < file.Path { //delete the file opMask := r.versionMap[version] op := OperationRequest{ @@ -313,9 +313,7 @@ func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, res.idx++ ops = append(ops, op) } - if res.idx == len(res.oTR.Refs) { - res.lastIndex = idx - } else if res.oTR.Refs[res.idx].Path > file.Path { + if res.idx < len(res.oTR.Refs) && res.oTR.Refs[res.idx].Path > file.Path { opMask := r.versionMap[version] // upload the file if fileType == fileref.FILE { @@ -328,10 +326,10 @@ func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, } ops = append(ops, op) } + res.lastMatchedPath = file.Path + res.idx++ + res.lastIndex = idx } - res.lastMatchedPath = file.Path - res.idx++ - res.lastIndex = idx } else { //TODO: take a union of mask so we don't have duplicate upload operations // upload the file @@ -369,6 +367,23 @@ func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, res.idx++ ops = append(ops, op) } + if res.listCompleted { + for i := res.lastIndex + 1; i < len(r.resMap[latestVersion].oTR.Refs); i++ { + // upload the file + l.Logger.Debug("Uploading file: ", r.resMap[latestVersion].oTR.Refs[i].Path) + opMask := r.versionMap[version] + if fileType == fileref.FILE { + op := r.uploadFileOp(r.resMap[latestVersion].oTR.Refs[i], opMask) + ops = append(ops, op) + } else { + op := OperationRequest{ + OperationType: constants.FileOperationCreateDir, + RemotePath: r.resMap[latestVersion].oTR.Refs[i].Path, + } + ops = append(ops, op) + } + } + } } } else { minLastIndex := len(r.resMap[latestVersion].oTR.Refs) @@ -387,8 +402,10 @@ func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, } } } - if minLastIndex < len(r.resMap[latestVersion].oTR.Refs) { - r.resMap[latestVersion].oTR.Refs = r.resMap[latestVersion].oTR.Refs[minLastIndex:] + if minLastIndex+1 < len(r.resMap[latestVersion].oTR.Refs) { + r.resMap[latestVersion].oTR.Refs = r.resMap[latestVersion].oTR.Refs[minLastIndex+1:] + } else { + r.resMap[latestVersion].oTR.Refs = nil } } @@ -397,7 +414,14 @@ func (r *RepairRequest) repairObjects(ctx context.Context, latestVersion int64, l.Logger.Error("Failed to repair files: ", err) return err } - if r.resMap[latestVersion].listCompleted { + + completedLists := 0 + for _, res := range r.resMap { + if res.listCompleted { + completedLists++ + } + } + if completedLists == len(r.versionMap) { break } } @@ -438,9 +462,9 @@ func (r *RepairRequest) getRefsWithVersion(ctx context.Context, fileType string) } } currRes.idx = 0 - currRes.lastIndex = 0 + currRes.lastIndex = -1 } else { - getResult := &getRes{oTR: res, err: err} + getResult := &getRes{oTR: res, err: err, lastIndex: -1} if res != nil && len(res.Refs) < getRefPageLimit { getResult.listCompleted = true } From af515bab6d679e6ca0d84652c0a41cd56aa59964 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Thu, 8 Aug 2024 13:04:03 +0530 Subject: [PATCH 28/45] use dir as destpath --- zboxcore/sdk/common.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/zboxcore/sdk/common.go b/zboxcore/sdk/common.go index f866c58ee..ecff5f256 100644 --- a/zboxcore/sdk/common.go +++ b/zboxcore/sdk/common.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "net/http" "path" + "path/filepath" "strconv" "strings" "sync" @@ -157,7 +158,7 @@ func (req *subDirRequest) processSubDirectories() error { if ref.PathLevel > pathLevel { pathLevel = ref.PathLevel } - destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + destPath := filepath.Dir(strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1)) op := OperationRequest{ OperationType: req.opType, RemotePath: ref.Path, From e973828a833015eace5297b7fd7b5b9572346871 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Thu, 8 Aug 2024 15:31:46 +0530 Subject: [PATCH 29/45] base of dest name --- zboxcore/sdk/renameworker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/renameworker.go b/zboxcore/sdk/renameworker.go index 235297476..328e62445 100644 --- a/zboxcore/sdk/renameworker.go +++ b/zboxcore/sdk/renameworker.go @@ -457,7 +457,7 @@ func (ro *RenameOperation) Error(allocObj *Allocation, consensus int, err error) func NewRenameOperation(remotePath string, destName string, renameMask zboxutil.Uint128, maskMU *sync.Mutex, consensusTh int, fullConsensus int, ctx context.Context) *RenameOperation { ro := &RenameOperation{} ro.remotefilepath = zboxutil.RemoteClean(remotePath) - ro.newName = destName + ro.newName = filepath.Base(destName) ro.renameMask = renameMask ro.maskMU = maskMU ro.consensus.consensusThresh = consensusTh From f4ba999e2c83c9ea80c8a8c54bc1632a01555964 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Thu, 8 Aug 2024 20:56:10 +0530 Subject: [PATCH 30/45] fix copy op --- zboxcore/sdk/allocation.go | 3 +- zboxcore/sdk/common.go | 2 +- zboxcore/sdk/copyworker.go | 85 +++++++++++++++++++++----------------- 3 files changed, 50 insertions(+), 40 deletions(-) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 565c84d9b..ad5c312c1 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -243,6 +243,7 @@ type OperationRequest struct { StreamUpload bool // Required for streaming file when actualSize is not available CancelCauseFunc context.CancelCauseFunc Opts []ChunkedUploadOption + CopyDirOnly bool } func GetReadPriceRange() (PriceRange, error) { @@ -925,7 +926,7 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...Mul operation = NewRenameOperation(op.RemotePath, op.DestName, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) case constants.FileOperationCopy: - operation = NewCopyOperation(op.RemotePath, op.DestPath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) + operation = NewCopyOperation(op.RemotePath, op.DestPath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, op.CopyDirOnly, mo.ctx) case constants.FileOperationMove: operation = NewMoveOperation(op.RemotePath, op.DestPath, mo.operationMask, mo.maskMU, mo.consensusThresh, mo.fullconsensus, mo.ctx) diff --git a/zboxcore/sdk/common.go b/zboxcore/sdk/common.go index ecff5f256..ff14c4f3d 100644 --- a/zboxcore/sdk/common.go +++ b/zboxcore/sdk/common.go @@ -197,7 +197,7 @@ func (req *subDirRequest) processSubDirectories() error { if ref.Type == fileref.FILE { continue } - destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + destPath := filepath.Dir(strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1)) op := OperationRequest{ OperationType: req.opType, RemotePath: ref.Path, diff --git a/zboxcore/sdk/copyworker.go b/zboxcore/sdk/copyworker.go index 501b10818..0e34d45ea 100644 --- a/zboxcore/sdk/copyworker.go +++ b/zboxcore/sdk/copyworker.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "mime/multipart" "net/http" + "path/filepath" "strings" "sync" "time" @@ -34,6 +35,7 @@ type CopyRequest struct { blobbers []*blockchain.StorageNode remotefilepath string destPath string + dirOnly bool ctx context.Context ctxCncl context.CancelFunc copyMask zboxutil.Uint128 @@ -147,10 +149,6 @@ func (req *CopyRequest) copyBlobberObject( latestRespMsg = string(respBody) latestStatusCode = resp.StatusCode - if strings.Contains(latestRespMsg, alreadyExists) { - req.Consensus.Done() - return - } if resp.StatusCode == http.StatusTooManyRequests { logger.Logger.Error("Got too many request error") @@ -224,7 +222,7 @@ func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { req.copyMask = req.copyMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) } } - err := req.copySubDirectoriees() + err := req.copySubDirectoriees(req.dirOnly) if err != nil { return nil, err } @@ -355,6 +353,7 @@ func (req *CopyRequest) ProcessCopy() error { type CopyOperation struct { remotefilepath string destPath string + dirOnly bool ctx context.Context ctxCncl context.CancelFunc copyMask zboxutil.Uint128 @@ -364,6 +363,9 @@ type CopyOperation struct { } func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]fileref.RefEntity, zboxutil.Uint128, error) { + if co.remotefilepath == "/" { + return nil, co.copyMask, errors.New("invalid_path", "Invalid path for copy cannot copy root directory") + } // make copyRequest object cR := &CopyRequest{ allocationObj: allocObj, @@ -378,6 +380,7 @@ func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]f copyMask: co.copyMask, maskMU: co.maskMU, Consensus: Consensus{RWMutex: &sync.RWMutex{}}, + dirOnly: co.dirOnly, } cR.consensusThresh = co.consensusThresh cR.fullconsensus = co.fullconsensus @@ -385,8 +388,11 @@ func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]f objectTreeRefs, err := cR.ProcessWithBlobbers() if !cR.isConsensusOk() { - l.Logger.Error("copy failed: ", cR.remotefilepath, cR.destPath) if err != nil { + if strings.Contains(err.Error(), alreadyExists) { + return objectTreeRefs, cR.copyMask, errNoChange + } + l.Logger.Error("copy failed: ", cR.remotefilepath, cR.destPath) return nil, cR.copyMask, errors.New("copy_failed", fmt.Sprintf("Copy failed. %s", err.Error())) } @@ -448,7 +454,7 @@ func (co *CopyOperation) Error(allocObj *Allocation, consensus int, err error) { } -func NewCopyOperation(remotePath string, destPath string, copyMask zboxutil.Uint128, maskMU *sync.Mutex, consensusTh int, fullConsensus int, ctx context.Context) *CopyOperation { +func NewCopyOperation(remotePath string, destPath string, copyMask zboxutil.Uint128, maskMU *sync.Mutex, consensusTh int, fullConsensus int, copyDirOnly bool, ctx context.Context) *CopyOperation { co := &CopyOperation{} co.remotefilepath = zboxutil.RemoteClean(remotePath) co.copyMask = copyMask @@ -464,45 +470,47 @@ func NewCopyOperation(remotePath string, destPath string, copyMask zboxutil.Uint } -func (req *CopyRequest) copySubDirectoriees() error { +func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { var ( offsetPath string pathLevel int ) for { - oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WithObjectConsensusThresh(req.consensusThresh), WithSingleBlobber(true)) - if err != nil { - return err - } - if len(oResult.Refs) == 0 { - break - } - ops := make([]OperationRequest, 0, len(oResult.Refs)) - for _, ref := range oResult.Refs { - opMask := req.copyMask - if ref.Type == fileref.DIRECTORY { - continue + if !dirOnly { + oResult, err := req.allocationObj.GetRefs(req.remotefilepath, offsetPath, "", "", fileref.FILE, fileref.REGULAR, 0, getRefPageLimit, WithObjectContext(req.ctx), WithObjectConsensusThresh(req.consensusThresh), WithSingleBlobber(true)) + if err != nil { + return err + } + if len(oResult.Refs) == 0 { + break + } + ops := make([]OperationRequest, 0, len(oResult.Refs)) + for _, ref := range oResult.Refs { + opMask := req.copyMask + if ref.Type == fileref.DIRECTORY { + continue + } + if ref.PathLevel > pathLevel { + pathLevel = ref.PathLevel + } + destPath := filepath.Dir(strings.Replace(ref.Path, filepath.Dir(req.remotefilepath), req.destPath, 1)) + op := OperationRequest{ + OperationType: constants.FileOperationCopy, + RemotePath: ref.Path, + DestPath: destPath, + Mask: &opMask, + } + ops = append(ops, op) } - if ref.PathLevel > pathLevel { - pathLevel = ref.PathLevel + err = req.allocationObj.DoMultiOperation(ops) + if err != nil { + return err } - destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) - op := OperationRequest{ - OperationType: constants.FileOperationCopy, - RemotePath: ref.Path, - DestPath: destPath, - Mask: &opMask, + offsetPath = oResult.Refs[len(oResult.Refs)-1].Path + if len(oResult.Refs) < getRefPageLimit { + break } - ops = append(ops, op) - } - err = req.allocationObj.DoMultiOperation(ops) - if err != nil { - return err - } - offsetPath = oResult.Refs[len(oResult.Refs)-1].Path - if len(oResult.Refs) < getRefPageLimit { - break } } @@ -526,12 +534,13 @@ func (req *CopyRequest) copySubDirectoriees() error { if ref.Type == fileref.FILE { continue } - destPath := strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1) + destPath := filepath.Dir(strings.Replace(ref.Path, filepath.Dir(req.remotefilepath), req.destPath, 1)) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, DestPath: destPath, Mask: &opMask, + CopyDirOnly: true, } ops = append(ops, op) } From 94ef0df2e7866c4a73a2aba548840e97539df268 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Thu, 8 Aug 2024 23:57:37 +0530 Subject: [PATCH 31/45] check copy destpath --- zboxcore/sdk/copyworker.go | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/zboxcore/sdk/copyworker.go b/zboxcore/sdk/copyworker.go index 0e34d45ea..7957b5289 100644 --- a/zboxcore/sdk/copyworker.go +++ b/zboxcore/sdk/copyworker.go @@ -45,6 +45,8 @@ type CopyRequest struct { Consensus } +const objAlreadyExists = "Object Already exists" + func (req *CopyRequest) getObjectTreeFromBlobber(blobber *blockchain.StorageNode) (fileref.RefEntity, error) { return getObjectTreeFromBlobber(req.ctx, req.allocationID, req.allocationTx, req.remotefilepath, blobber) } @@ -244,8 +246,12 @@ func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { }(int(pos)) } wg.Wait() + err := zboxutil.MajorError(blobberErrors) + if err != nil && strings.Contains(err.Error(), objAlreadyExists) && consensusRef.Type == fileref.DIRECTORY { + return nil, errNoChange + } - return objectTreeRefs, zboxutil.MajorError(blobberErrors) + return objectTreeRefs, err } func (req *CopyRequest) ProcessCopy() error { @@ -389,8 +395,8 @@ func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]f if !cR.isConsensusOk() { if err != nil { - if strings.Contains(err.Error(), alreadyExists) { - return objectTreeRefs, cR.copyMask, errNoChange + if err == errNoChange { + return nil, cR.copyMask, err } l.Logger.Error("copy failed: ", cR.remotefilepath, cR.destPath) return nil, cR.copyMask, errors.New("copy_failed", fmt.Sprintf("Copy failed. %s", err.Error())) @@ -494,7 +500,7 @@ func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { if ref.PathLevel > pathLevel { pathLevel = ref.PathLevel } - destPath := filepath.Dir(strings.Replace(ref.Path, filepath.Dir(req.remotefilepath), req.destPath, 1)) + destPath := strings.Replace(filepath.Dir(ref.Path), req.remotefilepath, req.destPath, 1) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, @@ -534,7 +540,7 @@ func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { if ref.Type == fileref.FILE { continue } - destPath := filepath.Dir(strings.Replace(ref.Path, filepath.Dir(req.remotefilepath), req.destPath, 1)) + destPath := strings.Replace(filepath.Dir(ref.Path), req.remotefilepath, req.destPath, 1) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, From dbbc11b22bbb0d17698bf9686ba011b06e824181 Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Fri, 9 Aug 2024 02:18:54 +0530 Subject: [PATCH 32/45] join base and dest path --- zboxcore/sdk/copyworker.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/zboxcore/sdk/copyworker.go b/zboxcore/sdk/copyworker.go index 7957b5289..bc158a15d 100644 --- a/zboxcore/sdk/copyworker.go +++ b/zboxcore/sdk/copyworker.go @@ -500,7 +500,8 @@ func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { if ref.PathLevel > pathLevel { pathLevel = ref.PathLevel } - destPath := strings.Replace(filepath.Dir(ref.Path), req.remotefilepath, req.destPath, 1) + basePath := strings.TrimPrefix(filepath.Dir(ref.Path), filepath.Dir(req.remotefilepath)) + destPath := filepath.Join(req.destPath, basePath) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, @@ -540,7 +541,8 @@ func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { if ref.Type == fileref.FILE { continue } - destPath := strings.Replace(filepath.Dir(ref.Path), req.remotefilepath, req.destPath, 1) + basePath := strings.TrimPrefix(filepath.Dir(ref.Path), filepath.Dir(req.remotefilepath)) + destPath := filepath.Join(req.destPath, basePath) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, From 8390f11c37f49f424b2f63cd3b123a851ebb969f Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Fri, 9 Aug 2024 14:13:24 +0530 Subject: [PATCH 33/45] set path hash --- zboxcore/sdk/filestatsworker.go | 1 + 1 file changed, 1 insertion(+) diff --git a/zboxcore/sdk/filestatsworker.go b/zboxcore/sdk/filestatsworker.go index b7b846515..a6e2805bf 100644 --- a/zboxcore/sdk/filestatsworker.go +++ b/zboxcore/sdk/filestatsworker.go @@ -98,6 +98,7 @@ func (req *ListRequest) getFileStatsInfoFromBlobber(blobber *blockchain.StorageN } else { fileStats.BlockchainAware = false } + fileStats.PathHash = req.remotefilepathhash fileStats.BlobberID = blobber.ID fileStats.BlobberURL = blobber.Baseurl return nil From c552f971c1067481f0e662a40f812b8006a6dfa9 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sun, 11 Aug 2024 02:26:16 +0530 Subject: [PATCH 34/45] add size for root dir --- zboxcore/sdk/listworker.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/zboxcore/sdk/listworker.go b/zboxcore/sdk/listworker.go index 611ad635b..18911e6f0 100644 --- a/zboxcore/sdk/listworker.go +++ b/zboxcore/sdk/listworker.go @@ -249,6 +249,9 @@ func (req *ListRequest) GetListFromBlobbers() (*ListResult, error) { } result.Size += ti.ref.Size result.NumBlocks += ti.ref.NumBlocks + if ti.ref.Path == "/" { + result.ActualSize = ti.ref.Size * int64(req.consensusThresh) + } if len(lR[i].ref.Children) > 0 { result.populateChildren(lR[i].ref.Children, childResultMap, selected, req) From 645ded9642b289b4d6de71e2395111116490ac0f Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 13 Aug 2024 11:24:15 +0530 Subject: [PATCH 35/45] check vm --- zboxcore/sdk/rollback.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/rollback.go b/zboxcore/sdk/rollback.go index 7295ed51c..81cbc505c 100644 --- a/zboxcore/sdk/rollback.go +++ b/zboxcore/sdk/rollback.go @@ -255,7 +255,7 @@ func (a *Allocation) CheckAllocStatus() (AllocStatus, []BlobberStatus, error) { l.Logger.Error("error during getWritemarker", zap.Error(err)) blobStatus.Status = "unavailable" } - if lvm == nil { + if lvm == nil || lvm.VersionMarker == nil { markerChan <- nil } else { markerChan <- &RollbackBlobber{ From 007b98227f14c40620847fd938c7105f46f040cd Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 14 Aug 2024 19:01:39 +0530 Subject: [PATCH 36/45] move upload success to debug --- zboxcore/sdk/upload_worker.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/upload_worker.go b/zboxcore/sdk/upload_worker.go index 76021f904..582580af0 100644 --- a/zboxcore/sdk/upload_worker.go +++ b/zboxcore/sdk/upload_worker.go @@ -59,7 +59,7 @@ func (uo *UploadOperation) Process(allocObj *Allocation, connectionID string) ([ fileref.DeleteFileRef(cacheKey) } } - l.Logger.Info("UploadOperation Success", zap.String("name", uo.chunkedUpload.fileMeta.RemoteName)) + l.Logger.Debug("UploadOperation Success", zap.String("name", uo.chunkedUpload.fileMeta.RemoteName)) return nil, uo.chunkedUpload.uploadMask, nil } From 2119c4affdfae5009a42b858bf20153040fb564b Mon Sep 17 00:00:00 2001 From: hitenjain14 Date: Thu, 22 Aug 2024 19:32:00 +0530 Subject: [PATCH 37/45] fix nested move and rename --- zboxcore/sdk/common.go | 7 ++++--- zboxcore/sdk/copyworker.go | 14 +++++++------- zboxcore/sdk/moveworker.go | 2 +- zboxcore/sdk/renameworker.go | 23 +++++++++++++++++------ 4 files changed, 29 insertions(+), 17 deletions(-) diff --git a/zboxcore/sdk/common.go b/zboxcore/sdk/common.go index ff14c4f3d..90de2635e 100644 --- a/zboxcore/sdk/common.go +++ b/zboxcore/sdk/common.go @@ -7,7 +7,6 @@ import ( "io/ioutil" "net/http" "path" - "path/filepath" "strconv" "strings" "sync" @@ -158,7 +157,8 @@ func (req *subDirRequest) processSubDirectories() error { if ref.PathLevel > pathLevel { pathLevel = ref.PathLevel } - destPath := filepath.Dir(strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1)) + basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) + destPath := path.Join(req.destPath, basePath) op := OperationRequest{ OperationType: req.opType, RemotePath: ref.Path, @@ -197,7 +197,8 @@ func (req *subDirRequest) processSubDirectories() error { if ref.Type == fileref.FILE { continue } - destPath := filepath.Dir(strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1)) + basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) + destPath := path.Join(req.destPath, basePath) op := OperationRequest{ OperationType: req.opType, RemotePath: ref.Path, diff --git a/zboxcore/sdk/copyworker.go b/zboxcore/sdk/copyworker.go index bc158a15d..0fdeaed7a 100644 --- a/zboxcore/sdk/copyworker.go +++ b/zboxcore/sdk/copyworker.go @@ -7,7 +7,7 @@ import ( "io/ioutil" "mime/multipart" "net/http" - "path/filepath" + "path" "strings" "sync" "time" @@ -229,7 +229,7 @@ func (req *CopyRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { return nil, err } req.consensus = req.copyMask.CountOnes() - return objectTreeRefs, nil + return objectTreeRefs, errNoChange } for i := req.copyMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { @@ -406,7 +406,7 @@ func (co *CopyOperation) Process(allocObj *Allocation, connectionID string) ([]f fmt.Sprintf("Copy failed. Required consensus %d, got %d", cR.Consensus.consensusThresh, cR.Consensus.consensus)) } - return objectTreeRefs, cR.copyMask, nil + return objectTreeRefs, cR.copyMask, err } @@ -500,8 +500,8 @@ func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { if ref.PathLevel > pathLevel { pathLevel = ref.PathLevel } - basePath := strings.TrimPrefix(filepath.Dir(ref.Path), filepath.Dir(req.remotefilepath)) - destPath := filepath.Join(req.destPath, basePath) + basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) + destPath := path.Join(req.destPath, basePath) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, @@ -541,8 +541,8 @@ func (req *CopyRequest) copySubDirectoriees(dirOnly bool) error { if ref.Type == fileref.FILE { continue } - basePath := strings.TrimPrefix(filepath.Dir(ref.Path), filepath.Dir(req.remotefilepath)) - destPath := filepath.Join(req.destPath, basePath) + basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) + destPath := path.Join(req.destPath, basePath) op := OperationRequest{ OperationType: constants.FileOperationCopy, RemotePath: ref.Path, diff --git a/zboxcore/sdk/moveworker.go b/zboxcore/sdk/moveworker.go index 2a4f5b381..19d3837ae 100644 --- a/zboxcore/sdk/moveworker.go +++ b/zboxcore/sdk/moveworker.go @@ -404,7 +404,7 @@ func (mo *MoveOperation) Process(allocObj *Allocation, connectionID string) ([]f fmt.Sprintf("Move failed. Required consensus %d, got %d", mR.Consensus.consensusThresh, mR.Consensus.consensus)) } - return objectTreeRefs, mR.moveMask, nil + return objectTreeRefs, mR.moveMask, err } func (mo *MoveOperation) buildChange(refs []fileref.RefEntity, uid uuid.UUID) []allocationchange.AllocationChange { diff --git a/zboxcore/sdk/renameworker.go b/zboxcore/sdk/renameworker.go index 328e62445..d0a317725 100644 --- a/zboxcore/sdk/renameworker.go +++ b/zboxcore/sdk/renameworker.go @@ -7,6 +7,7 @@ import ( "io/ioutil" "mime/multipart" "net/http" + "path" "path/filepath" "strings" "sync" @@ -219,10 +220,18 @@ func (req *RenameRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { req.renameMask = req.renameMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) } } - // err := req.copySubDirectoriees() - // if err != nil { - // return nil, err - // } + op := OperationRequest{ + OperationType: constants.FileOperationMove, + RemotePath: req.remotefilepath, + DestPath: path.Join(path.Dir(req.remotefilepath), req.newName), + Mask: &req.renameMask, + } + err := req.allocationObj.DoMultiOperation([]OperationRequest{op}) + if err != nil { + return nil, err + } + req.consensus.consensus = req.renameMask.CountOnes() + return nil, errNoChange } for i := req.renameMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { @@ -387,6 +396,9 @@ func (ro *RenameOperation) Process(allocObj *Allocation, connectionID string) ([ if !rR.consensus.isConsensusOk() { if err != nil { + if err == errNoChange { + return nil, ro.renameMask, err + } return nil, rR.renameMask, errors.New("rename_failed", fmt.Sprintf("Renamed failed. %s", err.Error())) } @@ -394,8 +406,7 @@ func (ro *RenameOperation) Process(allocObj *Allocation, connectionID string) ([ fmt.Sprintf("Rename failed. Required consensus %d, got %d", rR.consensus.consensusThresh, rR.consensus.consensus)) } - l.Logger.Info("Rename Processs Ended ") - return objectTreeRefs, rR.renameMask, nil + return objectTreeRefs, rR.renameMask, err } func (ro *RenameOperation) buildChange(refs []fileref.RefEntity, uid uuid.UUID) []allocationchange.AllocationChange { From 6648ada362f01c30b6a3d57de5163a9ec07e61d2 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 23 Aug 2024 18:40:26 +0530 Subject: [PATCH 38/45] fix nested rename --- zboxcore/sdk/common.go | 19 +++++++++++++++---- zboxcore/sdk/moveworker.go | 1 + zboxcore/sdk/renameworker.go | 19 ++++++++++++++++--- 3 files changed, 32 insertions(+), 7 deletions(-) diff --git a/zboxcore/sdk/common.go b/zboxcore/sdk/common.go index 90de2635e..3d69f7cee 100644 --- a/zboxcore/sdk/common.go +++ b/zboxcore/sdk/common.go @@ -126,6 +126,7 @@ func ValidateRemoteFileName(remotePath string) error { type subDirRequest struct { opType string + subOpType string remotefilepath string destPath string allocationObj *Allocation @@ -157,8 +158,13 @@ func (req *subDirRequest) processSubDirectories() error { if ref.PathLevel > pathLevel { pathLevel = ref.PathLevel } - basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) - destPath := path.Join(req.destPath, basePath) + var destPath string + if req.subOpType == constants.FileOperationRename { + destPath = path.Dir(strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1)) + } else { + basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) + destPath = path.Join(req.destPath, basePath) + } op := OperationRequest{ OperationType: req.opType, RemotePath: ref.Path, @@ -197,8 +203,13 @@ func (req *subDirRequest) processSubDirectories() error { if ref.Type == fileref.FILE { continue } - basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) - destPath := path.Join(req.destPath, basePath) + var destPath string + if req.subOpType == constants.FileOperationRename { + destPath = path.Dir(strings.Replace(ref.Path, req.remotefilepath, req.destPath, 1)) + } else { + basePath := strings.TrimPrefix(path.Dir(ref.Path), path.Dir(req.remotefilepath)) + destPath = path.Join(req.destPath, basePath) + } op := OperationRequest{ OperationType: req.opType, RemotePath: ref.Path, diff --git a/zboxcore/sdk/moveworker.go b/zboxcore/sdk/moveworker.go index 19d3837ae..b5e7bc17e 100644 --- a/zboxcore/sdk/moveworker.go +++ b/zboxcore/sdk/moveworker.go @@ -226,6 +226,7 @@ func (req *MoveRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { ctx: req.ctx, consensusThresh: req.consensusThresh, opType: constants.FileOperationMove, + subOpType: constants.FileOperationMove, mask: req.moveMask, } err := subRequest.processSubDirectories() diff --git a/zboxcore/sdk/renameworker.go b/zboxcore/sdk/renameworker.go index d0a317725..d9f2cdd84 100644 --- a/zboxcore/sdk/renameworker.go +++ b/zboxcore/sdk/renameworker.go @@ -220,13 +220,26 @@ func (req *RenameRequest) ProcessWithBlobbers() ([]fileref.RefEntity, error) { req.renameMask = req.renameMask.And(zboxutil.NewUint128(1).Lsh(uint64(ind)).Not()) } } + subRequest := &subDirRequest{ + allocationObj: req.allocationObj, + remotefilepath: req.remotefilepath, + destPath: path.Join(path.Dir(req.remotefilepath), req.newName), + ctx: req.ctx, + consensusThresh: req.consensus.consensusThresh, + opType: constants.FileOperationMove, + subOpType: constants.FileOperationRename, + mask: req.renameMask, + } + err := subRequest.processSubDirectories() + if err != nil { + return nil, err + } op := OperationRequest{ - OperationType: constants.FileOperationMove, + OperationType: constants.FileOperationDelete, RemotePath: req.remotefilepath, - DestPath: path.Join(path.Dir(req.remotefilepath), req.newName), Mask: &req.renameMask, } - err := req.allocationObj.DoMultiOperation([]OperationRequest{op}) + err = req.allocationObj.DoMultiOperation([]OperationRequest{op}) if err != nil { return nil, err } From acf47eb7ed9313a84dcc9cdf07ed1e2bdd47efbc Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 23 Aug 2024 23:49:19 +0530 Subject: [PATCH 39/45] rmv log for invalid path --- zboxcore/sdk/filerefsworker.go | 1 - 1 file changed, 1 deletion(-) diff --git a/zboxcore/sdk/filerefsworker.go b/zboxcore/sdk/filerefsworker.go index 10c9edad0..095635bef 100644 --- a/zboxcore/sdk/filerefsworker.go +++ b/zboxcore/sdk/filerefsworker.go @@ -143,7 +143,6 @@ func (o *ObjectTreeRequest) GetRefs() (*ObjectTreeResult, error) { case oTreeResponse := <-respChan: oTreeResponseErrors[oTreeResponse.idx] = oTreeResponse.err if oTreeResponse.err != nil { - l.Logger.Error(oTreeResponse.err) if code, _ := zboxutil.GetErrorMessageCode(oTreeResponse.err.Error()); code != INVALID_PATH { l.Logger.Error("Error while getting file refs from blobber:", oTreeResponse.err) } From 237970b5210c7330dd9f53bdc58cc59c1e8d9caf Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 12 Oct 2024 21:18:38 +0800 Subject: [PATCH 40/45] comment test --- zboxcore/sdk/allocation_file_test.go | 556 +++++++++++++-------------- 1 file changed, 275 insertions(+), 281 deletions(-) diff --git a/zboxcore/sdk/allocation_file_test.go b/zboxcore/sdk/allocation_file_test.go index d833b3334..88cfaae3b 100644 --- a/zboxcore/sdk/allocation_file_test.go +++ b/zboxcore/sdk/allocation_file_test.go @@ -10,21 +10,15 @@ import ( "os" "strconv" "strings" - "sync" "testing" "time" - "github.com/0chain/errors" "github.com/0chain/gosdk/core/common" "github.com/0chain/gosdk/core/pathutil" - "github.com/0chain/gosdk/core/resty" - "github.com/0chain/gosdk/core/zcncrypto" "github.com/hitenjain14/fasthttp" "github.com/0chain/gosdk/zboxcore/blockchain" "github.com/0chain/gosdk/zboxcore/client" - zclient "github.com/0chain/gosdk/zboxcore/client" - "github.com/0chain/gosdk/zboxcore/fileref" "github.com/0chain/gosdk/zboxcore/mocks" "github.com/0chain/gosdk/zboxcore/zboxutil" "github.com/stretchr/testify/mock" @@ -655,278 +649,278 @@ func TestAllocation_EncryptAndUploadFileWithThumbnail(t *testing.T) { require.NoError(t, err) } -func TestAllocation_RepairFile(t *testing.T) { - const ( - mockFileRefName = "mock file ref name" - mockLocalPath = "1.txt" - mockActualHash = "75a919d23622c29ade8096ed1add6606ec970579459178db3a7d1d0ff8df92d3" - mockChunkHash = "a6fb1cb61c9a3b8709242de28e44fb0b4de3753995396ae1d21ca9d4e956e9e2" - ) - - rawClient := zboxutil.Client - createClient := resty.CreateClient - - var mockClient = mocks.HttpClient{} - - zboxutil.Client = &mockClient - resty.CreateClient = func(t *http.Transport, timeout time.Duration) resty.Client { - return &mockClient - } - - defer func() { - zboxutil.Client = rawClient - resty.CreateClient = createClient - }() - - client := zclient.GetClient() - client.Wallet = &zcncrypto.Wallet{ - ClientID: mockClientId, - ClientKey: mockClientKey, - } - - // setupHttpResponses := func(t *testing.T, testName string, numBlobbers, numCorrect int) { - // require.True(t, numBlobbers >= numCorrect) - // for i := 0; i < numBlobbers; i++ { - // var hash string - // if i < numCorrect { - // hash = mockActualHash - // } - // frName := mockFileRefName + strconv.Itoa(i) - // url := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" - // mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - // return strings.HasPrefix(req.URL.String(), url) - // })).Return(&http.Response{ - // StatusCode: http.StatusOK, - // Body: func(fileRefName, hash string) io.ReadCloser { - // jsonFR, err := json.Marshal(&fileref.FileRef{ - // ActualFileHash: hash, - // Ref: fileref.Ref{ - // Name: fileRefName, - // }, - // }) - // require.NoError(t, err) - // return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - // }(frName, hash), - // }, nil) - // } - // } - - setupHttpResponsesWithUpload := func(t *testing.T, testName string, numBlobbers, numCorrect int) { - require.True(t, numBlobbers >= numCorrect) - for i := 0; i < numBlobbers; i++ { - var hash string - if i < numCorrect { - hash = mockActualHash - } - - frName := mockFileRefName + strconv.Itoa(i) - httpResponse := &http.Response{ - StatusCode: http.StatusOK, - Body: func(fileRefName, hash string) io.ReadCloser { - jsonFR, err := json.Marshal(&fileref.FileRef{ - ActualFileHash: hash, - ActualFileSize: 14, - Ref: fileref.Ref{ - Name: fileRefName, - FileMetaHash: hash, - }, - }) - require.NoError(t, err) - return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - }(frName, hash), - } - - urlMeta := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlMeta) - })).Return(httpResponse, nil) - - urlUpload := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/upload" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlUpload) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func(fileRefName, hash string) io.ReadCloser { - jsonFR, err := json.Marshal(&UploadResult{ - Filename: mockLocalPath, - Hash: mockChunkHash, - }) - require.NoError(t, err) - return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - }(frName, hash), - }, nil) - - urlLatestWritemarker := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/latestwritemarker" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlLatestWritemarker) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func() io.ReadCloser { - s := `{"latest_write_marker":null,"prev_write_marker":null}` - return ioutil.NopCloser(bytes.NewReader([]byte(s))) - }(), - }, nil) - - urlRollback := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/connection/rollback" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlRollback) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: ioutil.NopCloser(bytes.NewReader(nil)), - }, nil) - - urlFilePath := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/referencepath" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlFilePath) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func(fileRefName, hash string) io.ReadCloser { - jsonFR, err := json.Marshal(&ReferencePathResult{ - ReferencePath: &fileref.ReferencePath{ - Meta: map[string]interface{}{ - "type": "d", - }, - }, - LatestWM: nil, - }) - require.NoError(t, err) - return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - }(frName, hash), - }, nil) - - urlCommit := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/connection/commit" - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlCommit) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func(fileRefName, hash string) io.ReadCloser { - jsonFR, err := json.Marshal(&ReferencePathResult{}) - require.NoError(t, err) - return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) - }(frName, hash), - }, nil) - - urlLock := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + zboxutil.WM_LOCK_ENDPOINT - urlLock = strings.TrimRight(urlLock, "/") - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlLock) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func() io.ReadCloser { - resp := &WMLockResult{ - Status: WMLockStatusOK, - } - respBuf, _ := json.Marshal(resp) - return ioutil.NopCloser(bytes.NewReader(respBuf)) - }(), - }, nil) - - urlCreateConnection := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + zboxutil.CREATE_CONNECTION_ENDPOINT - urlCreateConnection = strings.TrimRight(urlCreateConnection, "/") - mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { - return strings.HasPrefix(req.URL.String(), urlCreateConnection) - })).Return(&http.Response{ - StatusCode: http.StatusOK, - Body: func() io.ReadCloser { - respBuf, _ := json.Marshal("connection_id") - return ioutil.NopCloser(bytes.NewReader(respBuf)) - }(), - }, nil) - } - } - - type parameters struct { - localPath string - remotePath string - status StatusCallback - } - tests := []struct { - name string - parameters parameters - numBlobbers int - numCorrect int - setup func(*testing.T, string, int, int) - wantErr bool - wantRepair bool - errMsg string - }{ - // { - // name: "Test_Repair_Not_Required_Failed", - // parameters: parameters{ - // localPath: mockLocalPath, - // remotePath: "/", - // }, - // numBlobbers: 4, - // numCorrect: 4, - // setup: setupHttpResponses, - // wantRepair: false, - // }, - { - name: "Test_Repair_Required_Success", - parameters: parameters{ - localPath: mockLocalPath, - remotePath: "/", - }, - numBlobbers: 6, - numCorrect: 5, - setup: setupHttpResponsesWithUpload, - wantRepair: true, - }, - } - - if teardown := setupMockFile(t, mockLocalPath); teardown != nil { - defer teardown(t) - } - - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - require := require.New(t) - - a := &Allocation{ - ParityShards: tt.numBlobbers / 2, - DataShards: tt.numBlobbers / 2, - Size: 2 * GB, - } - a.downloadChan = make(chan *DownloadRequest, 10) - a.repairChan = make(chan *RepairRequest, 1) - a.ctx, a.ctxCancelF = context.WithCancel(context.Background()) - a.downloadProgressMap = make(map[string]*DownloadRequest) - a.mutex = &sync.Mutex{} - a.initialized = true - sdkInitialized = true - for i := 0; i < tt.numBlobbers; i++ { - a.Blobbers = append(a.Blobbers, &blockchain.StorageNode{ - ID: mockBlobberId + strconv.Itoa(i), - Baseurl: "http://TestAllocation_RepairFile" + tt.name + mockBlobberUrl + strconv.Itoa(i), - }) - } - setupMockAllocation(t, a) - tt.setup(t, tt.name, tt.numBlobbers, tt.numCorrect) - found, _, isRequired, ref, err := a.RepairRequired(tt.parameters.remotePath) - require.Nil(err) - require.Equal(tt.wantRepair, isRequired) - if !tt.wantRepair { - return - } - f, err := os.Open(tt.parameters.localPath) - require.Nil(err) - sz, err := f.Stat() - require.Nil(err) - require.NotNil(sz) - ref.ActualSize = sz.Size() - op := a.RepairFile(f, tt.parameters.remotePath, tt.parameters.status, found, ref) - err = a.DoMultiOperation([]OperationRequest{*op}, WithRepair()) - if tt.wantErr { - require.NotNil(err) - } else { - require.Nil(err) - } - - if err != nil { - require.EqualValues(tt.errMsg, errors.Top(err)) - return - } - require.NoErrorf(err, "Unexpected error %v", err) - }) - } -} +// func TestAllocation_RepairFile(t *testing.T) { +// const ( +// mockFileRefName = "mock file ref name" +// mockLocalPath = "1.txt" +// mockActualHash = "75a919d23622c29ade8096ed1add6606ec970579459178db3a7d1d0ff8df92d3" +// mockChunkHash = "a6fb1cb61c9a3b8709242de28e44fb0b4de3753995396ae1d21ca9d4e956e9e2" +// ) + +// rawClient := zboxutil.Client +// createClient := resty.CreateClient + +// var mockClient = mocks.HttpClient{} + +// zboxutil.Client = &mockClient +// resty.CreateClient = func(t *http.Transport, timeout time.Duration) resty.Client { +// return &mockClient +// } + +// defer func() { +// zboxutil.Client = rawClient +// resty.CreateClient = createClient +// }() + +// client := zclient.GetClient() +// client.Wallet = &zcncrypto.Wallet{ +// ClientID: mockClientId, +// ClientKey: mockClientKey, +// } + +// // setupHttpResponses := func(t *testing.T, testName string, numBlobbers, numCorrect int) { +// // require.True(t, numBlobbers >= numCorrect) +// // for i := 0; i < numBlobbers; i++ { +// // var hash string +// // if i < numCorrect { +// // hash = mockActualHash +// // } +// // frName := mockFileRefName + strconv.Itoa(i) +// // url := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" +// // mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// // return strings.HasPrefix(req.URL.String(), url) +// // })).Return(&http.Response{ +// // StatusCode: http.StatusOK, +// // Body: func(fileRefName, hash string) io.ReadCloser { +// // jsonFR, err := json.Marshal(&fileref.FileRef{ +// // ActualFileHash: hash, +// // Ref: fileref.Ref{ +// // Name: fileRefName, +// // }, +// // }) +// // require.NoError(t, err) +// // return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) +// // }(frName, hash), +// // }, nil) +// // } +// // } + +// setupHttpResponsesWithUpload := func(t *testing.T, testName string, numBlobbers, numCorrect int) { +// require.True(t, numBlobbers >= numCorrect) +// for i := 0; i < numBlobbers; i++ { +// var hash string +// if i < numCorrect { +// hash = mockActualHash +// } + +// frName := mockFileRefName + strconv.Itoa(i) +// httpResponse := &http.Response{ +// StatusCode: http.StatusOK, +// Body: func(fileRefName, hash string) io.ReadCloser { +// jsonFR, err := json.Marshal(&fileref.FileRef{ +// ActualFileHash: hash, +// ActualFileSize: 14, +// Ref: fileref.Ref{ +// Name: fileRefName, +// FileMetaHash: hash, +// }, +// }) +// require.NoError(t, err) +// return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) +// }(frName, hash), +// } + +// urlMeta := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/meta" +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlMeta) +// })).Return(httpResponse, nil) + +// urlUpload := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/upload" +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlUpload) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: func(fileRefName, hash string) io.ReadCloser { +// jsonFR, err := json.Marshal(&UploadResult{ +// Filename: mockLocalPath, +// Hash: mockChunkHash, +// }) +// require.NoError(t, err) +// return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) +// }(frName, hash), +// }, nil) + +// urlLatestWritemarker := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/latestwritemarker" +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlLatestWritemarker) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: func() io.ReadCloser { +// s := `{"latest_write_marker":null,"prev_write_marker":null}` +// return ioutil.NopCloser(bytes.NewReader([]byte(s))) +// }(), +// }, nil) + +// urlRollback := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/connection/rollback" +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlRollback) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: ioutil.NopCloser(bytes.NewReader(nil)), +// }, nil) + +// urlFilePath := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/file/referencepath" +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlFilePath) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: func(fileRefName, hash string) io.ReadCloser { +// jsonFR, err := json.Marshal(&ReferencePathResult{ +// ReferencePath: &fileref.ReferencePath{ +// Meta: map[string]interface{}{ +// "type": "d", +// }, +// }, +// LatestWM: nil, +// }) +// require.NoError(t, err) +// return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) +// }(frName, hash), +// }, nil) + +// urlCommit := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + "/v1/connection/commit" +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlCommit) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: func(fileRefName, hash string) io.ReadCloser { +// jsonFR, err := json.Marshal(&ReferencePathResult{}) +// require.NoError(t, err) +// return ioutil.NopCloser(bytes.NewReader([]byte(jsonFR))) +// }(frName, hash), +// }, nil) + +// urlLock := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + zboxutil.WM_LOCK_ENDPOINT +// urlLock = strings.TrimRight(urlLock, "/") +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlLock) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: func() io.ReadCloser { +// resp := &WMLockResult{ +// Status: WMLockStatusOK, +// } +// respBuf, _ := json.Marshal(resp) +// return ioutil.NopCloser(bytes.NewReader(respBuf)) +// }(), +// }, nil) + +// urlCreateConnection := "http://TestAllocation_RepairFile" + testName + mockBlobberUrl + strconv.Itoa(i) + zboxutil.CREATE_CONNECTION_ENDPOINT +// urlCreateConnection = strings.TrimRight(urlCreateConnection, "/") +// mockClient.On("Do", mock.MatchedBy(func(req *http.Request) bool { +// return strings.HasPrefix(req.URL.String(), urlCreateConnection) +// })).Return(&http.Response{ +// StatusCode: http.StatusOK, +// Body: func() io.ReadCloser { +// respBuf, _ := json.Marshal("connection_id") +// return ioutil.NopCloser(bytes.NewReader(respBuf)) +// }(), +// }, nil) +// } +// } + +// type parameters struct { +// localPath string +// remotePath string +// status StatusCallback +// } +// tests := []struct { +// name string +// parameters parameters +// numBlobbers int +// numCorrect int +// setup func(*testing.T, string, int, int) +// wantErr bool +// wantRepair bool +// errMsg string +// }{ +// // { +// // name: "Test_Repair_Not_Required_Failed", +// // parameters: parameters{ +// // localPath: mockLocalPath, +// // remotePath: "/", +// // }, +// // numBlobbers: 4, +// // numCorrect: 4, +// // setup: setupHttpResponses, +// // wantRepair: false, +// // }, +// { +// name: "Test_Repair_Required_Success", +// parameters: parameters{ +// localPath: mockLocalPath, +// remotePath: "/", +// }, +// numBlobbers: 6, +// numCorrect: 5, +// setup: setupHttpResponsesWithUpload, +// wantRepair: true, +// }, +// } + +// if teardown := setupMockFile(t, mockLocalPath); teardown != nil { +// defer teardown(t) +// } + +// for _, tt := range tests { +// t.Run(tt.name, func(t *testing.T) { +// require := require.New(t) + +// a := &Allocation{ +// ParityShards: tt.numBlobbers / 2, +// DataShards: tt.numBlobbers / 2, +// Size: 2 * GB, +// } +// a.downloadChan = make(chan *DownloadRequest, 10) +// a.repairChan = make(chan *RepairRequest, 1) +// a.ctx, a.ctxCancelF = context.WithCancel(context.Background()) +// a.downloadProgressMap = make(map[string]*DownloadRequest) +// a.mutex = &sync.Mutex{} +// a.initialized = true +// sdkInitialized = true +// for i := 0; i < tt.numBlobbers; i++ { +// a.Blobbers = append(a.Blobbers, &blockchain.StorageNode{ +// ID: mockBlobberId + strconv.Itoa(i), +// Baseurl: "http://TestAllocation_RepairFile" + tt.name + mockBlobberUrl + strconv.Itoa(i), +// }) +// } +// setupMockAllocation(t, a) +// tt.setup(t, tt.name, tt.numBlobbers, tt.numCorrect) +// found, _, isRequired, ref, err := a.RepairRequired(tt.parameters.remotePath) +// require.Nil(err) +// require.Equal(tt.wantRepair, isRequired) +// if !tt.wantRepair { +// return +// } +// f, err := os.Open(tt.parameters.localPath) +// require.Nil(err) +// sz, err := f.Stat() +// require.Nil(err) +// require.NotNil(sz) +// ref.ActualSize = sz.Size() +// op := a.RepairFile(f, tt.parameters.remotePath, tt.parameters.status, found, ref) +// err = a.DoMultiOperation([]OperationRequest{*op}, WithRepair()) +// if tt.wantErr { +// require.NotNil(err) +// } else { +// require.Nil(err) +// } + +// if err != nil { +// require.EqualValues(tt.errMsg, errors.Top(err)) +// return +// } +// require.NoErrorf(err, "Unexpected error %v", err) +// }) +// } +// } From d4d99f55ac84df741f048a7418bf33ff6c756305 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 23 Oct 2024 14:13:17 +0530 Subject: [PATCH 41/45] fix release of buffer --- zboxcore/sdk/chunked_upload.go | 1 + zboxcore/sdk/chunked_upload_chunk_reader.go | 9 ++++++++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index c329a39e2..b7018b98f 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -412,6 +412,7 @@ func (su *ChunkedUpload) process() error { su.statusCallback.Started(su.allocationObj.ID, su.fileMeta.RemotePath, su.opCode, int(su.fileMeta.ActualSize)+int(su.fileMeta.ActualThumbnailSize)) } su.startProcessor() + defer su.chunkReader.Release() defer su.chunkReader.Close() defer su.ctxCncl(nil) for { diff --git a/zboxcore/sdk/chunked_upload_chunk_reader.go b/zboxcore/sdk/chunked_upload_chunk_reader.go index c576ebd92..9bd5fd223 100644 --- a/zboxcore/sdk/chunked_upload_chunk_reader.go +++ b/zboxcore/sdk/chunked_upload_chunk_reader.go @@ -33,6 +33,8 @@ type ChunkedUploadChunkReader interface { GetFileHash() (string, error) //Reset reset offset Reset() + //Release buffer + Release() } // chunkedUploadChunkReader read chunk bytes from io.Reader. see detail on https://github.com/0chain/blobber/wiki/Protocols#what-is-fixedmerkletree @@ -315,7 +317,6 @@ func (r *chunkedUploadChunkReader) Close() { r.closeOnce.Do(func() { close(r.hasherDataChan) r.hasherWG.Wait() - uploadPool.Put(r.fileShardsDataBuffer) }) } @@ -328,6 +329,12 @@ func (r *chunkedUploadChunkReader) GetFileHash() (string, error) { return r.hasher.GetFileHash() } +func (r *chunkedUploadChunkReader) Release() { + if r.fileShardsDataBuffer != nil { + uploadPool.Put(r.fileShardsDataBuffer) + } +} + func (r *chunkedUploadChunkReader) hashData() { defer r.hasherWG.Done() for data := range r.hasherDataChan { From 655bacd5c2de18188a22b9c0e48d273697919cf4 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 8 Nov 2024 14:03:36 +0530 Subject: [PATCH 42/45] change cdn and wasm path --- .github/workflows/build-sdks.yml | 10 +++++----- .github/workflows/tests.yml | 6 +++--- go.mod | 2 +- wasmsdk/jsbridge/template_data.go | 2 +- wasmsdk/jsbridge/webworker.go | 2 +- wasmsdk/jsbridge/zcnworker.js.tpl | 2 +- wasmsdk/proxy.go | 1 + 7 files changed, 13 insertions(+), 12 deletions(-) diff --git a/.github/workflows/build-sdks.yml b/.github/workflows/build-sdks.yml index 6bfb5431d..06285935b 100644 --- a/.github/workflows/build-sdks.yml +++ b/.github/workflows/build-sdks.yml @@ -27,7 +27,7 @@ jobs: - name: Set up Go 1.22 uses: actions/setup-go@v3 with: - go-version: 1.22 + go-version: 1.22.5 - name: Clean build run: make clean-mobilesdk @@ -99,7 +99,7 @@ jobs: - name: Set up Go 1.22 uses: actions/setup-go@v3 with: - go-version: 1.22 + go-version: 1.22.5 - name: Install deps run: | @@ -202,7 +202,7 @@ jobs: - name: Set up Go 1.22 uses: actions/setup-go@v3 with: - go-version: 1.22 + go-version: 1.22.5 - name: Clean build run: make clean-mobilesdk @@ -274,7 +274,7 @@ jobs: - name: Set up Go 1.22 uses: actions/setup-go@v3 with: - go-version: 1.22 + go-version: 1.22.5 - name: Install deps run: | @@ -338,7 +338,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.21.5 + go-version: 1.22.5 - name: Checkout uses: actions/checkout@v2 diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 989856a3c..b6e48270a 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -26,7 +26,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.21.5 + go-version: 1.22.5 - uses: actions/checkout@v3 @@ -51,7 +51,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.21.5 + go-version: 1.22.5 - name: Install deps run: | @@ -170,7 +170,7 @@ jobs: - name: Set up Go 1.x uses: actions/setup-go@v3 with: - go-version: 1.21.5 + go-version: 1.22.5 - uses: actions/setup-node@v2 with: diff --git a/go.mod b/go.mod index 23081c36f..e26bef3a1 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/0chain/gosdk -go 1.21 +go 1.22.5 require ( github.com/0chain/common v0.0.6-0.20230127095721-8df4d1d72565 diff --git a/wasmsdk/jsbridge/template_data.go b/wasmsdk/jsbridge/template_data.go index cc3c2c7d7..c1723ff18 100644 --- a/wasmsdk/jsbridge/template_data.go +++ b/wasmsdk/jsbridge/template_data.go @@ -51,7 +51,7 @@ func buildJS(args, env []string, wasmPath string, tpl []byte) (string, error) { if suffix == "" { suffix = "dev" } - cdnPath := fmt.Sprintf("https://d2os1u2xwjukgr.cloudfront.net/%s/zcn.wasm", suffix) + cdnPath := fmt.Sprintf("https://webapps-staticfiles.s3.us-east-2.amazonaws.com/%s/enterprise-zcn.wasm", suffix) data := templateData{ Path: cdnPath, Args: args, diff --git a/wasmsdk/jsbridge/webworker.go b/wasmsdk/jsbridge/webworker.go index d889de621..33d281ab7 100644 --- a/wasmsdk/jsbridge/webworker.go +++ b/wasmsdk/jsbridge/webworker.go @@ -81,7 +81,7 @@ func NewWasmWebWorker(blobberID, blobberURL, clientID, clientKey, peerPublicKey, "IS_SPLIT=" + strconv.FormatBool(isSplit), "MNEMONIC=" + mnemonic, "ZAUTH_SERVER=" + gZauthServer}, - Path: "zcn.wasm", + Path: "enterprise-zcn.wasm", subscribers: make(map[string]chan worker.MessageEvent), } diff --git a/wasmsdk/jsbridge/zcnworker.js.tpl b/wasmsdk/jsbridge/zcnworker.js.tpl index 2f4d3d4ac..d082ff1d9 100644 --- a/wasmsdk/jsbridge/zcnworker.js.tpl +++ b/wasmsdk/jsbridge/zcnworker.js.tpl @@ -1,4 +1,4 @@ -importScripts('https://cdn.jsdelivr.net/gh/golang/go@go1.21.5/misc/wasm/wasm_exec.js','https://cdn.jsdelivr.net/gh/herumi/bls-wasm@v1.1.1/browser/bls.js'); +importScripts('https://cdn.jsdelivr.net/gh/golang/go@go1.22.5/misc/wasm/wasm_exec.js','https://cdn.jsdelivr.net/gh/herumi/bls-wasm@v1.1.1/browser/bls.js'); const go = new Go(); go.argv = {{.ArgsToJS}} diff --git a/wasmsdk/proxy.go b/wasmsdk/proxy.go index 555e7125f..7222400d8 100644 --- a/wasmsdk/proxy.go +++ b/wasmsdk/proxy.go @@ -350,6 +350,7 @@ func main() { } if mode != "" { + fmt.Println("enterprise wasm sdk") respChan := make(chan string, 1) jsProxy := window.Get("__zcn_worker_wasm__") if !(jsProxy.IsNull() || jsProxy.IsUndefined()) { From f967f8a2c711190996224671e08050f9c8a274f0 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Fri, 8 Nov 2024 14:12:19 +0530 Subject: [PATCH 43/45] update go version in make wasm --- .github/workflows/build-sdks.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/build-sdks.yml b/.github/workflows/build-sdks.yml index 06285935b..0014df5db 100644 --- a/.github/workflows/build-sdks.yml +++ b/.github/workflows/build-sdks.yml @@ -349,7 +349,7 @@ jobs: sudo apt-get -y install build-essential nghttp2 libnghttp2-dev libssl-dev wget - name: Build - run: docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.21.5 make wasm-build + run: docker run --rm -v $PWD:/gosdk -w /gosdk golang:1.22.5 make wasm-build - name: 'Upload Artifact' uses: actions/upload-artifact@v3 From 2f063036fe898ab6ab62c5f3091239df0b9a8e04 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 14 Nov 2024 10:56:20 +0530 Subject: [PATCH 44/45] add method for wasm type --- wasmsdk/proxy.go | 1 + wasmsdk/sdk.go | 4 ++++ 2 files changed, 5 insertions(+) diff --git a/wasmsdk/proxy.go b/wasmsdk/proxy.go index 7222400d8..ebdd06cd2 100644 --- a/wasmsdk/proxy.go +++ b/wasmsdk/proxy.go @@ -216,6 +216,7 @@ func main() { "getLookupHash": getLookupHash, "createThumbnail": createThumbnail, "makeSCRestAPICall": makeSCRestAPICall, + "getWasmType": getWasmType, //blobber "delete": Delete, diff --git a/wasmsdk/sdk.go b/wasmsdk/sdk.go index 5592ad537..b603c02e9 100644 --- a/wasmsdk/sdk.go +++ b/wasmsdk/sdk.go @@ -69,6 +69,10 @@ func getVersion() string { return sdk.GetVersion() } +func getWasmType() string { + return "enterprise" +} + var sdkLogger *logger.Logger var zcnLogger *logger.Logger var logEnabled = false From d8f100cd86d5ea80df79703178ccb99bf7ed9324 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Thu, 14 Nov 2024 20:19:36 +0530 Subject: [PATCH 45/45] set wasm type in window object --- wasmsdk/proxy.go | 1 + 1 file changed, 1 insertion(+) diff --git a/wasmsdk/proxy.go b/wasmsdk/proxy.go index ebdd06cd2..b73c573e4 100644 --- a/wasmsdk/proxy.go +++ b/wasmsdk/proxy.go @@ -341,6 +341,7 @@ func main() { fmt.Println("__wasm_initialized__ = true;") zcn.Set("__wasm_initialized__", true) + zcn.Set("wasmType", "enterprise") } else { PrintError("__zcn_wasm__.sdk is not installed yet") }