From cfc74040818bd472de395809f475f399f23eef67 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Mon, 6 May 2024 20:52:19 +0530 Subject: [PATCH] fix empty chunk on eof --- zboxcore/sdk/chunked_upload.go | 10 ++--- zboxcore/sdk/chunked_upload_blobber.go | 15 ------- zboxcore/sdk/chunked_upload_form_builder.go | 46 ++++++++++----------- 3 files changed, 28 insertions(+), 43 deletions(-) diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index 0857a7fb1..06d4cdd1d 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -476,10 +476,7 @@ func (su *ChunkedUpload) process() error { return thrown.New("upload_failed", "Upload failed. Uploaded size does not match with actual size: "+fmt.Sprintf("%d != %d", su.fileMeta.ActualSize, su.progress.ReadLength)) } } - if chunks.totalReadSize == 0 { - logger.Logger.Info("isFinal: ", chunks.isFinal, " totalReadSize: ", su.progress.ReadLength) - break - } + //chunk has not be uploaded yet if chunks.chunkEndIndex > su.progress.ChunkIndex { err = su.processUpload( @@ -653,7 +650,10 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, wgErrors := make(chan error, len(su.blobbers)) if len(fileShards) == 0 { - return thrown.New("upload_failed", "Upload failed. No data to upload") + if !isFinal { + return thrown.New("upload_failed", "Upload failed. No data to upload") + } + fileShards = make([]blobberShards, len(su.blobbers)) } for i := su.uploadMask; !i.Equals64(0); i = i.And(zboxutil.NewUint128(1).Lsh(pos).Not()) { diff --git a/zboxcore/sdk/chunked_upload_blobber.go b/zboxcore/sdk/chunked_upload_blobber.go index 886464aff..c323549e2 100644 --- a/zboxcore/sdk/chunked_upload_blobber.go +++ b/zboxcore/sdk/chunked_upload_blobber.go @@ -53,21 +53,6 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest( } }() - if formData.FileBytesLen == 0 { - //fixed fileRef in last chunk on stream. io.EOF with nil bytes - if isFinal { - sb.fileRef.ChunkSize = su.chunkSize - sb.fileRef.Size = su.shardUploadedSize - sb.fileRef.Path = su.fileMeta.RemotePath - sb.fileRef.ActualFileHash = su.fileMeta.ActualHash - sb.fileRef.ActualFileSize = su.fileMeta.ActualSize - - sb.fileRef.EncryptedKey = encryptedKey - sb.fileRef.CalculateHash() - consensus.Done() - } - } - eg, _ := errgroup.WithContext(ctx) for dataInd := 0; dataInd < len(dataBuffers); dataInd++ { diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index 94cb956f5..e3c20a930 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -57,14 +57,13 @@ func (b *chunkedUploadFormBuilder) Build( var res blobberData - if len(fileChunksData) == 0 { - return res, nil - } - numBodies := len(fileChunksData) / MAX_BLOCKS if len(fileChunksData)%MAX_BLOCKS > 0 { numBodies++ } + if len(fileChunksData) == 0 { + numBodies = 1 + } dataBuffers := make([]*bytes.Buffer, 0, numBodies) contentSlice := make([]string, 0, numBodies) @@ -103,33 +102,34 @@ func (b *chunkedUploadFormBuilder) Build( body := bytes.NewBuffer(bodyBuf) formWriter := multipart.NewWriter(body) defer formWriter.Close() - - uploadFile, err := formWriter.CreateFormFile("uploadFile", formData.Filename) - if err != nil { - return res, err - } - - for _, chunkBytes := range fileChunksData[startRange:endRange] { - _, err = uploadFile.Write(chunkBytes) + if endRange > startRange { + uploadFile, err := formWriter.CreateFormFile("uploadFile", formData.Filename) if err != nil { return res, err } - err = hasher.WriteToFixedMT(chunkBytes) - if err != nil { - return res, err - } + for _, chunkBytes := range fileChunksData[startRange:endRange] { + _, err = uploadFile.Write(chunkBytes) + if err != nil { + return res, err + } - err = hasher.WriteToValidationMT(chunkBytes) - if err != nil { - return res, err - } + err = hasher.WriteToFixedMT(chunkBytes) + if err != nil { + return res, err + } + + err = hasher.WriteToValidationMT(chunkBytes) + if err != nil { + return res, err + } - metadata.FileBytesLen += len(chunkBytes) + metadata.FileBytesLen += len(chunkBytes) + } } if isFinal && i == numBodies-1 { - err = hasher.Finalize() + err := hasher.Finalize() if err != nil { return res, err } @@ -202,7 +202,7 @@ func (b *chunkedUploadFormBuilder) Build( formData.UploadOffset = formData.UploadOffset + chunkSize*int64(MAX_BLOCKS) } - err = formWriter.WriteField("connection_id", connectionID) + err := formWriter.WriteField("connection_id", connectionID) if err != nil { return res, err }