From 0ec7e4cabd0f4ab9c0041d045724388e3ae2355e Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 27 Apr 2024 23:50:33 +0530 Subject: [PATCH 01/12] add timings for upload --- wasmsdk/blobber.go | 16 ++++++++++++++++ wasmsdk/proxy.go | 1 + zboxcore/sdk/chunked_upload.go | 4 ++-- zboxcore/sdk/multi_operation_worker.go | 12 +++++++++++- 4 files changed, 30 insertions(+), 3 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index 031715717..f9903e0cc 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -924,3 +924,19 @@ func getBlobbers(stakable bool) ([]*sdk.Blobber, error) { } return blobbs, err } + +type timingRes struct { + UploadTime int64 `json:"upload_time"` + ReadTime int64 `json:"read_time"` + TotalTime int64 `json:"total_time"` +} + +func getUploadTiming() string { + res := timingRes{ + UploadTime: sdk.TotalUploadTime, + ReadTime: sdk.TotalReadTime, + TotalTime: sdk.TotalTime, + } + respBytes, _ := json.Marshal(res) + return string(respBytes) +} diff --git a/wasmsdk/proxy.go b/wasmsdk/proxy.go index fca2681f3..de3c6962b 100644 --- a/wasmsdk/proxy.go +++ b/wasmsdk/proxy.go @@ -187,6 +187,7 @@ func main() { "updateForbidAllocation": UpdateForbidAllocation, "send": send, "cancelUpload": cancelUpload, + "uploadTiming": getUploadTiming, // player "play": play, diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index aa4b9729a..11f576adb 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -440,9 +440,9 @@ func (su *ChunkedUpload) process() error { defer su.chunkReader.Close() defer su.ctxCncl(nil) for { - + now := time.Now() chunks, err := su.readChunks(su.chunkNumber) - + TotalReadTime += time.Since(now).Milliseconds() // chunk, err := su.chunkReader.Next() if err != nil { if su.statusCallback != nil { diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index ca71709a5..6efdff420 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -40,6 +40,12 @@ func WithRepair() MultiOperationOption { } } +var ( + TotalUploadTime int64 + TotalTime int64 + TotalReadTime int64 +) + type Operationer interface { Process(allocObj *Allocation, connectionID string) ([]fileref.RefEntity, zboxutil.Uint128, error) buildChange(refs []fileref.RefEntity, uid uuid.UUID) []allocationchange.AllocationChange @@ -160,6 +166,7 @@ func (mo *MultiOperation) createConnectionObj(blobberIdx int) (err error) { func (mo *MultiOperation) Process() error { l.Logger.Info("MultiOperation Process start") + TotalReadTime = 0 wg := &sync.WaitGroup{} mo.changes = make([][]allocationchange.AllocationChange, len(mo.operations)) ctx := mo.ctx @@ -168,6 +175,7 @@ func (mo *MultiOperation) Process() error { swg := sizedwaitgroup.New(BatchSize) errsSlice := make([]error, len(mo.operations)) mo.operationMask = zboxutil.NewUint128(0) + now := time.Now() for idx, op := range mo.operations { uid := util.GetNewUUID() swg.Add() @@ -196,7 +204,8 @@ func (mo *MultiOperation) Process() error { }(op, idx) } swg.Wait() - + logger.Logger.Info("[process]", time.Since(now).Milliseconds()) + TotalUploadTime = time.Since(now).Milliseconds() // Check consensus if mo.operationMask.CountOnes() < mo.consensusThresh || ctx.Err() != nil { majorErr := zboxutil.MajorError(errsSlice) @@ -353,6 +362,7 @@ func (mo *MultiOperation) Process() error { op.Completed(mo.allocationObj) } } + TotalTime = time.Since(now).Milliseconds() return nil From 50239a804853465c49f471b66ce6be3179c572bf Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 1 May 2024 20:04:43 +0530 Subject: [PATCH 02/12] update file reader and writer for wasm --- Makefile | 2 +- wasmsdk/blobber.go | 13 +++- wasmsdk/jsbridge/file_reader.go | 62 +++++++++++----- wasmsdk/jsbridge/file_writer.go | 78 +++++++++++++-------- zboxcore/sdk/chunked_upload_form_builder.go | 2 +- 5 files changed, 107 insertions(+), 50 deletions(-) diff --git a/Makefile b/Makefile index 3398c21ea..d0652973d 100644 --- a/Makefile +++ b/Makefile @@ -35,7 +35,7 @@ gosdk-build: gomod-download go build -x -v -tags bn256 ./... wasm-build: getrev - CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -buildvcs=false -o ./zcn.wasm ./wasmsdk + CGO_ENABLED=0 GOOS=js GOARCH=wasm go build -ldflags="-s -w" -buildvcs=false -o ./zcn.wasm ./wasmsdk wasm-test: wasm-build env -i $(shell go env) PATH="$(shell go env GOROOT)/misc/wasm:$(PATH)" CGO_ENABLED=0 GOOS=js GOARCH=wasm go test -v github.com/0chain/gosdk/wasmsdk/jsbridge/... diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index f9903e0cc..a64c2436b 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -631,8 +631,12 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) { wg.Add(1) encrypt := option.Encrypt remotePath := option.RemotePath - - fileReader := jsbridge.NewFileReader(option.ReadChunkFuncName, option.FileSize) + fileReader, err := jsbridge.NewFileReader(option.ReadChunkFuncName, option.FileSize, allocationObj.GetChunkReadSize(encrypt)) + if err != nil { + result.Error = "Error in file operation" + result.Success = false + return result, err + } mimeType := option.MimeType localPath := remotePath remotePath = zboxutil.RemoteClean(remotePath) @@ -725,7 +729,10 @@ func uploadWithJsFuncs(allocationID, remotePath string, readChunkFuncName string } wg.Add(1) - fileReader := jsbridge.NewFileReader(readChunkFuncName, fileSize) + fileReader, err := jsbridge.NewFileReader(readChunkFuncName, fileSize, allocationObj.GetChunkReadSize(encrypt)) + if err != nil { + return false, err + } localPath := remotePath diff --git a/wasmsdk/jsbridge/file_reader.go b/wasmsdk/jsbridge/file_reader.go index 33e8fc33f..22a88ad64 100644 --- a/wasmsdk/jsbridge/file_reader.go +++ b/wasmsdk/jsbridge/file_reader.go @@ -6,46 +6,74 @@ package jsbridge import ( "errors" "io" - "sync" "syscall/js" ) -var jsFileReaderMutex sync.Mutex - type FileReader struct { size int64 offset int64 readChunk js.Value + buf []byte + bufOffset int + endOfFile bool } -func NewFileReader(readChunkFuncName string, fileSize int64) *FileReader { - readChunk := js.Global().Get(readChunkFuncName) +const ( + bufferSize = 16 * 1024 * 1024 //16MB +) +func NewFileReader(readChunkFuncName string, fileSize, chunkReadSize int64) (*FileReader, error) { + readChunk := js.Global().Get(readChunkFuncName) + var buf []byte + if bufferSize > fileSize { + buf = make([]byte, fileSize) + } else { + bufSize := (chunkReadSize * (bufferSize / chunkReadSize)) + buf = make([]byte, bufSize) + } + result, err := Await(readChunk.Invoke(0, len(buf))) + if len(err) > 0 && !err[0].IsNull() { + return nil, errors.New("file_reader: " + err[0].String()) + } + chunk := result[0] + n := js.CopyBytesToGo(buf, chunk) + if n < len(buf) { + return nil, errors.New("file_reader: failed to read first chunk") + } return &FileReader{ size: fileSize, - offset: 0, + offset: int64(n), readChunk: readChunk, - } + buf: buf, + endOfFile: n == int(fileSize), + }, nil } func (r *FileReader) Read(p []byte) (int, error) { //js.Value doesn't work in parallel invoke - jsFileReaderMutex.Lock() - defer jsFileReaderMutex.Unlock() size := len(p) - result, err := Await(r.readChunk.Invoke(r.offset, size)) + if len(r.buf)-r.bufOffset < size && !r.endOfFile { + r.bufOffset = 0 //reset buffer offset + result, err := Await(r.readChunk.Invoke(r.offset, len(r.buf))) - if len(err) > 0 && !err[0].IsNull() { - return 0, errors.New("file_reader: " + err[0].String()) - } + if len(err) > 0 && !err[0].IsNull() { + return 0, errors.New("file_reader: " + err[0].String()) + } - chunk := result[0] + chunk := result[0] - n := js.CopyBytesToGo(p, chunk) - r.offset += int64(n) + n := js.CopyBytesToGo(r.buf, chunk) + r.offset += int64(n) + if n < len(r.buf) { + r.buf = r.buf[:n] + r.endOfFile = true + } + } - if n < size { + n := copy(p, r.buf[r.bufOffset:]) + r.bufOffset += n + if r.endOfFile && r.bufOffset == len(r.buf) { return n, io.EOF } diff --git a/wasmsdk/jsbridge/file_writer.go b/wasmsdk/jsbridge/file_writer.go index 9f0d2d55d..50e6721af 100644 --- a/wasmsdk/jsbridge/file_writer.go +++ b/wasmsdk/jsbridge/file_writer.go @@ -5,53 +5,74 @@ package jsbridge import ( "errors" + "io" "io/fs" - "sync" "syscall/js" ) -var jsFileWriterMutex sync.Mutex - type FileWriter struct { writableStream js.Value uint8Array js.Value fileHandle js.Value bufLen int + buf []byte + bufWriteOffset int } -func (w *FileWriter) Write(p []byte) (int, error) { - //js.Value doesn't work in parallel invoke - jsFileWriterMutex.Lock() - defer jsFileWriterMutex.Unlock() +const writeBufferSize = 4 * 1024 * 1024 //4MB - if w.bufLen != len(p) { - w.bufLen = len(p) - w.uint8Array = js.Global().Get("Uint8Array").New(w.bufLen) +// len(p) will always be <= 64KB +func (w *FileWriter) Write(p []byte) (int, error) { + //copy bytes to buf + if w.bufWriteOffset+len(p) > len(w.buf) { + return 0, io.ErrShortWrite } - js.CopyBytesToJS(w.uint8Array, p) - _, err := Await(w.writableStream.Call("write", w.uint8Array)) - if len(err) > 0 && !err[0].IsNull() { - return 0, errors.New("file_writer: " + err[0].String()) + n := copy(w.buf[w.bufWriteOffset:], p) + w.bufWriteOffset += n + if w.bufWriteOffset == len(w.buf) { + //write to file + if w.bufLen != len(w.buf) { + w.bufLen = len(w.buf) + w.uint8Array = js.Global().Get("Uint8Array").New(w.bufLen) + } + js.CopyBytesToJS(w.uint8Array, w.buf) + _, err := Await(w.writableStream.Call("write", w.uint8Array)) + if len(err) > 0 && !err[0].IsNull() { + return 0, errors.New("file_writer: " + err[0].String()) + } + //reset buffer + w.bufWriteOffset = 0 } return len(p), nil } -func (w *FileWriter) WriteAt(p []byte, offset int64) (int, error) { - uint8Array := js.Global().Get("Uint8Array").New(len(p)) - js.CopyBytesToJS(uint8Array, p) - options := js.Global().Get("Object").New() - options.Set("type", "write") - options.Set("position", offset) - options.Set("data", uint8Array) - options.Set("size", len(p)) - _, err := Await(w.writableStream.Call("write", options)) - if len(err) > 0 && !err[0].IsNull() { - return 0, errors.New("file_writer: " + err[0].String()) - } - return len(p), nil -} +// func (w *FileWriter) WriteAt(p []byte, offset int64) (int, error) { +// uint8Array := js.Global().Get("Uint8Array").New(len(p)) +// js.CopyBytesToJS(uint8Array, p) +// options := js.Global().Get("Object").New() +// options.Set("type", "write") +// options.Set("position", offset) +// options.Set("data", uint8Array) +// options.Set("size", len(p)) +// _, err := Await(w.writableStream.Call("write", options)) +// if len(err) > 0 && !err[0].IsNull() { +// return 0, errors.New("file_writer: " + err[0].String()) +// } +// return len(p), nil +// } func (w *FileWriter) Close() error { + + if w.bufWriteOffset > 0 { + w.buf = w.buf[:w.bufWriteOffset] + uint8Array := js.Global().Get("Uint8Array").New(len(w.buf)) + js.CopyBytesToJS(uint8Array, w.buf) + _, err := Await(w.writableStream.Call("write", uint8Array)) + if len(err) > 0 && !err[0].IsNull() { + return errors.New("file_writer: " + err[0].String()) + } + } + _, err := Await(w.writableStream.Call("close")) if len(err) > 0 && !err[0].IsNull() { return errors.New("file_writer: " + err[0].String()) @@ -99,5 +120,6 @@ func NewFileWriter(filename string) (*FileWriter, error) { return &FileWriter{ writableStream: writableStream[0], fileHandle: fileHandle[0], + buf: make([]byte, writeBufferSize), }, nil } diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index 94cb956f5..72a89cbab 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -42,7 +42,7 @@ func CreateChunkedUploadFormBuilder() ChunkedUploadFormBuilder { type chunkedUploadFormBuilder struct { } -const MAX_BLOCKS = 80 // 5MB(CHUNK_SIZE*80) +const MAX_BLOCKS = 240 // 5MB(CHUNK_SIZE*80) func (b *chunkedUploadFormBuilder) Build( fileMeta *FileMeta, hasher Hasher, connectionID string, From 63d4a9748c54ab35cd47eeb826c64e4439de6f68 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 1 May 2024 20:44:26 +0530 Subject: [PATCH 03/12] unset verify hash --- core/version/version.go | 2 +- wasmsdk/cache.go | 2 +- wasmsdk/demo/index.html | 28 ++-- wasmsdk/demo/zcn.js | 293 +++++++++++++++++++++------------------- 4 files changed, 177 insertions(+), 148 deletions(-) diff --git a/core/version/version.go b/core/version/version.go index 1244037bc..e6f92099c 100644 --- a/core/version/version.go +++ b/core/version/version.go @@ -2,5 +2,5 @@ //====== THIS IS AUTOGENERATED FILE. DO NOT MODIFY ======== package version -const VERSIONSTR = "v1.12.1-3-gbc68f654" +const VERSIONSTR = "v1.14.0-RC5-5-g0ec7e4ca" diff --git a/wasmsdk/cache.go b/wasmsdk/cache.go index 57a324fb9..c90835254 100644 --- a/wasmsdk/cache.go +++ b/wasmsdk/cache.go @@ -33,7 +33,7 @@ func getAllocation(allocationId string) (*sdk.Allocation, error) { if err != nil { return nil, err } - + sdk.SetShouldVerifyHash(false) it = &cachedAllocation{ Allocation: a, Expiration: time.Now().Add(5 * time.Minute), diff --git a/wasmsdk/demo/index.html b/wasmsdk/demo/index.html index d25c4178d..e5f8ea055 100644 --- a/wasmsdk/demo/index.html +++ b/wasmsdk/demo/index.html @@ -4,6 +4,7 @@ + @@ -171,10 +172,10 @@

please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates } const getWallet = () => { - const clientID = "b1d533fa60431a76014c4f94a7e8e19a3b1a7f34eebd4cacd29a8dd948b3844c" - const publicKey = "1f30a07b34146435cabc3244a1452fb8933f6982e0b33f384e5d25b9d6531e24e342003349990483f8481052a0748cbc72355d9cbda621ec914f7ed03c127791" - const privateKey = "fb98f46969be6921586e547b0f6f70c6b92d7823359f00bafa3900523910661a" - const mnemonic = "snake second property crush thrive monkey already lake fire sort cheap lake census adult this cloth panic filter taste punch pistol project rack obscure" + const clientID = "2ba00d37a162b3fe7783fd65c0507bca67418e9b2cac2a25244339f00ac30026" + const publicKey = "174cb917c5e5e5a7906b244bc73d03dc8adaf85a61d7c31989959ad1b4ffaa01884556b5e739f01bea08a329c14552e85540300484b24037962b730cca3dbd0c" + const privateKey = "8564a063420e98ee6826f56342b81a4ab3e5acab0b5bd5d9598b8c7207656415" + const mnemonic = "grunt human dog license demise black similar level satoshi hockey guide wrap drive chase lazy engage cruel fly unable verb point else minor where" return { clientID, publicKey, privateKey, mnemonic } @@ -211,7 +212,7 @@

please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates let network = query.get('network') if (!network || network == 'undefined') { - network = "dev.zus.network" + network = "demo.zus.network" } const blockWorker = 'https://' + network + '/dns'; @@ -441,13 +442,14 @@

please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates alert("please selection allocationID") return } - const { list = [] } = await goWasm.sdk.listObjects(allocationId, '/') + const { list = [] } = await goWasm.sdk.listObjects(allocationId, '/',0,100) files = list || [] bindFiles() }) onClick('btnUploadFile', async () => { const { files } = get('inputSelectedFile') + console.log("start upload") if (files && files.length > 0) { const objects = [] @@ -457,16 +459,26 @@

please download zcn.wasm from https://github.com/0chain/gosdk/releases/lates allocationId: allocationId, remotePath: `/${file.name}`, file: file, - thumbnailBytes: await readBytes(file),//only for demo, don't upload original file as thumbnail in production encrypt: false, webstreaming: false, isUpdate: false, isRepair: false, - numBlocks: 100, + numBlocks: 250, callback: function (totalBytes, completedBytes, error) { console.log(file.name + " " + completedBytes + "/" + totalBytes + " err:" + error) } }) + {{/* const startChunkSize = 3 * 64 * 1024; + let endOffset = startChunkSize + 335413248; + //Time this for loop + const startTime = new Date().getTime(); + for (let j = 0; j < endOffset; j += startChunkSize) { + console.log("readChunk",j) + const chunk = await readChunk(j, startChunkSize, file); + } */}} + + {{/* const endTime = new Date().getTime(); + console.log('Time taken: ', endTime - startTime); */}} } const results = await goWasm.bulkUpload(objects) console.log(JSON.stringify(results)) diff --git a/wasmsdk/demo/zcn.js b/wasmsdk/demo/zcn.js index 3f5fd8e73..6d463b0e1 100644 --- a/wasmsdk/demo/zcn.js +++ b/wasmsdk/demo/zcn.js @@ -15,68 +15,66 @@ * along with this program. If not, see . */ -'use strict' +'use strict'; -const g = window +const g = window; function hexStringToByte(str) { - if (!str) return new Uint8Array() + if (!str) return new Uint8Array(); - const a = [] + const a = []; for (let i = 0, len = str.length; i < len; i += 2) { - a.push(parseInt(str.substr(i, 2), 16)) + a.push(parseInt(str.substr(i, 2), 16)); } - return new Uint8Array(a) + return new Uint8Array(a); } function blsSign(hash, secretKey) { - const { jsProxy } = g.__zcn_wasm__ + const { jsProxy } = g.__zcn_wasm__; if (!jsProxy || !secretKey) { - const errMsg = 'err: bls.secretKey is not initialized' - console.warn(errMsg) - throw new Error(errMsg) + const errMsg = 'err: bls.secretKey is not initialized'; + console.warn(errMsg); + throw new Error(errMsg); } - const bytes = hexStringToByte(hash) - const sk = bls.deserializeHexStrToSecretKey(secretKey) - const sig = sk.sign(bytes) + const bytes = hexStringToByte(hash); + const sk = bls.deserializeHexStrToSecretKey(secretKey); + const sig = sk.sign(bytes); if (!sig) { - const errMsg = 'err: wasm blsSign function failed to sign transaction' - console.warn(errMsg) - throw new Error(errMsg) + const errMsg = 'err: wasm blsSign function failed to sign transaction'; + console.warn(errMsg); + throw new Error(errMsg); } - return sig.serializeToHexStr() + return sig.serializeToHexStr(); } async function createObjectURL(buf, mimeType) { - var blob = new Blob([buf], { type: mimeType }) - return URL.createObjectURL(blob) + var blob = new Blob([buf], { type: mimeType }); + return URL.createObjectURL(blob); } - const readChunk = (offset, chunkSize, file) => - new Promise((res,rej) => { - const fileReader = new FileReader() - const blob = file.slice(offset, chunkSize+offset) - fileReader.onload = e => { - const t = e.target + new Promise((res, rej) => { + const fileReader = new FileReader(); + const blob = file.slice(offset, chunkSize + offset); + fileReader.onload = (e) => { + const t = e.target; if (t.error == null) { res({ size: t.result.byteLength, - buffer: new Uint8Array(t.result) - }) - }else{ - rej(t.error) + buffer: new Uint8Array(t.result), + }); + } else { + rej(t.error); } - } - - fileReader.readAsArrayBuffer(blob) - }) + }; + fileReader.readAsArrayBuffer(blob); + }); /** * Sleep is used when awaiting for Go Wasm to initialize. @@ -89,12 +87,10 @@ const readChunk = (offset, chunkSize, file) => * completed. */ const sleep = (ms = 1000) => - new Promise(res => { - requestAnimationFrame(res) - setTimeout(res, ms) - }) - - + new Promise((res) => { + requestAnimationFrame(res); + setTimeout(res, ms); + }); /** * The maximum amount of time that we would expect Wasm to take to initialize. @@ -102,12 +98,12 @@ const sleep = (ms = 1000) => * Most likely something has gone wrong if it takes more than 3 seconds to * initialize. */ -const maxTime = 10 * 1000 +const maxTime = 10 * 1000; // Initialize __zcn_wasm__ g.__zcn_wasm__ = g.__zcn_wasm_ || { - glob:{ - index:0, + glob: { + index: 0, }, jsProxy: { secretKey: null, @@ -119,12 +115,12 @@ g.__zcn_wasm__ = g.__zcn_wasm_ || { sleep, }, sdk: {}, //proxy object for go to expose its methods -} +}; /** * bridge is an easier way to refer to the Go WASM object. */ -const bridge = g.__zcn_wasm__ +const bridge = g.__zcn_wasm__; // bulk upload files with FileReader // objects: the list of upload object @@ -138,99 +134,125 @@ const bridge = g.__zcn_wasm__ // - numBlocks: int // - callback: function(totalBytes,completedBytes,error) async function bulkUpload(options) { - const start = bridge.glob.index - const opts = options.map(obj=>{ + console.log('func called'); + const start = bridge.glob.index; + // let hasher = await hashwasm.createMD5(); + // hasher.init(); + const opts = options.map((obj) => { const i = bridge.glob.index; - bridge.glob.index++ - const readChunkFuncName = "__zcn_upload_reader_"+i.toString() - const callbackFuncName = "__zcn_upload_callback_"+i.toString() - g[readChunkFuncName] = async (offset,chunkSize) => { - console.log("multi_upload: read chunk remotePath:"+ obj.remotePath + " offset:"+offset+" chunkSize:"+chunkSize) - const chunk = await readChunk(offset,chunkSize,obj.file) - return chunk.buffer - } - - if(obj.callback) { - g[callbackFuncName] = async (totalBytes,completedBytes,error)=> obj.callback(totalBytes,completedBytes,error) + bridge.glob.index++; + const readChunkFuncName = '__zcn_upload_reader_' + i.toString(); + const callbackFuncName = '__zcn_upload_callback_' + i.toString(); + g[readChunkFuncName] = async (offset, chunkSize) => { + console.log( + 'multi_upload: read chunk remotePath:' + + obj.remotePath + + ' offset:' + + offset + + ' chunkSize:' + + chunkSize + ); + const chunk = await readChunk(offset, chunkSize, obj.file); + // hasher.update(chunk.buffer); + // if (chunk.size < chunkSize) { + // const hash = hasher.digest(); + // console.log('multi_upload: hash ' + hash); + // } + return chunk.buffer; + }; + + if (obj.callback) { + g[callbackFuncName] = async (totalBytes, completedBytes, error) => + obj.callback(totalBytes, completedBytes, error); } return { - allocationId:obj.allocationId, - remotePath:obj.remotePath, - readChunkFuncName:readChunkFuncName, + allocationId: obj.allocationId, + remotePath: obj.remotePath, + readChunkFuncName: readChunkFuncName, fileSize: obj.file.size, - thumbnailBytes:obj.thumbnailBytes?obj.thumbnailBytes.toString():"", - encrypt:obj.encrypt, - webstreaming:obj.webstreaming, - isUpdate:obj.isUpdate, - isRepair:obj.isRepair, - numBlocks:obj.numBlocks, - callbackFuncName:callbackFuncName - } - }) - - const end = bridge.glob.index - const result = await bridge.__proxy__.sdk.multiUpload(JSON.stringify(opts)) - for (let i=start; i { - const source = await (await resp).arrayBuffer() - return await WebAssembly.instantiate(source, importObject) - } + const source = await (await resp).arrayBuffer(); + return await WebAssembly.instantiate(source, importObject); + }; } const result = await WebAssembly.instantiateStreaming( await fetch('zcn.wasm'), go.importObject - ) + ); setTimeout(() => { if (g.__zcn_wasm__?.__wasm_initialized__ !== true) { console.warn( 'wasm window.__zcn_wasm__ (zcn.__wasm_initialized__) still not true after max time' - ) + ); } - }, maxTime) + }, maxTime); - go.run(result.instance) + go.run(result.instance); } async function createWasm() { if (bridge.__proxy__) { - return bridge.__proxy__ + return bridge.__proxy__; } - const go = new g.Go() + const go = new g.Go(); - loadWasm(go) + loadWasm(go); const sdkGet = (_, key) => @@ -274,74 +296,69 @@ async function createWasm() { // eslint-disable-next-line new Promise(async (resolve, reject) => { if (!go || go.exited) { - return reject(new Error('The Go instance is not active.')) + return reject(new Error('The Go instance is not active.')); } while (bridge.__wasm_initialized__ !== true) { - await sleep(1000) + await sleep(1000); } if (typeof bridge.sdk[key] !== 'function') { - resolve(bridge.sdk[key]) + resolve(bridge.sdk[key]); if (args.length !== 0) { reject( new Error( 'Retrieved value from WASM returned function type, however called with arguments.' ) - ) + ); } - return + return; } try { - let resp = bridge.sdk[key].apply(undefined, args) + let resp = bridge.sdk[key].apply(undefined, args); // support wasm.BindAsyncFunc if (resp && typeof resp.then === 'function') { - resp = await Promise.race([resp]) + resp = await Promise.race([resp]); } if (resp && resp.error) { - reject(resp.error) + reject(resp.error); } else { - resolve(resp) + resolve(resp); } } catch (e) { - reject(e) + reject(e); } - }) + }); const sdkProxy = new Proxy( - { - - }, + {}, { get: sdkGet, } - ) + ); const jsProxy = new Proxy( {}, { get: (_, key) => bridge.jsProxy[key], set: (_, key, value) => { - bridge.jsProxy[key] = value + bridge.jsProxy[key] = value; }, } - ) + ); const proxy = { bulkUpload: bulkUpload, setWallet: setWallet, sdk: sdkProxy, //expose sdk methods for js jsProxy, //expose js methods for go - } + }; - bridge.__proxy__ = proxy + bridge.__proxy__ = proxy; - return proxy + return proxy; } - - - From 7d0620beab8e092ba326b5a28ef82eaba2beb0eb Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 1 May 2024 20:48:26 +0530 Subject: [PATCH 04/12] add timing for read chunk --- wasmsdk/blobber.go | 14 ++++++++------ zboxcore/sdk/chunked_upload_chunk_reader.go | 3 +++ zboxcore/sdk/multi_operation_worker.go | 7 ++++--- 3 files changed, 15 insertions(+), 9 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index a64c2436b..2c4db6114 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -933,16 +933,18 @@ func getBlobbers(stakable bool) ([]*sdk.Blobber, error) { } type timingRes struct { - UploadTime int64 `json:"upload_time"` - ReadTime int64 `json:"read_time"` - TotalTime int64 `json:"total_time"` + UploadTime int64 `json:"upload_time"` + ReadTime int64 `json:"read_time"` + TotalTime int64 `json:"total_time"` + TotalReadChunkTime int64 `json:"total_read_chunk_time"` } func getUploadTiming() string { res := timingRes{ - UploadTime: sdk.TotalUploadTime, - ReadTime: sdk.TotalReadTime, - TotalTime: sdk.TotalTime, + UploadTime: sdk.TotalUploadTime, + ReadTime: sdk.TotalReadTime, + TotalTime: sdk.TotalTime, + TotalReadChunkTime: sdk.TotalReadChunkTime, } respBytes, _ := json.Marshal(res) return string(respBytes) diff --git a/zboxcore/sdk/chunked_upload_chunk_reader.go b/zboxcore/sdk/chunked_upload_chunk_reader.go index 38570ec98..ecba438c5 100644 --- a/zboxcore/sdk/chunked_upload_chunk_reader.go +++ b/zboxcore/sdk/chunked_upload_chunk_reader.go @@ -5,6 +5,7 @@ import ( "math" "strconv" "sync" + "time" "github.com/0chain/errors" "github.com/0chain/gosdk/constants" @@ -158,11 +159,13 @@ func (r *chunkedUploadChunkReader) Next() (*ChunkData, error) { readLen int err error ) + now := time.Now() for readLen < len(chunkBytes) && err == nil { var nn int nn, err = r.fileReader.Read(chunkBytes[readLen:]) readLen += nn } + TotalReadChunkTime += time.Since(now).Milliseconds() if err != nil { if !errors.Is(err, io.EOF) { diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index 6efdff420..c16280742 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -41,9 +41,10 @@ func WithRepair() MultiOperationOption { } var ( - TotalUploadTime int64 - TotalTime int64 - TotalReadTime int64 + TotalUploadTime int64 + TotalTime int64 + TotalReadTime int64 + TotalReadChunkTime int64 ) type Operationer interface { From 929c56c7e19b06ecf1380b98f76697dd7567cf89 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 1 May 2024 21:33:55 +0530 Subject: [PATCH 05/12] reset read chunk time --- wasmsdk/blobber.go | 4 ++-- zboxcore/sdk/multi_operation_worker.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index 2c4db6114..6c3ad4ebf 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -934,17 +934,17 @@ func getBlobbers(stakable bool) ([]*sdk.Blobber, error) { type timingRes struct { UploadTime int64 `json:"upload_time"` + TotalReadChunkTime int64 `json:"total_read_chunk_time"` ReadTime int64 `json:"read_time"` TotalTime int64 `json:"total_time"` - TotalReadChunkTime int64 `json:"total_read_chunk_time"` } func getUploadTiming() string { res := timingRes{ UploadTime: sdk.TotalUploadTime, + TotalReadChunkTime: sdk.TotalReadChunkTime, ReadTime: sdk.TotalReadTime, TotalTime: sdk.TotalTime, - TotalReadChunkTime: sdk.TotalReadChunkTime, } respBytes, _ := json.Marshal(res) return string(respBytes) diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index c16280742..fd742605e 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -168,6 +168,7 @@ func (mo *MultiOperation) createConnectionObj(blobberIdx int) (err error) { func (mo *MultiOperation) Process() error { l.Logger.Info("MultiOperation Process start") TotalReadTime = 0 + TotalReadChunkTime = 0 wg := &sync.WaitGroup{} mo.changes = make([][]allocationchange.AllocationChange, len(mo.operations)) ctx := mo.ctx From f20fdf11d8bf036c04748145dd179771ae84d085 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 4 May 2024 01:53:16 +0530 Subject: [PATCH 06/12] set max block to 80 --- zboxcore/sdk/chunked_upload_form_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index 72a89cbab..94cb956f5 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -42,7 +42,7 @@ func CreateChunkedUploadFormBuilder() ChunkedUploadFormBuilder { type chunkedUploadFormBuilder struct { } -const MAX_BLOCKS = 240 // 5MB(CHUNK_SIZE*80) +const MAX_BLOCKS = 80 // 5MB(CHUNK_SIZE*80) func (b *chunkedUploadFormBuilder) Build( fileMeta *FileMeta, hasher Hasher, connectionID string, From eb40bc4418bcdb3d12c24dddd8860b044779f615 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Sat, 4 May 2024 22:49:52 +0530 Subject: [PATCH 07/12] add form build and upload time --- wasmsdk/blobber.go | 4 ++++ zboxcore/sdk/allocation.go | 7 +++++++ zboxcore/sdk/chunked_upload.go | 5 ++++- zboxcore/sdk/multi_operation_worker.go | 10 ++++++---- 4 files changed, 21 insertions(+), 5 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index 6c3ad4ebf..e7425739b 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -937,6 +937,8 @@ type timingRes struct { TotalReadChunkTime int64 `json:"total_read_chunk_time"` ReadTime int64 `json:"read_time"` TotalTime int64 `json:"total_time"` + TotalBuildTime int64 `json:"total_build_time"` + TotalBlobberUpload int64 `json:"total_blobber_upload"` } func getUploadTiming() string { @@ -945,6 +947,8 @@ func getUploadTiming() string { TotalReadChunkTime: sdk.TotalReadChunkTime, ReadTime: sdk.TotalReadTime, TotalTime: sdk.TotalTime, + TotalBuildTime: sdk.TotalFormBuildTime, + TotalBlobberUpload: sdk.TotalUploadBlobberTime, } respBytes, _ := json.Marshal(res) return string(respBytes) diff --git a/zboxcore/sdk/allocation.go b/zboxcore/sdk/allocation.go index 83349ea6f..90b7f9e1e 100644 --- a/zboxcore/sdk/allocation.go +++ b/zboxcore/sdk/allocation.go @@ -813,6 +813,13 @@ func (a *Allocation) DoMultiOperation(operations []OperationRequest, opts ...Mul if !a.isInitialized() { return notInitialized } + TotalFormBuildTime = 0 // reset the time + TotalUploadTime = 0 // reset the time + TotalReadChunkTime = 0 // reset the time + TotalUploadBlobberTime = 0 // reset the time + TotalTime = 0 // reset the time + TotalReadTime = 0 // reset the time + connectionID := zboxutil.NewConnectionId() var mo MultiOperation for i := 0; i < len(operations); { diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index 11f576adb..0df71128d 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -643,7 +643,7 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, uploadBody: make([]blobberData, len(su.blobbers)), uploadLength: uploadLength, } - + now := time.Now() wgErrors := make(chan error, len(su.blobbers)) if len(fileShards) == 0 { return thrown.New("upload_failed", "Upload failed. No data to upload") @@ -697,6 +697,7 @@ func (su *ChunkedUpload) processUpload(chunkStartIndex, chunkEndIndex int, su.removeProgress() return thrown.New("upload_failed", fmt.Sprintf("Upload failed. %s", err)) } + TotalFormBuildTime += time.Since(now).Milliseconds() if !lastBufferOnly { su.uploadWG.Add(1) select { @@ -817,6 +818,7 @@ func (su *ChunkedUpload) uploadToBlobbers(uploadData UploadData) error { return context.Cause(su.ctx) default: } + now := time.Now() consensus := Consensus{ RWMutex: &sync.RWMutex{}, consensusThresh: su.consensus.consensusThresh, @@ -873,6 +875,7 @@ func (su *ChunkedUpload) uploadToBlobbers(uploadData UploadData) error { su.statusCallback.InProgress(su.allocationObj.ID, su.fileMeta.RemotePath, su.opCode, int(atomic.AddInt64(&su.progress.UploadLength, uploadLength)), nil) } } + atomic.AddInt64(&TotalUploadBlobberTime, time.Since(now).Milliseconds()) uploadData = UploadData{} // release memory return nil } diff --git a/zboxcore/sdk/multi_operation_worker.go b/zboxcore/sdk/multi_operation_worker.go index fd742605e..904116973 100644 --- a/zboxcore/sdk/multi_operation_worker.go +++ b/zboxcore/sdk/multi_operation_worker.go @@ -41,10 +41,12 @@ func WithRepair() MultiOperationOption { } var ( - TotalUploadTime int64 - TotalTime int64 - TotalReadTime int64 - TotalReadChunkTime int64 + TotalUploadTime int64 + TotalTime int64 + TotalReadTime int64 + TotalReadChunkTime int64 + TotalFormBuildTime int64 + TotalUploadBlobberTime int64 ) type Operationer interface { From 5b6a6bf7b418d4f5309b8aa8adfae595f8e18ba6 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Tue, 7 May 2024 16:35:38 +0530 Subject: [PATCH 08/12] use http2 --- zboxcore/sdk/chunked_upload_blobber.go | 82 +++++++++++++++----------- zboxcore/zboxutil/http.go | 9 +-- zboxcore/zboxutil/transport.go | 1 + 3 files changed, 51 insertions(+), 41 deletions(-) diff --git a/zboxcore/sdk/chunked_upload_blobber.go b/zboxcore/sdk/chunked_upload_blobber.go index 886464aff..1d021084b 100644 --- a/zboxcore/sdk/chunked_upload_blobber.go +++ b/zboxcore/sdk/chunked_upload_blobber.go @@ -9,7 +9,6 @@ import ( "mime/multipart" "net/http" "strings" - "syscall" "time" "github.com/0chain/errors" @@ -22,7 +21,6 @@ import ( "github.com/0chain/gosdk/zboxcore/logger" "github.com/0chain/gosdk/zboxcore/marker" "github.com/0chain/gosdk/zboxcore/zboxutil" - "github.com/hitenjain14/fasthttp" "golang.org/x/sync/errgroup" ) @@ -73,40 +71,47 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest( for dataInd := 0; dataInd < len(dataBuffers); dataInd++ { ind := dataInd eg.Go(func() error { - var ( - shouldContinue bool - ) - var req *fasthttp.Request + req, err := zboxutil.NewUploadRequestWithMethod( + sb.blobber.Baseurl, su.allocationObj.ID, su.allocationObj.Tx, dataBuffers[ind], su.httpMethod) + if err != nil { + return err + } + req.Header.Add("Content-Type", contentSlice[ind]) for i := 0; i < 3; i++ { - req, err = zboxutil.NewFastUploadRequest( - sb.blobber.Baseurl, su.allocationObj.ID, su.allocationObj.Tx, dataBuffers[ind].Bytes(), su.httpMethod) - if err != nil { - return err - } + err, shouldContinue := func() (err error, shouldContinue bool) { + reqCtx, ctxCncl := context.WithTimeout(ctx, su.uploadTimeOut) + var resp *http.Response + err = zboxutil.HttpDo(reqCtx, ctxCncl, req, func(r *http.Response, err error) error { + resp = r + return err + }) + defer ctxCncl() - req.Header.Add("Content-Type", contentSlice[ind]) - err, shouldContinue = func() (err error, shouldContinue bool) { - resp := fasthttp.AcquireResponse() - defer fasthttp.ReleaseResponse(resp) - err = zboxutil.FastHttpClient.DoTimeout(req, resp, su.uploadTimeOut) - fasthttp.ReleaseRequest(req) if err != nil { logger.Logger.Error("Upload : ", err) - if errors.Is(err, fasthttp.ErrConnectionClosed) || errors.Is(err, syscall.EPIPE) { - return err, true - } return fmt.Errorf("Error while doing reqeust. Error %s", err), false } - if resp.StatusCode() == http.StatusOK { + if resp.Body != nil { + defer resp.Body.Close() + } + if resp.StatusCode == http.StatusOK { + io.Copy(io.Discard, resp.Body) //nolint:errcheck return } + var r UploadResult + var respbody []byte + + respbody, err = io.ReadAll(resp.Body) + if err != nil { + logger.Logger.Error("Error: Resp ", err) + return fmt.Errorf("Error while reading body. Error %s", err), false + } - respbody := resp.Body() - if resp.StatusCode() == http.StatusTooManyRequests { + if resp.StatusCode == http.StatusTooManyRequests { logger.Logger.Error("Got too many request error") var r int - r, err = zboxutil.GetFastRateLimitValue(resp) + r, err = zboxutil.GetRateLimitValue(resp) if err != nil { logger.Logger.Error(err) return @@ -116,25 +121,32 @@ func (sb *ChunkedUploadBlobber) sendUploadRequest( return } - msg := string(respbody) - logger.Logger.Error(sb.blobber.Baseurl, - " Upload error response: ", resp.StatusCode(), - "err message: ", msg) - err = errors.Throw(constants.ErrBadRequest, msg) + if resp.StatusCode != http.StatusOK { + msg := string(respbody) + logger.Logger.Error(sb.blobber.Baseurl, + " Upload error response: ", resp.StatusCode, + "err message: ", msg) + err = errors.Throw(constants.ErrBadRequest, msg) + return + } + + err = json.Unmarshal(respbody, &r) + if err != nil { + logger.Logger.Error(sb.blobber.Baseurl, "Upload response parse error: ", err) + return + } return }() - if shouldContinue { - continue - } - if err != nil { return err } - + if shouldContinue { + continue + } break } - return err + return nil }) } err = eg.Wait() diff --git a/zboxcore/zboxutil/http.go b/zboxcore/zboxutil/http.go index fa7cfd032..ef9cfc012 100644 --- a/zboxcore/zboxutil/http.go +++ b/zboxcore/zboxutil/http.go @@ -644,18 +644,15 @@ func NewFastUploadRequest(baseURL, allocationID string, allocationTx string, bod return req, nil } -func NewUploadRequest(baseUrl, allocationID string, allocationTx string, body io.Reader, update bool) (*http.Request, error) { +func NewUploadRequest(baseUrl, allocationID string, allocationTx string, body io.Reader, method string) (*http.Request, error) { u, err := joinUrl(baseUrl, UPLOAD_ENDPOINT, allocationTx) if err != nil { return nil, err } var req *http.Request - if update { - req, err = http.NewRequest(http.MethodPut, u.String(), body) - } else { - req, err = http.NewRequest(http.MethodPost, u.String(), body) - } + req, err = http.NewRequest(method, u.String(), body) + if err != nil { return nil, err } diff --git a/zboxcore/zboxutil/transport.go b/zboxcore/zboxutil/transport.go index 7cf7d4d5b..c05ace567 100644 --- a/zboxcore/zboxutil/transport.go +++ b/zboxcore/zboxutil/transport.go @@ -22,4 +22,5 @@ var DefaultTransport = &http.Transport{ ExpectContinueTimeout: 1 * time.Second, MaxIdleConnsPerHost: 25, WriteBufferSize: 16 * 1024, + ForceAttemptHTTP2: true, } From 83cbe30dc2be9ebe9d3067819771b471f90ca722 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 8 May 2024 11:43:38 +0530 Subject: [PATCH 09/12] set http2 for wasm --- zboxcore/zboxutil/transport_wasm.go | 1 + 1 file changed, 1 insertion(+) diff --git a/zboxcore/zboxutil/transport_wasm.go b/zboxcore/zboxutil/transport_wasm.go index 553f6d807..89c3019e6 100644 --- a/zboxcore/zboxutil/transport_wasm.go +++ b/zboxcore/zboxutil/transport_wasm.go @@ -16,4 +16,5 @@ var DefaultTransport = &http.Transport{ TLSHandshakeTimeout: 10 * time.Second, ExpectContinueTimeout: 1 * time.Second, MaxIdleConnsPerHost: 100, + ForceAttemptHTTP2: true, } From 8e1b0280d335e59be06a1b6228b3bc78eb843bb2 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 8 May 2024 17:22:01 +0530 Subject: [PATCH 10/12] chan block num and mode --- wasmsdk/blobber.go | 8 ++++---- zboxcore/sdk/chunked_upload.go | 2 +- zboxcore/sdk/chunked_upload_form_builder.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/wasmsdk/blobber.go b/wasmsdk/blobber.go index e7425739b..8d240f069 100644 --- a/wasmsdk/blobber.go +++ b/wasmsdk/blobber.go @@ -667,10 +667,10 @@ func multiUpload(jsonBulkUploadOptions string) (MultiUploadResult, error) { RemoteName: fileName, RemotePath: fullRemotePath, } - numBlocks := option.NumBlocks - if numBlocks <= 1 { - numBlocks = 100 - } + numBlocks := 80 + // if numBlocks <= 1 { + // numBlocks = 100 + // } options := []sdk.ChunkedUploadOption{ sdk.WithThumbnail(option.ThumbnailBytes.Buffer), diff --git a/zboxcore/sdk/chunked_upload.go b/zboxcore/sdk/chunked_upload.go index 0df71128d..67a39f921 100644 --- a/zboxcore/sdk/chunked_upload.go +++ b/zboxcore/sdk/chunked_upload.go @@ -47,7 +47,7 @@ var ( ErrNoEnoughSpaceLeftInAllocation = errors.New("alloc: no enough space left in allocation") CancelOpCtx = make(map[string]context.CancelCauseFunc) cancelLock sync.Mutex - CurrentMode = UploadModeMedium + CurrentMode = UploadModeHigh ) // DefaultChunkSize default chunk size for file and thumbnail diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index 94cb956f5..f4001146d 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -42,7 +42,7 @@ func CreateChunkedUploadFormBuilder() ChunkedUploadFormBuilder { type chunkedUploadFormBuilder struct { } -const MAX_BLOCKS = 80 // 5MB(CHUNK_SIZE*80) +const MAX_BLOCKS = 60 // 5MB(CHUNK_SIZE*80) func (b *chunkedUploadFormBuilder) Build( fileMeta *FileMeta, hasher Hasher, connectionID string, From bf2b3d87716c1d3dbe23f39d820fd87b812a3cd4 Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 8 May 2024 17:41:21 +0530 Subject: [PATCH 11/12] revert blocks to 80 --- zboxcore/sdk/chunked_upload_form_builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/zboxcore/sdk/chunked_upload_form_builder.go b/zboxcore/sdk/chunked_upload_form_builder.go index f4001146d..94cb956f5 100644 --- a/zboxcore/sdk/chunked_upload_form_builder.go +++ b/zboxcore/sdk/chunked_upload_form_builder.go @@ -42,7 +42,7 @@ func CreateChunkedUploadFormBuilder() ChunkedUploadFormBuilder { type chunkedUploadFormBuilder struct { } -const MAX_BLOCKS = 60 // 5MB(CHUNK_SIZE*80) +const MAX_BLOCKS = 80 // 5MB(CHUNK_SIZE*80) func (b *chunkedUploadFormBuilder) Build( fileMeta *FileMeta, hasher Hasher, connectionID string, From 20b3c92aa2830b4a21915d51e112fc5a4d77eacc Mon Sep 17 00:00:00 2001 From: Hitenjain14 Date: Wed, 8 May 2024 20:03:55 +0530 Subject: [PATCH 12/12] fix file seek --- wasmsdk/jsbridge/file_reader.go | 16 +++++++++++++--- 1 file changed, 13 insertions(+), 3 deletions(-) diff --git a/wasmsdk/jsbridge/file_reader.go b/wasmsdk/jsbridge/file_reader.go index 22a88ad64..9d2da5630 100644 --- a/wasmsdk/jsbridge/file_reader.go +++ b/wasmsdk/jsbridge/file_reader.go @@ -5,6 +5,7 @@ package jsbridge import ( "errors" + "fmt" "io" "syscall/js" ) @@ -40,13 +41,15 @@ func NewFileReader(readChunkFuncName string, fileSize, chunkReadSize int64) (*Fi if n < len(buf) { return nil, errors.New("file_reader: failed to read first chunk") } - return &FileReader{ + fr := &FileReader{ size: fileSize, offset: int64(n), readChunk: readChunk, buf: buf, endOfFile: n == int(fileSize), - }, nil + } + fmt.Println("file_reader: created", fr.size, fr.offset, fr.endOfFile) + return fr, nil } func (r *FileReader) Read(p []byte) (int, error) { @@ -68,12 +71,15 @@ func (r *FileReader) Read(p []byte) (int, error) { if n < len(r.buf) { r.buf = r.buf[:n] r.endOfFile = true + fmt.Println("file_reader: reached end of file", r.offset, r.size) } } n := copy(p, r.buf[r.bufOffset:]) + fmt.Println("file_reader: read", r.offset, r.bufOffset, len(r.buf), len(p), n) r.bufOffset += n if r.endOfFile && r.bufOffset == len(r.buf) { + fmt.Println("file_reader: last call", r.offset, r.bufOffset, len(r.buf), len(p), n) return n, io.EOF } @@ -96,6 +102,10 @@ func (r *FileReader) Seek(offset int64, whence int) (int64, error) { if abs < 0 { return 0, errors.New("FileReader.Seek: negative position") } - r.offset = abs + if abs > int64(len(r.buf)) { + return 0, errors.New("FileReader.Seek: position out of bounds") + } + r.bufOffset = int(abs) + fmt.Println("file_reader: seek", r.bufOffset, abs) return abs, nil }