From 146116d2635ee9e86cd13e2166095eec305ff513 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Wed, 1 Jun 2022 16:57:27 +0800 Subject: [PATCH 1/4] fix: Correctly map deduplicated chunk indices when setting content --- src/EncryptedModels.ts | 41 +++++++++++++++++---------------- src/Etebase.test.ts | 51 +++++++++++++++++++++++++++++++++++++++++- 2 files changed, 72 insertions(+), 20 deletions(-) diff --git a/src/EncryptedModels.ts b/src/EncryptedModels.ts index 544c510..fe857db 100644 --- a/src/EncryptedModels.ts +++ b/src/EncryptedModels.ts @@ -309,31 +309,34 @@ class EncryptedRevision { chunks.push([hash, buf]); } - // Shuffle the items and save the ordering if we have more than one - if (chunks.length > 0) { + // Shuffle the items, deduplicate and save the ordering + if (chunks.length > 1) { const indices = shuffle(chunks); + const uniqueChunksMap = new Map(); - // Filter duplicates and construct the indice list. - const uidIndices = new Map(); - chunks = chunks.filter((chunk, i) => { + chunks.forEach((chunk) => { const uid = chunk[0]; - const previousIndex = uidIndices.get(uid); - if (previousIndex !== undefined) { - indices[i] = previousIndex; - return false; - } else { - uidIndices.set(uid, i); - return true; + if (!uniqueChunksMap.has(uid)) { + uniqueChunksMap.set(uid, chunk); } }); - // If we have more than one chunk we need to encode the mapping header in the last chunk - if (indices.length > 1) { - // We encode it in an array so we can extend it later on if needed - const buf = msgpackEncode([indices]); - const hash = toBase64(cryptoManager.calculateMac(buf)); - chunks.push([hash, buf]); - } + const chunkKeys = [...uniqueChunksMap.keys()]; + + // Change the original (shuffled) indices to point at the deduplicated chunks + const newIndices = indices.map((i) => { + const [id] = chunks[i]; + return chunkKeys.indexOf(id); + }); + + chunks = [...uniqueChunksMap.values()]; + + // We encode it in an array so we can extend it later on if needed + const buf = msgpackEncode([newIndices]); + const hash = toBase64(cryptoManager.calculateMac(buf)); + + // Append a chunk wth the mapping + chunks.push([hash, buf]); } // Encrypt all of the chunks diff --git a/src/Etebase.test.ts b/src/Etebase.test.ts index 85f7ed0..a771acb 100644 --- a/src/Etebase.test.ts +++ b/src/Etebase.test.ts @@ -5,7 +5,7 @@ import * as Etebase from "./Etebase"; import { USER, USER2, sessionStorageKey } from "./TestConstants"; import { Authenticator, PrefetchOption } from "./OnlineManagers"; -import { fromBase64, fromString, msgpackEncode, msgpackDecode, randomBytesDeterministic, toBase64 } from "./Helpers"; +import { fromBase64, fromString, msgpackEncode, msgpackDecode, randomBytesDeterministic, toBase64, toString } from "./Helpers"; const testApiBase = process.env.ETEBASE_TEST_API_URL ?? "http://localhost:8033"; @@ -1700,3 +1700,52 @@ it.skip("Login and password change", async () => { const etebase4 = await Etebase.Account.login(USER2.email, USER2.password, testApiBase); await etebase4.logout(); }, 30000); + +describe("Chunking files", () => { + it("Duplicate Chunks", async () => { + const collectionManager = etebase.getCollectionManager(); + const col = await collectionManager.create(colType, {}, ""); + + const buf = randomBytesDeterministic(10 * 1024, new Uint8Array(32)); // 10kb of psuedorandom data + const content = JSON.stringify([buf, buf, buf, buf]); + + await col.setContent(content); + + await collectionManager.transaction(col); + const decryptedContent = await col.getContent(); + + const out = toString(decryptedContent); + expect(out).toEqual(content); + }); + + it("Regular Chunks", async () => { + const collectionManager = etebase.getCollectionManager(); + const col = await collectionManager.create(colType, {}, ""); + + const buf = randomBytesDeterministic(100 * 1024, new Uint8Array(32)); + const content = JSON.stringify(buf); + + await col.setContent(content); + + await collectionManager.transaction(col); + const decryptedContent = await col.getContent(); + + const out = toString(decryptedContent); + expect(out).toEqual(content); + }); + + it("Small file, no chunks", async () => { + const collectionManager = etebase.getCollectionManager(); + const col = await collectionManager.create(colType, {}, ""); + + const content = JSON.stringify("foo"); + + await col.setContent(content); + + await collectionManager.transaction(col); + const decryptedContent = await col.getContent(); + + const out = toString(decryptedContent); + expect(out).toEqual(content); + }); +}); From 9422fb118e069131649b2a3f4ef52258b57e1255 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Sun, 5 Jun 2022 00:28:48 +0800 Subject: [PATCH 2/4] fix: simplify test --- src/Etebase.test.ts | 26 ++++++-------------------- 1 file changed, 6 insertions(+), 20 deletions(-) diff --git a/src/Etebase.test.ts b/src/Etebase.test.ts index a771acb..91e7d03 100644 --- a/src/Etebase.test.ts +++ b/src/Etebase.test.ts @@ -1705,32 +1705,22 @@ describe("Chunking files", () => { it("Duplicate Chunks", async () => { const collectionManager = etebase.getCollectionManager(); const col = await collectionManager.create(colType, {}, ""); - const buf = randomBytesDeterministic(10 * 1024, new Uint8Array(32)); // 10kb of psuedorandom data - const content = JSON.stringify([buf, buf, buf, buf]); - + const content = new Uint8Array([...buf, ...buf, ...buf, ...buf]); await col.setContent(content); - await collectionManager.transaction(col); - const decryptedContent = await col.getContent(); - - const out = toString(decryptedContent); + const out = await col.getContent(); expect(out).toEqual(content); }); it("Regular Chunks", async () => { const collectionManager = etebase.getCollectionManager(); const col = await collectionManager.create(colType, {}, ""); - const buf = randomBytesDeterministic(100 * 1024, new Uint8Array(32)); - const content = JSON.stringify(buf); - + const content = new Uint8Array(buf); await col.setContent(content); - await collectionManager.transaction(col); - const decryptedContent = await col.getContent(); - - const out = toString(decryptedContent); + const out = await col.getContent(); expect(out).toEqual(content); }); @@ -1738,14 +1728,10 @@ describe("Chunking files", () => { const collectionManager = etebase.getCollectionManager(); const col = await collectionManager.create(colType, {}, ""); - const content = JSON.stringify("foo"); - + const content = new Uint8Array([1]); await col.setContent(content); - await collectionManager.transaction(col); - const decryptedContent = await col.getContent(); - - const out = toString(decryptedContent); + const out = await col.getContent(); expect(out).toEqual(content); }); }); From 45353626d769768f1ad5f9e5c9446bdc13396d56 Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Tue, 7 Jun 2022 10:39:47 +0800 Subject: [PATCH 3/4] fix: lint --- package.json | 2 +- src/Etebase.test.ts | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/package.json b/package.json index 92ff38f..c9ea855 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "etebase", - "version": "0.43.0", + "version": "0.44.0", "description": "Etebase TypeScript API for the web and node", "type": "commonjs", "main": "dist/lib-cjs/Etebase.js", diff --git a/src/Etebase.test.ts b/src/Etebase.test.ts index 91e7d03..985a066 100644 --- a/src/Etebase.test.ts +++ b/src/Etebase.test.ts @@ -5,7 +5,7 @@ import * as Etebase from "./Etebase"; import { USER, USER2, sessionStorageKey } from "./TestConstants"; import { Authenticator, PrefetchOption } from "./OnlineManagers"; -import { fromBase64, fromString, msgpackEncode, msgpackDecode, randomBytesDeterministic, toBase64, toString } from "./Helpers"; +import { fromBase64, fromString, msgpackEncode, msgpackDecode, randomBytesDeterministic, toBase64 } from "./Helpers"; const testApiBase = process.env.ETEBASE_TEST_API_URL ?? "http://localhost:8033"; From 713bffcfe7e19477f9b4bd030f1a084c5fc37e5b Mon Sep 17 00:00:00 2001 From: Craig Cooper Date: Tue, 7 Jun 2022 19:37:47 +0800 Subject: [PATCH 4/4] fix: reset version --- package.json | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package.json b/package.json index c9ea855..92ff38f 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "etebase", - "version": "0.44.0", + "version": "0.43.0", "description": "Etebase TypeScript API for the web and node", "type": "commonjs", "main": "dist/lib-cjs/Etebase.js",