diff --git a/README.md b/README.md index fc2890eb..2d06bfc4 100644 --- a/README.md +++ b/README.md @@ -5,7 +5,7 @@ DAG based key value store. Sharded DAG that minimises traversals and work to build shards. -* 📖 [Read the SPEC](https://github.com/web3-storage/specs/blob/460b6511979a52ec9870f307695ee3f0b3860f81/kv.md). +* 📖 [Read the SPEC](https://github.com/web3-storage/specs/blob/4163e28d7e6a7c44cff68db9d9bffb9b37707dc6/pail.md). * 🎬 [Watch the Presentation](https://youtu.be/f-BrtpYKZfg). ## Install diff --git a/bench/put-x10_000.js b/bench/put-x10_000.js index f494f1aa..a50b07cd 100644 --- a/bench/put-x10_000.js +++ b/bench/put-x10_000.js @@ -1,4 +1,7 @@ -import { ShardBlock, put } from '../src/index.js' +// eslint-disable-next-line no-unused-vars +import * as API from '../src/api.js' +import { put } from '../src/index.js' +import { ShardBlock } from '../src/shard.js' import { MemoryBlockstore } from '../src/block.js' import { randomCID, randomString, randomInteger } from '../test/helpers.js' @@ -11,7 +14,7 @@ async function main () { const blocks = new MemoryBlockstore() await blocks.put(rootBlock.cid, rootBlock.bytes) - /** @type {Array<[string, import('multiformats').UnknownLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const kvs = [] for (let i = 0; i < NUM; i++) { @@ -22,7 +25,7 @@ async function main () { console.log('bench') console.time(`put x${NUM}`) - /** @type {import('../src/shard.js').ShardLink} */ + /** @type {API.ShardLink} */ let root = rootBlock.cid for (let i = 0; i < kvs.length; i++) { const result = await put(blocks, root, kvs[i][0], kvs[i][1]) diff --git a/cli.js b/cli.js index 60d4b498..1f092e0c 100755 --- a/cli.js +++ b/cli.js @@ -8,8 +8,10 @@ import { CID } from 'multiformats/cid' import { CarIndexedReader, CarReader, CarWriter } from '@ipld/car' import clc from 'cli-color' import archy from 'archy' -import { MaxShardSize, put, ShardBlock, get, del, entries } from './src/index.js' -import { ShardFetcher } from './src/shard.js' +// eslint-disable-next-line no-unused-vars +import * as API from './src/api.js' +import { put, get, del, entries } from './src/index.js' +import { ShardFetcher, ShardBlock, MaxShardSize } from './src/shard.js' import { difference } from './src/diff.js' import { merge } from './src/merge.js' @@ -21,11 +23,11 @@ cli.command('put ') .alias('set') .option('--max-shard-size', 'Maximum shard size in bytes.', MaxShardSize) .action(async (key, value, opts) => { - const blocks = await openPail(opts.path) - const roots = await blocks.getRoots() const maxShardSize = opts['max-shard-size'] ?? MaxShardSize + const blocks = await openPail(opts.path, { maxSize: maxShardSize }) + const roots = await blocks.getRoots() // @ts-expect-error - const { root, additions, removals } = await put(blocks, roots[0], key, CID.parse(value), { maxShardSize }) + const { root, additions, removals } = await put(blocks, roots[0], key, CID.parse(value)) await updatePail(opts.path, blocks, root, { additions, removals }) console.log(clc.red(`--- ${roots[0]}`)) @@ -95,7 +97,7 @@ cli.command('tree') /** @type {archy.Data} */ const archyRoot = { label: `Shard(${clc.yellow(rshard.cid.toString())}) ${rshard.bytes.length + 'b'}`, nodes: [] } - /** @param {import('./src/shard').ShardEntry} entry */ + /** @param {API.ShardEntry} entry */ const getData = async ([k, v]) => { if (!Array.isArray(v)) { return { label: `Key(${clc.magenta(k)})`, nodes: [{ label: `Value(${clc.cyan(v)})` }] } @@ -106,12 +108,12 @@ cli.command('tree') const blk = await shards.get(v[0]) data.nodes?.push({ label: `Shard(${clc.yellow(v[0])}) ${blk.bytes.length + 'b'}`, - nodes: await Promise.all(blk.value.map(e => getData(e))) + nodes: await Promise.all(blk.value.entries.map(e => getData(e))) }) return data } - for (const entry of rshard.value) { + for (const entry of rshard.value.entries) { archyRoot.nodes?.push(await getData(entry)) } @@ -125,7 +127,7 @@ cli.command('diff ') .action(async (path, opts) => { const [ablocks, bblocks] = await Promise.all([openPail(opts.path), openPail(path)]) const [aroot, broot] = await Promise.all([ablocks, bblocks].map(async blocks => { - return /** @type {import('./src/shard').ShardLink} */((await blocks.getRoots())[0]) + return /** @type {API.ShardLink} */((await blocks.getRoots())[0]) })) if (aroot.toString() === broot.toString()) return @@ -136,6 +138,7 @@ cli.command('diff ') return bblocks.get(cid) } } + // @ts-expect-error CarReader is not BlockFetcher const { shards: { additions, removals }, keys } = await difference(fetcher, aroot, broot) console.log(clc.red(`--- ${aroot}`)) @@ -160,7 +163,7 @@ cli.command('merge ') .action(async (path, opts) => { const [ablocks, bblocks] = await Promise.all([openPail(opts.path), openPail(path)]) const [aroot, broot] = await Promise.all([ablocks, bblocks].map(async blocks => { - return /** @type {import('./src/shard').ShardLink} */((await blocks.getRoots())[0]) + return /** @type {API.ShardLink} */((await blocks.getRoots())[0]) })) if (aroot.toString() === broot.toString()) return @@ -171,6 +174,7 @@ cli.command('merge ') return bblocks.get(cid) } } + // @ts-expect-error CarReader is not BlockFetcher const { root, additions, removals } = await merge(fetcher, aroot, [broot]) await updatePail(opts.path, ablocks, root, { additions, removals }) @@ -188,17 +192,16 @@ cli.parse(process.argv) /** * @param {string} path + * @param {{ maxSize?: number }} [config] * @returns {Promise} */ -async function openPail (path) { +async function openPail (path, config) { try { return await CarIndexedReader.fromFile(path) } catch (err) { if (err.code !== 'ENOENT') throw new Error('failed to open bucket', { cause: err }) - const rootblk = await ShardBlock.create() - // @ts-expect-error + const rootblk = await ShardBlock.create(config) const { writer, out } = CarWriter.create(rootblk.cid) - // @ts-expect-error writer.put(rootblk) writer.close() return CarReader.fromIterable(out) @@ -215,8 +218,8 @@ async function closePail (reader) { /** * @param {string} path * @param {import('@ipld/car/api').CarReader} reader - * @param {import('./src/shard').ShardLink} root - * @param {import('./src/index').ShardDiff} diff + * @param {API.ShardLink} root + * @param {API.ShardDiff} diff */ async function updatePail (path, reader, root, { additions, removals }) { // @ts-expect-error @@ -229,7 +232,6 @@ async function updatePail (path, reader, root, { additions, removals }) { // put new blocks for (const b of additions) { - // @ts-expect-error await writer.put(b) } // put old blocks without removals diff --git a/package-lock.json b/package-lock.json index 29505f81..557ed6fa 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9,11 +9,11 @@ "version": "0.3.4", "license": "Apache-2.0 OR MIT", "dependencies": { - "@ipld/car": "^5.0.1", - "@ipld/dag-cbor": "^9.0.0", + "@ipld/car": "^5.2.4", + "@ipld/dag-cbor": "^9.0.6", "archy": "^1.0.0", "cli-color": "^2.0.3", - "multiformats": "^12.1.1", + "multiformats": "^12.1.3", "sade": "^1.8.1" }, "bin": { @@ -176,12 +176,12 @@ "dev": true }, "node_modules/@ipld/car": { - "version": "5.2.2", - "resolved": "https://registry.npmjs.org/@ipld/car/-/car-5.2.2.tgz", - "integrity": "sha512-8IapvzPNB1Z2VwtA7n6olB3quhrLMbFxk4JaENIT4OlQ6YQNz1peY00qb2iJTC/kCDir7yb3TuNHkbdDzSKiXA==", + "version": "5.2.4", + "resolved": "https://registry.npmjs.org/@ipld/car/-/car-5.2.4.tgz", + "integrity": "sha512-YoVXE/o5HLXKi/Oqh9Nhcn423sdn9brRFKnbUid68/1D332/XINcoyCTvBluFcCw/9IeiTx+sEAV+onXZ/A4eA==", "dependencies": { "@ipld/dag-cbor": "^9.0.0", - "cborg": "^2.0.5", + "cborg": "^4.0.0", "multiformats": "^12.1.0", "varint": "^6.0.0" }, @@ -191,11 +191,11 @@ } }, "node_modules/@ipld/dag-cbor": { - "version": "9.0.4", - "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.4.tgz", - "integrity": "sha512-HBNVngk/47pKNLTAelN6ORWgKkjJtQj96Xb+jIBtRShJGCsXgghj1TzTynTTIp1dZxwPe5rVIL6yjZmvdyP2Wg==", + "version": "9.0.6", + "resolved": "https://registry.npmjs.org/@ipld/dag-cbor/-/dag-cbor-9.0.6.tgz", + "integrity": "sha512-3kNab5xMppgWw6DVYx2BzmFq8t7I56AGWfp5kaU1fIPkwHVpBRglJJTYsGtbVluCi/s/q97HZM3bC+aDW4sxbQ==", "dependencies": { - "cborg": "^2.0.1", + "cborg": "^4.0.0", "multiformats": "^12.0.1" }, "engines": { @@ -632,11 +632,11 @@ } }, "node_modules/cborg": { - "version": "2.0.5", - "resolved": "https://registry.npmjs.org/cborg/-/cborg-2.0.5.tgz", - "integrity": "sha512-xVW1rSIw1ZXbkwl2XhJ7o/jAv0vnVoQv/QlfQxV8a7V5PlA4UU/AcIiXqmpyybwNWy/GPQU1m/aBVNIWr7/T0w==", + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/cborg/-/cborg-4.0.5.tgz", + "integrity": "sha512-q8TAjprr8pn9Fp53rOIGp/UFDdFY6os2Nq62YogPSIzczJD9M6g2b6igxMkpCiZZKJ0kn/KzDLDvG+EqBIEeCg==", "bin": { - "cborg": "cli.js" + "cborg": "lib/bin.js" } }, "node_modules/chalk": { @@ -2909,9 +2909,9 @@ "dev": true }, "node_modules/multiformats": { - "version": "12.1.1", - "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.1.tgz", - "integrity": "sha512-GBSToTmri2vJYs8wqcZQ8kB21dCaeTOzHTIAlr8J06C1eL6UbzqURXFZ5Fl0EYm9GAFz1IlYY8SxGOs9G9NJRg==", + "version": "12.1.3", + "resolved": "https://registry.npmjs.org/multiformats/-/multiformats-12.1.3.tgz", + "integrity": "sha512-eajQ/ZH7qXZQR2AgtfpmSMizQzmyYVmCql7pdhldPuYQi4atACekbJaQplk6dWyIi10jCaFnd6pqvcEFXjbaJw==", "engines": { "node": ">=16.0.0", "npm": ">=7.0.0" diff --git a/package.json b/package.json index 4079777a..eee523e9 100644 --- a/package.json +++ b/package.json @@ -13,6 +13,9 @@ "dist/src/index.d.ts": [ "dist/src/index.d.ts" ], + "api": [ + "dist/src/api.d.ts" + ], "block": [ "dist/src/block.d.ts" ], @@ -41,6 +44,10 @@ "types": "./dist/src/index.d.ts", "import": "./src/index.js" }, + "./api": { + "types": "./dist/src/api.d.ts", + "import": "./src/api.js" + }, "./block": { "types": "./dist/src/block.d.ts", "import": "./src/block.js" @@ -94,11 +101,11 @@ "dist" ], "dependencies": { - "@ipld/car": "^5.0.1", - "@ipld/dag-cbor": "^9.0.0", + "@ipld/car": "^5.2.4", + "@ipld/dag-cbor": "^9.0.6", "archy": "^1.0.0", "cli-color": "^2.0.3", - "multiformats": "^12.1.1", + "multiformats": "^12.1.3", "sade": "^1.8.1" }, "devDependencies": { diff --git a/src/api.js b/src/api.js new file mode 100644 index 00000000..336ce12b --- /dev/null +++ b/src/api.js @@ -0,0 +1 @@ +export {} diff --git a/src/api.ts b/src/api.ts new file mode 100644 index 00000000..3aaa1324 --- /dev/null +++ b/src/api.ts @@ -0,0 +1,65 @@ +import { Link, UnknownLink, BlockView, Block, Version } from 'multiformats' +import { sha256 } from 'multiformats/hashes/sha2' +import * as dagCBOR from '@ipld/dag-cbor' + +export { Link, UnknownLink, BlockView, Block, Version } + +export type ShardEntryValueValue = UnknownLink + +export type ShardEntryLinkValue = [ShardLink] + +export type ShardEntryLinkAndValueValue = [ShardLink, UnknownLink] + +export type ShardValueEntry = [key: string, value: ShardEntryValueValue] + +export type ShardLinkEntry = [key: string, value: ShardEntryLinkValue | ShardEntryLinkAndValueValue] + +/** Single key/value entry within a shard. */ +export type ShardEntry = [key: string, value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue] + +/** Legacy shards are not self describing. */ +export type LegacyShard = ShardEntry[] + +export interface Shard { + entries: ShardEntry[] + /** Max key length (in UTF-8 encoded characters) - default 64. */ + maxKeyLength: number + /** Max encoded shard size in bytes - default 512 KiB. */ + maxSize: number +} + +export type ShardLink = Link + +export interface ShardBlockView extends BlockView { + prefix: string +} + +export interface ShardDiff { + additions: ShardBlockView[] + removals: ShardBlockView[] +} + +export interface BlockFetcher { + get (link: Link): + Promise | undefined> +} + +// Clock ////////////////////////////////////////////////////////////////////// + +export type EventLink = Link> + +export interface EventView { + parents: EventLink[] + data: T +} + +export interface EventBlockView extends BlockView> {} + +// CRDT /////////////////////////////////////////////////////////////////////// + +export interface EventData { + type: 'put'|'del' + key: string + value: UnknownLink + root: ShardLink +} diff --git a/src/block.js b/src/block.js index 9de492f5..4846b7c5 100644 --- a/src/block.js +++ b/src/block.js @@ -1,17 +1,14 @@ +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' import { parse } from 'multiformats/link' -/** - * @typedef {{ cid: import('./link').AnyLink, bytes: Uint8Array }} AnyBlock - * @typedef {{ get: (link: import('./link').AnyLink) => Promise }} BlockFetcher - */ - -/** @implements {BlockFetcher} */ +/** @implements {API.BlockFetcher} */ export class MemoryBlockstore { /** @type {Map} */ #blocks = new Map() /** - * @param {Array} [blocks] + * @param {Array} [blocks] */ constructor (blocks) { if (blocks) { @@ -19,10 +16,7 @@ export class MemoryBlockstore { } } - /** - * @param {import('./link').AnyLink} cid - * @returns {Promise} - */ + /** @type {API.BlockFetcher['get']} */ async get (cid) { const bytes = this.#blocks.get(cid.toString()) if (!bytes) return @@ -30,7 +24,7 @@ export class MemoryBlockstore { } /** - * @param {import('./link').AnyLink} cid + * @param {API.UnknownLink} cid * @param {Uint8Array} bytes */ async put (cid, bytes) { @@ -38,19 +32,19 @@ export class MemoryBlockstore { } /** - * @param {import('./link').AnyLink} cid + * @param {API.UnknownLink} cid * @param {Uint8Array} bytes */ putSync (cid, bytes) { this.#blocks.set(cid.toString(), bytes) } - /** @param {import('./link').AnyLink} cid */ + /** @param {API.UnknownLink} cid */ async delete (cid) { this.#blocks.delete(cid.toString()) } - /** @param {import('./link').AnyLink} cid */ + /** @param {API.UnknownLink} cid */ deleteSync (cid) { this.#blocks.delete(cid.toString()) } @@ -63,15 +57,15 @@ export class MemoryBlockstore { } export class MultiBlockFetcher { - /** @type {BlockFetcher[]} */ + /** @type {API.BlockFetcher[]} */ #fetchers - /** @param {BlockFetcher[]} fetchers */ + /** @param {API.BlockFetcher[]} fetchers */ constructor (...fetchers) { this.#fetchers = fetchers } - /** @param {import('./link').AnyLink} link */ + /** @type {API.BlockFetcher['get']} */ async get (link) { for (const f of this.#fetchers) { const v = await f.get(link) diff --git a/src/clock.js b/src/clock.js index 4d776f83..86b142a2 100644 --- a/src/clock.js +++ b/src/clock.js @@ -1,31 +1,18 @@ import { Block, encode, decode } from 'multiformats/block' import { sha256 } from 'multiformats/hashes/sha2' import * as cbor from '@ipld/dag-cbor' - -/** - * @template T - * @typedef {{ parents: EventLink[], data: T }} EventView - */ - -/** - * @template T - * @typedef {import('multiformats').BlockView>} EventBlockView - */ - -/** - * @template T - * @typedef {import('multiformats').Link>} EventLink - */ +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' /** * Advance the clock by adding an event. * * @template T - * @param {import('./block').BlockFetcher} blocks Block storage. - * @param {EventLink[]} head The head of the clock. - * @param {EventLink} event The event to add. + * @param {API.BlockFetcher} blocks Block storage. + * @param {API.EventLink[]} head The head of the clock. + * @param {API.EventLink} event The event to add. */ -export async function advance (blocks, head, event) { +export const advance = async (blocks, head, event) => { const events = new EventFetcher(blocks) const headmap = new Map(head.map(cid => [cid.toString(), cid])) if (headmap.has(event.toString())) return head @@ -55,13 +42,13 @@ export async function advance (blocks, head, event) { /** * @template T - * @extends {Block, typeof cbor.code, typeof sha256.code, 1>} - * @implements {EventBlockView} + * @extends {Block, typeof cbor.code, typeof sha256.code, 1>} + * @implements {API.EventBlockView} */ export class EventBlock extends Block { /** * @param {object} config - * @param {EventLink} config.cid + * @param {API.EventLink} config.cid * @param {Event} config.value * @param {Uint8Array} config.bytes * @param {string} config.prefix @@ -75,7 +62,7 @@ export class EventBlock extends Block { /** * @template T * @param {T} data - * @param {EventLink[]} [parents] + * @param {API.EventLink[]} [parents] */ static create (data, parents) { return encodeEventBlock({ data, parents: parents ?? [] }) @@ -84,15 +71,15 @@ export class EventBlock extends Block { /** @template T */ export class EventFetcher { - /** @param {import('./block').BlockFetcher} blocks */ + /** @param {API.BlockFetcher} blocks */ constructor (blocks) { /** @private */ this._blocks = blocks } /** - * @param {EventLink} link - * @returns {Promise>} + * @param {API.EventLink} link + * @returns {Promise>} */ async get (link) { const block = await this._blocks.get(link) @@ -103,10 +90,10 @@ export class EventFetcher { /** * @template T - * @param {EventView} value - * @returns {Promise>} + * @param {API.EventView} value + * @returns {Promise>} */ -export async function encodeEventBlock (value) { +export const encodeEventBlock = async (value) => { // TODO: sort parents const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 }) // @ts-expect-error @@ -116,9 +103,9 @@ export async function encodeEventBlock (value) { /** * @template T * @param {Uint8Array} bytes - * @returns {Promise>} + * @returns {Promise>} */ -export async function decodeEventBlock (bytes) { +export const decodeEventBlock = async (bytes) => { const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 }) // @ts-expect-error return new Block({ cid, value, bytes }) @@ -128,10 +115,10 @@ export async function decodeEventBlock (bytes) { * Returns true if event "a" contains event "b". Breadth first search. * @template T * @param {EventFetcher} events - * @param {EventLink} a - * @param {EventLink} b + * @param {API.EventLink} a + * @param {API.EventLink} b */ -async function contains (events, a, b) { +const contains = async (events, a, b) => { if (a.toString() === b.toString()) return true const [{ value: aevent }, { value: bevent }] = await Promise.all([events.get(a), events.get(b)]) const links = [...aevent.parents] @@ -150,18 +137,18 @@ async function contains (events, a, b) { /** * @template T - * @param {import('./block').BlockFetcher} blocks Block storage. - * @param {EventLink[]} head + * @param {API.BlockFetcher} blocks Block storage. + * @param {API.EventLink[]} head * @param {object} [options] - * @param {(b: EventBlockView) => string} [options.renderNodeLabel] + * @param {(b: API.EventBlockView) => string} [options.renderNodeLabel] */ -export async function * vis (blocks, head, options = {}) { +export const vis = async function * (blocks, head, options = {}) { const renderNodeLabel = options.renderNodeLabel ?? (b => shortLink(b.cid)) const events = new EventFetcher(blocks) yield 'digraph clock {' yield ' node [shape=point fontname="Courier"]; head;' const hevents = await Promise.all(head.map(link => events.get(link))) - /** @type {import('multiformats').Link>[]} */ + /** @type {import('multiformats').Link>[]} */ const links = [] const nodes = new Set() for (const e of hevents) { @@ -188,5 +175,5 @@ export async function * vis (blocks, head, options = {}) { yield '}' } -/** @param {import('./link').AnyLink} l */ +/** @param {API.UnknownLink} l */ const shortLink = l => `${String(l).slice(0, 4)}..${String(l).slice(-4)}` diff --git a/src/crdt.js b/src/crdt.js index 880a509b..603691ad 100644 --- a/src/crdt.js +++ b/src/crdt.js @@ -1,43 +1,38 @@ +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' import * as Clock from './clock.js' import { EventFetcher, EventBlock } from './clock.js' import * as Pail from './index.js' -import { ShardBlock } from './index.js' +import { ShardBlock } from './shard.js' import { MemoryBlockstore, MultiBlockFetcher } from './block.js' /** * @typedef {{ - * type: 'put'|'del' - * key: string - * value: import('./link').AnyLink - * root: import('./shard').ShardLink - * }} EventData - * @typedef {{ - * root: import('./shard').ShardLink - * head: import('./clock').EventLink[] - * event?: import('./clock').EventBlockView - * } & import('./index').ShardDiff} Result + * root: API.ShardLink + * head: API.EventLink[] + * event?: API.EventBlockView + * } & API.ShardDiff} Result */ /** * Put a value (a CID) for the given key. If the key exists it's value is * overwritten. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./clock').EventLink[]} head Merkle clock head. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.EventLink[]} head Merkle clock head. * @param {string} key The key of the value to put. - * @param {import('./link').AnyLink} value The value to put. - * @param {object} [options] + * @param {API.UnknownLink} value The value to put. * @returns {Promise} */ -export async function put (blocks, head, key, value, options) { +export const put = async (blocks, head, key, value) => { const mblocks = new MemoryBlockstore() blocks = new MultiBlockFetcher(mblocks, blocks) if (!head.length) { const shard = await ShardBlock.create() mblocks.putSync(shard.cid, shard.bytes) - const result = await Pail.put(blocks, shard.cid, key, value, options) - /** @type {EventData} */ + const result = await Pail.put(blocks, shard.cid, key, value) + /** @type {API.EventData} */ const data = { type: 'put', root: result.root, key, value } const event = await EventBlock.create(data, head) head = await Clock.advance(blocks, head, event.cid) @@ -50,7 +45,7 @@ export async function put (blocks, head, key, value, options) { } } - /** @type {EventFetcher} */ + /** @type {EventFetcher} */ const events = new EventFetcher(blocks) const ancestor = await findCommonAncestor(events, head) if (!ancestor) throw new Error('failed to find common ancestor event') @@ -59,9 +54,9 @@ export async function put (blocks, head, key, value, options) { let { root } = aevent.value.data const sorted = await findSortedEvents(events, head, ancestor) - /** @type {Map} */ + /** @type {Map} */ const additions = new Map() - /** @type {Map} */ + /** @type {Map} */ const removals = new Map() for (const { value: event } of sorted) { @@ -82,7 +77,7 @@ export async function put (blocks, head, key, value, options) { } } - const result = await Pail.put(blocks, root, key, value, options) + const result = await Pail.put(blocks, root, key, value) // if we didn't change the pail we're done if (result.root.toString() === root.toString()) { return { root, additions: [], removals: [], head } @@ -96,7 +91,7 @@ export async function put (blocks, head, key, value, options) { removals.set(r.cid.toString(), r) } - /** @type {EventData} */ + /** @type {API.EventData} */ const data = { type: 'put', root: result.root, key, value } const event = await EventBlock.create(data, head) mblocks.putSync(event.cid, event.bytes) @@ -123,13 +118,13 @@ export async function put (blocks, head, key, value, options) { * Delete the value for the given key from the bucket. If the key is not found * no operation occurs. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./clock').EventLink[]} head Merkle clock head. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.EventLink[]} head Merkle clock head. * @param {string} key The key of the value to delete. * @param {object} [options] * @returns {Promise} */ -export async function del (blocks, head, key, options) { +export const del = async (blocks, head, key, options) => { throw new Error('not implemented') } @@ -139,17 +134,17 @@ export async function del (blocks, head, key, options) { * Clocks with multiple head events may return blocks that were added or * removed while playing forward events from their common ancestor. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./clock').EventLink[]} head Merkle clock head. - * @returns {Promise<{ root: import('./shard').ShardLink } & import('./index').ShardDiff>} + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.EventLink[]} head Merkle clock head. + * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>} */ -export async function root (blocks, head) { +export const root = async (blocks, head) => { if (!head.length) throw new Error('cannot determine root of headless clock') const mblocks = new MemoryBlockstore() blocks = new MultiBlockFetcher(mblocks, blocks) - /** @type {EventFetcher} */ + /** @type {EventFetcher} */ const events = new EventFetcher(blocks) if (head.length === 1) { @@ -165,9 +160,9 @@ export async function root (blocks, head) { let { root } = aevent.value.data const sorted = await findSortedEvents(events, head, ancestor) - /** @type {Map} */ + /** @type {Map} */ const additions = new Map() - /** @type {Map} */ + /** @type {Map} */ const removals = new Map() for (const { value: event } of sorted) { @@ -204,11 +199,11 @@ export async function root (blocks, head) { } /** - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./clock').EventLink[]} head Merkle clock head. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.EventLink[]} head Merkle clock head. * @param {string} key The key of the value to retrieve. */ -export async function get (blocks, head, key) { +export const get = async (blocks, head, key) => { if (!head.length) return const result = await root(blocks, head) if (result.additions.length) { @@ -218,12 +213,12 @@ export async function get (blocks, head, key) { } /** - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./clock').EventLink[]} head Merkle clock head. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.EventLink[]} head Merkle clock head. * @param {object} [options] * @param {string} [options.prefix] */ -export async function * entries (blocks, head, options) { +export const entries = async function * (blocks, head, options) { if (!head.length) return const result = await root(blocks, head) if (result.additions.length) { @@ -236,10 +231,10 @@ export async function * entries (blocks, head, options) { * Find the common ancestor event of the passed children. A common ancestor is * the first single event in the DAG that _all_ paths from children lead to. * - * @param {import('./clock').EventFetcher} events - * @param {import('./clock').EventLink[]} children + * @param {EventFetcher} events + * @param {API.EventLink[]} children */ -async function findCommonAncestor (events, children) { +const findCommonAncestor = async (events, children) => { if (!children.length) return const candidates = children.map(c => [c]) while (true) { @@ -257,10 +252,10 @@ async function findCommonAncestor (events, children) { } /** - * @param {import('./clock').EventFetcher} events - * @param {import('./clock').EventLink} root + * @param {EventFetcher} events + * @param {API.EventLink} root */ -async function findAncestorCandidate (events, root) { +const findAncestorCandidate = async (events, root) => { const { value: event } = await events.get(root) if (!event.parents.length) return root return event.parents.length === 1 @@ -272,7 +267,7 @@ async function findAncestorCandidate (events, root) { * @template {{ toString: () => string }} T * @param {Array} arrays */ -function findCommonString (arrays) { +const findCommonString = (arrays) => { arrays = arrays.map(a => [...a]) for (const arr of arrays) { for (const item of arr) { @@ -289,13 +284,13 @@ function findCommonString (arrays) { /** * Find and sort events between the head(s) and the tail. - * @param {import('./clock').EventFetcher} events - * @param {import('./clock').EventLink[]} head - * @param {import('./clock').EventLink} tail + * @param {EventFetcher} events + * @param {API.EventLink[]} head + * @param {API.EventLink} tail */ -async function findSortedEvents (events, head, tail) { +const findSortedEvents = async (events, head, tail) => { // get weighted events - heavier events happened first - /** @type {Map, weight: number }>} */ + /** @type {Map, weight: number }>} */ const weights = new Map() const all = await Promise.all(head.map(h => findEvents(events, h, tail))) for (const arr of all) { @@ -310,7 +305,7 @@ async function findSortedEvents (events, head, tail) { } // group events into buckets by weight - /** @type {Map[]>} */ + /** @type {Map[]>} */ const buckets = new Map() for (const { event, weight } of weights.values()) { const bucket = buckets.get(weight) @@ -328,12 +323,12 @@ async function findSortedEvents (events, head, tail) { } /** - * @param {import('./clock').EventFetcher} events - * @param {import('./clock').EventLink} start - * @param {import('./clock').EventLink} end - * @returns {Promise, depth: number }>>} + * @param {EventFetcher} events + * @param {API.EventLink} start + * @param {API.EventLink} end + * @returns {Promise, depth: number }>>} */ -async function findEvents (events, start, end, depth = 0) { +const findEvents = async (events, start, end, depth = 0) => { const event = await events.get(start) const acc = [{ event, depth }] const { parents } = event.value diff --git a/src/diff.js b/src/diff.js index f6cc5f85..85e39b8a 100644 --- a/src/diff.js +++ b/src/diff.js @@ -1,19 +1,21 @@ +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' import { ShardFetcher } from './shard.js' /** * @typedef {string} K - * @typedef {[before: null, after: import('./link').AnyLink]} AddV - * @typedef {[before: import('./link').AnyLink, after: import('./link').AnyLink]} UpdateV - * @typedef {[before: import('./link').AnyLink, after: null]} DeleteV + * @typedef {[before: null, after: API.UnknownLink]} AddV + * @typedef {[before: API.UnknownLink, after: API.UnknownLink]} UpdateV + * @typedef {[before: API.UnknownLink, after: null]} DeleteV * @typedef {[key: K, value: AddV|UpdateV|DeleteV]} KV * @typedef {KV[]} KeysDiff - * @typedef {{ keys: KeysDiff, shards: import('./index').ShardDiff }} CombinedDiff + * @typedef {{ keys: KeysDiff, shards: API.ShardDiff }} CombinedDiff */ /** - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./shard').ShardLink} a Base DAG. - * @param {import('./shard').ShardLink} b Comparison DAG. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.ShardLink} a Base DAG. + * @param {API.ShardLink} b Comparison DAG. * @returns {Promise} */ export async function difference (blocks, a, b, prefix = '') { @@ -22,15 +24,15 @@ export async function difference (blocks, a, b, prefix = '') { const shards = new ShardFetcher(blocks) const [ashard, bshard] = await Promise.all([shards.get(a, prefix), shards.get(b, prefix)]) - const aents = new Map(ashard.value) - const bents = new Map(bshard.value) + const aents = new Map(ashard.value.entries) + const bents = new Map(bshard.value.entries) const keys = /** @type {Map} */(new Map()) const additions = new Map([[bshard.cid.toString(), bshard]]) const removals = new Map([[ashard.cid.toString(), ashard]]) // find shards removed in B - for (const [akey, aval] of ashard.value) { + for (const [akey, aval] of ashard.value.entries) { const bval = bents.get(akey) if (bval) continue if (!Array.isArray(aval)) { @@ -42,7 +44,7 @@ export async function difference (blocks, a, b, prefix = '') { keys.set(`${ashard.prefix}${akey}`, [aval[1], null]) } for await (const s of collect(shards, aval[0], `${ashard.prefix}${akey}`)) { - for (const [k, v] of s.value) { + for (const [k, v] of s.value.entries) { if (!Array.isArray(v)) { keys.set(`${s.prefix}${k}`, [v, null]) } else if (v[1] != null) { @@ -54,7 +56,7 @@ export async function difference (blocks, a, b, prefix = '') { } // find shards added or updated in B - for (const [bkey, bval] of bshard.value) { + for (const [bkey, bval] of bshard.value.entries) { const aval = aents.get(bkey) if (!Array.isArray(bval)) { if (!aval) { @@ -90,7 +92,7 @@ export async function difference (blocks, a, b, prefix = '') { keys.set(`${bshard.prefix}${bkey}`, [aval, bval[1]]) } for await (const s of collect(shards, bval[0], `${bshard.prefix}${bkey}`)) { - for (const [k, v] of s.value) { + for (const [k, v] of s.value.entries) { if (!Array.isArray(v)) { keys.set(`${s.prefix}${k}`, [null, v]) } else if (v[1] != null) { @@ -102,7 +104,7 @@ export async function difference (blocks, a, b, prefix = '') { } else { // added in B keys.set(`${bshard.prefix}${bkey}`, [null, bval[0]]) for await (const s of collect(shards, bval[0], `${bshard.prefix}${bkey}`)) { - for (const [k, v] of s.value) { + for (const [k, v] of s.value.entries) { if (!Array.isArray(v)) { keys.set(`${s.prefix}${k}`, [null, v]) } else if (v[1] != null) { @@ -129,20 +131,20 @@ export async function difference (blocks, a, b, prefix = '') { } /** - * @param {import('./link').AnyLink} a - * @param {import('./link').AnyLink} b + * @param {API.UnknownLink} a + * @param {API.UnknownLink} b */ const isEqual = (a, b) => a.toString() === b.toString() /** - * @param {import('./shard').ShardFetcher} shards - * @param {import('./shard').ShardLink} root - * @returns {AsyncIterableIterator} + * @param {import('./shard.js').ShardFetcher} shards + * @param {API.ShardLink} root + * @returns {AsyncIterableIterator} */ async function * collect (shards, root, prefix = '') { const shard = await shards.get(root, prefix) yield shard - for (const [k, v] of shard.value) { + for (const [k, v] of shard.value.entries) { if (!Array.isArray(v)) continue yield * collect(shards, v[0], `${prefix}${k}`) } diff --git a/src/index.js b/src/index.js index d7253caf..8fc8e57b 100644 --- a/src/index.js +++ b/src/index.js @@ -1,83 +1,79 @@ -import { - ShardFetcher, - ShardBlock, - encodeShardBlock, - decodeShardBlock, - putEntry, - findCommonPrefix -} from './shard.js' - -export { ShardBlock, encodeShardBlock, decodeShardBlock } - -/** - * @typedef {{ additions: import('./shard').ShardBlockView[], removals: import('./shard').ShardBlockView[] }} ShardDiff - */ - -export const MaxKeyLength = 64 -export const MaxShardSize = 512 * 1024 +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' +import { ShardFetcher } from './shard.js' +import * as Shard from './shard.js' /** * Put a value (a CID) for the given key. If the key exists it's value is * overwritten. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./shard').ShardLink} root CID of the root node of the bucket. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.ShardLink} root CID of the root node of the bucket. * @param {string} key The key of the value to put. - * @param {import('./link').AnyLink} value The value to put. - * @param {object} [options] - * @param {number} [options.maxShardSize] Maximum shard size in bytes. - * @returns {Promise<{ root: import('./shard').ShardLink } & ShardDiff>} + * @param {API.UnknownLink} value The value to put. + * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>} */ -export async function put (blocks, root, key, value, options = {}) { +export const put = async (blocks, root, key, value) => { const shards = new ShardFetcher(blocks) const rshard = await shards.get(root) const path = await traverse(shards, rshard, key) const target = path[path.length - 1] const skey = key.slice(target.prefix.length) // key within the shard - /** @type {import('./shard').ShardEntry} */ + /** @type {API.ShardEntry} */ let entry = [skey, value] - /** @type {import('./shard').ShardBlockView[]} */ + /** @type {API.ShardBlockView[]} */ const additions = [] // if the key in this shard is longer than allowed, then we need to make some // intermediate shards. - if (skey.length > MaxKeyLength) { - const pfxskeys = Array.from(Array(Math.ceil(skey.length / MaxKeyLength)), (_, i) => { - const start = i * MaxKeyLength + if (skey.length > target.value.maxKeyLength) { + const pfxskeys = Array.from(Array(Math.ceil(skey.length / target.value.maxKeyLength)), (_, i) => { + const start = i * target.value.maxKeyLength return { prefix: target.prefix + skey.slice(0, start), - skey: skey.slice(start, start + MaxKeyLength) + skey: skey.slice(start, start + target.value.maxKeyLength) } }) - let child = await encodeShardBlock([[pfxskeys[pfxskeys.length - 1].skey, value]], pfxskeys[pfxskeys.length - 1].prefix) + let child = await Shard.encodeBlock( + Shard.withEntries([[pfxskeys[pfxskeys.length - 1].skey, value]], target.value), + pfxskeys[pfxskeys.length - 1].prefix + ) additions.push(child) for (let i = pfxskeys.length - 2; i > 0; i--) { - child = await encodeShardBlock([[pfxskeys[i].skey, [child.cid]]], pfxskeys[i].prefix) + child = await Shard.encodeBlock( + Shard.withEntries([[pfxskeys[i].skey, [child.cid]]], target.value), + pfxskeys[i].prefix + ) additions.push(child) } entry = [pfxskeys[0].skey, [child.cid]] } - /** @type {import('./shard').Shard} */ - let shard = putEntry(target.value, entry) - let child = await encodeShardBlock(shard, target.prefix) + /** @type {API.Shard} */ + let shard = Shard.putEntry(target.value, entry) + let child = await Shard.encodeBlock(shard, target.prefix) - if (child.bytes.length > (options.maxShardSize ?? MaxShardSize)) { - const common = findCommonPrefix(shard, entry[0]) + if (child.bytes.length > shard.maxSize) { + const common = Shard.findCommonPrefix(shard, entry[0]) if (!common) throw new Error('shard limit reached') const { prefix, matches } = common - const block = await encodeShardBlock( - matches.filter(([k]) => k !== prefix).map(([k, v]) => [k.slice(prefix.length), v]), + const block = await Shard.encodeBlock( + Shard.withEntries( + matches + .filter(([k]) => k !== prefix) + .map(([k, v]) => [k.slice(prefix.length), v]), + shard + ), target.prefix + prefix ) additions.push(block) - /** @type {import('./shard').ShardEntryLinkValue | import('./shard').ShardEntryLinkAndValueValue} */ + /** @type {API.ShardEntryLinkValue | API.ShardEntryLinkAndValueValue} */ let value const pfxmatch = matches.find(([k]) => k === prefix) if (pfxmatch) { @@ -91,9 +87,9 @@ export async function put (blocks, root, key, value, options = {}) { value = [block.cid] } - shard = shard.filter(e => matches.every(m => e[0] !== m[0])) - shard = putEntry(shard, [prefix, value]) - child = await encodeShardBlock(shard, target.prefix) + shard.entries = shard.entries.filter(e => matches.every(m => e[0] !== m[0])) + shard = Shard.putEntry(shard, [prefix, value]) + child = await Shard.encodeBlock(shard, target.prefix) } // if no change in the target then we're done @@ -107,14 +103,17 @@ export async function put (blocks, root, key, value, options = {}) { for (let i = path.length - 2; i >= 0; i--) { const parent = path[i] const key = child.prefix.slice(parent.prefix.length) - const value = parent.value.map((entry) => { - const [k, v] = entry - if (k !== key) return entry - if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`) - return /** @type {import('./shard').ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]) - }) + const value = Shard.withEntries( + parent.value.entries.map((entry) => { + const [k, v] = entry + if (k !== key) return entry + if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`) + return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]) + }), + parent.value + ) - child = await encodeShardBlock(value, parent.prefix) + child = await Shard.encodeBlock(value, parent.prefix) additions.push(child) } @@ -125,18 +124,18 @@ export async function put (blocks, root, key, value, options = {}) { * Get the stored value for the given key from the bucket. If the key is not * found, `undefined` is returned. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./shard').ShardLink} root CID of the root node of the bucket. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.ShardLink} root CID of the root node of the bucket. * @param {string} key The key of the value to get. - * @returns {Promise} + * @returns {Promise} */ -export async function get (blocks, root, key) { +export const get = async (blocks, root, key) => { const shards = new ShardFetcher(blocks) const rshard = await shards.get(root) const path = await traverse(shards, rshard, key) const target = path[path.length - 1] const skey = key.slice(target.prefix.length) // key within the shard - const entry = target.value.find(([k]) => k === skey) + const entry = target.value.entries.find(([k]) => k === skey) if (!entry) return return Array.isArray(entry[1]) ? entry[1][1] : entry[1] } @@ -145,65 +144,73 @@ export async function get (blocks, root, key) { * Delete the value for the given key from the bucket. If the key is not found * no operation occurs. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./shard').ShardLink} root CID of the root node of the bucket. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.ShardLink} root CID of the root node of the bucket. * @param {string} key The key of the value to delete. - * @returns {Promise<{ root: import('./shard').ShardLink } & ShardDiff>} + * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>} */ -export async function del (blocks, root, key) { +export const del = async (blocks, root, key) => { const shards = new ShardFetcher(blocks) const rshard = await shards.get(root) const path = await traverse(shards, rshard, key) const target = path[path.length - 1] const skey = key.slice(target.prefix.length) // key within the shard - const entryidx = target.value.findIndex(([k]) => k === skey) + const entryidx = target.value.entries.findIndex(([k]) => k === skey) if (entryidx === -1) return { root, additions: [], removals: [] } - const entry = target.value[entryidx] + const entry = target.value.entries[entryidx] // cannot delete a shard (without data) - if (Array.isArray(entry[1]) && entry[1][1] == null) return { root, additions: [], removals: [] } + if (Array.isArray(entry[1]) && entry[1][1] == null) { + return { root, additions: [], removals: [] } + } - /** @type {import('./shard').ShardBlockView[]} */ + /** @type {API.ShardBlockView[]} */ const additions = [] - /** @type {import('./shard').ShardBlockView[]} */ + /** @type {API.ShardBlockView[]} */ const removals = [...path] - let shard = [...target.value] + let shard = Shard.withEntries([...target.value.entries], target.value) if (Array.isArray(entry[1])) { // remove the value from this link+value - shard[entryidx] = [entry[0], [entry[1][0]]] + shard.entries[entryidx] = [entry[0], [entry[1][0]]] } else { - shard.splice(entryidx, 1) + shard.entries.splice(entryidx, 1) // if now empty, remove from parent - while (!shard.length) { + while (!shard.entries.length) { const child = path[path.length - 1] const parent = path[path.length - 2] if (!parent) break path.pop() - shard = parent.value.filter(e => { - if (!Array.isArray(e[1])) return true - return e[1][0].toString() !== child.cid.toString() - }) + shard = Shard.withEntries( + parent.value.entries.filter(e => { + if (!Array.isArray(e[1])) return true + return e[1][0].toString() !== child.cid.toString() + }), + parent.value + ) } } - let child = await encodeShardBlock(shard, path[path.length - 1].prefix) + let child = await Shard.encodeBlock(shard, path[path.length - 1].prefix) additions.push(child) // path is root -> shard, so work backwards, propagating the new shard CID for (let i = path.length - 2; i >= 0; i--) { const parent = path[i] const key = child.prefix.slice(parent.prefix.length) - const value = parent.value.map((entry) => { - const [k, v] = entry - if (k !== key) return entry - if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`) - return /** @type {import('./shard').ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]) - }) + const value = Shard.withEntries( + parent.value.entries.map((entry) => { + const [k, v] = entry + if (k !== key) return entry + if (!Array.isArray(v)) throw new Error(`"${key}" is not a shard link in: ${parent.cid}`) + return /** @type {API.ShardEntry} */(v[1] == null ? [k, [child.cid]] : [k, [child.cid, v[1]]]) + }), + parent.value + ) - child = await encodeShardBlock(value, parent.prefix) + child = await Shard.encodeBlock(value, parent.prefix) additions.push(child) } @@ -213,21 +220,21 @@ export async function del (blocks, root, key) { /** * List entries in the bucket. * - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./shard').ShardLink} root CID of the root node of the bucket. + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.ShardLink} root CID of the root node of the bucket. * @param {object} [options] * @param {string} [options.prefix] - * @returns {AsyncIterableIterator} + * @returns {AsyncIterableIterator} */ -export async function * entries (blocks, root, options = {}) { +export const entries = async function * (blocks, root, options = {}) { const { prefix } = options const shards = new ShardFetcher(blocks) const rshard = await shards.get(root) yield * ( - /** @returns {AsyncIterableIterator} */ + /** @returns {AsyncIterableIterator} */ async function * ents (shard) { - for (const entry of shard.value) { + for (const entry of shard.value.entries) { const key = shard.prefix + entry[0] if (Array.isArray(entry[1])) { @@ -263,12 +270,12 @@ export async function * entries (blocks, root, options = {}) { * shard and ending with the target. * * @param {ShardFetcher} shards - * @param {import('./shard').ShardBlockView} shard + * @param {API.ShardBlockView} shard * @param {string} key - * @returns {Promise<[import('./shard').ShardBlockView, ...Array]>} + * @returns {Promise<[API.ShardBlockView, ...Array]>} */ -async function traverse (shards, shard, key) { - for (const [k, v] of shard.value) { +const traverse = async (shards, shard, key) => { + for (const [k, v] of shard.value.entries) { if (key === k) return [shard] if (key.startsWith(k) && Array.isArray(v)) { const path = await traverse(shards, await shards.get(v[0], shard.prefix + k), key.slice(k.length)) diff --git a/src/link.js b/src/link.js deleted file mode 100644 index cd358236..00000000 --- a/src/link.js +++ /dev/null @@ -1,2 +0,0 @@ -/** @typedef {import('multiformats').Link} AnyLink */ -export {} diff --git a/src/merge.js b/src/merge.js index 2a4ac356..38759f96 100644 --- a/src/merge.js +++ b/src/merge.js @@ -1,17 +1,19 @@ +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' import { difference } from './diff.js' import { put, del } from './index.js' /** - * @param {import('./block').BlockFetcher} blocks Bucket block storage. - * @param {import('./shard').ShardLink} base Merge base. Common parent of target DAGs. - * @param {import('./shard').ShardLink[]} targets Target DAGs to merge. - * @returns {Promise<{ root: import('./shard').ShardLink } & import('./index').ShardDiff>} + * @param {API.BlockFetcher} blocks Bucket block storage. + * @param {API.ShardLink} base Merge base. Common parent of target DAGs. + * @param {API.ShardLink[]} targets Target DAGs to merge. + * @returns {Promise<{ root: API.ShardLink } & API.ShardDiff>} */ export async function merge (blocks, base, targets) { const diffs = await Promise.all(targets.map(t => difference(blocks, base, t))) const additions = new Map() const removals = new Map() - /** @type {import('./block').BlockFetcher} */ + /** @type {API.BlockFetcher} */ const fetcher = { get: cid => additions.get(cid.toString()) ?? blocks.get(cid) } let root = base diff --git a/src/shard.js b/src/shard.js index 283462f2..2d276297 100644 --- a/src/shard.js +++ b/src/shard.js @@ -1,28 +1,21 @@ import { Block, encode, decode } from 'multiformats/block' import { sha256 } from 'multiformats/hashes/sha2' import * as cbor from '@ipld/dag-cbor' +// eslint-disable-next-line no-unused-vars +import * as API from './api.js' -/** - * @typedef {import('./link').AnyLink} ShardEntryValueValue - * @typedef {[ShardLink]} ShardEntryLinkValue - * @typedef {[ShardLink, import('./link').AnyLink]} ShardEntryLinkAndValueValue - * @typedef {[key: string, value: ShardEntryValueValue]} ShardValueEntry - * @typedef {[key: string, value: ShardEntryLinkValue | ShardEntryLinkAndValueValue]} ShardLinkEntry - * @typedef {[key: string, value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue]} ShardEntry - * @typedef {ShardEntry[]} Shard - * @typedef {import('multiformats').Link} ShardLink - * @typedef {import('multiformats').BlockView & { prefix: string }} ShardBlockView - */ +export const MaxKeyLength = 64 +export const MaxShardSize = 512 * 1024 /** - * @extends {Block} - * @implements {ShardBlockView} + * @extends {Block} + * @implements {API.ShardBlockView} */ export class ShardBlock extends Block { /** * @param {object} config - * @param {ShardLink} config.cid - * @param {Shard} config.value + * @param {API.ShardLink} config.cid + * @param {API.Shard} config.value * @param {Uint8Array} config.bytes * @param {string} config.prefix */ @@ -32,20 +25,38 @@ export class ShardBlock extends Block { this.prefix = prefix } - static create () { - return encodeShardBlock([]) + /** @param {{ maxSize?: number, maxKeyLength?: number }} [config] */ + static create (config) { + return encodeBlock(create(config)) } } -/** @type {WeakMap} */ +/** + * @param {{ maxSize?: number, maxKeyLength?: number }} [config] + * @returns {API.Shard} + */ +export const create = (config) => ({ + entries: [], + maxSize: config?.maxSize ?? MaxShardSize, + maxKeyLength: config?.maxKeyLength ?? MaxKeyLength +}) + +/** + * @param {API.ShardEntry[]} entries + * @param {{ maxSize?: number, maxKeyLength?: number }} [config] + * @returns {API.Shard} + */ +export const withEntries = (entries, config) => ({ ...create(config), entries }) + +/** @type {WeakMap} */ const decodeCache = new WeakMap() /** - * @param {Shard} value + * @param {API.Shard} value * @param {string} [prefix] - * @returns {Promise} + * @returns {Promise} */ -export async function encodeShardBlock (value, prefix) { +export const encodeBlock = async (value, prefix) => { const { cid, bytes } = await encode({ value, codec: cbor, hasher: sha256 }) const block = new ShardBlock({ cid, value, bytes, prefix: prefix ?? '' }) decodeCache.set(block.bytes, block) @@ -55,45 +66,55 @@ export async function encodeShardBlock (value, prefix) { /** * @param {Uint8Array} bytes * @param {string} [prefix] - * @returns {Promise} + * @returns {Promise} */ -export async function decodeShardBlock (bytes, prefix) { +export const decodeBlock = async (bytes, prefix) => { const block = decodeCache.get(bytes) if (block) return block const { cid, value } = await decode({ bytes, codec: cbor, hasher: sha256 }) - if (!Array.isArray(value)) throw new Error(`invalid shard: ${cid}`) + if (!isShard(value)) throw new Error(`invalid shard: ${cid}`) return new ShardBlock({ cid, value, bytes, prefix: prefix ?? '' }) } +/** + * @param {any} value + * @returns {value is API.Shard} + */ +const isShard = (value) => + value != null && + typeof value === 'object' && + Array.isArray(value.entries) && + typeof value.maxSize === 'number' && + typeof value.maxKeyLength === 'number' + export class ShardFetcher { - /** @param {import('./block').BlockFetcher} blocks */ + /** @param {API.BlockFetcher} blocks */ constructor (blocks) { this._blocks = blocks } /** - * @param {ShardLink} link + * @param {API.ShardLink} link * @param {string} [prefix] - * @returns {Promise} + * @returns {Promise} */ async get (link, prefix = '') { const block = await this._blocks.get(link) if (!block) throw new Error(`missing block: ${link}`) - return decodeShardBlock(block.bytes, prefix) + return decodeBlock(block.bytes, prefix) } } /** - * @param {Shard} target Shard to put to. - * @param {ShardEntry} entry - * @returns {Shard} + * @param {API.Shard} target Shard to put to. + * @param {API.ShardEntry} entry + * @returns {API.Shard} */ -export function putEntry (target, entry) { - if (!target.length) return [entry] +export const putEntry = (target, entry) => { + /** @type {API.Shard} */ + const shard = create(target) - /** @type {Shard} */ - const shard = [] - for (const [i, [k, v]] of target.entries()) { + for (const [i, [k, v]] of target.entries.entries()) { if (entry[0] === k) { // if new value is link to shard... if (Array.isArray(entry[1])) { @@ -102,64 +123,64 @@ export function putEntry (target, entry) { // and new value does not have link to data // then preserve old data if (Array.isArray(v) && v[1] != null && entry[1][1] == null) { - shard.push([k, [entry[1][0], v[1]]]) + shard.entries.push([k, [entry[1][0], v[1]]]) } else { - shard.push(entry) + shard.entries.push(entry) } } else { // shard as well as value? - /** @type {ShardEntry} */ + /** @type {API.ShardEntry} */ const newEntry = Array.isArray(v) ? [k, [v[0], entry[1]]] : entry - shard.push(newEntry) + shard.entries.push(newEntry) } - for (let j = i + 1; j < target.length; j++) { - shard.push(target[j]) + for (let j = i + 1; j < target.entries.length; j++) { + shard.entries.push(target.entries[j]) } return shard } if (i === 0 && entry[0] < k) { - shard.push(entry) - for (let j = i; j < target.length; j++) { - shard.push(target[j]) + shard.entries.push(entry) + for (let j = i; j < target.entries.length; j++) { + shard.entries.push(target.entries[j]) } return shard } - if (i > 0 && entry[0] > target[i - 1][0] && entry[0] < k) { - shard.push(entry) - for (let j = i; j < target.length; j++) { - shard.push(target[j]) + if (i > 0 && entry[0] > target.entries[i - 1][0] && entry[0] < k) { + shard.entries.push(entry) + for (let j = i; j < target.entries.length; j++) { + shard.entries.push(target.entries[j]) } return shard } - shard.push([k, v]) + shard.entries.push([k, v]) } - shard.push(entry) + shard.entries.push(entry) return shard } /** - * @param {import('./shard').Shard} shard + * @param {API.Shard} shard * @param {string} skey Shard key to use as a base. */ -export function findCommonPrefix (shard, skey) { - const startidx = shard.findIndex(([k]) => skey === k) +export const findCommonPrefix = (shard, skey) => { + const startidx = shard.entries.findIndex(([k]) => skey === k) if (startidx === -1) throw new Error(`key not found in shard: ${skey}`) let i = startidx /** @type {string} */ let pfx while (true) { - pfx = shard[i][0].slice(0, -1) + pfx = shard.entries[i][0].slice(0, -1) if (pfx.length) { while (true) { - const matches = shard.filter(entry => entry[0].startsWith(pfx)) + const matches = shard.entries.filter(entry => entry[0].startsWith(pfx)) if (matches.length > 1) return { prefix: pfx, matches } pfx = pfx.slice(0, -1) if (!pfx.length) break } } i++ - if (i >= shard.length) { + if (i >= shard.entries.length) { i = 0 } if (i === startidx) { diff --git a/test/clock.test.js b/test/clock.test.js index bf60d028..32c34e5f 100644 --- a/test/clock.test.js +++ b/test/clock.test.js @@ -1,5 +1,7 @@ import { describe, it } from 'mocha' import assert from 'node:assert' +// eslint-disable-next-line no-unused-vars +import * as API from '../src/api.js' import { advance, EventBlock, vis } from '../src/clock.js' import { Blockstore, randomCID } from './helpers.js' @@ -29,7 +31,7 @@ describe('clock', () => { const root = await EventBlock.create(await randomEventData()) await blocks.put(root.cid, root.bytes) - /** @type {import('../src/clock').EventLink[]} */ + /** @type {API.EventLink[]} */ let head = [root.cid] const event = await EventBlock.create(await randomEventData(), head) @@ -47,7 +49,7 @@ describe('clock', () => { const root = await EventBlock.create(await randomEventData()) await blocks.put(root.cid, root.bytes) - /** @type {import('../src/clock').EventLink[]} */ + /** @type {API.EventLink[]} */ let head = [root.cid] const parents = head @@ -70,7 +72,7 @@ describe('clock', () => { const root = await EventBlock.create(await randomEventData()) await blocks.put(root.cid, root.bytes) - /** @type {import('../src/clock').EventLink[]} */ + /** @type {API.EventLink[]} */ let head = [root.cid] const parents0 = head @@ -105,7 +107,7 @@ describe('clock', () => { const root = await EventBlock.create(await randomEventData()) await blocks.put(root.cid, root.bytes) - /** @type {import('../src/clock').EventLink[]} */ + /** @type {API.EventLink[]} */ let head = [root.cid] const parents0 = head @@ -147,7 +149,7 @@ describe('clock', () => { const root = await EventBlock.create(await randomEventData()) await blocks.put(root.cid, root.bytes) - /** @type {import('../src/clock').EventLink[]} */ + /** @type {API.EventLink[]} */ let head = [root.cid] const parents0 = head @@ -195,7 +197,7 @@ describe('clock', () => { const root = await EventBlock.create(await randomEventData()) await blocks.put(root.cid, root.bytes) - /** @type {import('../src/clock').EventLink[]} */ + /** @type {API.EventLink[]} */ let head = [root.cid] const event0 = await EventBlock.create(await randomEventData(), head) diff --git a/test/crdt.test.js b/test/crdt.test.js index 2d8c8f94..26ad2378 100644 --- a/test/crdt.test.js +++ b/test/crdt.test.js @@ -1,5 +1,7 @@ import { describe, it } from 'mocha' import assert from 'node:assert' +// eslint-disable-next-line no-unused-vars +import * as API from '../src/api.js' import { advance, vis } from '../src/clock.js' import { put, get, root, entries } from '../src/crdt.js' import { Blockstore, randomCID } from './helpers.js' @@ -46,7 +48,7 @@ describe('CRDT', () => { await alice.put('apple', await randomCID(32)) const bob = new TestPail(blocks, alice.head) - /** @type {Array<[string, import('../src/link').AnyLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const data = [ ['banana', await randomCID(32)], ['kiwi', await randomCID(32)], @@ -109,7 +111,7 @@ describe('CRDT', () => { await alice.put('apple', await randomCID(32)) const bob = new TestPail(blocks, alice.head) - /** @type {Array<[string, import('../src/link').AnyLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const data = [ ['banana', await randomCID(32)], ['kiwi', await randomCID(32)] @@ -133,7 +135,7 @@ describe('CRDT', () => { await alice.put('apple', await randomCID(32)) const bob = new TestPail(blocks, alice.head) - /** @type {Array<[string, import('../src/link').AnyLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const data = [ ['banana', await randomCID(32)], ['kiwi', await randomCID(32)] @@ -178,16 +180,16 @@ describe('CRDT', () => { class TestPail { /** * @param {Blockstore} blocks - * @param {import('../src/clock').EventLink[]} head + * @param {API.EventLink[]} head */ constructor (blocks, head) { this.blocks = blocks this.head = head - /** @type {import('../src/shard.js').ShardLink?} */ + /** @type {API.ShardLink?} */ this.root = null } - /** @param {import('../src/clock').EventLink} event */ + /** @param {API.EventLink} event */ async advance (event) { this.head = await advance(this.blocks, this.head, event) const result = await root(this.blocks, this.head) @@ -198,7 +200,7 @@ class TestPail { /** * @param {string} key - * @param {import('../src/link').AnyLink} value + * @param {API.UnknownLink} value */ async put (key, value) { const result = await put(this.blocks, this.head, key, value) @@ -211,13 +213,13 @@ class TestPail { /** * @param {string} key - * @param {import('../src/link').AnyLink} value + * @param {API.UnknownLink} value */ async putAndVis (key, value) { const result = await this.put(key, value) - /** @param {import('../src/link').AnyLink} l */ + /** @param {API.UnknownLink} l */ const shortLink = l => `${String(l).slice(0, 4)}..${String(l).slice(-4)}` - /** @type {(e: import('../src/clock').EventBlockView) => string} */ + /** @type {(e: API.EventBlockView) => string} */ const renderNodeLabel = event => { return event.value.data.type === 'put' ? `${shortLink(event.cid)}\\nput(${event.value.data.key}, ${shortLink(event.value.data.value)})` diff --git a/test/del.test.js b/test/del.test.js index da29cacc..463a6543 100644 --- a/test/del.test.js +++ b/test/del.test.js @@ -1,6 +1,7 @@ import { describe, it } from 'mocha' import assert from 'node:assert' -import { ShardBlock, put, get, del } from '../src/index.js' +import { put, get, del } from '../src/index.js' +import { ShardBlock } from '../src/shard.js' import { Blockstore, randomCID } from './helpers.js' describe('del', () => { diff --git a/test/diff.test.js b/test/diff.test.js index 6be7687e..def8246c 100644 --- a/test/diff.test.js +++ b/test/diff.test.js @@ -1,6 +1,9 @@ import { describe, it } from 'mocha' import assert from 'node:assert' -import { ShardBlock, put } from '../src/index.js' +// eslint-disable-next-line no-unused-vars +import * as API from '../src/api.js' +import { put } from '../src/index.js' +import { ShardBlock } from '../src/shard.js' import { difference } from '../src/diff.js' import { Blockstore, randomCID } from './helpers.js' @@ -10,12 +13,12 @@ describe('diff', () => { const blocks = new Blockstore() await blocks.put(empty.cid, empty.bytes) - /** @type {Array<[string, import('../src/link').AnyLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const testdata = [ ['a', await randomCID(32)] ] - /** @type {import('../src/shard').ShardLink} */ + /** @type {API.ShardLink} */ let root = empty.cid for (const [k, v] of testdata) { const res = await put(blocks, root, k, v) diff --git a/test/entries.test.js b/test/entries.test.js index 7027bc08..ca167b83 100644 --- a/test/entries.test.js +++ b/test/entries.test.js @@ -1,6 +1,9 @@ import { describe, it } from 'mocha' import assert from 'node:assert' -import { ShardBlock, put, entries } from '../src/index.js' +// eslint-disable-next-line no-unused-vars +import * as API from '../src/api.js' +import { put, entries } from '../src/index.js' +import { ShardBlock } from '../src/shard.js' import { Blockstore, randomCID } from './helpers.js' describe('entries', () => { @@ -9,7 +12,7 @@ describe('entries', () => { const blocks = new Blockstore() await blocks.put(empty.cid, empty.bytes) - /** @type {Array<[string, import('../src/link').AnyLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const testdata = [ ['c', await randomCID(32)], ['d', await randomCID(32)], @@ -17,7 +20,7 @@ describe('entries', () => { ['b', await randomCID(32)] ] - /** @type {import('../src/shard').ShardLink} */ + /** @type {API.ShardLink} */ let root = empty.cid for (const [k, v] of testdata) { const res = await put(blocks, root, k, v) @@ -42,7 +45,7 @@ describe('entries', () => { const blocks = new Blockstore() await blocks.put(empty.cid, empty.bytes) - /** @type {Array<[string, import('../src/link').AnyLink]>} */ + /** @type {Array<[string, API.UnknownLink]>} */ const testdata = [ ['cccc', await randomCID(32)], ['deee', await randomCID(32)], @@ -50,7 +53,7 @@ describe('entries', () => { ['beee', await randomCID(32)] ] - /** @type {import('../src/shard').ShardLink} */ + /** @type {API.ShardLink} */ let root = empty.cid for (const [k, v] of testdata) { const res = await put(blocks, root, k, v) diff --git a/test/get.test.js b/test/get.test.js index b5ab74c9..89671eaa 100644 --- a/test/get.test.js +++ b/test/get.test.js @@ -1,6 +1,7 @@ import { describe, it } from 'mocha' import assert from 'node:assert' -import { ShardBlock, put, get } from '../src/index.js' +import { put, get } from '../src/index.js' +import { ShardBlock } from '../src/shard.js' import { Blockstore, randomCID } from './helpers.js' describe('get', () => { diff --git a/test/helpers.js b/test/helpers.js index 710c4d97..07325437 100644 --- a/test/helpers.js +++ b/test/helpers.js @@ -3,7 +3,7 @@ import assert from 'node:assert' import * as Link from 'multiformats/link' import * as raw from 'multiformats/codecs/raw' import { sha256 } from 'multiformats/hashes/sha2' -import { decodeShardBlock } from '../src/index.js' +import { decodeBlock } from '../src/shard.js' import { MemoryBlockstore } from '../src/block.js' /** @@ -49,12 +49,12 @@ export async function randomBytes (size) { export class Blockstore extends MemoryBlockstore { /** - * @param {import('../src/shard').ShardLink} cid + * @param {import('../src/api.js').ShardLink} cid * @param {string} [prefix] */ async getShardBlock (cid, prefix) { const blk = await this.get(cid) assert(blk) - return decodeShardBlock(blk.bytes, prefix) + return decodeBlock(blk.bytes, prefix) } } diff --git a/test/put.test.js b/test/put.test.js index 6d13c305..c7df0895 100644 --- a/test/put.test.js +++ b/test/put.test.js @@ -1,8 +1,11 @@ import { describe, it } from 'mocha' import assert from 'node:assert' import { nanoid } from 'nanoid' -import { ShardBlock, put, MaxKeyLength, get, encodeShardBlock } from '../src/index.js' -import { putEntry } from '../src/shard.js' +// eslint-disable-next-line no-unused-vars +import * as API from '../src/api.js' +import { put, get } from '../src/index.js' +import { ShardBlock, MaxKeyLength } from '../src/shard.js' +import * as Shard from '../src/shard.js' import { Blockstore, randomCID } from './helpers.js' const maxShardSize = 1024 // tiny shard size for testing @@ -11,18 +14,17 @@ const maxShardSize = 1024 // tiny shard size for testing * Fill a shard until it exceeds the size limit. Returns the entry that will * cause the limit to exceed. * - * @param {import('../src/shard').Shard} shard - * @param {number} size - * @param {(i: number) => Promise} [mkentry] + * @param {API.Shard} shard + * @param {(i: number) => Promise} [mkentry] */ -async function fillShard (shard, size, mkentry) { +async function fillShard (shard, mkentry) { mkentry = mkentry ?? (async () => [nanoid(), await randomCID(32)]) let i = 0 while (true) { const entry = await mkentry(i) - const blk = await encodeShardBlock(putEntry(shard, entry)) - if (blk.bytes.length > size) return { shard, entry } - shard = putEntry(shard, entry) + const blk = await Shard.encodeBlock(Shard.putEntry(shard, entry)) + if (blk.bytes.length > shard.maxSize) return { shard, entry } + shard = Shard.putEntry(shard, entry) i++ } } @@ -39,9 +41,9 @@ describe('put', () => { assert.equal(result.removals.length, 1) assert.equal(result.removals[0].cid.toString(), root.cid.toString()) assert.equal(result.additions.length, 1) - assert.equal(result.additions[0].value.length, 1) - assert.equal(result.additions[0].value[0][0], 'test') - assert.equal(result.additions[0].value[0][1].toString(), dataCID.toString()) + assert.equal(result.additions[0].value.entries.length, 1) + assert.equal(result.additions[0].value.entries[0][0], 'test') + assert.equal(result.additions[0].value.entries[0][1].toString(), dataCID.toString()) }) it('put same value to existing key', async () => { @@ -55,9 +57,9 @@ describe('put', () => { assert.equal(result0.removals.length, 1) assert.equal(result0.removals[0].cid.toString(), root.cid.toString()) assert.equal(result0.additions.length, 1) - assert.equal(result0.additions[0].value.length, 1) - assert.equal(result0.additions[0].value[0][0], 'test') - assert.equal(result0.additions[0].value[0][1].toString(), dataCID.toString()) + assert.equal(result0.additions[0].value.entries.length, 1) + assert.equal(result0.additions[0].value.entries[0][0], 'test') + assert.equal(result0.additions[0].value.entries[0][1].toString(), dataCID.toString()) for (const b of result0.additions) { await blocks.put(b.cid, b.bytes) @@ -82,12 +84,12 @@ describe('put', () => { assert.equal(result.removals.length, 1) assert.equal(result.removals[0].cid.toString(), root.cid.toString()) assert.equal(result.additions.length, 2) - assert.equal(result.additions[0].value.length, 1) - assert.equal(result.additions[0].value[0][0], key.slice(-1)) - assert.equal(result.additions[0].value[0][1].toString(), dataCID.toString()) - assert.equal(result.additions[1].value.length, 1) - assert.equal(result.additions[1].value[0][0], key.slice(0, -1)) - assert.equal(result.additions[1].value[0][1][0].toString(), result.additions[0].cid.toString()) + assert.equal(result.additions[0].value.entries.length, 1) + assert.equal(result.additions[0].value.entries[0][0], key.slice(-1)) + assert.equal(result.additions[0].value.entries[0][1].toString(), dataCID.toString()) + assert.equal(result.additions[1].value.entries.length, 1) + assert.equal(result.additions[1].value.entries[0][0], key.slice(0, -1)) + assert.equal(result.additions[1].value.entries[0][1][0].toString(), result.additions[0].cid.toString()) }) it('auto-shards on super long key', async () => { @@ -102,15 +104,15 @@ describe('put', () => { assert.equal(result.removals.length, 1) assert.equal(result.removals[0].cid.toString(), root.cid.toString()) assert.equal(result.additions.length, 3) - assert.equal(result.additions[0].value.length, 1) - assert.equal(result.additions[0].value[0][0], key.slice(-1)) - assert.equal(result.additions[0].value[0][1].toString(), dataCID.toString()) - assert.equal(result.additions[1].value.length, 1) - assert.equal(result.additions[1].value[0][0], key.slice(MaxKeyLength, MaxKeyLength * 2)) - assert.equal(result.additions[1].value[0][1][0].toString(), result.additions[0].cid.toString()) - assert.equal(result.additions[2].value.length, 1) - assert.equal(result.additions[2].value[0][0], key.slice(0, MaxKeyLength)) - assert.equal(result.additions[2].value[0][1][0].toString(), result.additions[1].cid.toString()) + assert.equal(result.additions[0].value.entries.length, 1) + assert.equal(result.additions[0].value.entries[0][0], key.slice(-1)) + assert.equal(result.additions[0].value.entries[0][1].toString(), dataCID.toString()) + assert.equal(result.additions[1].value.entries.length, 1) + assert.equal(result.additions[1].value.entries[0][0], key.slice(MaxKeyLength, MaxKeyLength * 2)) + assert.equal(result.additions[1].value.entries[0][1][0].toString(), result.additions[0].cid.toString()) + assert.equal(result.additions[2].value.entries.length, 1) + assert.equal(result.additions[2].value.entries[0][0], key.slice(0, MaxKeyLength)) + assert.equal(result.additions[2].value.entries[0][1][0].toString(), result.additions[1].cid.toString()) }) // TODO: deep shard propagates to root @@ -118,13 +120,13 @@ describe('put', () => { it('shards at size limit', async () => { const blocks = new Blockstore() const pfx = 'test/' - const { shard, entry: [k, v] } = await fillShard([], maxShardSize, async () => { + const { shard, entry: [k, v] } = await fillShard(Shard.create({ maxSize: maxShardSize }), async () => { return [pfx + nanoid(), await randomCID(1)] }) - const rootblk0 = await encodeShardBlock(shard) + const rootblk0 = await Shard.encodeBlock(shard) await blocks.put(rootblk0.cid, rootblk0.bytes) - const { root, additions, removals } = await put(blocks, rootblk0.cid, k, v, { maxShardSize }) + const { root, additions, removals } = await put(blocks, rootblk0.cid, k, v) assert.notEqual(root.toString(), rootblk0.cid.toString()) assert.equal(removals.length, 1) @@ -136,11 +138,11 @@ describe('put', () => { const rootblk1 = await blocks.getShardBlock(root) - const entry = rootblk1.value.find(([, v]) => Array.isArray(v)) + const entry = rootblk1.value.entries.find(([, v]) => Array.isArray(v)) assert(entry, 'should find a shard entry') assert(entry[0].startsWith(pfx)) - for (const [k, v] of rootblk0.value) { + for (const [k, v] of rootblk0.value.entries) { const value = await get(blocks, rootblk1.cid, k) assert(value) assert.equal(value.toString(), v.toString())