feat!: self describing (#26)

Updates data format to match the spec: https://github.com/web3-storage/specs/blob/4163e28d7e6a7c44cff68db9d9bffb9b37707dc6/pail.md#data-format

README.md

@@ -5,7 +5,7 @@
 
 DAG based key value store. Sharded DAG that minimises traversals and work to build shards.
 
-* 📖 [Read the SPEC](https://github.com/web3-storage/specs/blob/460b6511979a52ec9870f307695ee3f0b3860f81/kv.md).
+* 📖 [Read the SPEC](https://github.com/web3-storage/specs/blob/4163e28d7e6a7c44cff68db9d9bffb9b37707dc6/pail.md).
 * 🎬 [Watch the Presentation](https://youtu.be/f-BrtpYKZfg).
 
 ## Install

bench/put-x10_000.js

@@ -1,4 +1,7 @@
-import { ShardBlock, put } from '../src/index.js'
+// eslint-disable-next-line no-unused-vars
+import * as API from '../src/api.js'
+import { put } from '../src/index.js'
+import { ShardBlock } from '../src/shard.js'
 import { MemoryBlockstore } from '../src/block.js'
 import { randomCID, randomString, randomInteger } from '../test/helpers.js'
 
@@ -11,7 +14,7 @@ async function main () {
   const blocks = new MemoryBlockstore()
   await blocks.put(rootBlock.cid, rootBlock.bytes)
 
-  /** @type {Array<[string, import('multiformats').UnknownLink]>} */
+  /** @type {Array<[string, API.UnknownLink]>} */
   const kvs = []
 
   for (let i = 0; i < NUM; i++) {
@@ -22,7 +25,7 @@ async function main () {
 
   console.log('bench')
   console.time(`put x${NUM}`)
-  /** @type {import('../src/shard.js').ShardLink} */
+  /** @type {API.ShardLink} */
   let root = rootBlock.cid
   for (let i = 0; i < kvs.length; i++) {
     const result = await put(blocks, root, kvs[i][0], kvs[i][1])

cli.js

@@ -8,8 +8,10 @@ import { CID } from 'multiformats/cid'
 import { CarIndexedReader, CarReader, CarWriter } from '@ipld/car'
 import clc from 'cli-color'
 import archy from 'archy'
-import { MaxShardSize, put, ShardBlock, get, del, entries } from './src/index.js'
-import { ShardFetcher } from './src/shard.js'
+// eslint-disable-next-line no-unused-vars
+import * as API from './src/api.js'
+import { put, get, del, entries } from './src/index.js'
+import { ShardFetcher, ShardBlock, MaxShardSize } from './src/shard.js'
 import { difference } from './src/diff.js'
 import { merge } from './src/merge.js'
 
@@ -21,11 +23,11 @@ cli.command('put <key> <value>')
   .alias('set')
   .option('--max-shard-size', 'Maximum shard size in bytes.', MaxShardSize)
   .action(async (key, value, opts) => {
-    const blocks = await openPail(opts.path)
-    const roots = await blocks.getRoots()
     const maxShardSize = opts['max-shard-size'] ?? MaxShardSize
+    const blocks = await openPail(opts.path, { maxSize: maxShardSize })
+    const roots = await blocks.getRoots()
     // @ts-expect-error
-    const { root, additions, removals } = await put(blocks, roots[0], key, CID.parse(value), { maxShardSize })
+    const { root, additions, removals } = await put(blocks, roots[0], key, CID.parse(value))
     await updatePail(opts.path, blocks, root, { additions, removals })
 
     console.log(clc.red(`--- ${roots[0]}`))
@@ -95,7 +97,7 @@ cli.command('tree')
     /** @type {archy.Data} */
     const archyRoot = { label: `Shard(${clc.yellow(rshard.cid.toString())}) ${rshard.bytes.length + 'b'}`, nodes: [] }
 
-    /** @param {import('./src/shard').ShardEntry} entry */
+    /** @param {API.ShardEntry} entry */
     const getData = async ([k, v]) => {
       if (!Array.isArray(v)) {
         return { label: `Key(${clc.magenta(k)})`, nodes: [{ label: `Value(${clc.cyan(v)})` }] }
@@ -106,12 +108,12 @@ cli.command('tree')
       const blk = await shards.get(v[0])
       data.nodes?.push({
         label: `Shard(${clc.yellow(v[0])}) ${blk.bytes.length + 'b'}`,
-        nodes: await Promise.all(blk.value.map(e => getData(e)))
+        nodes: await Promise.all(blk.value.entries.map(e => getData(e)))
       })
       return data
     }
 
-    for (const entry of rshard.value) {
+    for (const entry of rshard.value.entries) {
       archyRoot.nodes?.push(await getData(entry))
     }
 
@@ -125,7 +127,7 @@ cli.command('diff <path>')
   .action(async (path, opts) => {
     const [ablocks, bblocks] = await Promise.all([openPail(opts.path), openPail(path)])
     const [aroot, broot] = await Promise.all([ablocks, bblocks].map(async blocks => {
-      return /** @type {import('./src/shard').ShardLink} */((await blocks.getRoots())[0])
+      return /** @type {API.ShardLink} */((await blocks.getRoots())[0])
     }))
     if (aroot.toString() === broot.toString()) return
 
@@ -136,6 +138,7 @@ cli.command('diff <path>')
         return bblocks.get(cid)
       }
     }
+    // @ts-expect-error CarReader is not BlockFetcher
     const { shards: { additions, removals }, keys } = await difference(fetcher, aroot, broot)
 
     console.log(clc.red(`--- ${aroot}`))
@@ -160,7 +163,7 @@ cli.command('merge <path>')
   .action(async (path, opts) => {
     const [ablocks, bblocks] = await Promise.all([openPail(opts.path), openPail(path)])
     const [aroot, broot] = await Promise.all([ablocks, bblocks].map(async blocks => {
-      return /** @type {import('./src/shard').ShardLink} */((await blocks.getRoots())[0])
+      return /** @type {API.ShardLink} */((await blocks.getRoots())[0])
     }))
     if (aroot.toString() === broot.toString()) return
 
@@ -171,6 +174,7 @@ cli.command('merge <path>')
         return bblocks.get(cid)
       }
     }
+    // @ts-expect-error CarReader is not BlockFetcher
     const { root, additions, removals } = await merge(fetcher, aroot, [broot])
 
     await updatePail(opts.path, ablocks, root, { additions, removals })
@@ -188,17 +192,16 @@ cli.parse(process.argv)
 
 /**
  * @param {string} path
+ * @param {{ maxSize?: number }} [config]
  * @returns {Promise<import('@ipld/car/api').CarReader>}
  */
-async function openPail (path) {
+async function openPail (path, config) {
   try {
     return await CarIndexedReader.fromFile(path)
   } catch (err) {
     if (err.code !== 'ENOENT') throw new Error('failed to open bucket', { cause: err })
-    const rootblk = await ShardBlock.create()
-    // @ts-expect-error
+    const rootblk = await ShardBlock.create(config)
     const { writer, out } = CarWriter.create(rootblk.cid)
-    // @ts-expect-error
     writer.put(rootblk)
     writer.close()
     return CarReader.fromIterable(out)
@@ -215,8 +218,8 @@ async function closePail (reader) {
 /**
  * @param {string} path
  * @param {import('@ipld/car/api').CarReader} reader
- * @param {import('./src/shard').ShardLink} root
- * @param {import('./src/index').ShardDiff} diff
+ * @param {API.ShardLink} root
+ * @param {API.ShardDiff} diff
  */
 async function updatePail (path, reader, root, { additions, removals }) {
   // @ts-expect-error
@@ -229,7 +232,6 @@ async function updatePail (path, reader, root, { additions, removals }) {
 
   // put new blocks
   for (const b of additions) {
-    // @ts-expect-error
     await writer.put(b)
   }
   // put old blocks without removals

package.json

@@ -13,6 +13,9 @@
       "dist/src/index.d.ts": [
         "dist/src/index.d.ts"
       ],
+      "api": [
+        "dist/src/api.d.ts"
+      ],
       "block": [
         "dist/src/block.d.ts"
       ],
@@ -41,6 +44,10 @@
       "types": "./dist/src/index.d.ts",
       "import": "./src/index.js"
     },
+    "./api": {
+      "types": "./dist/src/api.d.ts",
+      "import": "./src/api.js"
+    },
     "./block": {
       "types": "./dist/src/block.d.ts",
       "import": "./src/block.js"
@@ -94,11 +101,11 @@
     "dist"
   ],
   "dependencies": {
-    "@ipld/car": "^5.0.1",
-    "@ipld/dag-cbor": "^9.0.0",
+    "@ipld/car": "^5.2.4",
+    "@ipld/dag-cbor": "^9.0.6",
     "archy": "^1.0.0",
     "cli-color": "^2.0.3",
-    "multiformats": "^12.1.1",
+    "multiformats": "^12.1.3",
     "sade": "^1.8.1"
   },
   "devDependencies": {

src/api.js

@@ -0,0 +1 @@
+export {}

src/api.ts

@@ -0,0 +1,65 @@
+import { Link, UnknownLink, BlockView, Block, Version } from 'multiformats'
+import { sha256 } from 'multiformats/hashes/sha2'
+import * as dagCBOR from '@ipld/dag-cbor'
+
+export { Link, UnknownLink, BlockView, Block, Version }
+
+export type ShardEntryValueValue = UnknownLink
+
+export type ShardEntryLinkValue = [ShardLink]
+
+export type ShardEntryLinkAndValueValue = [ShardLink, UnknownLink]
+
+export type ShardValueEntry = [key: string, value: ShardEntryValueValue]
+
+export type ShardLinkEntry = [key: string, value: ShardEntryLinkValue | ShardEntryLinkAndValueValue]
+
+/** Single key/value entry within a shard. */
+export type ShardEntry = [key: string, value: ShardEntryValueValue | ShardEntryLinkValue | ShardEntryLinkAndValueValue]
+
+/** Legacy shards are not self describing. */
+export type LegacyShard = ShardEntry[]
+
+export interface Shard {
+  entries: ShardEntry[]
+  /** Max key length (in UTF-8 encoded characters) - default 64. */
+  maxKeyLength: number
+  /** Max encoded shard size in bytes - default 512 KiB. */
+  maxSize: number
+}
+
+export type ShardLink = Link<Shard, typeof dagCBOR.code, typeof sha256.code, 1>
+
+export interface ShardBlockView extends BlockView<Shard, typeof dagCBOR.code, typeof sha256.code, 1> {
+  prefix: string
+}
+
+export interface ShardDiff {
+  additions: ShardBlockView[]
+  removals: ShardBlockView[]
+}
+
+export interface BlockFetcher {
+  get<T = unknown, C extends number = number, A extends number = number, V extends Version = 1> (link: Link<T, C, A, V>):
+    Promise<Block<T, C, A, V> | undefined>
+}
+
+// Clock //////////////////////////////////////////////////////////////////////
+
+export type EventLink<T> = Link<EventView<T>>
+
+export interface EventView<T> {
+  parents: EventLink<T>[]
+  data: T
+}
+
+export interface EventBlockView<T> extends BlockView<EventView<T>> {}
+
+// CRDT ///////////////////////////////////////////////////////////////////////
+
+export interface EventData {
+  type: 'put'|'del'
+  key: string
+  value: UnknownLink
+  root: ShardLink
+}