From 91a9897195a2b16065f0986f92db17d9095a5318 Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Mon, 30 Jan 2023 18:28:49 +0100
Subject: [PATCH 1/7] feat: add spec for KV storage aka PAIL

---
 kv.md | 367 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 367 insertions(+)
 create mode 100644 kv.md

diff --git a/kv.md b/kv.md
new file mode 100644
index 0000000..35e24d3
--- /dev/null
+++ b/kv.md
@@ -0,0 +1,367 @@
+# KV/DAG
+
+## Editors
+
+* [Alan Shaw](https://github.com/alanshaw), [DAG House](https://dag.house/)
+
+## Authors
+
+* [Alan Shaw](https://github.com/alanshaw), [DAG House](https://dag.house/)
+
+## Language
+
+The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "SHOULD NOT", "RECOMMENDED", "MAY", and "OPTIONAL" in this document are to be interpreted as described in [RFC 2119](https://datatracker.ietf.org/doc/html/rfc2119).
+
+## Abstract
+
+This specification describes a method of key/value storage implemented as an IPLD DAG. It details the format, encoding and mechanisms to mutate the storage.
+
+This method of key/value storage is designed to allow fast _ordered_ value lookups by key _prefix_.
+
+## Data Format
+
+IPLD Schema
+
+```ipldsch
+# A shard is just a list of entries
+type Shard [ShardEntry]
+
+# Single key/value entry within a shard
+type ShardEntry struct {
+  key String
+  value ShardValue
+} representation tuple
+
+# User data (any CID to any data) or shard link
+type ShardValue union {
+  | &UserData link
+  | ShardLinkValue list
+} representation kinded
+
+# A link to another shard, and optional user data
+type ShardLinkValue struct {
+  link &Shard
+  data optional &UserData
+} representation tuple
+
+# User data - any CID to any data
+type UserData = Any
+```
+
+Typsecript
+
+```ts
+import { Link } from 'multiformats/link'
+
+/** A shard is just a list of entries */
+type Shard = ShardEntry[]
+
+/** Single key/value entry within a shard */
+type ShardEntry = [
+  key: string,
+  value: ShardValue
+]
+
+type ShardValue = UserData | [ShardLink, UserData?]
+
+/** A link to another shard */
+type ShardLink = Link<Shard>
+
+/** User data - any CID to any data */
+type UserData = Link<any>
+```
+
+### Shard
+
+The storage is made up of shards. They are blocks of IPLD data. Shards must be [dag-cbor](https://ipld.io/specs/codecs/dag-cbor/spec/) encoded and must not exceed `512KiB` in size (post encode).
+
+A shard is an ordered list of [shard entries](#Shard-Entry). Shard entries must always be ordered lexicographically by key within a shard.
+
+### Shard Entry
+
+A key/value pair whose value corresponds to [user data](#User-Data) or a [shard link](#Shard-Link).
+
+### Key
+
+A UTF-8 encoded string.
+
+The key length must not exceed 64 characters. Putting a key whose length is greater than 64 characters must create a new shard(s) to accomodate the additional length. See [Long Keys](#Long-Keys).
+
+### Value
+
+#### User Data
+
+An IPLD [CID](https://github.com/multiformats/cid) to any data that has explicitly been put to the storage by a user.
+
+For example (dag-json encoded):
+
+```javascript
+{ '/': 'bafkreiem4twkqzsq2aj4shbycd4yvoj2cx72vezicletlhi7dijjciqpui' }
+```
+
+#### Shard Link
+
+An IPLD [CID](https://github.com/multiformats/cid) link to another shard in the storage.
+
+Shard link values must be encoded as an array (tuple) in order to differentiate them from [user data](#User-Data).
+
+If the value is a shard link value, the first item in the array must be an IPLD [CID](https://github.com/multiformats/cid) link to another shard in the storage. If the array contains a second item, the item is [user data](#User-Data).
+
+Shard link values must contain one or two elements. The first element (the shard link) is required (not nullable).
+
+For example, a shard link _without_ user data (dag-json encoded):
+
+```javascript
+[{ '/': 'bafyreibq6w6xgqluv7ubskavehlfsnvodmh2gbc2q4c3d4ijlf7gva2day' }]
+```
+
+For example, a shard link _with_ user data (dag-json encoded):
+
+```javascript
+[
+  { '/': 'bafyreibq6w6xgqluv7ubskavehlfsnvodmh2gbc2q4c3d4ijlf7gva2day' },
+  { '/': 'bafkreiem4twkqzsq2aj4shbycd4yvoj2cx72vezicletlhi7dijjciqpui' }
+]
+```
+
+## Operations
+
+### Put
+
+The "put" operation adds a new value or updates an existing value for a given key in the storage.
+
+The storage must first be [traversed](#Shard-Traversal) to identify the target shard where the value should be placed, as well as the key within the shard that should be used.
+
+Any changes made must be [propagated to the root shard](#Propagating-Changes).
+
+#### New Value
+
+If no value exists in the shard for the shard key then a new user data entry should be added to the shard at the correct lexicographical index.
+
+For example, putting a key `b` and value `bafyvalueb` to a shard with existing keys `a` and `c` (dag-json encoded):
+
+Before:
+```javascript
+[
+  ['a', { '/': 'bafyvaluea' }],
+  ['c', { '/': 'bafyvaluec' }]
+]
+```
+
+After:
+```javascript
+[
+  ['a', { '/': 'bafyvaluea' }],
+  ['b', { '/': 'bafyvalueb' }], // <- new entry
+  ['c', { '/': 'bafyvaluec' }]
+]
+```
+
+#### Existing User Data Value
+
+If a value exists in the shard for the shard key and the value is user data, then the entry must be updated.
+
+For example, putting a key `a` and value `bafyvalueaaa` to a shard with existing key `a` and value `bafyvaluea` (dag-json encoded):
+
+Before:
+```javascript
+[['a', { '/': 'bafyvaluea' }]]
+```
+
+After:
+```javascript
+[['a', { '/': 'bafyvalueaaa' }]]
+```
+
+#### Existing Shard Link Value
+
+If a value exists in the shard for the shard key and the value is a shard link, then the value must be placed at index 1 of the shard link array.
+
+For example, putting a key `a` and value `bafyvaluea` to a shard with existing key `a` with a shard link value `bafyshard` (dag-json encoded):
+
+Before:
+```javascript
+[['a', [{ '/': 'bafyshard' }]]]
+```
+
+After:
+```javascript
+[['a', [{ '/': 'bafyshard' }, { '/': 'bafyvaluea' }]]]
+```
+
+For example, putting a key `a` and value `bafyvalueaaa` to a shard with existing key `a` with a shard link value `bafyshard`, with user data `bafyvaluea` (dag-json encoded):
+
+Before:
+```javascript
+[['a', [{ '/': 'bafyshard' }, { '/': 'bafyvaluea' }]]]
+```
+
+After:
+```javascript
+[['a', [{ '/': 'bafyshard' }, { '/': 'bafyvalueaaa' }]]]
+```
+
+#### Long Keys
+
+If the shard key is longer than 64 characters a new shard(s) must be created to acommodate the new length. The first 64 characters must be added as a new entry in the shard, along with a value that is a link to a new shard with the next 64 characters of the key. This is repeated until the key has less than 64 characters. The value for the entry for the key with less than 64 characters must be set as the value for the put operation.
+
+For example, putting a key `ax64...bx64...cx10...` and value `bafyvalue` in an empty shard (dag-json encoded):
+
+```javascript
+[['ax64...', [{ '/': 'bafyshard1' }]]]
+```
+```javascript
+// bafyshard1
+[['bx64...', [{ '/': 'bafyshard0' }]]]
+```
+```javascript
+// bafyshard0
+[['cx10...', { '/': 'bafyvalue' }]]
+```
+
+#### Sharding
+
+After putting a value to the shard, it must be encoded and it's size measured. If the byte size of the encoded shard exceeds `512KiB`, it must be sharded.
+
+Sharding involves taking two or more keys from the shard and moving them into a new shard. To select keys for sharding, the longest common prefix (LCP) must be found, using the newly inserted shard key as the base. Work backwards through the string until one or more other keys within the shard share the same prefix. Move to the next key in the shard as the base if no other keys in the shard match any substring of the inserted shard key.
+
+The following is pseudocode of the algorithm for creating a new shard when a shard exceeds the size limit:
+
+1. Find longest common prefix using insert key as base
+2. IF common prefix for > 1 entries exists
+    1. Create new shard with suffixes for entries that match common prefix
+    1. Remove entries with common prefix from shard
+    1. Add entry for common prefix, linking new shard
+    1. FINISH
+3. ELSE
+    1. Find longest common prefix using adjacent key as base
+    1. GOTO 2
+
+For example:
+
+```
+abel
+foobarbaz
+foobarwooz
+food
+somethingelse
+```
+
+Put "foobarboz" and exceed shard size limit:
+```
+abel
+foobarbaz
+<- foobarboz
+foobarwooz
+food
+somethingelse
+```
+
+Find "foobarb" as longest common prefix, create shard:
+```
+abel
+foobarb -> az
+           oz
+foobarwooz
+food
+somethingelse
+```
+
+Put "foopey", exceeding shard size:
+```
+abel
+foobarb -> az
+           oz
+foobarwooz
+food
+<- foopey
+somethingelse
+```
+
+Find "foo" as longest common prefix, create shard:
+```
+abel
+foo -> barb -> az
+               oz
+       barwooz
+       d
+       pey
+somethingelse
+```
+
+### Delete
+
+The "delete" operation removes a value for a given key in the storage.
+
+The storage must first be [traversed](#Shard-Traversal) to identify the target shard where the value should be removed from, as well as the key within the shard that should be used.
+
+Any changes made must be [propagated to the root shard](#Propagating-Changes).
+
+Deleting the last remaining key in a non-root shard must remove the shard entirely and it's entry in it's parent shard. That is unless the entry in the parent shard contains user data. In this case the value in the parent shard is updated from a shard link (with user data) to user data.
+
+For example, deleting a key `a` from a root shard (dag-json encoded):
+
+Before:
+```javascript
+[['a', { '/': 'bafyvaluea' }]]
+```
+
+After:
+```javascript
+[]
+```
+
+For example, deleting a key `abba` from a non-root shard (dag-json encoded):
+
+Before:
+```javascript
+[['abb', [{ '/': 'bafyshard' }]]]
+```
+```javascript
+// bafyshard
+[['a', { '/': 'bafyvalue' }]]
+```
+
+After:
+```javascript
+[]
+```
+
+For example, deleting a key `abba` from a non-root shard with user data in key `abb` (dag-json encoded):
+
+Before:
+```javascript
+[['abb', [{ '/': 'bafyshard' }, { '/': 'bafyvalueabb' }]]]
+```
+```javascript
+// bafyshard
+[['a', { '/': 'bafyvalue' }]]
+```
+
+After:
+```javascript
+[['abb', { '/': 'bafyvalueabb' }]]
+```
+
+## Propagating Changes
+
+Any changes made to a shard will result in it's CID changing. If the shard is not the root shard, the change must be propagated to the root.
+
+## Shard Traversal
+
+Given a key `k` it is often necessary to locate the shard the value is stored in or should be stored in for the purpose of adding, updating or removing the value from the storage.
+
+The root shard must first be loaded. Then `k` must be matched exactly with an existing key or prefixed by an existing key whose value is a link to another shard. In the former case the shard has been identified. In the latter case, the linked shard must be loaded and `k` shortened, removing the prefix. The process is then repeated in the linked shard. If no match is found for `k` then traversal has finished.
+
+The following is pseudocode of an algorithm for traversing the storage to identify the shard a key should be placed/found in:
+
+1. Let `link` be the CID of the root shard
+2. Retrieve and decode the shard for `link`
+3. LOOP over all entries in the shard
+    1. IF key of `entry` _equals_ `k` BREAK
+    2. IF key of `entry` _starts with_ `k` AND value of `entry` is a shard link
+        1. Set `link` to be `entry` shard link
+        2. Set `k` to the substring of `k` starting _after_ the key of `entry`
+        3. GOTO 2
+
+Traversal should return enough information for a caller to easily identify the key _within_ a shard that should be used to place their value.

From 42de5e8b1724df4c9e80188b0abae8097160e076 Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Mon, 18 Dec 2023 16:52:07 +0000
Subject: [PATCH 2/7] refactor: self describing

---
 kv.md => pail.md | 28 ++++++++++++++++++++--------
 1 file changed, 20 insertions(+), 8 deletions(-)
 rename kv.md => pail.md (94%)

diff --git a/kv.md b/pail.md
similarity index 94%
rename from kv.md
rename to pail.md
index 35e24d3..d8f86c9 100644
--- a/kv.md
+++ b/pail.md
@@ -1,4 +1,4 @@
-# KV/DAG
+# Pail
 
 ## Editors
 
@@ -23,8 +23,14 @@ This method of key/value storage is designed to allow fast _ordered_ value looku
 IPLD Schema
 
 ```ipldsch
-# A shard is just a list of entries
-type Shard [ShardEntry]
+# A shard is just a list of entries, and some config
+type Shard struct {
+  # Max key length (in UTF-8 encoded characters) - default 64
+  maxKeyLength: Int
+  # Max encoded shard size in bytes - default 512 KiB
+  maxSize: Int
+  entries: [ShardEntry]
+}
 
 # Single key/value entry within a shard
 type ShardEntry struct {
@@ -34,8 +40,8 @@ type ShardEntry struct {
 
 # User data (any CID to any data) or shard link
 type ShardValue union {
-  | &UserData link
-  | ShardLinkValue list
+  | &UserData Link
+  | ShardLinkValue List
 } representation kinded
 
 # A link to another shard, and optional user data
@@ -45,7 +51,7 @@ type ShardLinkValue struct {
 } representation tuple
 
 # User data - any CID to any data
-type UserData = Any
+type UserData Any
 ```
 
 Typsecript
@@ -53,8 +59,14 @@ Typsecript
 ```ts
 import { Link } from 'multiformats/link'
 
-/** A shard is just a list of entries */
-type Shard = ShardEntry[]
+/** A shard is just a list of entries, and some config */
+interface Shard {
+  /** Max key length (in UTF-8 encoded characters) - default 64 */
+  maxKeyLength: number
+  /** Max encoded shard size in bytes - default 512 KiB */
+  maxSize: number
+  entries: ShardEntry[]
+}
 
 /** Single key/value entry within a shard */
 type ShardEntry = [

From 4163e28d7e6a7c44cff68db9d9bffb9b37707dc6 Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Mon, 18 Dec 2023 16:55:47 +0000
Subject: [PATCH 3/7] chore: appease linter

---
 .github/workflows/words-to-ignore.txt | 3 +++
 pail.md                               | 6 +++---
 2 files changed, 6 insertions(+), 3 deletions(-)

diff --git a/.github/workflows/words-to-ignore.txt b/.github/workflows/words-to-ignore.txt
index 3c54472..56009cd 100644
--- a/.github/workflows/words-to-ignore.txt
+++ b/.github/workflows/words-to-ignore.txt
@@ -90,6 +90,7 @@ RFC6376
 UTF-8
 UCAN-IPLD
 DAG-JSON
+dag-json
 installationOraclePrincipal
 flowAuthorityPrincipal
 capabilitiesVerifierComponent
@@ -134,3 +135,5 @@ Irakli
 Gozalishvili
 Vasco
 invoker
+lexicographically
+nullable
diff --git a/pail.md b/pail.md
index d8f86c9..a284cc1 100644
--- a/pail.md
+++ b/pail.md
@@ -16,7 +16,7 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S
 
 This specification describes a method of key/value storage implemented as an IPLD DAG. It details the format, encoding and mechanisms to mutate the storage.
 
-This method of key/value storage is designed to allow fast _ordered_ value lookups by key _prefix_.
+This method of key/value storage is designed to allow fast _ordered_ value lookup by key _prefix_.
 
 ## Data Format
 
@@ -54,7 +54,7 @@ type ShardLinkValue struct {
 type UserData Any
 ```
 
-Typsecript
+Typescript
 
 ```ts
 import { Link } from 'multiformats/link'
@@ -97,7 +97,7 @@ A key/value pair whose value corresponds to [user data](#User-Data) or a [shard
 
 A UTF-8 encoded string.
 
-The key length must not exceed 64 characters. Putting a key whose length is greater than 64 characters must create a new shard(s) to accomodate the additional length. See [Long Keys](#Long-Keys).
+The key length must not exceed 64 characters. Putting a key whose length is greater than 64 characters must create a new shard(s) to accommodate the additional length. See [Long Keys](#Long-Keys).
 
 ### Value
 

From 5a90e46f44bd2c1d3d74586cf011185ed01a938b Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Mon, 18 Dec 2023 17:01:18 +0000
Subject: [PATCH 4/7] chore: appease linter

---
 pail.md | 50 ++++++++++++++++++++++++++++++++++++--------------
 1 file changed, 36 insertions(+), 14 deletions(-)

diff --git a/pail.md b/pail.md
index a284cc1..6eab815 100644
--- a/pail.md
+++ b/pail.md
@@ -87,17 +87,17 @@ type UserData = Link<any>
 
 The storage is made up of shards. They are blocks of IPLD data. Shards must be [dag-cbor](https://ipld.io/specs/codecs/dag-cbor/spec/) encoded and must not exceed `512KiB` in size (post encode).
 
-A shard is an ordered list of [shard entries](#Shard-Entry). Shard entries must always be ordered lexicographically by key within a shard.
+A shard is an ordered list of [shard entries](#shard-entry). Shard entries must always be ordered lexicographically by key within a shard.
 
 ### Shard Entry
 
-A key/value pair whose value corresponds to [user data](#User-Data) or a [shard link](#Shard-Link).
+A key/value pair whose value corresponds to [user data](#user-data) or a [shard link](#shard-link).
 
 ### Key
 
 A UTF-8 encoded string.
 
-The key length must not exceed 64 characters. Putting a key whose length is greater than 64 characters must create a new shard(s) to accommodate the additional length. See [Long Keys](#Long-Keys).
+The key length must not exceed 64 characters. Putting a key whose length is greater than 64 characters must create a new shard(s) to accommodate the additional length. See [Long Keys](#long-keys).
 
 ### Value
 
@@ -115,9 +115,9 @@ For example (dag-json encoded):
 
 An IPLD [CID](https://github.com/multiformats/cid) link to another shard in the storage.
 
-Shard link values must be encoded as an array (tuple) in order to differentiate them from [user data](#User-Data).
+Shard link values must be encoded as an array (tuple) in order to differentiate them from [user data](#user-data).
 
-If the value is a shard link value, the first item in the array must be an IPLD [CID](https://github.com/multiformats/cid) link to another shard in the storage. If the array contains a second item, the item is [user data](#User-Data).
+If the value is a shard link value, the first item in the array must be an IPLD [CID](https://github.com/multiformats/cid) link to another shard in the storage. If the array contains a second item, the item is [user data](#user-data).
 
 Shard link values must contain one or two elements. The first element (the shard link) is required (not nullable).
 
@@ -142,9 +142,9 @@ For example, a shard link _with_ user data (dag-json encoded):
 
 The "put" operation adds a new value or updates an existing value for a given key in the storage.
 
-The storage must first be [traversed](#Shard-Traversal) to identify the target shard where the value should be placed, as well as the key within the shard that should be used.
+The storage must first be [traversed](#shard-traversal) to identify the target shard where the value should be placed, as well as the key within the shard that should be used.
 
-Any changes made must be [propagated to the root shard](#Propagating-Changes).
+Any changes made must be [propagated to the root shard](#propagating-changes).
 
 #### New Value
 
@@ -153,6 +153,7 @@ If no value exists in the shard for the shard key then a new user data entry sho
 For example, putting a key `b` and value `bafyvalueb` to a shard with existing keys `a` and `c` (dag-json encoded):
 
 Before:
+
 ```javascript
 [
   ['a', { '/': 'bafyvaluea' }],
@@ -161,6 +162,7 @@ Before:
 ```
 
 After:
+
 ```javascript
 [
   ['a', { '/': 'bafyvaluea' }],
@@ -176,11 +178,13 @@ If a value exists in the shard for the shard key and the value is user data, the
 For example, putting a key `a` and value `bafyvalueaaa` to a shard with existing key `a` and value `bafyvaluea` (dag-json encoded):
 
 Before:
+
 ```javascript
 [['a', { '/': 'bafyvaluea' }]]
 ```
 
 After:
+
 ```javascript
 [['a', { '/': 'bafyvalueaaa' }]]
 ```
@@ -192,11 +196,13 @@ If a value exists in the shard for the shard key and the value is a shard link,
 For example, putting a key `a` and value `bafyvaluea` to a shard with existing key `a` with a shard link value `bafyshard` (dag-json encoded):
 
 Before:
+
 ```javascript
 [['a', [{ '/': 'bafyshard' }]]]
 ```
 
 After:
+
 ```javascript
 [['a', [{ '/': 'bafyshard' }, { '/': 'bafyvaluea' }]]]
 ```
@@ -204,11 +210,13 @@ After:
 For example, putting a key `a` and value `bafyvalueaaa` to a shard with existing key `a` with a shard link value `bafyshard`, with user data `bafyvaluea` (dag-json encoded):
 
 Before:
+
 ```javascript
 [['a', [{ '/': 'bafyshard' }, { '/': 'bafyvaluea' }]]]
 ```
 
 After:
+
 ```javascript
 [['a', [{ '/': 'bafyshard' }, { '/': 'bafyvalueaaa' }]]]
 ```
@@ -222,10 +230,12 @@ For example, putting a key `ax64...bx64...cx10...` and value `bafyvalue` in an e
 ```javascript
 [['ax64...', [{ '/': 'bafyshard1' }]]]
 ```
+
 ```javascript
 // bafyshard1
 [['bx64...', [{ '/': 'bafyshard0' }]]]
 ```
+
 ```javascript
 // bafyshard0
 [['cx10...', { '/': 'bafyvalue' }]]
@@ -251,7 +261,7 @@ The following is pseudocode of the algorithm for creating a new shard when a sha
 
 For example:
 
-```
+```text
 abel
 foobarbaz
 foobarwooz
@@ -260,7 +270,8 @@ somethingelse
 ```
 
 Put "foobarboz" and exceed shard size limit:
-```
+
+```text
 abel
 foobarbaz
 <- foobarboz
@@ -270,7 +281,8 @@ somethingelse
 ```
 
 Find "foobarb" as longest common prefix, create shard:
-```
+
+```text
 abel
 foobarb -> az
            oz
@@ -280,7 +292,8 @@ somethingelse
 ```
 
 Put "foopey", exceeding shard size:
-```
+
+```text
 abel
 foobarb -> az
            oz
@@ -291,7 +304,8 @@ somethingelse
 ```
 
 Find "foo" as longest common prefix, create shard:
-```
+
+```text
 abel
 foo -> barb -> az
                oz
@@ -305,20 +319,22 @@ somethingelse
 
 The "delete" operation removes a value for a given key in the storage.
 
-The storage must first be [traversed](#Shard-Traversal) to identify the target shard where the value should be removed from, as well as the key within the shard that should be used.
+The storage must first be [traversed](#shard-traversal) to identify the target shard where the value should be removed from, as well as the key within the shard that should be used.
 
-Any changes made must be [propagated to the root shard](#Propagating-Changes).
+Any changes made must be [propagated to the root shard](#propagating-changes).
 
 Deleting the last remaining key in a non-root shard must remove the shard entirely and it's entry in it's parent shard. That is unless the entry in the parent shard contains user data. In this case the value in the parent shard is updated from a shard link (with user data) to user data.
 
 For example, deleting a key `a` from a root shard (dag-json encoded):
 
 Before:
+
 ```javascript
 [['a', { '/': 'bafyvaluea' }]]
 ```
 
 After:
+
 ```javascript
 []
 ```
@@ -326,15 +342,18 @@ After:
 For example, deleting a key `abba` from a non-root shard (dag-json encoded):
 
 Before:
+
 ```javascript
 [['abb', [{ '/': 'bafyshard' }]]]
 ```
+
 ```javascript
 // bafyshard
 [['a', { '/': 'bafyvalue' }]]
 ```
 
 After:
+
 ```javascript
 []
 ```
@@ -342,15 +361,18 @@ After:
 For example, deleting a key `abba` from a non-root shard with user data in key `abb` (dag-json encoded):
 
 Before:
+
 ```javascript
 [['abb', [{ '/': 'bafyshard' }, { '/': 'bafyvalueabb' }]]]
 ```
+
 ```javascript
 // bafyshard
 [['a', { '/': 'bafyvalue' }]]
 ```
 
 After:
+
 ```javascript
 [['abb', { '/': 'bafyvalueabb' }]]
 ```

From 40114dc29cdf189a150f0585f81f2ab4e6e3434c Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Mon, 18 Dec 2023 17:06:19 +0000
Subject: [PATCH 5/7] chore: appease linter

---
 .github/workflows/words-to-ignore.txt | 6 ++++++
 pail.md                               | 2 +-
 2 files changed, 7 insertions(+), 1 deletion(-)

diff --git a/.github/workflows/words-to-ignore.txt b/.github/workflows/words-to-ignore.txt
index 56009cd..400ce4c 100644
--- a/.github/workflows/words-to-ignore.txt
+++ b/.github/workflows/words-to-ignore.txt
@@ -137,3 +137,9 @@ Vasco
 invoker
 lexicographically
 nullable
+substring
+LCP
+pseudocode
+foobarboz
+foobarb
+foopey
diff --git a/pail.md b/pail.md
index 6eab815..50b0a10 100644
--- a/pail.md
+++ b/pail.md
@@ -223,7 +223,7 @@ After:
 
 #### Long Keys
 
-If the shard key is longer than 64 characters a new shard(s) must be created to acommodate the new length. The first 64 characters must be added as a new entry in the shard, along with a value that is a link to a new shard with the next 64 characters of the key. This is repeated until the key has less than 64 characters. The value for the entry for the key with less than 64 characters must be set as the value for the put operation.
+If the shard key is longer than 64 characters a new shard(s) must be created to accommodate the new length. The first 64 characters must be added as a new entry in the shard, along with a value that is a link to a new shard with the next 64 characters of the key. This is repeated until the key has less than 64 characters. The value for the entry for the key with less than 64 characters must be set as the value for the put operation.
 
 For example, putting a key `ax64...bx64...cx10...` and value `bafyvalue` in an empty shard (dag-json encoded):
 

From 9109c7047f98e4ddde31a4ec6e26d4736959df95 Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Thu, 4 Apr 2024 22:37:02 +0200
Subject: [PATCH 6/7] feat: update for latest version

---
 pail.md | 334 ++++++++++++++++++++++++++++++++++++--------------------
 1 file changed, 216 insertions(+), 118 deletions(-)

diff --git a/pail.md b/pail.md
index 50b0a10..68f815b 100644
--- a/pail.md
+++ b/pail.md
@@ -16,7 +16,7 @@ The key words "MUST", "MUST NOT", "REQUIRED", "SHALL", "SHALL NOT", "SHOULD", "S
 
 This specification describes a method of key/value storage implemented as an IPLD DAG. It details the format, encoding and mechanisms to mutate the storage.
 
-This method of key/value storage is designed to allow fast _ordered_ value lookup by key _prefix_.
+This method of key/value storage is optimized for fast _ordered_ value lookup by key _prefix_ or _range_.
 
 ## Data Format
 
@@ -25,10 +25,16 @@ IPLD Schema
 ```ipldsch
 # A shard is just a list of entries, and some config
 type Shard struct {
-  # Max key length (in UTF-8 encoded characters) - default 64
-  maxKeyLength: Int
-  # Max encoded shard size in bytes - default 512 KiB
-  maxSize: Int
+  # Shard compatibility version.
+  version: Int
+  # Characters allowed in keys, referring to a known character set.
+  # e.g. "ascii" refers to the printable ASCII characters in the code range 32-126.
+  keyChars: String
+  # Max key size in bytes - default 4096 bytes.
+  maxKeySize: Int
+  # The key prefix from the root to this shard.
+  prefix: String
+  # The entries in this shard.
   entries: [ShardEntry]
 }
 
@@ -61,10 +67,18 @@ import { Link } from 'multiformats/link'
 
 /** A shard is just a list of entries, and some config */
 interface Shard {
-  /** Max key length (in UTF-8 encoded characters) - default 64 */
-  maxKeyLength: number
-  /** Max encoded shard size in bytes - default 512 KiB */
-  maxSize: number
+  /** Shard compatibility version. */
+  version: number
+  /**
+   * Characters allowed in keys, referring to a known character set.
+   * e.g. "ascii" refers to the printable ASCII characters in the code range 32-126.
+   */
+  keyChars: string
+  /** Max key size in bytes - default 4096 bytes. */
+  maxKeySize: number
+  /** The key prefix from the root to this shard. */
+  prefix: string
+  /** The entries in this shard. */
   entries: ShardEntry[]
 }
 
@@ -85,10 +99,21 @@ type UserData = Link<any>
 
 ### Shard
 
-The storage is made up of shards. They are blocks of IPLD data. Shards must be [dag-cbor](https://ipld.io/specs/codecs/dag-cbor/spec/) encoded and must not exceed `512KiB` in size (post encode).
+The storage is made up of shards. They are blocks of IPLD data. Shards are recommended to be [dag-cbor](https://ipld.io/specs/codecs/dag-cbor/spec/) encoded.
 
 A shard is an ordered list of [shard entries](#shard-entry). Shard entries must always be ordered lexicographically by key within a shard.
 
+The maximum encoded shard size is controlled by the creator of the pail by specifying 2 options:
+
+* `keyChars` - the characters allowed in keys (default ASCII only).
+* `maxKeySize` - maximum size in bytes a UTF-8 encoded key is allowed to be (default 4096 bytes).
+
+A good estimate for the max size of a shard is the number of characters allowed in keys multiplied by the maximum key size. This is because there cannot be more than one key with the same first character in a shard (see section on [Sharding](#sharding) for further explanation). For ASCII keys of 4096 bytes the biggest possible shard will stay well within the maximum block size allowed by libp2p.
+
+Note: The default max key size is the same as "`MAX_PATH`" - the maximum filename+path size on most Windows/Unix systems so should be sufficient for most purposes.
+
+Note: Even if you use unicode key characters it would be difficult to exceed the max libp2p block size, but it is not impossible.
+
 ### Shard Entry
 
 A key/value pair whose value corresponds to [user data](#user-data) or a [shard link](#shard-link).
@@ -97,13 +122,11 @@ A key/value pair whose value corresponds to [user data](#user-data) or a [shard
 
 A UTF-8 encoded string.
 
-The key length must not exceed 64 characters. Putting a key whose length is greater than 64 characters must create a new shard(s) to accommodate the additional length. See [Long Keys](#long-keys).
-
 ### Value
 
 #### User Data
 
-An IPLD [CID](https://github.com/multiformats/cid) to any data that has explicitly been put to the storage by a user.
+An IPLD [Link](https://github.com/multiformats/cid) to any data that has explicitly been put to the storage by a user.
 
 For example (dag-json encoded):
 
@@ -113,7 +136,7 @@ For example (dag-json encoded):
 
 #### Shard Link
 
-An IPLD [CID](https://github.com/multiformats/cid) link to another shard in the storage.
+An IPLD [Link](https://github.com/multiformats/cid) link to another shard in the storage.
 
 Shard link values must be encoded as an array (tuple) in order to differentiate them from [user data](#user-data).
 
@@ -221,100 +244,6 @@ After:
 [['a', [{ '/': 'bafyshard' }, { '/': 'bafyvalueaaa' }]]]
 ```
 
-#### Long Keys
-
-If the shard key is longer than 64 characters a new shard(s) must be created to accommodate the new length. The first 64 characters must be added as a new entry in the shard, along with a value that is a link to a new shard with the next 64 characters of the key. This is repeated until the key has less than 64 characters. The value for the entry for the key with less than 64 characters must be set as the value for the put operation.
-
-For example, putting a key `ax64...bx64...cx10...` and value `bafyvalue` in an empty shard (dag-json encoded):
-
-```javascript
-[['ax64...', [{ '/': 'bafyshard1' }]]]
-```
-
-```javascript
-// bafyshard1
-[['bx64...', [{ '/': 'bafyshard0' }]]]
-```
-
-```javascript
-// bafyshard0
-[['cx10...', { '/': 'bafyvalue' }]]
-```
-
-#### Sharding
-
-After putting a value to the shard, it must be encoded and it's size measured. If the byte size of the encoded shard exceeds `512KiB`, it must be sharded.
-
-Sharding involves taking two or more keys from the shard and moving them into a new shard. To select keys for sharding, the longest common prefix (LCP) must be found, using the newly inserted shard key as the base. Work backwards through the string until one or more other keys within the shard share the same prefix. Move to the next key in the shard as the base if no other keys in the shard match any substring of the inserted shard key.
-
-The following is pseudocode of the algorithm for creating a new shard when a shard exceeds the size limit:
-
-1. Find longest common prefix using insert key as base
-2. IF common prefix for > 1 entries exists
-    1. Create new shard with suffixes for entries that match common prefix
-    1. Remove entries with common prefix from shard
-    1. Add entry for common prefix, linking new shard
-    1. FINISH
-3. ELSE
-    1. Find longest common prefix using adjacent key as base
-    1. GOTO 2
-
-For example:
-
-```text
-abel
-foobarbaz
-foobarwooz
-food
-somethingelse
-```
-
-Put "foobarboz" and exceed shard size limit:
-
-```text
-abel
-foobarbaz
-<- foobarboz
-foobarwooz
-food
-somethingelse
-```
-
-Find "foobarb" as longest common prefix, create shard:
-
-```text
-abel
-foobarb -> az
-           oz
-foobarwooz
-food
-somethingelse
-```
-
-Put "foopey", exceeding shard size:
-
-```text
-abel
-foobarb -> az
-           oz
-foobarwooz
-food
-<- foopey
-somethingelse
-```
-
-Find "foo" as longest common prefix, create shard:
-
-```text
-abel
-foo -> barb -> az
-               oz
-       barwooz
-       d
-       pey
-somethingelse
-```
-
 ### Delete
 
 The "delete" operation removes a value for a given key in the storage.
@@ -344,12 +273,12 @@ For example, deleting a key `abba` from a non-root shard (dag-json encoded):
 Before:
 
 ```javascript
-[['abb', [{ '/': 'bafyshard' }]]]
+[['a', [{ '/': 'bafyshard' }]]]
 ```
 
 ```javascript
 // bafyshard
-[['a', { '/': 'bafyvalue' }]]
+[['bba', { '/': 'bafyvalue' }]]
 ```
 
 After:
@@ -358,29 +287,25 @@ After:
 []
 ```
 
-For example, deleting a key `abba` from a non-root shard with user data in key `abb` (dag-json encoded):
+For example, deleting a key `abba` from a non-root shard with user data in key `a` (dag-json encoded):
 
 Before:
 
 ```javascript
-[['abb', [{ '/': 'bafyshard' }, { '/': 'bafyvalueabb' }]]]
+[['a', [{ '/': 'bafyshard' }, { '/': 'bafyvalueabb' }]]]
 ```
 
 ```javascript
 // bafyshard
-[['a', { '/': 'bafyvalue' }]]
+[['bba', { '/': 'bafyvalue' }]]
 ```
 
 After:
 
 ```javascript
-[['abb', { '/': 'bafyvalueabb' }]]
+[['a', { '/': 'bafyvalueabb' }]]
 ```
 
-## Propagating Changes
-
-Any changes made to a shard will result in it's CID changing. If the shard is not the root shard, the change must be propagated to the root.
-
 ## Shard Traversal
 
 Given a key `k` it is often necessary to locate the shard the value is stored in or should be stored in for the purpose of adding, updating or removing the value from the storage.
@@ -399,3 +324,176 @@ The following is pseudocode of an algorithm for traversing the storage to identi
         3. GOTO 2
 
 Traversal should return enough information for a caller to easily identify the key _within_ a shard that should be used to place their value.
+
+## Sharding
+
+When putting a value it is sometimes necessary to create one or more shards in order to accomodate a new value within the storage.
+
+The storage must first be [traversed](#shard-traversal) to identify the target shard where the value should be placed, as well as the key within the shard that should be used. This will be referred to as the "sharded insertion key" from here on.
+
+The traversal process ensures there is no further traversal possible for the provided key.
+
+If any existing key is equal to the sharded insertion key, no sharding needs to happen - an existing value just needs to be replaced.
+
+Otherwise, the first UTF-8 character of each existing key should be compared to the first UTF-8 character of the sharded insertion key. If no match exists, the key and value are simply added to the shard at the correct lexicographical index.
+
+If there is a match, a new shard must be created. The existing key is replaced with the first character of the existing key, and it's value becomes a pointer to the new shard. The remainder of the existing key (and it's value) is moved into the new shard. The process is then repeated in the new shard, using the sharded insertion key, minus the matched first character.
+
+For example, given a pail with 2 keys:
+
+<!--
+car
+train
+-->
+
+```mermaid
+flowchart LR
+  subgraph s0 [root shard]
+    direction RL
+    car
+    train
+  end
+```
+
+Put "bus":
+
+<!--
+bus
+car
+train
+-->
+
+```mermaid
+flowchart LR
+  subgraph s0 [root shard]
+    direction RL
+    bus
+    car
+    train
+  end
+```
+
+Put "truck":
+
+<!--
+bus
+car
+t   -> r -> ain
+            uck
+-->
+
+```mermaid
+flowchart LR
+  subgraph s0 [root shard]
+    direction RL
+    bus
+    car
+    t
+  end
+  subgraph s1 [shard 1]
+    direction RL
+    r
+  end
+  subgraph s2 [shard 2]
+    direction RL
+    ain
+    uck
+  end
+  t-->s1
+  r-->s2
+```
+
+Put "trailer":
+
+<!--
+bus
+car
+t   -> r -> a   -> i -> ler
+                        n
+            uck
+-->
+
+```mermaid
+flowchart LR
+  subgraph s0 [root shard]
+    direction RL
+    bus
+    car
+    t
+  end
+  subgraph s1 [shard 1]
+    direction RL
+    r
+  end
+  subgraph s2 [shard 2]
+    direction RL
+    a
+    uck
+  end
+  subgraph s3 [shard 3]
+    direction RL
+    i
+  end
+  subgraph s4 [shard 4]
+    direction RL
+    ler
+    n
+  end
+  t-->s1
+  r-->s2
+  a-->s3
+  i-->s4
+```
+
+Put "trunk":
+
+<!--
+bus
+car
+t   -> r -> a -> i -> ler
+                      n
+            u -> ck
+                 nk
+-->
+
+```mermaid
+flowchart LR
+  subgraph s0 [root shard]
+    direction RL
+    bus
+    car
+    t
+  end
+  subgraph s1 [shard 1]
+    direction RL
+    r
+  end
+  subgraph s5 [shard 5]
+    direction RL
+    ck
+    nk
+  end
+  subgraph s2 [shard 2]
+    direction RL
+    a
+    u
+  end
+  subgraph s3 [shard 3]
+    direction RL
+    i
+  end
+  subgraph s4 [shard 4]
+    direction RL
+    ler
+    n
+  end
+  t-->s1
+  r-->s2
+  a-->s3
+  i-->s4
+  u-->s5
+```
+
+## Propagating Changes
+
+Any changes made to a shard will result in a change to it's CID. If the shard is not the root shard, the change must be propagated to the root.

From fb69b700c0115f094a502ddc51d88f4c3d26612f Mon Sep 17 00:00:00 2001
From: Alan Shaw <alan.shaw@protocol.ai>
Date: Thu, 4 Apr 2024 22:44:00 +0200
Subject: [PATCH 7/7] fix: ipldsch

---
 pail.md | 10 +++++-----
 1 file changed, 5 insertions(+), 5 deletions(-)

diff --git a/pail.md b/pail.md
index 68f815b..22aec9d 100644
--- a/pail.md
+++ b/pail.md
@@ -26,16 +26,16 @@ IPLD Schema
 # A shard is just a list of entries, and some config
 type Shard struct {
   # Shard compatibility version.
-  version: Int
+  version Int
   # Characters allowed in keys, referring to a known character set.
   # e.g. "ascii" refers to the printable ASCII characters in the code range 32-126.
-  keyChars: String
+  keyChars String
   # Max key size in bytes - default 4096 bytes.
-  maxKeySize: Int
+  maxKeySize Int
   # The key prefix from the root to this shard.
-  prefix: String
+  prefix String
   # The entries in this shard.
-  entries: [ShardEntry]
+  entries [ShardEntry]
 }
 
 # Single key/value entry within a shard