Verkle Implementation: Build out Trie Processing

electra.yaml

@@ -0,0 +1,21 @@
+participants:
+  - el_type: ethereumjs
+    el_image: ethereumjs:local
+    cl_type: grandine
+    cl_image: ethpandaops/grandine:feature-electra
+
+  - el_type: ethereumjs
+    el_image: ethereumjs:local
+    cl_type: grandine
+    cl_image: ethpandaops/grandine:feature-electra
+
+network_params:
+  electra_fork_epoch: 1
+  electra_fork_version: "0x50000038"
+    #global_log_level: debug
+additional_services:
+  - dora
+  - tx_spammer
+  - blob_spammer
+
+snooper_enabled: true

packages/verkle/src/node/internalNode.ts

@@ -1,11 +1,12 @@
-import { equalsBytes } from '@ethereumjs/util'
+import { BIGINT_0, bytesToBigInt, equalsBytes } from '@ethereumjs/util'
 
 import { POINT_IDENTITY } from '../util/crypto.js'
 
 import { BaseVerkleNode } from './baseVerkleNode.js'
 import { LeafNode } from './leafNode.js'
 import { NODE_WIDTH, VerkleNodeType } from './types.js'
 
+import type { VerkleCrypto } from '../types.js'
 import type { VerkleNode, VerkleNodeOptions } from './types.js'
 
 export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
@@ -14,9 +15,9 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
   public copyOnWrite: Record<string, Uint8Array>
   public type = VerkleNodeType.Internal
 
-  /* TODO: options.children is not actually used here */
   constructor(options: VerkleNodeOptions[VerkleNodeType.Internal]) {
     super(options)
+    // TODO: Decide whether to fill with null or not - personal opinion - should never use null
     this.children = options.children ?? new Array(NODE_WIDTH).fill(null)
     this.copyOnWrite = options.copyOnWrite ?? {}
   }
@@ -63,19 +64,30 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
     return this.children?.[index] ?? null
   }
 
-  insert(key: Uint8Array, value: Uint8Array, resolver: () => void): void {
+  insert(
+    key: Uint8Array,
+    value: Uint8Array,
+    resolver: () => void,
+    verkleCrypto?: VerkleCrypto
+  ): void {
     const values = new Array<Uint8Array>(NODE_WIDTH)
     values[key[31]] = value
-    this.insertStem(key.slice(0, 31), values, resolver)
+    this.insertStem(key.slice(0, 31), values, resolver, verkleCrypto!)
   }
 
-  insertStem(stem: Uint8Array, values: Uint8Array[], resolver: () => void): void {
+  insertStem(
+    stem: Uint8Array,
+    values: Uint8Array[],
+    resolver: () => void,
+    verkleCrypto: VerkleCrypto
+  ): void {
     // Index of the child pointed by the next byte in the key
     const childIndex = stem[this.depth]
 
     const child = this.children[childIndex]
 
     if (child instanceof LeafNode) {
+      // TODO: Understand the intent of what cowChild is suppoded to do
       this.cowChild(childIndex)
       if (equalsBytes(child.stem, stem)) {
         return child.insertMultiple(stem, values)
@@ -93,19 +105,26 @@ export class InternalNode extends BaseVerkleNode<VerkleNodeType.Internal> {
 
       const nextByteInInsertedKey = stem[this.depth + 1]
       if (nextByteInInsertedKey === nextByteInExistingKey) {
-        return newBranch.insertStem(stem, values, resolver)
+        return newBranch.insertStem(stem, values, resolver, verkleCrypto)
       }
 
       // Next word differs, so this was the last level.
       // Insert it directly into its final slot.
-      const leafNode = LeafNode.create(stem, values)
+      // TODO: Determine if this is how to create the correct commitment
+      let commitment = verkleCrypto.zeroCommitment
+      for (const [idx, value] of values.entries()) {
+        if (bytesToBigInt(value) > BIGINT_0)
+          commitment = verkleCrypto.updateCommitment(commitment, idx, new Uint8Array(32), value)
+      }
+      const leafNode = LeafNode.create(stem, values, this.depth + 1, commitment)
 
+      // TODO - Why is the leaf node set at depth + 2 instead of + 1)?
       leafNode.setDepth(this.depth + 2)
       newBranch.cowChild(nextByteInInsertedKey)
       newBranch.children[nextByteInInsertedKey] = leafNode
     } else if (child instanceof InternalNode) {
       this.cowChild(childIndex)
-      return child.insertStem(stem, values, resolver)
+      return child.insertStem(stem, values, resolver, verkleCrypto)
     } else {
       throw new Error('Invalid node type')
     }

packages/verkle/src/node/leafNode.ts

@@ -1,4 +1,3 @@
-/* eslint-disable @typescript-eslint/no-unused-vars */
 import { BaseVerkleNode } from './baseVerkleNode.js'
 import { NODE_WIDTH, VerkleNodeType } from './types.js'
 
@@ -7,9 +6,9 @@ import type { VerkleNodeOptions } from './types.js'
 
 export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
   public stem: Uint8Array
-  public values: Uint8Array[]
-  public c1: Point
-  public c2: Point
+  public values: Uint8Array[] // Array of 256 possible values represented as 32 byte Uint8Arrays
+  public c1?: Point
+  public c2?: Point
   public type = VerkleNodeType.Leaf
 
   constructor(options: VerkleNodeOptions[VerkleNodeType.Leaf]) {
@@ -21,8 +20,13 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
     this.c2 = options.c2
   }
 
-  static create(stem: Uint8Array, values: Uint8Array[]): LeafNode {
-    throw new Error('Not implemented')
+  static create(
+    stem: Uint8Array,
+    values: Uint8Array[],
+    depth: number,
+    commitment: Uint8Array
+  ): LeafNode {
+    return new LeafNode({ stem, values, depth, commitment })
   }
 
   static fromRawNode(rawNode: Uint8Array[], depth: number): LeafNode {
@@ -73,8 +77,8 @@ export class LeafNode extends BaseVerkleNode<VerkleNodeType.Leaf> {
       new Uint8Array([VerkleNodeType.Leaf]),
       this.stem,
       this.commitment,
-      this.c1.bytes(),
-      this.c2.bytes(),
+      this.c1?.bytes() ?? new Uint8Array(),
+      this.c2?.bytes() ?? new Uint8Array(),
       ...this.values,
     ]
   }

packages/verkle/src/node/types.ts

@@ -37,8 +37,8 @@ interface VerkleInternalNodeOptions extends BaseVerkleNodeOptions {
 interface VerkleLeafNodeOptions extends BaseVerkleNodeOptions {
   stem: Uint8Array
   values: Uint8Array[]
-  c1: Point
-  c2: Point
+  c1?: Point
+  c2?: Point
 }
 
 export interface VerkleNodeOptions {

packages/verkle/src/util/keys.ts

@@ -94,3 +94,7 @@ export const getTreeKeyForStorageSlot = async (
 
   return concatBytes(getStem(verkleCrypto, address, treeIndex), toBytes(subIndex))
 }
+
+export const verifyKeyLength = (key: Uint8Array) => {
+  if (key.length !== 32) throw new Error(`expected key with length 32l; got ${key.length}`)
+}

packages/verkle/src/verkleTree.ts

@@ -13,9 +13,10 @@ import {
   type VerkleTreeOptsWithDefaults,
 } from './types.js'
 import { WalkController, matchingBytesLength } from './util/index.js'
+import { verifyKeyLength } from './util/keys.js'
 
 import type { VerkleNode } from './node/types.js'
-import type { FoundNodeFunction } from './types.js'
+import type { FoundNodeFunction, VerkleCrypto } from './types.js'
 import type { BatchDBOp, DB, PutBatch } from '@ethereumjs/util'
 
 interface Path {
@@ -43,7 +44,7 @@ export class VerkleTree {
   protected _lock = new Lock()
   protected _root: Uint8Array
 
-  protected verkleCrypto: any
+  protected verkleCrypto: VerkleCrypto
   /**
    * Creates a new verkle tree.
    * @param opts Options for instantiating the verkle tree
@@ -153,14 +154,16 @@ export class VerkleTree {
    * @returns A Promise that resolves to `Uint8Array` if a value was found or `null` if no value was found.
    */
   async get(key: Uint8Array, throwIfMissing = false): Promise<Uint8Array | null> {
-    const node = await this.findLeafNode(key, throwIfMissing)
-    if (node !== null) {
-      const keyLastByte = key[key.length - 1]
-
-      // The retrieved leaf node contains an array of 256 possible values.
-      // The index of the value we want is at the key's last byte
-      return node.values?.[keyLastByte] ?? null
-    }
+    verifyKeyLength(key)
+    const node = await this._db.get(key)
+    if (node !== undefined && node instanceof LeafNode)
+      if (node instanceof LeafNode) {
+        const keyLastByte = key[key.length - 1]
+
+        // The retrieved leaf node contains an array of 256 possible values.
+        // The index of the value we want is at the key's last byte
+        return node.values?.[keyLastByte] ?? null
+      }
 
     return null
   }
@@ -173,14 +176,22 @@ export class VerkleTree {
    * @returns A Promise that resolves once value is stored.
    */
   async put(key: Uint8Array, value: Uint8Array): Promise<void> {
+    verifyKeyLength(key)
     await this._db.put(key, value)
 
     // Find or create the leaf node
-    const leafNode = await this.findLeafNode(key, false)
-    if (leafNode === null) {
+    let leafNode = await this.findLeafNode(key, false)
+    if (!(leafNode instanceof LeafNode)) {
       // If leafNode is missing, create it
-      // leafNode = LeafNode.create()
-      throw new Error('Not implemented')
+      const values = new Array(256) as Uint8Array[] // Create new empty array of 256 values
+      const suffix = key[31]
+      values[suffix] = value // Set value at key suffix
+
+      // Generate a commitment for the new leaf node, using the zero commitment as a base
+      const commitment = this.verkleCrypto.zeroCommitment
+      // TODO: Confirm the old/new scalar values we're passing in here are correct
+      this.verkleCrypto.updateCommitment(commitment, suffix, new Uint8Array(32), value)
+      leafNode = LeafNode.create(key.slice(0, 31), values, leafNode.length, commitment)
     }
 
     // Walk up the tree and update internal nodes
@@ -280,13 +291,14 @@ export class VerkleTree {
    * @param key - the search key
    * @param throwIfMissing - if true, throws if any nodes are missing. Used for verifying proofs. (default: false)
    */
-  async findLeafNode(key: Uint8Array, throwIfMissing = false): Promise<LeafNode | null> {
-    const { node } = await this.findPath(key, throwIfMissing)
+  async findLeafNode(key: Uint8Array, throwIfMissing = false): Promise<LeafNode | VerkleNode[]> {
+    const { node, stack } = await this.findPath(key, throwIfMissing)
     if (!(node instanceof LeafNode)) {
       if (throwIfMissing) {
         throw new Error('leaf node not found')
       }
-      return null
+      // return depth of remaining nodes
+      return stack
     }
     return node
   }

packages/verkle/test/leafNode.spec.ts

@@ -1,10 +1,11 @@
-import { equalsBytes, randomBytes } from '@ethereumjs/util'
+import { equalsBytes, hexToBytes, randomBytes } from '@ethereumjs/util'
 import { assert, describe, it } from 'vitest'
 
 import { VerkleNodeType } from '../src/node/index.js'
 import { LeafNode } from '../src/node/leafNode.js'
 
 import type { Point } from '../src/types.js'
+import type { PrefixedHexString } from '@ethereumjs/util'
 
 describe('verkle node - leaf', () => {
   it('constructor should create an leaf node', async () => {
@@ -38,5 +39,18 @@ describe('verkle node - leaf', () => {
     assert.equal(node.depth, depth, 'depth should be set')
   })
 
-  it.todo('create method should create an leaf node')
+  it('create method should create an leaf node', () => {
+    const presentKeys = ['0x318dea512b6f3237a2d4763cf49bf26de3b617fb0cabe38a97807a5549df4d01'].map(
+      (key) => hexToBytes(key as PrefixedHexString)
+    )
+
+    // Corresponding values for the present keys
+    const values = ['0x320122e8584be00d000000000000000000000000000000000000000000000000'].map(
+      (key) => hexToBytes(key as PrefixedHexString)
+    )
+    const stem = presentKeys[0].slice(0, 31)
+    const nodeData = values
+    const node = LeafNode.create(stem, nodeData, 0, new Uint8Array(32))
+    assert.ok(node instanceof LeafNode)
+  })
 })

packages/verkle/test/verkle.spec.ts

@@ -1,4 +1,5 @@
-import { equalsBytes, hexToBytes } from '@ethereumjs/util'
+import { MapDB, equalsBytes, hexToBytes } from '@ethereumjs/util'
+import { loadVerkleCrypto } from 'verkle-cryptography-wasm'
 import { assert, describe, it } from 'vitest'
 
 import { VerkleTree } from '../src/verkleTree.js'
@@ -46,8 +47,13 @@ const absentKeys = [
 ].map((key) => hexToBytes(key as PrefixedHexString))
 
 describe('Verkle tree', () => {
-  it.todo('should insert and retrieve values', async () => {
-    const tree = new VerkleTree()
+  it.only('should insert and retrieve values', async () => {
+    const verkleCrypto = await loadVerkleCrypto()
+    const tree = await VerkleTree.create({
+      verkleCrypto,
+    })
+    const db = new MapDB<Uint8Array, Uint8Array>()
+    tree.database(db)
     for (let i = 0; i < presentKeys.length; i++) {
       await tree.put(presentKeys[i], values[i])
     }