Skip to content

Add Triekey to reduce allocations #8007

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Closed
benaadams wants to merge 10 commits into from
Closed

Add Triekey to reduce allocations #8007

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
10 commits
Select commit Hold shift + click to select a range
535bd0d
TrieNodeKey
benaadams Jan 4, 2025
c0a1d4e
TrieKey
benaadams Jan 4, 2025
e3742a8
Change to struct
benaadams Jan 4, 2025
336bffd
Add comments
benaadams Jan 4, 2025
355eda8
Add tests
benaadams Jan 4, 2025
75e003b
Fix tests
benaadams Jan 5, 2025
624f334
Formatting
benaadams Jan 5, 2025
9a3c3fe
Optimize CommonPrefixLength
benaadams Jan 5, 2025
e1231a3
Correct memory size
benaadams Jan 5, 2025
f4d1b7d
Fix memory tests
benaadams Jan 5, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion src/Nethermind/Nethermind.Synchronization.Test/Trie/HealingTreeTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -81,7 +81,7 @@ private void recovery_works<T>(
ITrieStore trieStore = Substitute.For<ITrieStore>();
trieStore.FindCachedOrUnknown(null, TreePath.Empty, _key).Returns(
k => throw new MissingTrieNodeException("", new TrieNodeException("", _key), path, 1),
k => new TrieNode(NodeType.Leaf) { Key = path });
k => new TrieNode(NodeType.Leaf, key: path));
trieStore.GetTrieStore(Arg.Any<Hash256?>())
.Returns((callInfo) => new ScopedTrieStore(trieStore, (Hash256?)callInfo[0]));
TestMemDb db = new();
Expand Down
23 changes: 9 additions & 14 deletions src/Nethermind/Nethermind.Trie.Test/Pruning/TreeStoreTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -198,7 +198,7 @@ public void FindCachedOrUnknown_CorrectlyCalculatedMemoryUsedByDirtyCache()
long startSize = trieStore.MemoryUsedByDirtyCache;
trieStore.FindCachedOrUnknown(null, TreePath.Empty, TestItem.KeccakA);
TrieNode trieNode = new(NodeType.Leaf, Keccak.Zero);
long oneKeccakSize = trieNode.GetMemorySize(false) + ExpectedPerNodeKeyMemorySize - MemorySizes.SmallObjectOverhead;
long oneKeccakSize = trieNode.GetMemorySize(false) + ExpectedPerNodeKeyMemorySize - MemorySizes.SmallObjectOverhead - MemorySizes.RefSize - MemorySizes.RefSize;
Assert.That(trieStore.MemoryUsedByDirtyCache, Is.EqualTo(startSize + oneKeccakSize));
trieStore.FindCachedOrUnknown(null, TreePath.Empty, TestItem.KeccakB);
Assert.That(trieStore.MemoryUsedByDirtyCache, Is.EqualTo(2 * oneKeccakSize + startSize));
Expand Down Expand Up @@ -253,7 +253,7 @@ public void Memory_with_concurrent_commits_is_correct()
tree.Commit();
}

fullTrieStore.MemoryUsedByDirtyCache.Should().Be(_scheme == INodeStorage.KeyScheme.Hash ? 540560L : 610708L);
fullTrieStore.MemoryUsedByDirtyCache.Should().Be(_scheme == INodeStorage.KeyScheme.Hash ? 556672L : 626820L);
fullTrieStore.CommittedNodesCount.Should().Be(1349);
}

Expand Down Expand Up @@ -617,10 +617,9 @@ public void Will_store_storage_on_snapshot()
TreePath emptyPath = TreePath.Empty;
storage1.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode a = new(NodeType.Leaf);
TrieNode a = new(NodeType.Leaf, key: Nibbles.BytesToNibbleBytes(TestItem.KeccakA.BytesToArray()));
Account account = new(1, 1, storage1.Keccak, Keccak.OfAnEmptyString);
a.Value = _accountDecoder.Encode(account).Bytes;
a.Key = Nibbles.BytesToNibbleBytes(TestItem.KeccakA.BytesToArray());
a.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

MemDb memDb = new();
Expand Down Expand Up @@ -667,10 +666,9 @@ public void Will_drop_transient_storage()
TreePath emptyPath = TreePath.Empty;
storage1.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode a = new(NodeType.Leaf);
TrieNode a = new(NodeType.Leaf, key: Bytes.FromHexString("abc"));
Account account = new(1, 1, storage1.Keccak, Keccak.OfAnEmptyString);
a.Value = _accountDecoder.Encode(account).Bytes;
a.Key = Bytes.FromHexString("abc");
a.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode b = new(NodeType.Leaf, new byte[1]);
Expand Down Expand Up @@ -729,19 +727,17 @@ public void Will_combine_same_storage()
TreePath emptyPath = TreePath.Empty;
storage1.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode a = new(NodeType.Leaf);
TrieNode a = new(NodeType.Leaf, key: storage1Nib[1..]);
Account account = new(1, 1, storage1.Keccak, Keccak.OfAnEmptyString);
a.Value = _accountDecoder.Encode(account).Bytes;
a.Key = storage1Nib[1..];
a.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode storage2 = new(NodeType.Leaf, new byte[32]);
storage2.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode b = new(NodeType.Leaf);
TrieNode b = new(NodeType.Leaf, key: storage2Nib[1..]);
Account accountB = new(2, 1, storage2.Keccak, Keccak.OfAnEmptyString);
b.Value = _accountDecoder.Encode(accountB).Bytes;
b.Key = storage2Nib[1..];
b.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

TrieNode branch = new(NodeType.Branch);
Expand Down Expand Up @@ -861,10 +857,9 @@ void CheckChildren()
[TestCase(false)]
public void ReadOnly_store_returns_copies(bool pruning)
{
TrieNode node = new(NodeType.Leaf);
TrieNode node = new(NodeType.Leaf, key: Nibbles.BytesToNibbleBytes(TestItem.KeccakA.BytesToArray()));
Account account = new(1, 1, TestItem.KeccakA, Keccak.OfAnEmptyString);
node.Value = _accountDecoder.Encode(account).Bytes;
node.Key = Nibbles.BytesToNibbleBytes(TestItem.KeccakA.BytesToArray());
TreePath emptyPath = TreePath.Empty;
node.ResolveKey(NullTrieNodeResolver.Instance, ref emptyPath, true);

Expand All @@ -890,7 +885,7 @@ public void ReadOnly_store_returns_copies(bool pruning)
var readOnlyRlp = readOnlyNode.FullRlp;
readOnlyRlp.Should().BeEquivalentTo(origRlp);

readOnlyNode.Key?.ToString().Should().Be(originalNode.Key?.ToString());
readOnlyNode.Key.ToString().Should().Be(originalNode.Key.ToString());
}

private long ExpectedPerNodeKeyMemorySize => _scheme == INodeStorage.KeyScheme.Hash ? 0 : TrieStoreDirtyNodesCache.Key.MemoryUsage;
Expand Down Expand Up @@ -1017,7 +1012,7 @@ public async Task Will_Not_RemovePastKeys_OnSnapshot_DuringFullPruning()

memDb.Count.Should().Be(61);
fullTrieStore.Prune();
fullTrieStore.MemoryUsedByDirtyCache.Should().Be(_scheme == INodeStorage.KeyScheme.Hash ? 552 : 708);
fullTrieStore.MemoryUsedByDirtyCache.Should().Be(_scheme == INodeStorage.KeyScheme.Hash ? 600 : 756);
}

[Test]
Expand Down
11 changes: 6 additions & 5 deletions src/Nethermind/Nethermind.Trie.Test/TreePathTests.cs
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
using Nethermind.Core.Crypto;
using Nethermind.Core.Extensions;
using NUnit.Framework;
using System;

namespace Nethermind.Trie.Test;

Expand Down Expand Up @@ -45,7 +46,7 @@ public void TestIndexRead()
[Test]
public void TestAppendArray()
{
byte[] nibbles = new byte[64];
Span<byte> nibbles = new byte[64];
for (int i = 0; i < 64; i++)
{
nibbles[i] = (byte)(i % 16);
Expand All @@ -66,7 +67,7 @@ public void TestAppendArray()
[TestCase(41)]
public void TestAppendArrayDivided(int partition)
{
byte[] nibbles = new byte[64];
Span<byte> nibbles = new byte[64];
for (int i = 0; i < 64; i++)
{
nibbles[i] = (byte)(i % 16);
Expand Down Expand Up @@ -156,12 +157,12 @@ public void TestScopedAppend()
{
TreePath path = TreePath.Empty;

using (path.ScopedAppend(new byte[] { 1, 2, 3, 4 }))
using (path.ScopedAppend(new Span<byte>([1, 2, 3, 4])))
{
path.Length.Should().Be(4);
path.Path.ToString().Should().Be("0x1234000000000000000000000000000000000000000000000000000000000000");

using (path.ScopedAppend(new byte[] { 5, 6, 7 }))
using (path.ScopedAppend(new Span<byte>([5, 6, 7])))
{
path.Length.Should().Be(7);
path.Path.ToString().Should().Be("0x1234567000000000000000000000000000000000000000000000000000000000");
Expand All @@ -178,7 +179,7 @@ private static TreePath CreateFullTreePath()
TreePath path = new TreePath();
for (int i = 0; i < 64; i++)
{
path = path.Append((byte)(i % 16));
path = path.Append((int)(i % 16));
}

return path;
Expand Down
Loading