From 13be2c7aaf2b1f7cda3cd760265dd4f747bcbf4d Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Mon, 2 Sep 2024 16:20:59 +0200 Subject: [PATCH 1/5] add new contract addresses --- docker-compose.chiado.yml | 6 +++--- docker-compose.gnosis.yml | 6 +++--- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/docker-compose.chiado.yml b/docker-compose.chiado.yml index d30f0ea..b1be5c4 100644 --- a/docker-compose.chiado.yml +++ b/docker-compose.chiado.yml @@ -35,9 +35,9 @@ services: - .env environment: - V1_HUB_ADDRESS=0xdbf22d4e8962db3b2f1d9ff55be728a887e47710 - - V2_HUB_ADDRESS=0xEddc960D3c78692BF38577054cb0a35114AE35e0 - - V2_NAME_REGISTRY_ADDRESS=0x5525cbF9ad01a4E805ed1b40723D6377b336eCcf - - V2_STANDARD_TREASURY_ADDRESS=0x66A024F2055fa84b40f27c2f3Eb68A848276A641 + - V2_HUB_ADDRESS=0xb80feeDfEce647dDc709777D5094fACD157BA001 + - V2_NAME_REGISTRY_ADDRESS=0x24b3fDCdD9fef844fB3094ef43c0A6Ac23a6dF9E + - V2_STANDARD_TREASURY_ADDRESS=0xC06ADED7950429FdF2023e139B076f6BaFf9Fe1C - POSTGRES_CONNECTION_STRING=Server=postgres-chiado;Port=5432;Database=postgres;User Id=${POSTGRES_USER};Password=${POSTGRES_PASSWORD}; postgres-chiado: diff --git a/docker-compose.gnosis.yml b/docker-compose.gnosis.yml index d150e12..b564f74 100644 --- a/docker-compose.gnosis.yml +++ b/docker-compose.gnosis.yml @@ -37,9 +37,9 @@ services: - .env environment: - V1_HUB_ADDRESS=0x29b9a7fBb8995b2423a71cC17cf9810798F6C543 - - V2_HUB_ADDRESS=0x7bC1F123089Bc1f384b6379d0587968d1CD5830a - - V2_NAME_REGISTRY_ADDRESS=0xb95eF3f3E693531d9588815bcA954dC8dce30937 - - V2_STANDARD_TREASURY_ADDRESS=0x2434151eB40Af648AbcF73a6C9F1711FfF0F498B + - V2_HUB_ADDRESS=0xa5c7ADAE2fd3844f12D52266Cb7926f8649869Da + - V2_NAME_REGISTRY_ADDRESS=0x738fFee24770d0DE1f912adf2B48b0194780E9AD + - V2_STANDARD_TREASURY_ADDRESS=0xbb76CF35ec106c5c7a447246257dcfCB7244cA04 - POSTGRES_CONNECTION_STRING=Server=postgres-gnosis;Port=5432;Database=postgres;User Id=${POSTGRES_USER};Password=${POSTGRES_PASSWORD}; - START_BLOCK=12000000 From ce5d81b0c7ca55bf799a2743754828bb7fcee6b4 Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Mon, 2 Sep 2024 21:15:41 +0200 Subject: [PATCH 2/5] Added a IEnumerable ParseTransaction(Block block, int transactionIndex, Transaction transaction) parse transaction method --- Circles.Index.CirclesV1/LogParser.cs | 7 +- .../LogParser.cs | 7 +- .../LogParser.cs | 7 +- Circles.Index.CirclesV2/LogParser.cs | 81 +++++++++++++------ Circles.Index.Common/ILogParser.cs | 5 +- Circles.Index/BlockIndexer.cs | 40 ++++++++- 6 files changed, 118 insertions(+), 29 deletions(-) diff --git a/Circles.Index.CirclesV1/LogParser.cs b/Circles.Index.CirclesV1/LogParser.cs index 9f5c01f..c33d050 100644 --- a/Circles.Index.CirclesV1/LogParser.cs +++ b/Circles.Index.CirclesV1/LogParser.cs @@ -18,7 +18,12 @@ public class LogParser(Address v1HubAddress) : ILogParser private readonly Hash256 _hubTransferTopic = new(DatabaseSchema.HubTransfer.Topic); private readonly Hash256 _trustTopic = new(DatabaseSchema.Trust.Topic); - public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex) + public IEnumerable ParseTransaction(Block block, int transactionIndex, Transaction transaction) + { + return Enumerable.Empty(); + } + + public IEnumerable ParseLog(Block block, Transaction transaction, TxReceipt receipt, LogEntry log, int logIndex) { List events = new(); if (log.Topics.Length == 0) diff --git a/Circles.Index.CirclesV2.NameRegistry/LogParser.cs b/Circles.Index.CirclesV2.NameRegistry/LogParser.cs index 9cd68a0..07006a8 100644 --- a/Circles.Index.CirclesV2.NameRegistry/LogParser.cs +++ b/Circles.Index.CirclesV2.NameRegistry/LogParser.cs @@ -11,7 +11,12 @@ public class LogParser(Address nameRegistryAddress) : ILogParser private readonly Hash256 _updateMetadataDigestTopic = new(DatabaseSchema.UpdateMetadataDigest.Topic); private readonly Hash256 _cidV0Topic = new(DatabaseSchema.CidV0.Topic); - public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex) + public IEnumerable ParseTransaction(Block block, int transactionIndex, Transaction transaction) + { + return Enumerable.Empty(); + } + + public IEnumerable ParseLog(Block block, Transaction transaction, TxReceipt receipt, LogEntry log, int logIndex) { if (log.Topics.Length == 0) { diff --git a/Circles.Index.CirclesV2.StandardTreasury/LogParser.cs b/Circles.Index.CirclesV2.StandardTreasury/LogParser.cs index d42eca3..a0a30bb 100644 --- a/Circles.Index.CirclesV2.StandardTreasury/LogParser.cs +++ b/Circles.Index.CirclesV2.StandardTreasury/LogParser.cs @@ -18,7 +18,12 @@ public class LogParser(Address standardTreasuryAddress) : ILogParser private readonly Hash256 _groupRedeemCollateralReturnTopic = new(DatabaseSchema.GroupRedeemCollateralReturn.Topic); private readonly Hash256 _groupRedeemCollateralBurnTopic = new(DatabaseSchema.GroupRedeemCollateralBurn.Topic); - public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex) + public IEnumerable ParseTransaction(Block block, int transactionIndex, Transaction transaction) + { + return Enumerable.Empty(); + } + + public IEnumerable ParseLog(Block block, Transaction transaction, TxReceipt receipt, LogEntry log, int logIndex) { if (log.Topics.Length == 0) { diff --git a/Circles.Index.CirclesV2/LogParser.cs b/Circles.Index.CirclesV2/LogParser.cs index 063ec03..07d39c0 100644 --- a/Circles.Index.CirclesV2/LogParser.cs +++ b/Circles.Index.CirclesV2/LogParser.cs @@ -12,8 +12,10 @@ namespace Circles.Index.CirclesV2; public class LogParser(Address v2HubAddress) : ILogParser { private readonly Hash256 _stoppedTopic = new(DatabaseSchema.Stopped.Topic); + private readonly Hash256 _trustTopic = new(DatabaseSchema.Trust.Topic); - private readonly Hash256 _inviteHumanTopic = new(DatabaseSchema.InviteHuman.Topic); + + // private readonly Hash256 _inviteHumanTopic = new(DatabaseSchema.InviteHuman.Topic); private readonly Hash256 _personalMintTopic = new(DatabaseSchema.PersonalMint.Topic); private readonly Hash256 _registerHumanTopic = new(DatabaseSchema.RegisterHuman.Topic); private readonly Hash256 _registerGroupTopic = new(DatabaseSchema.RegisterGroup.Topic); @@ -32,7 +34,60 @@ public class LogParser(Address v2HubAddress) : ILogParser public static readonly ConcurrentDictionary Erc20WrapperAddresses = new(); - public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex) + private readonly byte[] _registerHumanFunctionSignature = + Keccak.Compute("registerHuman(address,bytes32)").Bytes[..4].ToArray(); + + public IEnumerable ParseTransaction(Block block, int transactionIndex, Transaction transaction) + { + if (transaction.To != v2HubAddress) + { + yield break; + } + + if (transaction.Data == null || transaction.Data.Value.Length < 68) + { + // 68 is the size of a complete registerHuman call with arguments + yield break; + } + + Console.WriteLine($"Parsing tx {transaction.Hash}. Data length: {transaction.Data.Value.Length}"); + + // Parse the whole call data for a `registerHuman` call to get the inviter address and + // create a InviteHuman event from it. + // TODO: This is only for v0.3.6 and will be replace with an additional parameter on the InviteHuman event in the next version + // Because we cannot know how the contract was called (e.g. via a safe), we simply search for the function signature. + int callLength = 4; + for (int i = 0; i < transaction.Data.Value.Length - callLength; i++) + { + if (!transaction.Data.Value[i..(i + 4)].Span.SequenceEqual(_registerHumanFunctionSignature)) + { + continue; + } + + // Extract the bytes for the following call signature: + // function registerHuman(address _inviter, bytes32 _metadataDigest) external + var inviterAddressOffset = i + callLength; + var inviterAddressWithoutPaddingOffset = inviterAddressOffset + 12; + var inviterAddress = new Address(transaction.Data.Value[inviterAddressWithoutPaddingOffset.. + (inviterAddressWithoutPaddingOffset + 20)].ToArray()); + + // TODO: Usually the sender is the invitee, but if the call comes e.g. from a safe, this is no necessarily true. + var invitee = transaction.SenderAddress!; + if (inviterAddress == Address.Zero) + { + break; + } + + yield return new InviteHuman(block.Number, (long)block.Timestamp, transactionIndex, -1, + transaction.Hash!.ToString(), + inviterAddress.ToString(), invitee.ToString()); + + break; + } + } + + public IEnumerable ParseLog(Block block, Transaction transaction, TxReceipt receipt, LogEntry log, + int logIndex) { if (log.Topics.Length == 0) { @@ -53,11 +108,6 @@ public IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntr yield return CrcV2Trust(block, receipt, log, logIndex); } - if (topic == _inviteHumanTopic) - { - yield return CrcV2InviteHuman(block, receipt, log, logIndex); - } - if (topic == _personalMintTopic) { yield return CrcV2PersonalMint(block, receipt, log, logIndex); @@ -317,23 +367,6 @@ private PersonalMint CrcV2PersonalMint(Block block, TxReceipt receipt, LogEntry endPeriod); } - private InviteHuman CrcV2InviteHuman(Block block, TxReceipt receipt, LogEntry log, int logIndex) - { - string inviterAddress = - "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); - string inviteeAddress = - "0x" + log.Topics[2].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); - - return new InviteHuman( - block.Number, - (long)block.Timestamp, - receipt.Index, - logIndex, - receipt.TxHash!.ToString(), - inviterAddress, - inviteeAddress); - } - private Trust CrcV2Trust(Block block, TxReceipt receipt, LogEntry log, int logIndex) { string userAddress = "0x" + log.Topics[1].ToString().Substring(Consts.AddressEmptyBytesPrefixLength); diff --git a/Circles.Index.Common/ILogParser.cs b/Circles.Index.Common/ILogParser.cs index ea96468..282ef55 100644 --- a/Circles.Index.Common/ILogParser.cs +++ b/Circles.Index.Common/ILogParser.cs @@ -4,5 +4,8 @@ namespace Circles.Index.Common; public interface ILogParser { - IEnumerable ParseLog(Block block, TxReceipt receipt, LogEntry log, int logIndex); + IEnumerable ParseTransaction(Block block, int transactionIndex, Transaction transaction); + + IEnumerable ParseLog(Block block, Transaction transaction, TxReceipt receipt, LogEntry log, + int logIndex); } \ No newline at end of file diff --git a/Circles.Index/BlockIndexer.cs b/Circles.Index/BlockIndexer.cs index bf77e8a..2dd1c64 100644 --- a/Circles.Index/BlockIndexer.cs +++ b/Circles.Index/BlockIndexer.cs @@ -4,6 +4,7 @@ using Nethermind.Blockchain; using Nethermind.Blockchain.Receipts; using Nethermind.Core; +using Nethermind.Core.Crypto; namespace Circles.Index; @@ -96,6 +97,19 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) TransformBlock)> parserBlock = new( blockWithReceipts => { + Dictionary transactionsByHash = new(); + Dictionary transactionIndexByHash = new(); + + for (var i = 0; i < blockWithReceipts.Block.Transactions.Length; i++) + { + var tx = blockWithReceipts.Block.Transactions[i]; + if (tx.Hash != null) + { + transactionsByHash[tx.Hash] = tx; + transactionIndexByHash[tx.Hash] = i; + } + } + List events = []; foreach (var receipt in blockWithReceipts.Receipts) { @@ -105,6 +119,28 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) return (blockWithReceipts, events); } + var transaction = transactionsByHash[receipt.TxHash!]; + var transactionIndex = transactionIndexByHash[receipt.TxHash!]; + try + { + foreach (var parser in context.LogParsers) + { + var parsedEvents = parser.ParseTransaction(blockWithReceipts.Block, + transactionIndex, transaction); + + events.AddRange(parsedEvents); + } + } + catch (Exception e) + { + context.Logger.Error($"Error parsing transaction {transaction}"); + context.Logger.Error($"Block: {blockWithReceipts.Block.Number}"); + context.Logger.Error($"Receipt.TxHash: {receipt.TxHash}"); + context.Logger.Error(e.Message); + context.Logger.Error(e.StackTrace); + throw; + } + for (int i = 0; i < receipt.Logs?.Length; i++) { LogEntry log = receipt.Logs[i]; @@ -112,7 +148,9 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) { try { - var parsedEvents = parser.ParseLog(blockWithReceipts.Block, receipt, log, i); + var parsedEvents = parser.ParseLog(blockWithReceipts.Block, + transaction, receipt, log, i); + events.AddRange(parsedEvents); } catch (Exception e) From 938d5ce7ccbdffbe7d90ec97a13f9ffc4bdc3bcc Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Mon, 2 Sep 2024 21:23:00 +0200 Subject: [PATCH 3/5] refine inviteHuman call parsing from tx --- Circles.Index.CirclesV2/LogParser.cs | 14 +++++++++++--- 1 file changed, 11 insertions(+), 3 deletions(-) diff --git a/Circles.Index.CirclesV2/LogParser.cs b/Circles.Index.CirclesV2/LogParser.cs index 07d39c0..b868acc 100644 --- a/Circles.Index.CirclesV2/LogParser.cs +++ b/Circles.Index.CirclesV2/LogParser.cs @@ -50,8 +50,6 @@ public IEnumerable ParseTransaction(Block block, int transactionInd yield break; } - Console.WriteLine($"Parsing tx {transaction.Hash}. Data length: {transaction.Data.Value.Length}"); - // Parse the whole call data for a `registerHuman` call to get the inviter address and // create a InviteHuman event from it. // TODO: This is only for v0.3.6 and will be replace with an additional parameter on the InviteHuman event in the next version @@ -64,6 +62,12 @@ public IEnumerable ParseTransaction(Block block, int transactionInd continue; } + // The next 12 bytes must be zero padding. + if (!transaction.Data.Value[(i + callLength)..(i + callLength + 12)].Span.SequenceEqual(new byte[12])) + { + continue; + } + // Extract the bytes for the following call signature: // function registerHuman(address _inviter, bytes32 _metadataDigest) external var inviterAddressOffset = i + callLength; @@ -71,8 +75,12 @@ public IEnumerable ParseTransaction(Block block, int transactionInd var inviterAddress = new Address(transaction.Data.Value[inviterAddressWithoutPaddingOffset.. (inviterAddressWithoutPaddingOffset + 20)].ToArray()); - // TODO: Usually the sender is the invitee, but if the call comes e.g. from a safe, this is no necessarily true. + // TODO: Usually the sender is the invitee, but this will fail e.g. in case of relayers var invitee = transaction.SenderAddress!; + + // Console.WriteLine( + // $"Found `InviteHuman` call in tx {transaction.Hash}. Inviter: {inviterAddress}, Invitee: {invitee}"); + if (inviterAddress == Address.Zero) { break; From ae7aca4052a62c8561830be28f4692f61e802e13 Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:16:58 +0200 Subject: [PATCH 4/5] Add 'IsNull' and 'IsNotNull' filter predicate --- Circles.Index.Query/FilterPredicate.cs | 6 ++++-- Circles.Index.Query/FilterType.cs | 8 +++++++- 2 files changed, 11 insertions(+), 3 deletions(-) diff --git a/Circles.Index.Query/FilterPredicate.cs b/Circles.Index.Query/FilterPredicate.cs index 4266a2b..7e44c1c 100644 --- a/Circles.Index.Query/FilterPredicate.cs +++ b/Circles.Index.Query/FilterPredicate.cs @@ -26,6 +26,8 @@ public ParameterizedSql ToSql(IDatabaseUtils database) FilterType.NotIn => ConvertToEnumerable(Value)?.Any() ?? false ? $"{database.QuoteIdentifier(Column)} NOT IN ({FormatArrayParameter(Value, parameterName)})" : "1=0 /* empty 'not in' filter */", + FilterType.IsNotNull => $"{database.QuoteIdentifier(Column)} IS NOT NULL", + FilterType.IsNull => $"{database.QuoteIdentifier(Column)} IS NULL", _ => throw new NotImplementedException() }; @@ -65,7 +67,7 @@ private IEnumerable ConvertToEnumerable(object? value) private IEnumerable CreateParameters(IDatabaseUtils database, string parameterName, object? value) { var values = ConvertToEnumerable(value).ToArray(); - + return values.Select((v, index) => database.CreateParameter($"{parameterName}_{index}", v)).ToList(); } -} +} \ No newline at end of file diff --git a/Circles.Index.Query/FilterType.cs b/Circles.Index.Query/FilterType.cs index 2b4960e..b3e0977 100644 --- a/Circles.Index.Query/FilterType.cs +++ b/Circles.Index.Query/FilterType.cs @@ -32,5 +32,11 @@ public enum FilterType In, [JsonConverter(typeof(JsonStringEnumConverter))] - NotIn + NotIn, + + [JsonConverter(typeof(JsonStringEnumConverter))] + IsNotNull, + + [JsonConverter(typeof(JsonStringEnumConverter))] + IsNull } \ No newline at end of file From 82eaaec7626db9380f4f28da6dcb328414d2d16d Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Mon, 2 Sep 2024 23:17:40 +0200 Subject: [PATCH 5/5] v1.7.0 --- Circles.Index/Circles.Index.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Circles.Index/Circles.Index.csproj b/Circles.Index/Circles.Index.csproj index 35631ba..d66052d 100644 --- a/Circles.Index/Circles.Index.csproj +++ b/Circles.Index/Circles.Index.csproj @@ -8,8 +8,8 @@ Daniel Janz (Gnosis Service GmbH) Gnosis Service GmbH Circles - 1.6.4 - 1.6.4 + 1.7.0 + 1.7.0