From feae28c48fd2a28fa2a3b9d6e6c18c163b87b73c Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:23:47 +0200 Subject: [PATCH 1/4] nethermind/nethermind:1.27.1 --- Circles.Index/Circles.Index.csproj | 2 +- arm64.Dockerfile | 2 +- x64.Dockerfile | 2 +- x64.debug.Dockerfile | 2 +- x64.debug.spaceneth.Dockerfile | 2 +- 5 files changed, 5 insertions(+), 5 deletions(-) diff --git a/Circles.Index/Circles.Index.csproj b/Circles.Index/Circles.Index.csproj index b0258ec..6ef367f 100644 --- a/Circles.Index/Circles.Index.csproj +++ b/Circles.Index/Circles.Index.csproj @@ -22,7 +22,7 @@ - + diff --git a/arm64.Dockerfile b/arm64.Dockerfile index 0a510cf..5e1b364 100644 --- a/arm64.Dockerfile +++ b/arm64.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Release -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins diff --git a/x64.Dockerfile b/x64.Dockerfile index 57c8ad0..4c204ea 100644 --- a/x64.Dockerfile +++ b/x64.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Release -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins diff --git a/x64.debug.Dockerfile b/x64.debug.Dockerfile index 1d8b6a7..2f671fb 100644 --- a/x64.debug.Dockerfile +++ b/x64.debug.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Debug -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins diff --git a/x64.debug.spaceneth.Dockerfile b/x64.debug.spaceneth.Dockerfile index cd2c480..7d076ab 100644 --- a/x64.debug.spaceneth.Dockerfile +++ b/x64.debug.spaceneth.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Debug -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins From eb6d05f20dd10927019fd38e26888026f7f2e4f9 Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:23:56 +0200 Subject: [PATCH 2/4] Only initially check for gaps --- Circles.Index/StateMachine.cs | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/Circles.Index/StateMachine.cs b/Circles.Index/StateMachine.cs index d196a2c..9ffa567 100644 --- a/Circles.Index/StateMachine.cs +++ b/Circles.Index/StateMachine.cs @@ -37,7 +37,18 @@ private record LeaveState : IEvent; private State CurrentState { get; set; } = State.New; - private long LastIndexHeight => context.Database.FirstGap() ?? context.Database.LatestBlock() ?? 0; + private bool _checkedForGaps; + + private long LastIndexHeight + { + get + { + // Only initially check for gaps + var value = (!_checkedForGaps ? context.Database.FirstGap() : null) ?? context.Database.LatestBlock() ?? 0; + _checkedForGaps = true; + return value; + } + } public async Task HandleEvent(IEvent e) { From d868841c83f32640f969192d2400476931bdc094 Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Sun, 4 Aug 2024 18:24:20 +0200 Subject: [PATCH 3/4] 1.4.1 --- Circles.Index/Circles.Index.csproj | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/Circles.Index/Circles.Index.csproj b/Circles.Index/Circles.Index.csproj index 6ef367f..c4ccaa6 100644 --- a/Circles.Index/Circles.Index.csproj +++ b/Circles.Index/Circles.Index.csproj @@ -8,8 +8,8 @@ Daniel Janz (Gnosis Service GmbH) Gnosis Service GmbH Circles - 1.4.0 - 1.4.0 + 1.4.1 + 1.4.1 From 4e587df1ada9e884fcfa673dd6c8ff94b23f0387 Mon Sep 17 00:00:00 2001 From: daniel <4954577+jaensen@users.noreply.github.com> Date: Sun, 4 Aug 2024 20:29:27 +0200 Subject: [PATCH 4/4] Add exponential backoff and infinite restart attempts in case of state machine errors. --- Circles.Index/BlockIndexer.cs | 2 +- Circles.Index/StateMachine.cs | 58 +++++++++++++++++------------------ 2 files changed, 30 insertions(+), 30 deletions(-) diff --git a/Circles.Index/BlockIndexer.cs b/Circles.Index/BlockIndexer.cs index 96e6f09..7e844fa 100644 --- a/Circles.Index/BlockIndexer.cs +++ b/Circles.Index/BlockIndexer.cs @@ -91,7 +91,7 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) receiptsSourceBlock.LinkTo(parserBlock, new DataflowLinkOptions { PropagateCompletion = true }); ActionBlock<(BlockWithReceipts, IEnumerable)> sinkBlock = new(Sink, - CreateOptions(cancellationToken, 50000, 1)); + CreateOptions(cancellationToken, 64 * 1024, 1)); parserBlock.LinkTo(sinkBlock, new DataflowLinkOptions { PropagateCompletion = true }); return (sourceBlock, sinkBlock); diff --git a/Circles.Index/StateMachine.cs b/Circles.Index/StateMachine.cs index 9ffa567..35f1c9c 100644 --- a/Circles.Index/StateMachine.cs +++ b/Circles.Index/StateMachine.cs @@ -37,19 +37,6 @@ private record LeaveState : IEvent; private State CurrentState { get; set; } = State.New; - private bool _checkedForGaps; - - private long LastIndexHeight - { - get - { - // Only initially check for gaps - var value = (!_checkedForGaps ? context.Database.FirstGap() : null) ?? context.Database.LatestBlock() ?? 0; - _checkedForGaps = true; - return value; - } - } - public async Task HandleEvent(IEvent e) { try @@ -64,8 +51,11 @@ public async Task HandleEvent(IEvent e) { case EnterState: { - // Initially delete all events for which no "System_Block" exists - await TransitionTo(State.Reorg, LastIndexHeight); + context.Logger.Info("Initializing: Finding the last persisted block..."); + var lastPersistedBlock = context.Database.FirstGap() ?? context.Database.LatestBlock() ?? 0; + + context.Logger.Info($"Initializing: Last persisted block is {lastPersistedBlock}. Deleting all events from this block onwards..."); + await TransitionTo(State.Reorg, lastPersistedBlock); return; } } @@ -90,8 +80,8 @@ public async Task HandleEvent(IEvent e) switch (e) { case NewHead newHead: - context.Logger.Info($"New head received: {newHead.Head}"); - if (newHead.Head <= LastIndexHeight) + context.Logger.Debug($"New head received: {newHead.Head}"); + if (newHead.Head <= context.Database.LatestBlock()) { await TransitionTo(State.Reorg, newHead.Head); return; @@ -108,8 +98,8 @@ public async Task HandleEvent(IEvent e) { case EnterState enterSyncing: var importedBlockRange = await Sync(enterSyncing.Arg); - context.Logger.Info($"Imported blocks from {importedBlockRange.Min} " + - $"to {importedBlockRange.Max}"); + context.Logger.Debug($"Imported blocks from {importedBlockRange.Min} " + + $"to {importedBlockRange.Max}"); Errors.Clear(); await TransitionTo(State.NotifySubscribers, importedBlockRange); @@ -146,15 +136,25 @@ public async Task HandleEvent(IEvent e) switch (e) { case EnterState: - if (Errors.Count >= 3) + // Exponential backoff based on the number of errors + var delay = Errors.Count * Errors.Count * 1000; + + // If the delay is larger than 60 sec, clear the oldest errors + if (delay > 60000) { - // If we have tried 3 times, give up - await TransitionTo(State.End); - return; + Errors.RemoveAt(0); } - // Otherwise, wait for a bit before retrying - await Task.Delay(Errors.Count * Errors.Count * 1000, cancellationToken); + // Add some jitter to the delay + var jitter = new Random((int)DateTime.Now.TimeOfDay.TotalSeconds).Next(0, 1000); + delay += jitter; + + // Wait 'delay' ms + context.Logger.Info($"Waiting {delay} ms before retrying after an error..."); + await Task.Delay(delay, cancellationToken); + + // Retry + context.Logger.Info("Transitioning to 'Initial' state after an error..."); await TransitionTo(State.Initial); return; case LeaveState: @@ -167,7 +167,7 @@ public async Task HandleEvent(IEvent e) return; } - context.Logger.Debug($"Unhandled event {e} in state {CurrentState}"); + context.Logger.Trace($"Unhandled event {e} in state {CurrentState}"); } catch (Exception ex) { @@ -180,7 +180,7 @@ public async Task HandleEvent(IEvent e) private async Task TransitionTo(State newState, TArgument? argument) { - context.Logger.Info($"Transitioning from {CurrentState} to {newState}"); + context.Logger.Debug($"Transitioning from {CurrentState} to {newState}"); if (newState is not State.Error) { await HandleEvent(new LeaveState()); @@ -198,7 +198,7 @@ public async Task TransitionTo(State newState) private async IAsyncEnumerable GetBlocksToSync(long toBlock) { - long lastIndexHeight = LastIndexHeight; + long lastIndexHeight = context.Database.LatestBlock() ?? 0; if (lastIndexHeight == toBlock) { context.Logger.Info("No blocks to sync."); @@ -206,7 +206,7 @@ private async IAsyncEnumerable GetBlocksToSync(long toBlock) } var nextBlock = lastIndexHeight + 1; - context.Logger.Info($"Enumerating blocks to sync from {nextBlock} (LastIndexHeight + 1) to {toBlock}"); + context.Logger.Debug($"Enumerating blocks to sync from {nextBlock} (LastIndexHeight + 1) to {toBlock}"); for (long i = nextBlock; i <= toBlock; i++) {