diff --git a/Circles.Index/BlockIndexer.cs b/Circles.Index/BlockIndexer.cs index 96e6f09..7e844fa 100644 --- a/Circles.Index/BlockIndexer.cs +++ b/Circles.Index/BlockIndexer.cs @@ -91,7 +91,7 @@ private async Task Sink((BlockWithReceipts, IEnumerable) data) receiptsSourceBlock.LinkTo(parserBlock, new DataflowLinkOptions { PropagateCompletion = true }); ActionBlock<(BlockWithReceipts, IEnumerable)> sinkBlock = new(Sink, - CreateOptions(cancellationToken, 50000, 1)); + CreateOptions(cancellationToken, 64 * 1024, 1)); parserBlock.LinkTo(sinkBlock, new DataflowLinkOptions { PropagateCompletion = true }); return (sourceBlock, sinkBlock); diff --git a/Circles.Index/Circles.Index.csproj b/Circles.Index/Circles.Index.csproj index b0258ec..c4ccaa6 100644 --- a/Circles.Index/Circles.Index.csproj +++ b/Circles.Index/Circles.Index.csproj @@ -8,8 +8,8 @@ Daniel Janz (Gnosis Service GmbH) Gnosis Service GmbH Circles - 1.4.0 - 1.4.0 + 1.4.1 + 1.4.1 @@ -22,7 +22,7 @@ - + diff --git a/Circles.Index/StateMachine.cs b/Circles.Index/StateMachine.cs index d196a2c..35f1c9c 100644 --- a/Circles.Index/StateMachine.cs +++ b/Circles.Index/StateMachine.cs @@ -37,8 +37,6 @@ private record LeaveState : IEvent; private State CurrentState { get; set; } = State.New; - private long LastIndexHeight => context.Database.FirstGap() ?? context.Database.LatestBlock() ?? 0; - public async Task HandleEvent(IEvent e) { try @@ -53,8 +51,11 @@ public async Task HandleEvent(IEvent e) { case EnterState: { - // Initially delete all events for which no "System_Block" exists - await TransitionTo(State.Reorg, LastIndexHeight); + context.Logger.Info("Initializing: Finding the last persisted block..."); + var lastPersistedBlock = context.Database.FirstGap() ?? context.Database.LatestBlock() ?? 0; + + context.Logger.Info($"Initializing: Last persisted block is {lastPersistedBlock}. Deleting all events from this block onwards..."); + await TransitionTo(State.Reorg, lastPersistedBlock); return; } } @@ -79,8 +80,8 @@ public async Task HandleEvent(IEvent e) switch (e) { case NewHead newHead: - context.Logger.Info($"New head received: {newHead.Head}"); - if (newHead.Head <= LastIndexHeight) + context.Logger.Debug($"New head received: {newHead.Head}"); + if (newHead.Head <= context.Database.LatestBlock()) { await TransitionTo(State.Reorg, newHead.Head); return; @@ -97,8 +98,8 @@ public async Task HandleEvent(IEvent e) { case EnterState enterSyncing: var importedBlockRange = await Sync(enterSyncing.Arg); - context.Logger.Info($"Imported blocks from {importedBlockRange.Min} " + - $"to {importedBlockRange.Max}"); + context.Logger.Debug($"Imported blocks from {importedBlockRange.Min} " + + $"to {importedBlockRange.Max}"); Errors.Clear(); await TransitionTo(State.NotifySubscribers, importedBlockRange); @@ -135,15 +136,25 @@ public async Task HandleEvent(IEvent e) switch (e) { case EnterState: - if (Errors.Count >= 3) + // Exponential backoff based on the number of errors + var delay = Errors.Count * Errors.Count * 1000; + + // If the delay is larger than 60 sec, clear the oldest errors + if (delay > 60000) { - // If we have tried 3 times, give up - await TransitionTo(State.End); - return; + Errors.RemoveAt(0); } - // Otherwise, wait for a bit before retrying - await Task.Delay(Errors.Count * Errors.Count * 1000, cancellationToken); + // Add some jitter to the delay + var jitter = new Random((int)DateTime.Now.TimeOfDay.TotalSeconds).Next(0, 1000); + delay += jitter; + + // Wait 'delay' ms + context.Logger.Info($"Waiting {delay} ms before retrying after an error..."); + await Task.Delay(delay, cancellationToken); + + // Retry + context.Logger.Info("Transitioning to 'Initial' state after an error..."); await TransitionTo(State.Initial); return; case LeaveState: @@ -156,7 +167,7 @@ public async Task HandleEvent(IEvent e) return; } - context.Logger.Debug($"Unhandled event {e} in state {CurrentState}"); + context.Logger.Trace($"Unhandled event {e} in state {CurrentState}"); } catch (Exception ex) { @@ -169,7 +180,7 @@ public async Task HandleEvent(IEvent e) private async Task TransitionTo(State newState, TArgument? argument) { - context.Logger.Info($"Transitioning from {CurrentState} to {newState}"); + context.Logger.Debug($"Transitioning from {CurrentState} to {newState}"); if (newState is not State.Error) { await HandleEvent(new LeaveState()); @@ -187,7 +198,7 @@ public async Task TransitionTo(State newState) private async IAsyncEnumerable GetBlocksToSync(long toBlock) { - long lastIndexHeight = LastIndexHeight; + long lastIndexHeight = context.Database.LatestBlock() ?? 0; if (lastIndexHeight == toBlock) { context.Logger.Info("No blocks to sync."); @@ -195,7 +206,7 @@ private async IAsyncEnumerable GetBlocksToSync(long toBlock) } var nextBlock = lastIndexHeight + 1; - context.Logger.Info($"Enumerating blocks to sync from {nextBlock} (LastIndexHeight + 1) to {toBlock}"); + context.Logger.Debug($"Enumerating blocks to sync from {nextBlock} (LastIndexHeight + 1) to {toBlock}"); for (long i = nextBlock; i <= toBlock; i++) { diff --git a/arm64.Dockerfile b/arm64.Dockerfile index 0a510cf..5e1b364 100644 --- a/arm64.Dockerfile +++ b/arm64.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Release -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins diff --git a/x64.Dockerfile b/x64.Dockerfile index 57c8ad0..4c204ea 100644 --- a/x64.Dockerfile +++ b/x64.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Release -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins diff --git a/x64.debug.Dockerfile b/x64.debug.Dockerfile index 1d8b6a7..2f671fb 100644 --- a/x64.debug.Dockerfile +++ b/x64.debug.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Debug -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins diff --git a/x64.debug.spaceneth.Dockerfile b/x64.debug.spaceneth.Dockerfile index cd2c480..7d076ab 100644 --- a/x64.debug.spaceneth.Dockerfile +++ b/x64.debug.spaceneth.Dockerfile @@ -6,7 +6,7 @@ COPY . . RUN dotnet restore RUN dotnet publish -c Debug -o /circles-nethermind-plugin -FROM nethermind/nethermind:1.27.0 AS base +FROM nethermind/nethermind:1.27.1 AS base # dotnet libs COPY --from=build /circles-nethermind-plugin/Circles.Index.deps.json /nethermind/plugins