feat: use new chainsyncer

keyper: add contract-addresses to config

keyper-core: implement DB chaincache

gnosis-keyper: use new chainsyncer

gnosis-keyper: remove old sync-handler

gnosisaccessnode: use new chainsyncer

gnosisaccessnode: move storage to package

gnosis-keyper: Rename some functions and types

gnosis-keyper: refactor keyperCore init func

gnosis-keyper: Pass dbpool to keyper core

keyper: implement SyncStartBlock / DatabaseChainCache initial syncing

gnosis-keyper: Add int64 overflow checks

gnosis-keyper: add new synchandler

chore: avoid var shadow

feat(chainsync): add retry in initial block fetch

chore: rename keypersetmanager synchandler

chore: re-use the gnosis eth-client in keyper-core

chore: refine comments

chore: remove unnecessary import

chore: fix generate, gci, gofumpt

Author: ezdac

PR.md

@@ -0,0 +1,78 @@
+# 1/4 - Remove Optimism package.
+
+- This is part of a series of 4 Pull Requests that all aim to refactor our old
+  chainsync mechanism. The reason to split this up is to provide more digestible
+  and grouped parts of the refactoring effort. I am aware this is a big change,
+  but especially for refactoring an integral part of the system, and designing
+  it at the same time, at least for me it is impossible to provide smaller
+  increments of work that don't leave the code in an uncompilable,
+  non-functional state. \*
+
+We currently don't use the optimism package and for the following PRs I did not
+have the time to adapt the new chainsync mechanism and sync-handlers to the
+optimism package. If this package is chosen to continue working on, it can be
+re-integratated and the old state of the package can be retrieved from the git
+history and then adapted to the changes. Inspiration on how to integrate the new
+chainsync mechanism can be taken from the follow up PRs.
+
+# 2/4 - Add new chainsync package
+
+- This is part of a series of 4 Pull Requests that all aim to refactor our old
+  chainsync mechanism. The reason to split this up is to provide more digestible
+  and grouped parts of the refactoring effort. I am aware this is a big change,
+  but especially for refactoring an integral part of the system, and designing
+  it at the same time, at least for me it is impossible to provide smaller
+  increments of work that don't leave the code in an uncompilable,
+  non-functional state. \*
+
+This adds a new `chainsync` package that receives new latest-block updates from
+an Ethereum JSON RPC server and backtracks a locally cached chain-segment of
+previous blocks in comparison with the updated header information. It does this
+in order to recognize reorgs and being able to invalidate all chain-state that
+was inserted for a reorged-away chain-segment. The information of the difference
+of old chain and new chain in case of reorgs is omitted by Ethereum JSON RPC
+servers, since it is possible that only the new latest header is received by the
+client, and intermediary sections of the new reorg chain are not passively
+received. Therefore the client itself has to keep track of the most recent chain
+and what gaps between the last common block of a potential new chain segment and
+it's old chain-state's segment have to be filled.
+
+In addition to this active syncing mechanism, the `chainsync` package provides
+an easy way to register handler implementations that retrieve specific
+contract-event updates for the newly received chain-segments and the context of
+newly "reorged away" chain-segments are available in the same handler. That way,
+each handler can define behavior on how to remove old state-updates due to newly
+removed events and how to insert new state updates due to newly emitted events.
+
+There is also the possibility to define and execute "chain-update" handler.
+Those will not only receive updates for when the newly synced to chain-segment
+contains a certain contract event, but they will be executed for every new chain
+update and will provide the same chain-update context as the contract-event
+handler.
+
+# 3/4 - Implement event-handler
+
+- This is part of a series of 4 Pull Requests that all aim to refactor our old
+  chainsync mechanism. The reason to split this up is to provide more digestible
+  and grouped parts of the refactoring effort. I am aware this is a big change,
+  but especially for refactoring an integral part of the system, and designing
+  it at the same time, at least for me it is impossible to provide smaller
+  increments of work that don't leave the code in an uncompilable,
+  non-functional state. \*
+
+This will re-implement the event handling logic of the keyper and
+keyperimpl/gnosis packages for the new chainsync mechanism.
+
+# 4/4 - Integrate the new chainsync handlers in the keyper packages
+
+- This is part of a series of 4 Pull Requests that all aim to refactor our old
+  chainsync mechanism. The reason to split this up is to provide more digestible
+  and grouped parts of the refactoring effort. I am aware this is a big change,
+  but especially for refactoring an integral part of the system, and designing
+  it at the same time, at least for me it is impossible to provide smaller
+  increments of work that don't leave the code in an uncompilable,
+  non-functional state. \*
+
+This will remove the old blockchain handling in the keyper and keyperimpl/gnosis
+packages and use the newly added re-implementation of this blockchain handling
+in the keyper packages.

rolling-shutter/chainobserver/db/keyper/keyper.sqlc.gen.go

@@ -18,4 +18,42 @@ ORDER BY activation_block_number DESC LIMIT 1;
 
 -- name: GetKeyperSets :many
 SELECT * FROM keyper_set
-ORDER BY activation_block_number ASC;
+ORDER BY activation_block_number ASC;
+
+-- name: InsertSyncedBlock :exec
+INSERT INTO recent_block (
+       block_hash,
+       block_number,
+       parent_hash,
+       timestamp,
+       header
+) VALUES (
+    $1, $2, $3, $4, $5
+) ON CONFLICT DO UPDATE SET
+       block_hash = $1,
+       block_number =$2 ,
+       parent_hash =$3,
+       timestamp =$4 ,
+       header =$5
+;
+
+-- name: GetSyncedBlockByHash :one
+SELECT * FROM recent_block
+WHERE block_hash = $1;
+
+-- name: DeleteSyncedBlockByHash :exec
+DELETE FROM recent_block
+WHERE block_hash = $1;
+
+-- name: GetSyncedBlocks :many
+SELECT * FROM recent_block
+ORDER BY block_number DESC;
+
+-- name: GetLatestSyncedBlocks :many
+SELECT * FROM recent_block
+ORDER BY block_number DESC
+LIMIT $1;
+
+-- name: EvictSyncedBlocksBefore :exec
+DELETE FROM recent_block
+WHERE block_number < $1;

rolling-shutter/chainobserver/db/keyper/sql/queries/keyper.sql

@@ -29,6 +29,8 @@ const (
 )
 
 // EonKeyPublisher is a service that publishes eon keys via a eon key publisher contract.
+// E.g. in the Gnosis keyper implementation this is used to publish keys onchain
+// instead of broadcasting it on the p2p-network.
 type EonKeyPublisher struct {
 	dbpool           *pgxpool.Pool
 	client           *ethclient.Client

rolling-shutter/eonkeypublisher/eonkeypublisher.go

@@ -2,20 +2,17 @@ package gnosisaccessnode
 
 import (
 	"context"
+	"fmt"
 
+	"github.com/ethereum/go-ethereum/ethclient"
 	"github.com/pkg/errors"
-	"github.com/rs/zerolog/log"
 
-	"github.com/shutter-network/shutter/shlib/shcrypto"
-
-	obskeyperdatabase "github.com/shutter-network/rolling-shutter/rolling-shutter/chainobserver/db/keyper"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/gnosisaccessnode/storage"
-	chainsync "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/legacychainsync"
-	syncevent "github.com/shutter-network/rolling-shutter/rolling-shutter/medley/legacychainsync/event"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/gnosisaccessnode/synchandler"
+	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/chainsync"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/metricsserver"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/medley/service"
 	"github.com/shutter-network/rolling-shutter/rolling-shutter/p2p"
-	"github.com/shutter-network/rolling-shutter/rolling-shutter/shdb"
 )
 
 type GnosisAccessNode struct {
@@ -40,59 +37,39 @@ func (node *GnosisAccessNode) Start(ctx context.Context, runner service.Runner)
 	messageSender.AddMessageHandler(NewDecryptionKeysHandler(node.config, node.storage))
 	services = append(services, messageSender)
 
-	chainSyncClient, err := chainsync.NewClient(
-		ctx,
-		chainsync.WithClientURL(node.config.GnosisNode.EthereumURL),
-		chainsync.WithKeyperSetManager(node.config.Contracts.KeyperSetManager),
-		chainsync.WithKeyBroadcastContract(node.config.Contracts.KeyBroadcastContract),
-		chainsync.WithSyncNewKeyperSet(node.onNewKeyperSet),
-		chainsync.WithSyncNewEonKey(node.onNewEonKey),
+	ethClient, err := ethclient.DialContext(ctx, node.config.GnosisNode.EthereumURL)
+	if err != nil {
+		return err
+	}
+	keyperSetAdded, err := synchandler.NewKeyperSetAdded(
+		ethClient,
+		node.storage,
+		node.config.Contracts.KeyperSetManager,
 	)
 	if err != nil {
-		return errors.Wrap(err, "failed to initialize chain sync client")
+		return err
 	}
-	services = append(services, chainSyncClient)
-
+	eonKeyBroadcast, err := synchandler.NewEonKeyBroadcast(
+		node.storage,
+		node.config.Contracts.KeyBroadcastContract,
+	)
+	if err != nil {
+		return err
+	}
+	chainsyncOpts := []chainsync.Option{
+		chainsync.WithClient(ethClient),
+		chainsync.WithContractEventHandler(keyperSetAdded),
+		chainsync.WithContractEventHandler(eonKeyBroadcast),
+	}
+	chainsyncer, err := chainsync.New(chainsyncOpts...)
+	if err != nil {
+		return fmt.Errorf("can't instantiate chainsync: %w", err)
+	}
+	services = append(services, chainsyncer)
 	if node.config.Metrics.Enabled {
 		metricsServer := metricsserver.New(node.config.Metrics)
 		services = append(services, metricsServer)
 	}
 
 	return runner.StartService(services...)
 }
-
-func (node *GnosisAccessNode) onNewKeyperSet(_ context.Context, keyperSet *syncevent.KeyperSet) error {
-	obsKeyperSet := obskeyperdatabase.KeyperSet{
-		KeyperConfigIndex:     int64(keyperSet.Eon),
-		ActivationBlockNumber: int64(keyperSet.ActivationBlock),
-		Keypers:               shdb.EncodeAddresses(keyperSet.Members),
-		Threshold:             int32(keyperSet.Threshold),
-	}
-	log.Info().
-		Uint64("keyper-config-index", keyperSet.Eon).
-		Uint64("activation-block-number", keyperSet.ActivationBlock).
-		Int("num-keypers", len(keyperSet.Members)).
-		Uint64("threshold", keyperSet.Threshold).
-		Msg("adding keyper set")
-	node.storage.AddKeyperSet(keyperSet.Eon, &obsKeyperSet)
-	return nil
-}
-
-func (node *GnosisAccessNode) onNewEonKey(_ context.Context, eonKey *syncevent.EonPublicKey) error {
-	key := new(shcrypto.EonPublicKey)
-	err := key.Unmarshal(eonKey.Key)
-	if err != nil {
-		log.Error().
-			Err(err).
-			Hex("key", eonKey.Key).
-			Int("keyper-config-index", int(eonKey.Eon)).
-			Msg("received invalid eon key")
-		return nil
-	}
-	log.Info().
-		Int("keyper-config-index", int(eonKey.Eon)).
-		Hex("key", eonKey.Key).
-		Msg("adding eon key")
-	node.storage.AddEonKey(eonKey.Eon, key)
-	return nil
-}