From bcad2d7f9355c02c47fa60c0ff96d116cd2ae3ef Mon Sep 17 00:00:00 2001 From: k-kaddal Date: Wed, 3 Apr 2024 13:17:07 +0100 Subject: [PATCH 1/7] feat(blobstorage): allow get_blob api to return blob data --- packages/blobstorage/pkg/http/get_blob.go | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) diff --git a/packages/blobstorage/pkg/http/get_blob.go b/packages/blobstorage/pkg/http/get_blob.go index c01a00e7a0..89d0266355 100644 --- a/packages/blobstorage/pkg/http/get_blob.go +++ b/packages/blobstorage/pkg/http/get_blob.go @@ -15,8 +15,9 @@ type getBlobResponse struct { } type blobData struct { - Blob string `bson:"blob_hash" json:"blob_hash"` + BlobHash string `bson:"blob_hash" json:"blob_hash"` KzgCommitment string `bson:"kzg_commitment" json:"kzg_commitment"` + Blob string `bson:"blob" json:"blob"` } // GetBlob @@ -37,6 +38,7 @@ func (srv *Server) GetBlob(c echo.Context) error { } data, err := srv.getBlobData(strings.Split(blobHashes, ",")) + if err != nil { return webutils.LogAndRenderErrors(c, http.StatusBadRequest, err) } @@ -48,8 +50,9 @@ func (srv *Server) GetBlob(c echo.Context) error { // Convert data to the correct type for _, d := range data { response.Data = append(response.Data, blobData{ - Blob: d.Blob, + BlobHash: d.BlobHash, KzgCommitment: d.KzgCommitment, + Blob: d.Blob, }, ) } @@ -69,15 +72,17 @@ func (srv *Server) getBlobData(blobHashes []string) ([]blobData, error) { if err != nil { if err == gorm.ErrRecordNotFound { // Handle case where blob hash is not found - result.Blob = "NOT_FOUND" + result.BlobHash = "NOT_FOUND" result.KzgCommitment = "NOT_FOUND" + result.Blob = "NOT_FOUND" } else { // Return error for other types of errors return nil, err } } else { - result.Blob = bh.BlobHash + result.BlobHash = bh.BlobHash result.KzgCommitment = bh.KzgCommitment + result.Blob = bh.BlobData results = append(results, result) } From 08c0f15b998d605f60425d702c493113e28fa30c Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 3 Apr 2024 10:13:32 -0700 Subject: [PATCH 2/7] add lots --- packages/blobstorage/indexer/beaconclient.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/packages/blobstorage/indexer/beaconclient.go b/packages/blobstorage/indexer/beaconclient.go index ea2548aeeb..8e5ca075bc 100644 --- a/packages/blobstorage/indexer/beaconclient.go +++ b/packages/blobstorage/indexer/beaconclient.go @@ -8,6 +8,8 @@ import ( "net/http" "strconv" "time" + + "golang.org/x/exp/slog" ) var ( @@ -64,6 +66,8 @@ func NewBeaconClient(cfg *Config, timeout time.Duration) (*BeaconClient, error) return nil, err } + slog.Info("beaconClientInfo", "secondsPerSlot", secondsPerSlotUint64, "genesisTime", genesisTime) + return &BeaconClient{ beaconURL: cfg.BeaconURL, genesisTime: genesisTime, From 529532321ae4a2d48f74429ce021749880fca43e Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 3 Apr 2024 10:19:45 -0700 Subject: [PATCH 3/7] add timestamp to log --- packages/blobstorage/indexer/indexer.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/packages/blobstorage/indexer/indexer.go b/packages/blobstorage/indexer/indexer.go index 6d5752bce6..f9723dc8ac 100644 --- a/packages/blobstorage/indexer/indexer.go +++ b/packages/blobstorage/indexer/indexer.go @@ -289,7 +289,12 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp return err } - slog.Info("blockProposed event found", "blockID", blockID, "emittedIn", event.Raw.BlockNumber, "blobUsed", event.Meta.BlobUsed) + slog.Info("blockProposed event found", + "blockID", blockID, + "emittedIn", event.Raw.BlockNumber, + "blobUsed", event.Meta.BlobUsed, + "timesStamp", event.Meta.Timestamp, + ) if !event.Meta.BlobUsed { return nil From 19de6d567ebf035a37f9683966e54a51eacc740d Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 3 Apr 2024 10:20:53 -0700 Subject: [PATCH 4/7] add slog for blobsresponse --- packages/blobstorage/indexer/indexer.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/packages/blobstorage/indexer/indexer.go b/packages/blobstorage/indexer/indexer.go index f9723dc8ac..748fd88a27 100644 --- a/packages/blobstorage/indexer/indexer.go +++ b/packages/blobstorage/indexer/indexer.go @@ -305,6 +305,8 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp return err } + slog.Info("blobsResponse", "length", len(blobsResponse.Data)) + for _, data := range blobsResponse.Data { data.KzgCommitmentHex = common.FromHex(data.KzgCommitment) From 54e52d6b1e63ad2cd3c28e15c2bb1e7755171687 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 3 Apr 2024 10:26:40 -0700 Subject: [PATCH 5/7] rm block timestamp, its unused --- packages/blobstorage/blob_hash.go | 2 -- packages/blobstorage/indexer/indexer.go | 13 +++---------- .../migrations/16667_create_blob_hashes_table.sql | 1 - packages/blobstorage/pkg/repo/blob_hash.go | 1 - 4 files changed, 3 insertions(+), 14 deletions(-) diff --git a/packages/blobstorage/blob_hash.go b/packages/blobstorage/blob_hash.go index d8e7ab4408..5b4d9a4bd3 100644 --- a/packages/blobstorage/blob_hash.go +++ b/packages/blobstorage/blob_hash.go @@ -3,7 +3,6 @@ package blobstorage type BlobHash struct { BlobHash string KzgCommitment string - BlockTimestamp uint64 BlobData string BlockID uint64 EmittedBlockID uint64 @@ -12,7 +11,6 @@ type BlobHash struct { type SaveBlobHashOpts struct { BlobHash string KzgCommitment string - BlockTimestamp uint64 BlobData string BlockID uint64 EmittedBlockID uint64 diff --git a/packages/blobstorage/indexer/indexer.go b/packages/blobstorage/indexer/indexer.go index 748fd88a27..1dcf5e499d 100644 --- a/packages/blobstorage/indexer/indexer.go +++ b/packages/blobstorage/indexer/indexer.go @@ -313,15 +313,9 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp metaBlobHash := common.BytesToHash(event.Meta.BlobHash[:]) // Comparing the hex strings of meta.blobHash (blobHash) if calculateBlobHash(data.KzgCommitment) == metaBlobHash { - blockTs, err := i.getBlockTimestamp(i.cfg.RPCURL, new(big.Int).SetUint64(blockID)) - if err != nil { - slog.Error("error getting block timestamp", "error", err) - return err - } - - slog.Info("blockHash", "blobHash", metaBlobHash.String()) + slog.Info("blobHash", "blobHash", metaBlobHash.String()) - err = i.storeBlobInDB(metaBlobHash.String(), data.KzgCommitment, data.Blob, blockTs, event.BlockId.Uint64(), event.Raw.BlockNumber) + err = i.storeBlobInDB(metaBlobHash.String(), data.KzgCommitment, data.Blob, event.BlockId.Uint64(), event.Raw.BlockNumber) if err != nil { slog.Error("Error storing blob in DB", "error", err) return err @@ -334,13 +328,12 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp return errors.New("BLOB not found") } -func (i *Indexer) storeBlobInDB(blobHashInMeta, kzgCommitment, blob string, blockTs uint64, blockID uint64, emittedBlockID uint64) error { +func (i *Indexer) storeBlobInDB(blobHashInMeta, kzgCommitment, blob string, blockID uint64, emittedBlockID uint64) error { return i.blobHashRepo.Save(blobstorage.SaveBlobHashOpts{ BlobHash: blobHashInMeta, KzgCommitment: kzgCommitment, BlockID: blockID, BlobData: blob, - BlockTimestamp: blockTs, EmittedBlockID: emittedBlockID, }) } diff --git a/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql b/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql index a2b657017a..be1b4b63c9 100644 --- a/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql +++ b/packages/blobstorage/migrations/16667_create_blob_hashes_table.sql @@ -6,7 +6,6 @@ CREATE TABLE IF NOT EXISTS blob_hashes ( emitted_block_id BIGINT NOT NULL, blob_hash VARCHAR(100) NOT NULL, kzg_commitment LONGTEXT NOT NULL, - block_timestamp BIGINT NOT NULL, blob_data LONGTEXT NOT NULL, created_at TIMESTAMP DEFAULT CURRENT_TIMESTAMP , updated_at DATETIME DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP diff --git a/packages/blobstorage/pkg/repo/blob_hash.go b/packages/blobstorage/pkg/repo/blob_hash.go index eaeb04010d..273bd906b9 100644 --- a/packages/blobstorage/pkg/repo/blob_hash.go +++ b/packages/blobstorage/pkg/repo/blob_hash.go @@ -27,7 +27,6 @@ func (r *BlobHashRepository) Save(opts blobstorage.SaveBlobHashOpts) error { b := &blobstorage.BlobHash{ BlobHash: opts.BlobHash, KzgCommitment: opts.KzgCommitment, - BlockTimestamp: opts.BlockTimestamp, BlobData: opts.BlobData, BlockID: opts.BlockID, EmittedBlockID: opts.EmittedBlockID, From 31a727bcdef6bc97d6a3412fe2e1adc1666c9301 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 3 Apr 2024 10:27:29 -0700 Subject: [PATCH 6/7] rm log, add more useful log --- packages/blobstorage/indexer/indexer.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/packages/blobstorage/indexer/indexer.go b/packages/blobstorage/indexer/indexer.go index 1dcf5e499d..b3819eab7d 100644 --- a/packages/blobstorage/indexer/indexer.go +++ b/packages/blobstorage/indexer/indexer.go @@ -305,15 +305,13 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp return err } - slog.Info("blobsResponse", "length", len(blobsResponse.Data)) - for _, data := range blobsResponse.Data { data.KzgCommitmentHex = common.FromHex(data.KzgCommitment) metaBlobHash := common.BytesToHash(event.Meta.BlobHash[:]) // Comparing the hex strings of meta.blobHash (blobHash) if calculateBlobHash(data.KzgCommitment) == metaBlobHash { - slog.Info("blobHash", "blobHash", metaBlobHash.String()) + slog.Info("storing blobHash in db", "blobHash", metaBlobHash.String()) err = i.storeBlobInDB(metaBlobHash.String(), data.KzgCommitment, data.Blob, event.BlockId.Uint64(), event.Raw.BlockNumber) if err != nil { From ff4f23d4bff735f489aabe1bef213a03c479c708 Mon Sep 17 00:00:00 2001 From: Jeffery Walsh Date: Wed, 3 Apr 2024 10:56:47 -0700 Subject: [PATCH 7/7] blockID => slot --- packages/blobstorage/.default.indexer.env | 4 ++-- packages/blobstorage/indexer/indexer.go | 6 +++--- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/packages/blobstorage/.default.indexer.env b/packages/blobstorage/.default.indexer.env index c228b59292..a6cf895557 100644 --- a/packages/blobstorage/.default.indexer.env +++ b/packages/blobstorage/.default.indexer.env @@ -1,5 +1,5 @@ -RPC_URL=wss://l1ws.internal.taiko.xyz -BEACON_URL=https://l1beacon.internal.taiko.xyz +RPC_URL=wss://l1ws.hekla.taiko.xyz +BEACON_URL=https://l1beacon.hekla.taiko.xyz TAIKO_L1_CONTRACT_ADDRESS=0xC069c3d2a9f2479F559AD34485698ad5199C555f DATABASE_HOST=localhost DATABASE_PORT=3306 diff --git a/packages/blobstorage/indexer/indexer.go b/packages/blobstorage/indexer/indexer.go index b3819eab7d..cee4fb18e1 100644 --- a/packages/blobstorage/indexer/indexer.go +++ b/packages/blobstorage/indexer/indexer.go @@ -284,13 +284,13 @@ func (i *Indexer) checkReorg(ctx context.Context, event *taikol1.TaikoL1BlockPro } func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProposed) error { - blockID, err := i.beaconClient.timeToSlot(event.Meta.Timestamp) + slot, err := i.beaconClient.timeToSlot(event.Meta.Timestamp) if err != nil { return err } slog.Info("blockProposed event found", - "blockID", blockID, + "slot", slot, "emittedIn", event.Raw.BlockNumber, "blobUsed", event.Meta.BlobUsed, "timesStamp", event.Meta.Timestamp, @@ -300,7 +300,7 @@ func (i *Indexer) storeBlob(ctx context.Context, event *taikol1.TaikoL1BlockProp return nil } - blobsResponse, err := i.beaconClient.getBlobs(ctx, blockID) + blobsResponse, err := i.beaconClient.getBlobs(ctx, slot) if err != nil { return err }