From 9359b65cf8184d34f246bf989a3bf3c949477bce Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Mon, 29 May 2023 05:06:27 -0500 Subject: [PATCH 001/388] feat!: bump app v1.0.0-rc0 (#2253) Co-authored-by: Rootul Patel Co-authored-by: Wondertan --- api/gateway/share.go | 6 +- api/gateway/share_test.go | 133 +++-------- core/eds.go | 13 +- core/eds_test.go | 12 +- core/exchange_test.go | 2 +- core/testing.go | 3 +- core/testing_grpc.go | 2 +- das/coordinator_test.go | 2 +- das/daser_test.go | 4 +- das/metrics.go | 4 +- das/worker.go | 6 +- go.mod | 56 ++--- go.sum | 116 +++++----- nodebuilder/tests/swamp/config.go | 6 +- nodebuilder/tests/swamp/swamp.go | 2 +- nodebuilder/tests/swamp/swamp_tx.go | 2 +- share/availability/cache/availability_test.go | 2 +- share/availability/light/availability.go | 2 +- share/availability/light/availability_test.go | 209 +++--------------- .../light/testdata/sample-block.json | 60 ----- share/eds/byzantine/bad_encoding.go | 18 +- share/eds/byzantine/bad_encoding_test.go | 2 +- share/eds/byzantine/byzantine.go | 2 +- share/eds/byzantine/share_proof_test.go | 2 +- share/eds/eds.go | 6 +- share/eds/eds_test.go | 65 ++++-- share/eds/retriever.go | 10 +- share/eds/retriever_quadrant.go | 2 +- share/eds/retriever_test.go | 2 +- share/eds/store.go | 2 +- share/eds/testdata/example-root.json | 32 +-- share/eds/testdata/example.car | Bin 55235 -> 74051 bytes share/empty.go | 10 +- share/get_test.go | 8 +- share/getter.go | 2 +- share/getters/getter_test.go | 19 +- share/getters/ipld.go | 2 +- share/getters/shrex_test.go | 7 +- share/getters/store.go | 2 +- share/getters/utils.go | 6 +- share/ipld/nmt.go | 14 +- share/ipld/nmt_test.go | 11 +- share/p2p/shrexnd/exchange_test.go | 9 +- share/share.go | 2 +- share/test_helpers.go | 15 +- state/integration_test.go | 4 +- 46 files changed, 337 insertions(+), 559 deletions(-) delete mode 100755 share/availability/light/testdata/sample-block.json diff --git a/api/gateway/share.go b/api/gateway/share.go index 13903fd14b..36a9778f1a 100644 --- a/api/gateway/share.go +++ b/api/gateway/share.go @@ -123,7 +123,11 @@ func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID } func dataFromShares(input []share.Share) (data [][]byte, err error) { - sequences, err := shares.ParseShares(input) + appShares, err := shares.FromBytes(input) + if err != nil { + return nil, err + } + sequences, err := shares.ParseShares(appShares, false) if err != nil { return nil, err } diff --git a/api/gateway/share_test.go b/api/gateway/share_test.go index 92b7a2c853..423d08682b 100644 --- a/api/gateway/share_test.go +++ b/api/gateway/share_test.go @@ -1,129 +1,52 @@ package gateway import ( - "bytes" _ "embed" "encoding/base64" "encoding/json" "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/celestia-app/pkg/shares" ) func Test_dataFromShares(t *testing.T) { - type testCase struct { - name string - input [][]byte - want [][]byte - wantErr bool + testData := [][]byte{ + []byte("beep"), + []byte("beeap"), + []byte("BEEEEAHP"), } - smallTxInput := padShare([]uint8{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id - 0x1, // info byte - 0x0, 0x0, 0x0, 0x2, // 1 byte (unit) + 1 byte (unit length) = 2 bytes sequence length - 0x0, 0x0, 0x0, 17, // reserved bytes - 0x1, // unit length of first transaction - 0xa, // data of first transaction - }) - smallTxData := []byte{0x1, 0xa} - - largeTxInput := [][]byte{ - fillShare([]uint8{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id - 0x1, // info byte - 0x0, 0x0, 0x2, 0x2, // 512 (unit) + 2 (unit length) = 514 sequence length - 0x0, 0x0, 0x0, 17, // reserved bytes - 128, 4, // unit length of transaction is 512 - }, 0xc), // data of transaction - padShare(append([]uint8{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x1, // namespace id - 0x0, // info byte - 0x0, 0x0, 0x0, 0x0, // reserved bytes - }, bytes.Repeat([]byte{0xc}, 19)..., // continuation data of transaction - )), - } - largeTxData := []byte{128, 4} - largeTxData = append(largeTxData, bytes.Repeat([]byte{0xc}, 512)...) - - largePfbTxInput := [][]byte{ - fillShare([]uint8{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, // namespace id - 0x1, // info byte - 0x0, 0x0, 0x2, 0x2, // 512 (unit) + 2 (unit length) = 514 sequence length - 0x0, 0x0, 0x0, 17, // reserved bytes - 128, 4, // unit length of transaction is 512 - }, 0xc), // data of transaction - padShare(append([]uint8{ - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x4, // namespace id - 0x0, // info byte - 0x0, 0x0, 0x0, 0x0, // reserved bytes - }, bytes.Repeat([]byte{0xc}, 19)..., // continuation data of transaction - )), + ns := namespace.RandomBlobNamespace() + sss := shares.NewSparseShareSplitter() + for _, data := range testData { + b := coretypes.Blob{ + Data: data, + NamespaceID: ns.ID, + NamespaceVersion: ns.Version, + ShareVersion: appconsts.ShareVersionZero, + } + err := sss.Write(b) + require.NoError(t, err) } - largePfbTxData := []byte{128, 4} - largePfbTxData = append(largePfbTxData, bytes.Repeat([]byte{0xc}, 512)...) - testCases := []testCase{ - { - name: "empty", - input: [][]byte{}, - want: nil, - wantErr: false, - }, - { - name: "returns an error when shares contain two different namespaces", - input: [][]byte{ - {0, 0, 0, 0, 0, 0, 0, 1}, - {0, 0, 0, 0, 0, 0, 0, 2}, - }, - want: nil, - wantErr: true, - }, - { - name: "returns raw data of a single tx share", - input: [][]byte{smallTxInput}, - want: [][]byte{smallTxData}, - wantErr: false, - }, - { - name: "returns raw data of a large tx that spans two shares", - input: largeTxInput, - want: [][]byte{largeTxData}, - wantErr: false, - }, - { - name: "returns raw data of a large PFB tx that spans two shares", - input: largePfbTxInput, - want: [][]byte{largePfbTxData}, - wantErr: false, - }, - } + sssShares := sss.Export() - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := dataFromShares(tc.input) - if tc.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, tc.want, got) - }) + rawSSSShares := make([][]byte, len(sssShares)) + for i := 0; i < len(sssShares); i++ { + d := sssShares[i].ToBytes() + rawSSSShares[i] = d } -} -// padShare returns a share padded with trailing zeros. -func padShare(share []byte) (paddedShare []byte) { - return fillShare(share, 0) -} + parsedSSSShares, err := dataFromShares(rawSSSShares) + require.NoError(t, err) -// fillShare returns a share filled with filler so that the share length -// is equal to appconsts.ShareSize. -func fillShare(share []byte, filler byte) (paddedShare []byte) { - return append(share, bytes.Repeat([]byte{filler}, appconsts.ShareSize-len(share))...) + require.Equal(t, testData, parsedSSSShares) } // sharesBase64JSON is the base64 encoded share data from Blockspace Race @@ -137,6 +60,8 @@ var sharesBase64JSON string // // https://github.com/celestiaorg/celestia-app/issues/1816 func Test_dataFromSharesBSR(t *testing.T) { + t.Skip("skip until sharesBase64JSON is regenerated with v1 compatibility") + var sharesBase64 []string err := json.Unmarshal([]byte(sharesBase64JSON), &sharesBase64) assert.NoError(t, err) diff --git a/core/eds.go b/core/eds.go index e37fa5d960..c435f0e649 100644 --- a/core/eds.go +++ b/core/eds.go @@ -7,8 +7,9 @@ import ( "github.com/filecoin-project/dagstore" "github.com/tendermint/tendermint/types" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" - appshares "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/pkg/square" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -22,12 +23,18 @@ func extendBlock(data types.Data) (*rsmt2d.ExtendedDataSquare, error) { if len(data.Txs) == 0 && data.SquareSize == uint64(1) { return nil, nil } - shares, err := appshares.Split(data, true) + + sqr, err := square.Construct(data.Txs.ToSliceOfBytes(), appconsts.MaxSquareSize) if err != nil { return nil, err } - return da.ExtendShares(data.SquareSize, appshares.ToBytes(shares)) + shares := make([][]byte, len(sqr)) + for i, s := range sqr { + shares[i] = s.ToBytes() + } + + return da.ExtendShares(shares) } // storeEDS will only store extended block if it is not empty and doesn't already exist. diff --git a/core/eds_test.go b/core/eds_test.go index d67f5005f9..6bc04c96c4 100644 --- a/core/eds_test.go +++ b/core/eds_test.go @@ -18,7 +18,6 @@ import ( func TestTrulyEmptySquare(t *testing.T) { data := types.Data{ Txs: []types.Tx{}, - Blobs: []types.Blob{}, SquareSize: 1, } @@ -27,17 +26,20 @@ func TestTrulyEmptySquare(t *testing.T) { assert.Nil(t, eds) } -// TestNonEmptySquareWithZeroTxs tests that a non-empty square with no -// transactions or blobs computes the correct data root (not the minimum DAH). +// TestNonZeroSquareSize tests that the DAH hash of a block with no transactions +// is equal to the DAH hash for an empty root even if SquareSize is set to +// something non-zero. Technically, this block data is invalid because the +// construction of the square is deterministic, and the rules which dictate the +// square size do not allow for empty block data. However, should that ever +// occur, we need to ensure that the correct data root is generated. func TestNonEmptySquareWithZeroTxs(t *testing.T) { data := types.Data{ Txs: []types.Tx{}, - Blobs: []types.Blob{}, SquareSize: 16, } eds, err := extendBlock(data) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(eds) - assert.NotEqual(t, share.EmptyRoot().Hash(), dah.Hash()) + assert.Equal(t, share.EmptyRoot().Hash(), dah.Hash()) } diff --git a/core/exchange_test.go b/core/exchange_test.go index 928d15a9e5..f6302b5742 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -10,7 +10,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/testutil/testnode" + "github.com/celestiaorg/celestia-app/test/util/testnode" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share/eds" diff --git a/core/testing.go b/core/testing.go index 88154dc7d9..393ec62c09 100644 --- a/core/testing.go +++ b/core/testing.go @@ -12,7 +12,7 @@ import ( tmrand "github.com/tendermint/tendermint/libs/rand" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - "github.com/celestiaorg/celestia-app/testutil/testnode" + "github.com/celestiaorg/celestia-app/test/util/testnode" ) // TestConfig encompasses all the configs required to run test Tendermint + Celestia App tandem. @@ -79,7 +79,6 @@ func StartTestNodeWithConfig(t *testing.T, cfg *TestConfig) testnode.Context { state, kr, "private", - nil, ) require.NoError(t, err) diff --git a/core/testing_grpc.go b/core/testing_grpc.go index 2306b61723..d831bc0724 100644 --- a/core/testing_grpc.go +++ b/core/testing_grpc.go @@ -14,7 +14,7 @@ import ( "google.golang.org/grpc" "google.golang.org/grpc/credentials/insecure" - "github.com/celestiaorg/celestia-app/testutil/testnode" + "github.com/celestiaorg/celestia-app/test/util/testnode" ) /* diff --git a/das/coordinator_test.go b/das/coordinator_test.go index e7e2d5ceb2..188cb0d222 100644 --- a/das/coordinator_test.go +++ b/das/coordinator_test.go @@ -431,7 +431,7 @@ func (m *mockSampler) discover(ctx context.Context, newHeight uint64, emit liste emit(ctx, &header.ExtendedHeader{ Commit: &types.Commit{}, RawHeader: header.RawHeader{Height: int64(newHeight)}, - DAH: &header.DataAvailabilityHeader{RowsRoots: make([][]byte, 0)}, + DAH: &header.DataAvailabilityHeader{RowRoots: make([][]byte, 0)}, }) } diff --git a/das/daser_test.go b/das/daser_test.go index 2f1c494309..7398310a6b 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -345,7 +345,7 @@ type benchGetterStub struct { func newBenchGetter() benchGetterStub { return benchGetterStub{header: &header.ExtendedHeader{ - DAH: &header.DataAvailabilityHeader{RowsRoots: make([][]byte, 0)}}} + DAH: &header.DataAvailabilityHeader{RowRoots: make([][]byte, 0)}}} } func (m benchGetterStub) GetByHeight(context.Context, uint64) (*header.ExtendedHeader, error) { @@ -362,7 +362,7 @@ func (m getterStub) GetByHeight(_ context.Context, height uint64) (*header.Exten return &header.ExtendedHeader{ Commit: &types.Commit{}, RawHeader: header.RawHeader{Height: int64(height)}, - DAH: &header.DataAvailabilityHeader{RowsRoots: make([][]byte, 0)}}, nil + DAH: &header.DataAvailabilityHeader{RowRoots: make([][]byte, 0)}}, nil } func (m getterStub) GetRangeByHeight(context.Context, uint64, uint64) ([]*header.ExtendedHeader, error) { diff --git a/das/metrics.go b/das/metrics.go index fd72fe5e4e..1dcf5c8165 100644 --- a/das/metrics.go +++ b/das/metrics.go @@ -154,13 +154,13 @@ func (m *metrics) observeSample( } m.sampleTime.Record(ctx, sampleTime.Seconds(), attribute.Bool(failedLabel, err != nil), - attribute.Int(headerWidthLabel, len(h.DAH.RowsRoots)), + attribute.Int(headerWidthLabel, len(h.DAH.RowRoots)), attribute.String(jobTypeLabel, string(jobType)), ) m.sampled.Add(ctx, 1, attribute.Bool(failedLabel, err != nil), - attribute.Int(headerWidthLabel, len(h.DAH.RowsRoots)), + attribute.Int(headerWidthLabel, len(h.DAH.RowRoots)), attribute.String(jobTypeLabel, string(jobType)), ) diff --git a/das/worker.go b/das/worker.go index 06585e49b0..746324ec48 100644 --- a/das/worker.go +++ b/das/worker.go @@ -120,7 +120,7 @@ func (w *worker) sample(ctx context.Context, timeout time.Duration, height uint6 "type", w.state.jobType, "height", h.Height(), "hash", h.Hash(), - "square width", len(h.DAH.RowsRoots), + "square width", len(h.DAH.RowRoots), "data root", h.DAH.String(), "err", err, "finished (s)", time.Since(start), @@ -150,7 +150,7 @@ func (w *worker) sample(ctx context.Context, timeout time.Duration, height uint6 "type", w.state.jobType, "height", h.Height(), "hash", h.Hash(), - "square width", len(h.DAH.RowsRoots), + "square width", len(h.DAH.RowRoots), "data root", h.DAH.String(), "finished (s)", time.Since(start), ) @@ -179,7 +179,7 @@ func (w *worker) getHeader(ctx context.Context, height uint64) (*header.Extended "got header from header store", "height", h.Height(), "hash", h.Hash(), - "square width", len(h.DAH.RowsRoots), + "square width", len(h.DAH.RowRoots), "data root", h.DAH.String(), "finished (s)", time.Since(start), ) diff --git a/go.mod b/go.mod index 431a78f519..abdc7b6afc 100644 --- a/go.mod +++ b/go.mod @@ -9,13 +9,13 @@ require ( github.com/BurntSushi/toml v1.2.1 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.0 - github.com/celestiaorg/celestia-app v0.13.3 + github.com/celestiaorg/celestia-app v1.0.0-rc0 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 github.com/celestiaorg/go-libp2p-messenger v0.2.0 - github.com/celestiaorg/nmt v0.15.0 - github.com/celestiaorg/rsmt2d v0.8.0 - github.com/cosmos/cosmos-sdk v0.46.7 + github.com/celestiaorg/nmt v0.16.0 + github.com/celestiaorg/rsmt2d v0.9.0 + github.com/cosmos/cosmos-sdk v0.46.11 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 github.com/dgraph-io/badger/v2 v2.2007.4 @@ -60,8 +60,8 @@ require ( github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.1 - github.com/tendermint/tendermint v0.35.4 + github.com/stretchr/testify v1.8.3 + github.com/tendermint/tendermint v0.34.24 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.34.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 @@ -99,7 +99,6 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect - github.com/btcsuite/btcd v0.22.1 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect github.com/celestiaorg/quantum-gravity-bridge v1.3.0 // indirect @@ -109,6 +108,7 @@ require ( github.com/chzyer/readline v1.5.0 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect + github.com/cometbft/cometbft-db v0.7.0 // indirect github.com/confio/ics23/go v0.9.0 // indirect github.com/containerd/cgroups v1.0.4 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect @@ -117,16 +117,17 @@ require ( github.com/cosmos/go-bip39 v1.0.0 // indirect github.com/cosmos/gogoproto v1.4.2 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect - github.com/cosmos/iavl v0.19.4 // indirect + github.com/cosmos/iavl v0.19.5 // indirect github.com/cosmos/ibc-go/v6 v6.1.0 // indirect - github.com/cosmos/ledger-cosmos-go v0.12.1 // indirect + github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect - github.com/deckarep/golang-set v1.8.0 // indirect + github.com/deckarep/golang-set/v2 v2.1.0 // indirect github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect @@ -135,7 +136,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/go-ethereum v1.10.26 // indirect + github.com/ethereum/go-ethereum v1.11.6 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -151,7 +152,7 @@ require ( github.com/go-openapi/jsonreference v0.19.4 // indirect github.com/go-openapi/spec v0.19.11 // indirect github.com/go-openapi/swag v0.19.11 // indirect - github.com/go-stack/stack v1.8.0 // indirect + github.com/go-stack/stack v1.8.1 // indirect github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect @@ -159,8 +160,8 @@ require ( github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.2 // indirect - github.com/golang/snappy v0.0.4 // indirect - github.com/google/btree v1.0.1 // indirect + github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect + github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect @@ -186,10 +187,13 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect + github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect github.com/huin/goupnp v1.0.3 // indirect github.com/iancoleman/orderedmap v0.1.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect + github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-bitswap v0.12.0 // indirect github.com/ipfs/go-block-format v0.1.1 // indirect @@ -212,7 +216,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.15.12 // indirect + github.com/klauspost/compress v1.15.15 // indirect github.com/klauspost/cpuid/v2 v2.2.3 // indirect github.com/klauspost/reedsolomon v1.11.1 // indirect github.com/koron/go-ssdp v0.0.3 // indirect @@ -261,8 +265,8 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.3.0 // indirect - github.com/prometheus/common v0.37.0 // indirect - github.com/prometheus/procfs v0.8.0 // indirect + github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/procfs v0.9.0 // indirect github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.2.1 // indirect @@ -274,7 +278,6 @@ require ( github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect github.com/rivo/uniseg v0.4.2 // indirect - github.com/rjeczalik/notify v0.9.1 // indirect github.com/rs/cors v1.8.2 // indirect github.com/rs/zerolog v1.27.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect @@ -286,19 +289,18 @@ require ( github.com/spf13/viper v1.14.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect - github.com/tendermint/btcd v0.1.1 // indirect - github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 // indirect + github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tendermint/tm-db v0.6.7 // indirect + github.com/tidwall/btree v1.5.0 // indirect github.com/tklauser/go-sysconf v0.3.10 // indirect github.com/tklauser/numcpus v0.4.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect - github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/zondax/hid v0.9.1 // indirect - github.com/zondax/ledger-go v0.14.0 // indirect + github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect @@ -307,13 +309,13 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.16.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 // indirect - golang.org/x/mod v0.8.0 // indirect + golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect + golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.8.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sys v0.6.0 // indirect golang.org/x/term v0.6.0 // indirect - golang.org/x/tools v0.6.0 // indirect + golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.103.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -328,8 +330,8 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.8.0-sdk-v0.46.7 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.15.0-tm-v0.34.23 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.21.0-tm-v0.34.27 ) diff --git a/go.sum b/go.sum index a7feaf83ab..9eca643fef 100644 --- a/go.sum +++ b/go.sum @@ -229,6 +229,7 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= @@ -320,6 +321,7 @@ github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/i github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= +github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -327,7 +329,6 @@ github.com/btcsuite/btclog v0.0.0-20170628155309-84c8d2346e9f/go.mod h1:TdznJufo github.com/btcsuite/btcutil v0.0.0-20190207003914-4c204d697803/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v0.0.0-20190425235716-9e5f4b9a998d/go.mod h1:+5NJ2+qvTyV9exUAL/rxXi3DcLg2Ts+ymUAY5y4NvMg= github.com/btcsuite/btcutil v1.0.2/go.mod h1:j9HUFwoQRsZL3V4n+qG+CUnEGHOarIxfC3Le2Yhbcts= -github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce h1:YtWJF7RHm2pYCvA5t0RPmAaLUhREsKuKd+SLhxFbFeQ= github.com/btcsuite/btcutil v1.0.3-0.20201208143702-a53e38424cce/go.mod h1:0DVlHczLPewLcPGEIeUEzfOJhqGPQ0mJJRDBtD307+o= github.com/btcsuite/go-socks v0.0.0-20170105172521-4720035b7bfd/go.mod h1:HHNXQzUsZCxOoE+CPiyCTO6x34Zs86zZUiwtpXoGdtg= github.com/btcsuite/goleveldb v0.0.0-20160330041536-7834afc9e8cd/go.mod h1:F+uVaaLLH7j4eDXPRvw78tMflu7Ie2bzYOH4Y8rRKBY= @@ -340,12 +341,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v0.13.3 h1:lJQLBzKALrKe0lLe3Gf8Si7+CF0eqRuNhjIjj7pM2fk= -github.com/celestiaorg/celestia-app v0.13.3/go.mod h1:OcPBfWDyowJgoEQ89NB2LgLOm9LSwloCgCzdZKjmi78= -github.com/celestiaorg/celestia-core v1.15.0-tm-v0.34.23 h1:BHvn41IHOtvHeX1VZqO/xBFIHj93llcw9ZQfNxyVRlI= -github.com/celestiaorg/celestia-core v1.15.0-tm-v0.34.23/go.mod h1:nL+vkAMKy/A8wWemWqMwBy4pOGWYYbboAVTEe3N5gIU= -github.com/celestiaorg/cosmos-sdk v1.8.0-sdk-v0.46.7 h1:EADZy33ufskVIy6Rj6jbi3SOVCeYYo26zUi7iYx+QR0= -github.com/celestiaorg/cosmos-sdk v1.8.0-sdk-v0.46.7/go.mod h1:vg3Eza9adJJ5Mdx6boz5MpZsZcTZyrfTVYZHyi2zLm4= +github.com/celestiaorg/celestia-app v1.0.0-rc0 h1:wpuP5fTIEbLCP+U5pGwKfSzXUTE/bE8oqKECFN5yoO0= +github.com/celestiaorg/celestia-app v1.0.0-rc0/go.mod h1:C8pNwFQWBLYIGpdrFesO1uezthrKjv0H5meecYQc1ek= +github.com/celestiaorg/celestia-core v1.21.0-tm-v0.34.27 h1:EdkqFRBypVEq/nX2ZE7KQ6dTlN8j3rEYe+WGahWuSUk= +github.com/celestiaorg/celestia-core v1.21.0-tm-v0.34.27/go.mod h1:GVo91Wifg9KL/nFx9nPkpl0UIFdvvs4fhnly9GhGxZU= +github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 h1:Rd5EvJx1nG3KurBspVN51RVmvif0Lp2UVURbG2ad3Cs= +github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11/go.mod h1:xCG6OUkJy5KUMEg20Zk010lra9XjkmKS3+bk0wp7bd8= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= @@ -358,12 +359,12 @@ github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.15.0 h1:ID9QlMIeP6WK/iiGcfnYLu2qqVIq0UYe/dc3TVPt6EA= -github.com/celestiaorg/nmt v0.15.0/go.mod h1:GfwIvQPhUakn1modWxJ+rv8dUjJzuXg5H+MLFM1o7nY= +github.com/celestiaorg/nmt v0.16.0 h1:4CX6d1Uwf1C+tGcAWskPve0HCDTnI4Ey8ffjiDwcGH0= +github.com/celestiaorg/nmt v0.16.0/go.mod h1:GfwIvQPhUakn1modWxJ+rv8dUjJzuXg5H+MLFM1o7nY= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= -github.com/celestiaorg/rsmt2d v0.8.0 h1:ZUxTCELZCM9zMGKNF3cT+rUqMddXMeiuyleSJPZ3Wn4= -github.com/celestiaorg/rsmt2d v0.8.0/go.mod h1:hhlsTi6G3+X5jOP/8Lb/d7i5y2XNFmnyMddYbFSmrgo= +github.com/celestiaorg/rsmt2d v0.9.0 h1:kon78I748ZqjNzI8OAqPN+2EImuZuanj/6gTh8brX3o= +github.com/celestiaorg/rsmt2d v0.9.0/go.mod h1:E06nDxfoeBDltWRvTR9dLviiUZI5/6mLXAuhSJzz3Iw= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= @@ -408,11 +409,17 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= +github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= +github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA= github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M= +github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo= +github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0= github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4= github.com/confio/ics23/go v0.9.0/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -451,12 +458,12 @@ github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAY github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= -github.com/cosmos/iavl v0.19.4 h1:t82sN+Y0WeqxDLJRSpNd8YFX5URIrT+p8n6oJbJ2Dok= -github.com/cosmos/iavl v0.19.4/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= +github.com/cosmos/iavl v0.19.5 h1:rGA3hOrgNxgRM5wYcSCxgQBap7fW82WZgY78V9po/iY= +github.com/cosmos/iavl v0.19.5/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= github.com/cosmos/ibc-go/v6 v6.1.0 h1:o7oXws2vKkKfOFzJI+oNylRn44PCNt5wzHd/zKQKbvQ= github.com/cosmos/ibc-go/v6 v6.1.0/go.mod h1:CY3zh2HLfetRiW8LY6kVHMATe90Wj/UOoY8T6cuB0is= -github.com/cosmos/ledger-cosmos-go v0.12.1 h1:sMBxza5p/rNK/06nBSNmsI/WDqI0pVJFVNihy1Y984w= -github.com/cosmos/ledger-cosmos-go v0.12.1/go.mod h1:dhO6kj+Y+AHIOgAe4L9HL/6NDdyyth4q238I9yFpD2g= +github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA= +github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -482,8 +489,9 @@ github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSs github.com/davidlazar/go-crypto v0.0.0-20170701192655-dcfb0a7ac018/go.mod h1:rQYf4tfk5sSwFsnDg3qYaBxSjsD9S8+59vW0dKUgme4= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c h1:pFUpOrbxDR6AkioZ1ySsx5yxlDQZ8stG2b88gTPxgJU= github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6UhI8N9EjYm1c2odKpFpAYeR8dsBeM7PtzQhRgxRr9U= -github.com/deckarep/golang-set v1.8.0 h1:sk9/l/KqpunDwP7pSjUg0keiOOLEnOBHzykLrsPppp4= github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= +github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= +github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= @@ -491,6 +499,7 @@ github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1 github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= +github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= github.com/deepmap/oapi-codegen v1.8.2/go.mod h1:YLgSKSDv/bZQB7N4ws6luhozi3cEdRktEqrX88CvjIw= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f h1:U5y3Y5UE0w7amNe7Z5G/twsBW0KEalRQXZzf8ufSh9I= github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f/go.mod h1:xH/i4TFMt8koVQZ6WFms69WAsDWr2XsYL3Hkl7jkoLE= @@ -555,8 +564,8 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWS github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= -github.com/ethereum/go-ethereum v1.10.26 h1:i/7d9RBBwiXCEuyduBQzJw/mKmnvzsN14jqBmytw72s= -github.com/ethereum/go-ethereum v1.10.26/go.mod h1:EYFyF19u3ezGLD4RqOkLq+ZCXzYbLoNDdZlMt7kyKFg= +github.com/ethereum/go-ethereum v1.11.6 h1:2VF8Mf7XiSUfmoNOy3D+ocfl9Qu8baQBrCNbo2CXQ8E= +github.com/ethereum/go-ethereum v1.11.6/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= @@ -594,6 +603,7 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -658,8 +668,9 @@ github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7a github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= -github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= +github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= @@ -675,6 +686,7 @@ github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/gateway v1.1.0 h1:u0SuhL9+Il+UbjM9VIE3ntfRujKbvVpFvNB4HbjeVQ0= github.com/gogo/gateway v1.1.0/go.mod h1:S7rR8FRQyG3QFESeSv4l2WnsyzlCLG0CzBbUUo/mbic= @@ -726,13 +738,14 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.4 h1:yAGX7huGHXlcLOEtBnF4w7FQwA26wojNCwOYAEhLjQM= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= +github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golangci/lint-1 v0.0.0-20181222135242-d2cdd8c08219/go.mod h1:/X8TswGSh1pIozq4ZwCfxS0WA5JGXguxk94ar/4c87Y= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= -github.com/google/btree v1.0.1 h1:gK4Kx5IaGY9CD5sPJ36FHiBJ6ZXl0kilRiiCj+jdYp4= -github.com/google/btree v1.0.1/go.mod h1:xXMiIv4Fb/0kKde4SpL7qlzvu5cMJDRkFDxJfI9uaxA= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -902,8 +915,9 @@ github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 h1:aSV github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3/go.mod h1:5PC6ZNPde8bBqU/ewGZig35+UIZtw9Ytxez8/q5ZyFE= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= -github.com/holiman/uint256 v1.2.0 h1:gpSYcPLWGv4sG43I2mVLiDZCNDh/EpGjSk8tmtxitHM= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= +github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= +github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= @@ -927,10 +941,13 @@ github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLf github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= +github.com/influxdata/influxdb-client-go/v2 v2.12.2 h1:uYABKdrEKlYm+++qfKdbgaHKBPmoWR5wpbmj6MBB/2g= +github.com/influxdata/influxdb-client-go/v2 v2.12.2/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU= github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo= github.com/influxdata/influxql v1.1.1-0.20200828144457-65d3ef77d385/go.mod h1:gHp9y86a/pxhjJ+zMjNXiQAA197Xk9wLxaz+fGG+kWk= github.com/influxdata/line-protocol v0.0.0-20180522152040-32c6aa80de5e/go.mod h1:4kt73NQhadE3daL3WhR5EJ/J2ocX0PZzwxQ0gXJ7oFE= github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= +github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 h1:vilfsDSy7TDxedi9gyBkMvAirat/oRcL0lFdJBf6tdM= github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097/go.mod h1:xaLFMmpvUxqXtVkUJfg9QmT88cDaCJ3ZKgdZ78oO8Qo= github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19ybifQhZoQNF5D8= github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= @@ -1126,8 +1143,8 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.15.12 h1:YClS/PImqYbn+UILDnqxQCZ3RehC9N318SU3kElDUEM= -github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= +github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1157,6 +1174,7 @@ github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0SNc= github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= @@ -1742,8 +1760,8 @@ github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= -github.com/prometheus/common v0.37.0 h1:ccBbHCgIiT9uSoFY0vX8H3zsNR5eLt17/RQLUvn8pXE= -github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= +github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= +github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1754,9 +1772,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.8.0 h1:ODq8ZFEaYeCaZOJlZZdJA2AbQR98dSHSM1KW/You5mo= -github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/tsdb v0.7.1 h1:YZcsG11NqnK4czYLrWd9mpEuAJIHVQLwdrleYfszMAA= +github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= +github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pyroscope-io/client v0.7.0 h1:LWuuqPQ1oa6x7BnmUOuo/aGwdX85QGhWZUBYWWW3zdk= github.com/pyroscope-io/client v0.7.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= @@ -1790,7 +1807,6 @@ github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= -github.com/rjeczalik/notify v0.9.1 h1:CLCKso/QK1snAlnhNR/CNvNiFU2saUtjV0bx3EwNeCE= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -1889,8 +1905,8 @@ github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DM github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= -github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4 h1:Gb2Tyox57NRNuZ2d3rmvB3pcmbu7O1RS3m8WRx7ilrg= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= +github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -1909,22 +1925,23 @@ github.com/stretchr/testify v1.6.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= -github.com/stretchr/testify v1.8.1 h1:w7B6lhMri9wdJUVmEZPGGhZzrYTPvgJArz7wNPgYKsk= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= +github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tendermint/btcd v0.1.1 h1:0VcxPfflS2zZ3RiOAHkBiFUcPvbtRj5O7zHmcJWHV7s= -github.com/tendermint/btcd v0.1.1/go.mod h1:DC6/m53jtQzr/NFmMNEu0rxf18/ktVoVtMrnDD5pN+U= -github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15 h1:hqAk8riJvK4RMWx1aInLzndwxKalgi5rTqgfXxOxbEI= -github.com/tendermint/crypto v0.0.0-20191022145703-50d29ede1e15/go.mod h1:z4YtwM70uOnk8h0pjJYlj3zdYwi9l03By6iAIF5j/Pk= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= +github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= +github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= +github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= @@ -1948,8 +1965,8 @@ github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1 github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= -github.com/tyler-smith/go-bip39 v1.0.2 h1:+t3w+KwLXO6154GNJY+qUtIxLTmFjfUmpguQT1OlOT8= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= +github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -1963,14 +1980,12 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.10.2 h1:x3p8awjp/2arX+Nl/G2040AZpOCHS/eMJJ1/a+mye4Y= +github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49uaYMPRU= github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= -github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3 h1:zMsHhfK9+Wdl1F7sIKLyx3wrOFofpb3rWFbA4HgcK5k= -github.com/vivint/infectious v0.0.0-20200605153912-25a574ae18a3/go.mod h1:R0Gbuw7ElaGSLOZUSwBm/GgVwMd30jWxBDdAyMOeTuc= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= @@ -2008,8 +2023,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= -github.com/zondax/ledger-go v0.14.0 h1:dlMC7aO8Wss1CxBq2I96kZ69Nh1ligzbs8UWOtq/AsA= -github.com/zondax/ledger-go v0.14.0/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= +github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN4c= +github.com/zondax/ledger-go v0.14.1/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= @@ -2156,8 +2171,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2 h1:5sPMf9HJXrvBWIamTw+rTST0bZ3Mho2n1p58M0+W99c= -golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= +golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2189,8 +2204,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.8.0 h1:LUYupSeNrTNCGzR/hVBk2NHZO4hXcVaW1k4Qx7rjPx8= -golang.org/x/mod v0.8.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= +golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2473,7 +2488,7 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2546,8 +2561,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.6.0 h1:BOw41kyTf3PuCW1pVQf8+Cyg8pMlkYB1oo9iJ6D/lKM= -golang.org/x/tools v0.6.0/go.mod h1:Xwgl3UAJ/d3gWutnCtw505GrjyAbvKui8lOU390QaIU= +golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= +golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2823,6 +2838,7 @@ gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= +gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= diff --git a/nodebuilder/tests/swamp/config.go b/nodebuilder/tests/swamp/config.go index f14243d342..630920609f 100644 --- a/nodebuilder/tests/swamp/config.go +++ b/nodebuilder/tests/swamp/config.go @@ -15,8 +15,8 @@ type Config struct { // 100ms func DefaultConfig() *Config { cfg := core.DefaultTestConfig() - // timeout commits faster than this tend to be flakier - cfg.Tendermint.Consensus.TimeoutCommit = 200 * time.Millisecond + // target height duration lower than this tend to be flakier + cfg.Tendermint.Consensus.TargetHeightDuration = 200 * time.Millisecond return &Config{ cfg, } @@ -31,7 +31,7 @@ func WithBlockTime(t time.Duration) Option { // for empty block c.Tendermint.Consensus.CreateEmptyBlocksInterval = t // for filled block - c.Tendermint.Consensus.TimeoutCommit = t + c.Tendermint.Consensus.TargetHeightDuration = t c.Tendermint.Consensus.SkipTimeoutCommit = false } } diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 58ff807a4d..5b99b577b1 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -17,7 +17,7 @@ import ( "github.com/stretchr/testify/require" "go.uber.org/fx" - "github.com/celestiaorg/celestia-app/testutil/testnode" + "github.com/celestiaorg/celestia-app/test/util/testnode" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" libhead "github.com/celestiaorg/go-header" diff --git a/nodebuilder/tests/swamp/swamp_tx.go b/nodebuilder/tests/swamp/swamp_tx.go index 956c8cf7c4..656b2f341d 100644 --- a/nodebuilder/tests/swamp/swamp_tx.go +++ b/nodebuilder/tests/swamp/swamp_tx.go @@ -6,7 +6,7 @@ import ( "github.com/cosmos/cosmos-sdk/client/flags" - "github.com/celestiaorg/celestia-app/testutil/testnode" + "github.com/celestiaorg/celestia-app/test/util/testnode" ) // FillBlocks produces the given amount of contiguous blocks with customizable size. diff --git a/share/availability/cache/availability_test.go b/share/availability/cache/availability_test.go index 47df434c06..03df47f848 100644 --- a/share/availability/cache/availability_test.go +++ b/share/availability/cache/availability_test.go @@ -60,7 +60,7 @@ func TestCacheAvailability(t *testing.T) { } var invalidHeader = da.DataAvailabilityHeader{ - RowsRoots: [][]byte{{1, 2}}, + RowRoots: [][]byte{{1, 2}}, } // TestCacheAvailability_Failed tests to make sure a failed diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index 07a3e801d9..761671b955 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -48,7 +48,7 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Roo "err", err) panic(err) } - samples, err := SampleSquare(len(dah.RowsRoots), int(la.params.SampleAmount)) + samples, err := SampleSquare(len(dah.RowRoots), int(la.params.SampleAmount)) if err != nil { return err } diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 146cb8554a..4980af031e 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -1,23 +1,16 @@ package light import ( - "bytes" "context" - "crypto/rand" _ "embed" - "encoding/hex" - "encoding/json" - mrand "math/rand" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" - core "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/da" - appshares "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" @@ -87,16 +80,16 @@ func TestService_GetSharesByNamespace(t *testing.T) { for _, tt := range tests { t.Run("size: "+strconv.Itoa(tt.squareSize), func(t *testing.T) { getter, bServ := EmptyGetter() - n := tt.squareSize * tt.squareSize - randShares := share.RandShares(t, n) - idx1 := (n - 1) / 2 - idx2 := n / 2 + totalShares := tt.squareSize * tt.squareSize + randShares := share.RandShares(t, totalShares) + idx1 := (totalShares - 1) / 2 + idx2 := totalShares / 2 if tt.expectedShareCount > 1 { // make it so that two rows have the same namespace ID - copy(randShares[idx2][:8], randShares[idx1][:8]) + copy(randShares[idx2][:namespace.NamespaceSize], randShares[idx1][:namespace.NamespaceSize]) } root := availability_test.FillBS(t, bServ, randShares) - randNID := randShares[idx1][:8] + randNID := randShares[idx1][:namespace.NamespaceSize] shares, err := getter.GetSharesByNamespace(context.Background(), root, randNID) require.NoError(t, err) @@ -112,6 +105,21 @@ func TestService_GetSharesByNamespace(t *testing.T) { assert.Equal(t, randShares[idx2], flattened[1]) } }) + t.Run("last two rows of a 4x4 square that have the same namespace ID have valid NMT proofs", func(t *testing.T) { + squareSize := 4 + totalShares := squareSize * squareSize + getter, bServ := EmptyGetter() + randShares := share.RandShares(t, totalShares) + lastNID := randShares[totalShares-1][:namespace.NamespaceSize] + for i := totalShares / 2; i < totalShares; i++ { + copy(randShares[i][:namespace.NamespaceSize], lastNID) + } + root := availability_test.FillBS(t, bServ, randShares) + + shares, err := getter.GetSharesByNamespace(context.Background(), root, lastNID) + require.NoError(t, err) + require.NoError(t, shares.Verify(root, lastNID)) + }) } } @@ -131,9 +139,9 @@ func TestGetShares(t *testing.T) { func TestService_GetSharesByNamespaceNotFound(t *testing.T) { getter, root := GetterWithRandSquare(t, 1) - root.RowsRoots = nil + root.RowRoots = nil - _, err := getter.GetSharesByNamespace(context.Background(), root, []byte{1, 1, 1, 1, 1, 1, 1, 1}) + _, err := getter.GetSharesByNamespace(context.Background(), root, namespace.RandomNamespace().Bytes()) assert.ErrorIs(t, err, share.ErrNamespaceNotFound) } @@ -150,8 +158,8 @@ func BenchmarkService_GetSharesByNamespace(b *testing.B) { b.Run(strconv.Itoa(tt.amountShares), func(b *testing.B) { t := &testing.T{} getter, root := GetterWithRandSquare(t, tt.amountShares) - randNID := root.RowsRoots[(len(root.RowsRoots)-1)/2][:8] - root.RowsRoots[(len(root.RowsRoots) / 2)] = root.RowsRoots[(len(root.RowsRoots)-1)/2] + randNID := root.RowRoots[(len(root.RowRoots)-1)/2][:8] + root.RowRoots[(len(root.RowRoots) / 2)] = root.RowRoots[(len(root.RowRoots)-1)/2] b.ResetTimer() for i := 0; i < b.N; i++ { _, err := getter.GetSharesByNamespace(context.Background(), root, randNID) @@ -160,168 +168,3 @@ func BenchmarkService_GetSharesByNamespace(b *testing.B) { }) } } - -func TestSharesRoundTrip(t *testing.T) { - getter, store := EmptyGetter() - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - var pb tmproto.Block - err := json.Unmarshal([]byte(sampleBlock), &pb) - require.NoError(t, err) - - b, err := core.BlockFromProto(&pb) - require.NoError(t, err) - - namespace, err := hex.DecodeString("00001337BEEF0000") - require.NoError(t, err) - namespaceBefore, err := hex.DecodeString("0000000000000123") - require.NoError(t, err) - namespaceAfter, err := hex.DecodeString("1234000000000123") - require.NoError(t, err) - - type testCase struct { - name string - messages [][]byte - namespaces [][]byte - } - - cases := []testCase{ - { - "original test case", - [][]byte{b.Data.Blobs[0].Data}, - [][]byte{namespace}}, - { - "one short message", - [][]byte{{1, 2, 3, 4}}, - [][]byte{namespace}}, - { - "one short before other namespace", - [][]byte{{1, 2, 3, 4}, {4, 5, 6, 7}}, - [][]byte{namespace, namespaceAfter}, - }, - { - "one short after other namespace", - [][]byte{{1, 2, 3, 4}, {4, 5, 6, 7}}, - [][]byte{namespaceBefore, namespace}, - }, - { - "two short messages", - [][]byte{{1, 2, 3, 4}, {4, 5, 6, 7}}, - [][]byte{namespace, namespace}, - }, - { - "two short messages before other namespace", - [][]byte{{1, 2, 3, 4}, {4, 5, 6, 7}, {7, 8, 9}}, - [][]byte{namespace, namespace, namespaceAfter}, - }, - { - "two short messages after other namespace", - [][]byte{{1, 2, 3, 4}, {4, 5, 6, 7}, {7, 8, 9}}, - [][]byte{namespaceBefore, namespace, namespace}, - }, - } - randBytes := func(n int) []byte { - bs := make([]byte, n) - _, _ = rand.Read(bs) - return bs - } - for i := 128; i < 4192; i += mrand.Intn(256) { - l := strconv.Itoa(i) - cases = append(cases, testCase{ - "one " + l + " bytes message", - [][]byte{randBytes(i)}, - [][]byte{namespace}, - }) - cases = append(cases, testCase{ - "one " + l + " bytes before other namespace", - [][]byte{randBytes(i), randBytes(1 + mrand.Intn(i))}, - [][]byte{namespace, namespaceAfter}, - }) - cases = append(cases, testCase{ - "one " + l + " bytes after other namespace", - [][]byte{randBytes(1 + mrand.Intn(i)), randBytes(i)}, - [][]byte{namespaceBefore, namespace}, - }) - cases = append(cases, testCase{ - "two " + l + " bytes messages", - [][]byte{randBytes(i), randBytes(i)}, - [][]byte{namespace, namespace}, - }) - cases = append(cases, testCase{ - "two " + l + " bytes messages before other namespace", - [][]byte{randBytes(i), randBytes(i), randBytes(1 + mrand.Intn(i))}, - [][]byte{namespace, namespace, namespaceAfter}, - }) - cases = append(cases, testCase{ - "two " + l + " bytes messages after other namespace", - [][]byte{randBytes(1 + mrand.Intn(i)), randBytes(i), randBytes(i)}, - [][]byte{namespaceBefore, namespace, namespace}, - }) - } - - for _, tc := range cases { - tc := tc - t.Run(tc.name, func(t *testing.T) { - // prepare data - b.Data.Blobs = make([]core.Blob, len(tc.messages)) - b.SquareSize = 16 - var msgsInNamespace [][]byte - require.Equal(t, len(tc.namespaces), len(tc.messages)) - for i := range tc.messages { - b.Data.Blobs[i] = core.Blob{NamespaceID: tc.namespaces[i], Data: tc.messages[i]} - if bytes.Equal(tc.namespaces[i], namespace) { - msgsInNamespace = append(msgsInNamespace, tc.messages[i]) - } - } - - // TODO: set useShareIndexes to true. This requires updating the - // transaction data in this test to include share indexes. - shares, err := appshares.Split(b.Data, false) - if err != nil { - t.Fatal(err) - } - - // test round trip using only encoding, without IPLD - { - myShares := make([][]byte, 0) - for _, sh := range shares { - if bytes.Equal(namespace, sh[:8]) { - myShares = append(myShares, sh) - } - } - blobs, err := appshares.ParseBlobs(myShares) - require.NoError(t, err) - assert.Len(t, blobs, len(msgsInNamespace)) - for i := range blobs { - assert.Equal(t, msgsInNamespace[i], blobs[i].Data) - } - } - - // test full round trip - with IPLD + decoding shares - { - extSquare, err := share.AddShares(ctx, appshares.ToBytes(shares), store) - require.NoError(t, err) - - dah := da.NewDataAvailabilityHeader(extSquare) - shares, err := getter.GetSharesByNamespace(ctx, &dah, namespace) - require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, namespace)) - require.NotEmpty(t, shares) - - blobs, err := appshares.ParseBlobs(shares.Flatten()) - require.NoError(t, err) - assert.Len(t, blobs, len(msgsInNamespace)) - for i := range blobs { - assert.Equal(t, namespace, []byte(blobs[i].NamespaceID)) - assert.Equal(t, msgsInNamespace[i], blobs[i].Data) - } - } - }) - } -} - -// this is a sample block -// -//go:embed "testdata/sample-block.json" -var sampleBlock string diff --git a/share/availability/light/testdata/sample-block.json b/share/availability/light/testdata/sample-block.json deleted file mode 100755 index 2bb8d35bfb..0000000000 --- a/share/availability/light/testdata/sample-block.json +++ /dev/null @@ -1,60 +0,0 @@ -{ - "header": { - "version": { - "block": 11 - }, - "chain_id": "7sUWGE", - "height": 12, - "time": "2023-01-19T01:41:20.109585974Z", - "last_block_id": { - "hash": "MfZsPUTgVOBRak+3xnKdwWEr2NQ1v095T4BYaWcdYTs=", - "part_set_header": { - "total": 1, - "hash": "/qyxxHNcuOyYBcaGw90PqAa2GdFErMHkkbH66BDI1hI=" - } - }, - "last_commit_hash": "sQXqALyCQ+tHFKOjfQhB3+lQuud0d3C2ohuOgrO6YX4=", - "data_hash": "Pu8CIJqgqCFYkaSvfeS5jvb6+ThXtJkXeRDZPnb1auM=", - "validators_hash": "X7yBLGyFWnZvyakBElaaaqyV0iCSNTINfm+puFxcX/g=", - "next_validators_hash": "X7yBLGyFWnZvyakBElaaaqyV0iCSNTINfm+puFxcX/g=", - "consensus_hash": "BICRvH3cKD93v7+R1zxE2ljD34qcvIZ0Bdi389qtoi8=", - "app_hash": "pqd8GP5/icLOzlysLtq9IupqfLmoxNVySLWNarJssbE=", - "last_results_hash": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", - "evidence_hash": "47DEQpj8HBSa+/TImW+5JCeuQeRkm5NMpJWZG3hSuFU=", - "proposer_address": "oFwwfI48D9wGpICbXtGZNlx6xOI=" - }, - "data": { - "txs": [ - "CpsCCnsKeQoTL2Jsb2IuTXNnUGF5Rm9yQmxvYhJiCi1jb3Ntb3Mxemdwamp4bWZhNm00ZjU5cTJsNjgzOGp1c2RjcmZ4eHc1ajI4ZG0SCF62zWQ6Pw9TGgLBGyIgo5vwRkbYLo3+Xm1IpV8weKHdFYfqkfDyt8a1vfthQQJCAQASWgpOCkYKHy9jb3Ntb3MuY3J5cHRvLnNlY3AyNTZrMS5QdWJLZXkSIwohA64OVjUldXAMb1uSZqrBNuYaatsk+CMGyQKNzY2vM5T+EgQKAggBEggQgIDpg7HeFhpAM88quyAFPvk6/4fuiolAJf3SmOLNv471av8uir6yHaJPdsusvOUQ6gK+/gwoksZ2Lytk48k3Gf4F+tcJO06ZBxIBBBoESU5EWA==" - ], - "blobs": [ - { - "namespace_id": "XrbNZDo/D1M=", - "data": "Jh6q4+BLrF+bOReK6omOI45dajuFKbRSc7FQR/jY67EHnUdz67HWrjZRie7EoF37n+OHbhnI4X72BLNdKpXYVy7J5mtlihMznSVU9FyvUDTy2tKmN2Xwc7bqTbSPKscM5j6r9fpWsD1cycNBMe6ZifQm+qu3+XNDSPdOzG1t318ByRuTAn5ttHFVynZfW3FWYVGSWNzaJ1ECzktus14FLIPIv5hvpia+VEWiBerbP0Oij2ckZWDLFIJsLC142JfPi1iGillBGNOR/fJGYfeoT3W+p0LaUF6BLNS/IbnZS+TrivnUu1zuw9lx+HwwaqKsnNWQpvuytihTKMlebPb+Bmk7FaAJchnnVYbmQhAbWC44xIykgTTQnZZpac00KCrBzCOECyt0eIEBClIqHs7z1nKQiMifqjFrTx95fJ/5wr28iLeJl/5zTuirh5XQbcqpJnXsSWvPIY69VsLunr4/MVVLLre0v4uPD/sZTgNR0Q0qMWp0G1ja2yS8H0PtENOkslxjD2QA66fvQRgwgHLcDduAVR39tyRLcjK0ofnyD9EBm/q8NS/+kgcZkz/5nA+1JM2HjNFfuqwd1+l+pi2Y+11DbPJloTZyk18CewdZVhVclREb/0+ih1DDD2N/RAZG2Thkfx0vge4mwV8MSSDxnHDVLABSkjVGArj2JOwkbQeRWXMp94hZl/O9f1lRxqibcMhme/OsoaqxAErCulqbckaRiUjvrBnAyIM5Bpr13i3ZFxHxJTV7NkXEVbgKfeN3F9ewj4CGCT0Kcq/oWdzJ+RjV+fsQa9/RBLUPwdTfwPDH0FjMlBcuvMfU4Kx3LOHNjr+Gs+BoEf5AX0IRJu4C2KpxpFUl8QYQLK2q4+NKC3KHOcp810V+4auinkLFWMkK1sHruqcQ6sFwxiYQUEkmAavTQTEhJaK0fFTNImnAqPXzz8KB4oGcFnXgGBpsP7aMIVfuI+vY5H5p4Stv6Oa4V953KBOZbAMT3Bo5br7wmmeXa5BAWUSAg7xxLxJqTkx0ZC9tm6Z4UImLpL3g/ytnD5b4fdiJwpzbAx/q9zHolTPz9lK/vAt/MqejRqu5YqkQMO3TY5hNLUWUh6Ty3+XZybfS68xNE/hUpz1AhSKsKX9Mp32rQTauXGXT0BveV+IyNC11Am0pUjRX6vcf3CwN+LRTdNkAZLTEJTChFNSh7a3wEsqxJ1jpEgiSiWDfGcR0aDI5Q/9WXFUvpSlJ3KUHx/o+wmF6SpNmjDI3bq+PAe9/wgpWiciepxcytMTOj3p7TNmK9fCstLGvlaLQczrN0iZdQ93ZvFzA0pQjBSo/42johGZSI9Z/07+BEavhxPOT1zOUJcq9t3VFkLsyYM3BmwKRJ8VvV9aHnE+wKK55A3BB67s++mAkbWB0ImMNlybEUgLq4nm/NC86BZrxUIf98cgEEVEOXpLP9bYGxe+/qIJIlGzwkhk7Uwbzk+0S0IFJnuPSPV6CWfoMnWEsqY+zuti/0HQzms2hGeQsIeGG8lL6mqhorg7TAzShrLF5d55TiT2bY4UZJHxHXjY7CbLOxsQBA+/moo4u1l1Kk93CkNtLGMG5yQiTZsfejYIlpZ4fnPM50sxZG5cbm/MLTW9/yL/Xjp5tt4Rle/feK/CvXAg60oxqXQGq9BDxpsi+dRJ3Q8qyaM7fzIhtfHcGTzWxaqGyAD2XobcqdV3D1gOqzslUwu07NbR+JztroAt7hs5sbkPCSOoRC1LbaXEZ6FZu9j8DbJ2/lnaZTXiOjqLDI0cbQCXVul09nRBldILXCN7baPAXqr0RcW7pLiTEtYkmP+3Y1MU0Epz3xDUF/TteROV4C354ta4L5J7JfWBXEJ5JslWXWTbPddDAA9tdCXa55UhMWWRzLUB6pnpMZRMih85AXKAzsLJeuAPrUBUg4nuFmK/NUALe9jVrFuJsPwZB3ILIwwLvd2T4OZf5a9kH19JMEfWm+x6NcoESus83WmczGzOwjgIhe6mVHAuyzN1cTBoEg7PCRhOyhplRheGC6XqpfA2xvio0+y2WxVOrVvyrqGMTeocqB0NcrMtyZQSZTYteGHYD01uVWe8vbpr1ARpF+Diy9J7Cuud9OeRTtSiUU4LgRVkbVkgtcH24zYUrC8uBkkirOTD8HiEQVUbKm7UO/BXhZ3SGtv6kYISRdiQo0BSdQxwKmThGdF2zdj7OhdAzSXjY2RtrWMAsivd91PBLnDcQGGr+r9Va9CvkBmeB6VmdZ8IF/DUwrD87JLukz2KsTDJM6wVkq5bYYuWhKxloauwr0lvYAtLMQov78sAZm1I89py8GH3cZbja8ctZ0Wz0eaO0ziHBEy3DM0bVrfBwtpY1pyadwgfLVxRkmqMgTR3zaVVQdOkP0CwuR3FdNJJqNda5UZD72boY6vhJdZie4TXf06LBIt5Ptxjh5MxYJBn8hS2HONaMNxofHfb9O+zRYpWznpLSq41F91llLGteBequJCWjw8Skdsztfu/Mvcu+KeliY6VZ4fcT/xST73f7N0qV5ZvBIZ6v2wpf4pJrklyHVCdcdKG3jUUJ5DQNiDzQCA5Ocmjj5k7OTRrE7zBwcfFtVQcOFEncyzPC4c9xAANaeDNz5WNUcPTESvqRpzZxgETo4smZsJMTwCmGoyTm4qHZ3T92y08smZuZO3JuMqtyxRGM2IC3wsFQqrChHcVY7ND0SMc7X5LWVc2j7UwQ6UbnBd6grVfgd/bfHzvPGm31R0uF3//xPGs0iE3f/FVsJE0dScmS0iXx8c8bwamThzD9YBRCNJge7nJIF+eL1Kf+pqgwzfKX6eb8eUIXP2+EJNvGUBhr/BImeULf1edrtqjYWCDQWYn3ISLpKZcv0EVFU/Clv1EkU1wXmGEeTY9xOh3wevSwUAxJ0At5J4qjRFAlFuFD8dqb4PoxeIePaIEpI2SWn9hgnhtTeCWPcMK4HCqdUSUvJJ4ilc5378uG4WrO/ulxicPYAWrocsqlRHmNit0gKD32YwlRjq5W1m1eunr0SJc299kjFm+Jhg6lQsPcw8BhMnTipse3K5n6uiwa8cwpeHN6mraR+ZwFmZhCNTHsaYk2hbnFGCH4h6W7RT9QH01Xq0F8N130oegxOEsI4GWaJS5KrBa8SqCHtSI2O/4eISdv98tkwx84h50hCU4kQ2o3mEmrTYCfmY80MtjZdE+vX+iXKlmX1nZxPxxBb/loU/mKcXPvapeBx3hkrxjnFBKDDgg8NEDj8IU3SRSewVXGBivKbyk/zUZl+Ow49+636IzQNtH0RSac1ox5FqVOxidaQ04si5lCROJaLuuhfo1CqgYmN502KL3u6gLQdk9pvwoPOJ7iYEx63VwBy8205lHB38o7bO2FmzaSpnVsCOxcK32ExSY92l659n+Wrxjj0dSqDj8VaKiGnJn9FKRNZLGMzONuXgCLikM7ChiGHRO1AjnJmHnunsQcbYdcHhyPlRim19oX0JhmDxrcqhAZbD0a3PAL/MBs671MDQ4GL0blXIYBzSAxZgyEl4Zn8FMC4xeOeMChLJ/aaKmhJx8k+S6uUCQCXuHktXLGq0COjEIx5cuvYmS+hVgqWv+UrHj7VkYbLPaufFfKe1neR7pG8SK4nRSwa6KB5IXUfBATvT7PU4jiXEo2JPuDQUIh7v1uloDjOm9OdMLZc7Kfcz6RI0yTzlmCTjWXRgp9WzztZsldb1obgW7t2XMEgKBR8OOVKUKbeqo1nWUMKp4Tz5jQnos0ixufGJ5iac57rpvATGkIAPhIMNTj35dRInpn4qI21TpPpOTYRnJVL4e0l6JjPTvy+JzS6Qec9JqJy0HWEmQmyw9/+vCGWR50tFyfhtPZzW+OvpbSs+sF5Xd4J//SWL5Y1iC3XhnNa6Uc2buY3LyLGZxHPdjCJCZ/I+7gdz5xUNnvzFtfUoLM5qoRsyDzJmdfL3fFHq5eDA1MkDqSypkixT3mkLnNtN5nnGIsa/2wKLoBSGEwCjsQGLesVYRibJ236rI78yCslTqfgrEaVlBnIPFWKdnl9YlR12G3WlKjplqbCo3CMSUcfVOR+f1CLM+IOclEwDszPmvwuJc/p8sh+Xhsbu9bOTuYo7S4HV73+M6Ct6Y4WuJEw/xmTxlNjhMOJi/lZvygzYFT8vC2S2FA5HLthkQkwE+xMFKBS4i0/hq8Mwi/bqLShkD+N0nCbo+jT9SlN8rkEhTzSgu3aaStd33N0/56OYee4oscdGOD5Io7RH636aMQpq80X9irkb7E4jTAK3aGQJMGfwz8BtOJ9rDJ4yv3uJHmwlQhzIwwQENPOnOFipkbWBpalmnYr/WoxgoGwR/ZOHXtA1J/beDHtjLB6Y3P5KO5ud7LeX2QGu3b7BAc19UeW+XfED1chzUyz1O65149H+NQaw0vVajpwYaRuAkZ+6B1W5pLP+5k5Gyc6TTmQR+4EucAy1R9dh6p29S2pBNWvHzaxKK1i/R32wjxBRr6M1X2Ym+k4DyZIeFnqH0a/UKqStyP31Z/qNCa//Nbcx2XxM9RgUJk4LH2mZRmaKUkesxwmQ+WB+dNohq+7uxDmTOyjcKOClcgcaSlv62htt0RiqkYSW2+qji9My4fHMQpPtKXWkbWwhXmg7ZgtB3Hx2ogyV5HVR5gLJ6Ydaq5aYQVLEVCYdr0pVmfzgHlDCZwj36S0EFZaMGWB8vIfWgfjop+VE4=" - } - ], - "square_size": 4, - "hash": "Pu8CIJqgqCFYkaSvfeS5jvb6+ThXtJkXeRDZPnb1auM=" - }, - "evidence": { - "evidence": [] - }, - "last_commit": { - "height": 11, - "block_id": { - "hash": "MfZsPUTgVOBRak+3xnKdwWEr2NQ1v095T4BYaWcdYTs=", - "part_set_header": { - "total": 1, - "hash": "/qyxxHNcuOyYBcaGw90PqAa2GdFErMHkkbH66BDI1hI=" - } - }, - "signatures": [ - { - "block_id_flag": 2, - "validator_address": "oFwwfI48D9wGpICbXtGZNlx6xOI=", - "timestamp": "2023-01-19T01:41:20.109585974Z", - "signature": "mPRPdLbp8cQcZ2w6CmWlmE+4lVzX9YgxzxOBp7lYTT8jYmehJC62sC/Nt/X/ezVW7u9OCXZjGG2oIDYmabKsAA==" - } - ] - } -} \ No newline at end of file diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 9ca481d552..4e673ed8b8 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -5,13 +5,13 @@ import ( "errors" "fmt" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/go-fraud" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" pb "github.com/celestiaorg/celestia-node/share/eds/byzantine/pb" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -118,7 +118,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { if header.Height() != int64(p.BlockHeight) { return errors.New("fraud: incorrect block height") } - merkleRowRoots := header.DAH.RowsRoots + merkleRowRoots := header.DAH.RowRoots merkleColRoots := header.DAH.ColumnRoots if len(merkleRowRoots) != len(merkleColRoots) { // NOTE: This should never happen as callers of this method should not feed it with a @@ -157,7 +157,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { } odsWidth := uint64(len(merkleRowRoots) / 2) - codec := appconsts.DefaultCodec() + codec := share.DefaultRSMT2DCodec() // rebuild a row or col. rebuiltShares, err := codec.Decode(shares) @@ -172,11 +172,19 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { tree := wrapper.NewErasuredNamespacedMerkleTree(odsWidth, uint(p.Index)) for _, share := range rebuiltShares { - tree.Push(share) + err = tree.Push(share) + if err != nil { + return err + } + } + + expectedRoot, err := tree.Root() + if err != nil { + return err } // comparing rebuilt Merkle Root of bad row/col with respective Merkle Root of row/col from block. - if bytes.Equal(tree.Root(), root) { + if bytes.Equal(expectedRoot, root) { return errors.New("fraud: invalid proof: recomputed Merkle root matches the DAH's row/column root") } diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 6604ea1e17..6e413b17a2 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -34,7 +34,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { // get an arbitrary row row := uint(squareSize / 2) rowShares := eds.Row(row) - rowRoot := dah.RowsRoots[row] + rowRoot := dah.RowRoots[row] shareProofs, err := GetProofsForShares(ctx, bServ, ipld.MustCidFromNamespacedSha256(rowRoot), rowShares) require.NoError(t, err) diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index 67a6ee54c2..b9c8ef414f 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -36,7 +36,7 @@ func NewErrByzantine( errByz *rsmt2d.ErrByzantineData, ) *ErrByzantine { root := [][][]byte{ - dah.RowsRoots, + dah.RowRoots, dah.ColumnRoots, }[errByz.Axis][errByz.Index] sharesWithProof, err := GetProofsForShares( diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go index 501f4b40d9..9cffe6eb18 100644 --- a/share/eds/byzantine/share_proof_test.go +++ b/share/eds/byzantine/share_proof_test.go @@ -31,7 +31,7 @@ func TestGetProof(t *testing.T) { var tests = []struct { roots [][]byte }{ - {dah.RowsRoots}, + {dah.RowRoots}, {dah.ColumnRoots}, } diff --git a/share/eds/eds.go b/share/eds/eds.go index b183731a18..4e96fd684e 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -20,6 +20,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -219,11 +220,12 @@ func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte { // prependNamespace adds the namespace to the passed share if in the first quadrant, // otherwise it adds the ParitySharesNamespace to the beginning. func prependNamespace(quadrant int, share []byte) []byte { + namespacedShare := make([]byte, 0, appconsts.NamespaceSize+appconsts.ShareSize) switch quadrant { case 0: - return append(share[:ipld.NamespaceSize], share...) + return append(append(namespacedShare, share[:ipld.NamespaceSize]...), share...) case 1, 2, 3: - return append(appconsts.ParitySharesNamespaceID, share...) + return append(append(namespacedShare, namespace.ParitySharesNamespace.Bytes()...), share...) default: panic("invalid quadrant") } diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index 5eac847ac2..8df05d7d53 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -14,9 +14,11 @@ import ( carv1 "github.com/ipld/go-car" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/rand" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -30,26 +32,47 @@ var exampleRoot string var f embed.FS func TestQuadrantOrder(t *testing.T) { - // TODO: add more test cases - nID := []byte{0, 0, 0, 0, 0, 0, 0, 0} - parity := append(appconsts.ParitySharesNamespaceID, nID...) //nolint - doubleNID := append(nID, nID...) //nolint - result, _ := rsmt2d.ComputeExtendedDataSquare([][]byte{ - append(nID, 1), append(nID, 2), - append(nID, 3), append(nID, 4), - }, rsmt2d.NewRSGF8Codec(), rsmt2d.NewDefaultTree) - // {{1}, {2}, {7}, {13}}, - // {{3}, {4}, {13}, {31}}, - // {{5}, {14}, {19}, {41}}, - // {{9}, {26}, {47}, {69}}, - require.Equal(t, - [][]byte{ - append(doubleNID, 1), append(doubleNID, 2), append(doubleNID, 3), append(doubleNID, 4), - append(parity, 7), append(parity, 13), append(parity, 13), append(parity, 31), - append(parity, 5), append(parity, 14), append(parity, 9), append(parity, 26), - append(parity, 19), append(parity, 41), append(parity, 47), append(parity, 69), - }, quadrantOrder(result), - ) + testCases := []struct { + name string + squareSize int + }{ + {"smol", 2}, + {"still smol", 8}, + {"default mainnet", appconsts.DefaultGovMaxSquareSize}, + {"max", appconsts.MaxSquareSize}, + } + + testShareSize := 64 + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + shares := make([][]byte, tc.squareSize*tc.squareSize) + + for i := 0; i < tc.squareSize*tc.squareSize; i++ { + shares[i] = rand.Bytes(testShareSize) + } + + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), rsmt2d.NewDefaultTree) + require.NoError(t, err) + + res := quadrantOrder(eds) + for _, s := range res { + require.Len(t, s, testShareSize+namespace.NamespaceSize) + } + + for q := 0; q < 4; q++ { + for i := 0; i < tc.squareSize; i++ { + for j := 0; j < tc.squareSize; j++ { + resIndex := q*tc.squareSize*tc.squareSize + i*tc.squareSize + j + edsRow := q/2*tc.squareSize + i + edsCol := (q%2)*tc.squareSize + j + + assert.Equal(t, res[resIndex], prependNamespace(q, eds.Row(uint(edsRow))[edsCol])) + } + } + } + }) + } } func TestWriteEDS(t *testing.T) { @@ -164,7 +187,7 @@ func TestReadEDS(t *testing.T) { loaded, err := ReadEDS(context.Background(), f, dah.Hash()) require.NoError(t, err, "error reading EDS from file") - require.Equal(t, dah.RowsRoots, loaded.RowRoots()) + require.Equal(t, dah.RowRoots, loaded.RowRoots()) require.Equal(t, dah.ColumnRoots, loaded.ColRoots()) } diff --git a/share/eds/retriever.go b/share/eds/retriever.go index 5d3c61cabb..b2dcc4ff7a 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -63,11 +63,11 @@ func (r *Retriever) Retrieve(ctx context.Context, dah *da.DataAvailabilityHeader ctx, span := tracer.Start(ctx, "retrieve-square") defer span.End() span.SetAttributes( - attribute.Int("size", len(dah.RowsRoots)), + attribute.Int("size", len(dah.RowRoots)), attribute.String("data_hash", dah.String()), ) - log.Debugw("retrieving data square", "data_hash", dah.String(), "size", len(dah.RowsRoots)) + log.Debugw("retrieving data square", "data_hash", dah.String(), "size", len(dah.RowRoots)) ses, err := r.newSession(ctx, dah) if err != nil { return nil, err @@ -121,7 +121,7 @@ type retrievalSession struct { // newSession creates a new retrieval session and kicks off requesting process. func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHeader) (*retrievalSession, error) { - size := len(dah.RowsRoots) + size := len(dah.RowRoots) treeFn := func(_ rsmt2d.Axis, index uint) rsmt2d.Tree { tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(size)/2, index) return &tree @@ -169,12 +169,12 @@ func (rs *retrievalSession) Reconstruct(ctx context.Context) (*rsmt2d.ExtendedDa defer span.End() // and try to repair with what we have - err := rs.square.Repair(rs.dah.RowsRoots, rs.dah.ColumnRoots) + err := rs.square.Repair(rs.dah.RowRoots, rs.dah.ColumnRoots) if err != nil { span.RecordError(err) return nil, err } - log.Infow("data square reconstructed", "data_hash", rs.dah.String(), "size", len(rs.dah.RowsRoots)) + log.Infow("data square reconstructed", "data_hash", rs.dah.String(), "size", len(rs.dah.RowRoots)) close(rs.squareDn) return rs.square, nil } diff --git a/share/eds/retriever_quadrant.go b/share/eds/retriever_quadrant.go index 8b8037ce85..3d616e9cd4 100644 --- a/share/eds/retriever_quadrant.go +++ b/share/eds/retriever_quadrant.go @@ -52,7 +52,7 @@ type quadrant struct { func newQuadrants(dah *da.DataAvailabilityHeader) []*quadrant { // combine all the roots into one slice, so they can be easily accessible by index daRoots := [][][]byte{ - dah.RowsRoots, + dah.RowRoots, dah.ColumnRoots, } // create a quadrant slice for each source(row;col) diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index 1c98c133c4..e90216de13 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -75,7 +75,7 @@ func TestRetriever_ByzantineError(t *testing.T) { require.NoError(t, err) // corrupt shares so that eds erasure coding does not match - copy(shares[14][8:], shares[15][8:]) + copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) // import corrupted eds batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) diff --git a/share/eds/store.go b/share/eds/store.go index 8222d04dd6..f01e96a24b 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -280,7 +280,7 @@ func dahFromCARHeader(carHeader *carv1.CarHeader) *header.DataAvailabilityHeader rootBytes = append(rootBytes, ipld.NamespacedSha256FromCID(root)) } return &header.DataAvailabilityHeader{ - RowsRoots: rootBytes[:rootCount/2], + RowRoots: rootBytes[:rootCount/2], ColumnRoots: rootBytes[rootCount/2:], } } diff --git a/share/eds/testdata/example-root.json b/share/eds/testdata/example-root.json index b687e3cbeb..999d6301b6 100644 --- a/share/eds/testdata/example-root.json +++ b/share/eds/testdata/example-root.json @@ -1,22 +1,22 @@ { "row_roots": [ -"A3I8MUKAXaEf7JDjUxllL5CGD0/uS8k2l3KpbN+u5fa3tGL1h9FSqZhCJFcZhzAT", -"IA74YwcC1tFMkQfD9sE2NO7hwDtjo37O1KiiptYPCgAQWQnv/G7bxu6mUJ2s3Ljk", -"ZskuRtu6YfK9pTqLAOsfLXbvjrvDD+80xoj/850Z4YGvmAaRH5vFXraZNWgEQNyr", -"w9Tz/Q7t/JfX7McKTWvNjhqi784y+nGCYXrQtu4jy90kx0irPJcVTXTQ7v6Xnjdf", -"/////////////////////1MCHZjf0ySoNlpt+h8Vb6QFm7BHYJXI14SMiNRgwA0K", -"/////////////////////2dOgrZG2ZT+KIrfjoPmTcGpqXdsdFZE6ZHcF0l+fL6i", -"/////////////////////7OJEeOYA6VsagglTnpIysAtyK3m3u4d8Zd66FuqX0+M", -"/////////////////////+hYZyJ10gx7Brp05g/MLtSV4z7SLLwGcMsXZI/Fgw0J" +"AAAAAAAAAAAAAAAAAAAAAAAAABPYEuDlO9Dz69oAAAAAAAAAAAAAAAAAAAAAAAAAMcklN0h38T4b/UBC/Cmr5YWmjmmxvi1e35vZBW14b8gDHBoTFVvY6H4J", +"AAAAAAAAAAAAAAAAAAAAAAAAADxyZecUZD41W5IAAAAAAAAAAAAAAAAAAAAAAAAAh8vQUZ38PaWyeUs7dQhphIuRIKiGaTr4KFwEhMRhejTd6/4NHdnKTDyY", +"AAAAAAAAAAAAAAAAAAAAAAAAAKDQatbQSwQ9uJsAAAAAAAAAAAAAAAAAAAAAAAAArtdqXCSsM1OlVCRZqqfZDnEO9eC5cwlgy5MQHb2g4NLr7nZYTruiOoz7", +"AAAAAAAAAAAAAAAAAAAAAAAAAMeUhM8LZBo9sWwAAAAAAAAAAAAAAAAAAAAAAAAA8PtvJpbDc4APKOK6MT1k61HuQXwauWw3nFWwr9pSljiYMv6jjjdLDF8o", +"/////////////////////////////////////////////////////////////////////////////xnHmhDh4Y8vfJrgewAcvLWpvI5XOyATj1IQDkCwvIEh", +"/////////////////////////////////////////////////////////////////////////////+qngp0AfoykfXwsMBukRtYxNA/bzW0+F3J7Q/+S1YZJ", +"/////////////////////////////////////////////////////////////////////////////4WNPrME/2MLrIZgAUoKaVx2GzJqDcYGrBg+sudPKUDy", +"/////////////////////////////////////////////////////////////////////////////6HdebpaHl7iTpLvmuPvtQNnkHfNOPyEhahxbVnIB2d1" ], "column_roots": [ -"A3I8MUKAXaHD1PP9Du38l5n9/iXzaDzWlVrLnsRE5zp+rEeyH2A0is+czwU/aHUn", -"D7uVznpBCw/S2nVDrzgGx9Ditcfzy/cL2Zdn+MBV31zeOgzCv+zXA0QP3SQRWXlb", -"FQDYNIesVPTUn9q9Ybq0VqDgprQbzKsMGrX+7kTlnbdOu434iOmx4P+vZVGjXAsm", -"H+yQ41MZZS/X7McKTWvNjoZId6AiHkxnGRsmYKuh4EvQCy+eEj19Q6Z5eQoMGWZL", -"/////////////////////1wS3guBpJRGNMLQMkEwgVOuXrTG3gUwJwI+nybYxTRF", -"/////////////////////1Eg48mqxsQejtwUIJkwrjEYCPvDYCRYm+dPXVitBCjl", -"/////////////////////zDz519rhl2FfQp6S3hR887dRSi6zN+Qs4PwIFdRqbxP", -"/////////////////////zARSBQBFxvGNpLt8I1qtbBOyEfFOXD4V/beDdU1kXPm" +"AAAAAAAAAAAAAAAAAAAAAAAAABPYEuDlO9Dz69oAAAAAAAAAAAAAAAAAAAAAAAAAx5SEzwtkGj2xbESyOeamsjGWUBQdAQoiSl+rMtNMo1wEtfGQnFS/g+K+", +"AAAAAAAAAAAAAAAAAAAAAAAAAC3uK6nhCxHTfBwAAAAAAAAAAAAAAAAAAAAAAAAA1fxnqHyO6qV39pcUQ8MuTfJ7RBhbSVWf0aamUP27KRY0II55oJoY6Ng6", +"AAAAAAAAAAAAAAAAAAAAAAAAAC6DkYeeBY/kKvAAAAAAAAAAAAAAAAAAAAAAAAAA47rxk8hoCnWGM+CX47TlYWBeE2unvRhA/j3EvHdxeL1rFRkaYfAd5eg7", +"AAAAAAAAAAAAAAAAAAAAAAAAADHJJTdId/E+G/0AAAAAAAAAAAAAAAAAAAAAAAAA8PtvJpbDc4APKAk5QPSH59HECE2sf/CDLKAZJjWo9DD4sLXJQ4jTZoH6", +"/////////////////////////////////////////////////////////////////////////////4lKCT3K11RnNIuLNfY+SfDZCYAE2iW0hjQHIVBpoN0q", +"/////////////////////////////////////////////////////////////////////////////1NpYcgayEVenbFeEO5LJ1j1/1sD+PvZWHDv+jqT1dLR", +"/////////////////////////////////////////////////////////////////////////////8FOWVuCU0rTzUW9tP2R47RmTBvwXX8ycKrMhgKEi1xa", +"/////////////////////////////////////////////////////////////////////////////7K5SoZ3HF5QgPvIXpKSr9eT4Xfiokc3PUMmXE4pBDTf" ] } \ No newline at end of file diff --git a/share/eds/testdata/example.car b/share/eds/testdata/example.car index 487bca4653922f3b43ea1e6639470f051c7a2534..4d33c0ef33300d68c8f3a05f7e09c65409c30b9d 100644 GIT binary patch literal 74051 zcmb@NV{m0%yLRKGV>{{Cww;dcq+>fB+h)hMZQHhO+fKgsUw@}gSKfL~J?rnNYhAl) zk2%+>x$o@*lVfOWZEbJ&dx=6-9{|w%0?_9IP#xfZUr0-cTl<_-x2G%rn*q=d6SIms zUGreQ^9j6>m+l7?M4DE2P-|_bE`wS+TMq$aV&7l1hyoGkK9OB321Gy z|55Y#f4(pVQ{cC^qiEezw%h=)+vFjoPEQ`b*7j z%XRFK38X_H<5uEs-M+4l7Ls{kJ38M7cVC;+FXwi16bww=E1dl!&i_{PXFvH%&7XDi zjh*i<=y19ZLaNN`%8x1wgM_*0j_#?depoPg*2w4Of7JY+m(6F0_&4s6MMYz#urTHa z9o7X%-K3!q$@YejU<^ ziK$$S-FDi^5n<&PB>5>#4#Ko)>}Y6fXKHN)_zCiVed_#KTEDQp*~#s+8E?%ikPQSK zDx-?C$WRT=umvG((MK;KWo86tXR)RdR?|cYdjRAcR-{GBH@1AGa))<)Hw_TWN z6U0i`3pI*WVNlB7MSeyX=_ppGg|sEJXZxBk0+xSfl&LA*RYHrG?=B| zZzht`pOxBzKp0c`vXU!J^!b!@%nUtMa?ca6FAfGE6IO53`0x*uXnWj)0B>p!QT&G* zVe^@D4M{87KIw0}E98Q1&Jp}*v=~k_lFHff#Ip`Z-@ZfN?V&Jxip6X(ElIf9J>9Fs zHLUmpGnX;Ml_j~&?jwVPa0cltVti^|;1=o<408^}sLqsMs$eBvzAh|6>NWM!;P>C@ z>uM^t!|zreAG(76UW8;D<|ogH+elY5ET6rQaC1C--2s*Yu-iRcA9zf3b=hiToqjOx zfs80=z!0vb_zu`img)qL;tmhNeVgOq71Iw;OD?X`7!cNx#Is|2vCN7WlQ>aKT%Lvm zzd)Pg={XD+e>X>KqZv)hn-I!?v|HcF34lkFZWdCqq#oCM%XEYFrGQ`VtA8LLcE3;< z7bE(Xs*9GXU})2$^giHQG|hF!)(bCD#+Kk;9Qzag63%)HLnfU{evW*86PPlF?h6O^ zB0xsXB8?6tJYITan|`(}0zFRw47EJIPS4b@?GNIx4jOnAKC)j`#VC~z6?sHF-y12> z^kf@@lqrnnsD5A^Q!bDkoi(6#fmXtp#i)F9ZRu)SV1ABxBWldd32VgQ$pUktAoaq) zmmehn-XtH37eChkp}urTh@#ezA?=nVIoU*bigoW*|Ut{K7?&X^e8g8P289PX+CbkzN|RE!KdQZS ztNu7*T|QU?LT6*$1kT|@31|>0IQpng%>`_Rbmj~kiZRr8Ug;XrCz7>5g&8iZ$g5ELeWLn5+h7Z|d;EwONPTlVdn9eXkxBxcDBOm-W zd2+VNS-TOy$b(hPX6*LbS=#O#Z|spuZo&fI13d#tL(CZ0=ox-BmdL|1eUk)Jg)Mle z072xjUH<(RKc)oM78&=Py3tTUCY=Xslm&L_#cCa@H5|5N3c^8fVj!G|X%GaT6&3-E zfQ;@U26ZX1aGD9X^khUU63<(6K$k7Ojr_$p(}hM{`DH7MdY(qF=7;7EFEq117IT

uV63KN<4CdSW_vibBAyP55LvrQkQL-J<;&bqeNo1z>))&to zBoCU{l5>67!g*?9s@HSyYcSCu7?LLiSJdJ}tP!Wnd3ugXhrm#48Nw{#%h;SJZKzuz zH(r&9cB8%&@XL3xKhTR=c$(B0)qGg;hZ?nP;LQV(X$-xr8sgT9I{aAs0pMR8`xE{W zGNp+fVel3LGHk!LHBK2DzlJ)Jt;0j~soqMKu-~JJJ0R#y>}Rsg-s0x_K=Yf()fH1B z)}sR9@zGevLqY)+#nGds+p?u*0B;aBR^v};APAOqeT4kyZ3}okE7X{>Op0^CG`=X9 zB0k|;hW!{`$VlVwECK?QGpu(vc5}@n;wU7ei&cj_1DMX`YR~}%GtD;HfEcadSK;2v z8lC>KGpxcbgx0=2+Y85S;kB1gUp4LV(J-C0!>v^i1Kl%0-tLfVg(=H5c*~USY1g$S z)-DCK_Nc8?!yV1(IkUJHXM%%(l7Svv8xd=(6T!xnuY8f!|BOEUZP(-90pZ{16}I7_>uQ~ng@Lg-TVwf0Ap)sA*GIy&>sLDFQ* zSb?Ybs>NcKYP>rkZm{W2d#n?AmZy#(x2$slI|$+Fq>k*W_hDp-H1_L^ROwidaTV@GYkTWb&MeJWUkdo;k-#6Alj0p%Qu~e6*{NvM8a%M0uCI;JIU5?2q|dJ} zYfZ}GFOK~Qe+j2PL$NYJAVmbB*7-CCr-SxJ=#x_k-`TN?{5hRKPbcB`;s8+zMZ?@m z%@(z-y;|2N=a{H?wX|@qHr;2g@iSJYFLQb}=CY{9^Q(lh(W;plL>oKnoXz_D0-Y$- zcGNdKkfY>~av@F%>-F(Cj1|9aVk3amBQ&g_a}R1tTH(%a8!rnsD2aB@v$IC3TKP3e z`YJ0&#TygN#jtR**-hk0~?} ztP&DH3X5RLCk1`CnkqX(x&uM2=i&syb-j+Fyn%A|)#Y1(rx5Pg-u^C(^ouoilN}EW zV7L<8e=gCWym4{crl6#z=Lu6}S13HDHx4;uRn}}aK$H(*^MW7C%D9_%BS8>^*+w>~ z=g5F9*R+w-M9a*dCSt3tD{bps24C<-DG~@B1d6 z00g<(!k$={7YN%Ospd{r#A*i=BpS-S|N3J-25AXjY`&d%fO`~d(!Ad3X;sGNKx)uc zLa}x)j%>9)H<@sL<$nQM>hCS55Y@z)ZP5wYGf4MrMI-EySRt)1!(YNY#LYkfBZ>6cxR17dB~Wzgi`N5IfVo9o z`pD@))M*}08aJu$^;LxXu@sG|#6aE?LA$6}Nk7~mJf%>fWzMcQq6y3*E)iFLgU*wv zT?bQBq;aE&TlTbPhaxXN6|u3}J5Zsr?rmMdVch5Qcy}kR^qU-VjsIZ!gb|_!Q)z29 zSBxmgS#o(;KTSC+r6{uUROK4v8{P~-&?VXzx@(xOj2CAu&1?uaa!&>=OLPx?ru#e; zuL5!a1Fnd*rQ<)ELK6#>A;qny_#}sYXNXLbb*E7wDW4X-98c-u79YSuz_3mM06Ap< zGi2pH#=N}^ z*s2Ve!!UPr&Sh_?#c2)B-P%fa)a0P@n|6=Xkav;WcPVbFo8;X%4C-_II1KT73|Yvk z%u&?T-{QB4&IVlZjTr+rx$n=Gp7OufKGhN(thJ5&MAUVpSegitORPp!#MzaE64ST% z5{VoEGiwk!^}0l9FV)i{$oMA#jFKvUVXYl5!@6*;Ojx;Og#$PFQ@{x{*36{1!ZG zKEyJi$K4qjk}Wf&q6(cpx{?Wr*#y^p&|M{M9Zv!AOI4$Yx=KvqaGwv`b$|9&yT8v2 z=JHgE77pzxxL#1~OD8^>Cp+gpvR;zT%n`dTQnBlsX2Nd9U9| z<*DyErRyY1nYSZBnDwR6{Kq8QyKlcRTXyL`KO^@Q$AVP_o?U)YZaB-mg|z0#wdw}LKxU$z5d+i( zZzHp?9r_OP4!~OmySKhRyIH78T4?=9T!;lJ$IB%pOt_`r#Bn^fT((QzWW`-1mXL~p zlZ#EdE!MyLIDi|%v)r`~(0RC?<9vir>YPhU3{`vPEsaF4A${Ce1bXR*L_}k=QvZ3h zWv4;!z^wBCq<>?-xQuBk|HMCxVB#kWq*%rgjRp(cT`wKtl)g{v@K|BUo_lCq5Jc~5 z&+^b$1Jgik^`(G+4&MKw$sd^L^4rpkiM7)h#ifXN=<$?S{AtW}MY#)D^qv{LYESIT z@Rtw^#|#b*^f?6|?=b~QqI3uzP+RvfX{V9OC6-4vs=E@-SU5^Bry+-C0J@CI*)_uT z17EH8qHrp;wm=x0&SL>Vg1D}o#k}O$=R31|S-W~9p27^rN~?gU_8L33Qw{GKSTQ7E zIFVmMRpA}Qf}CNXO|T)Y!e$jaY>HO{3ADdHGL?u8L7juY#3vY#D;MmN z$2)F|#@Yt693TW;0>xnQ-$nZO3awnh3~S_+e_|u8$02hIv(+uEGWc-ioOzHT6^U24 z*Rx4s!90EpJuR~U^(|%oTq~`5_Zzm8MF2Mj{zT z2MH%egp++A61teLUR^U7xI+w9@#O#Yk)fU?#bEqUPKhJ|_V_HFe%_69$y!@H%XSdR z%VBaUnAs1a$Rx`GvV@8x;^A;G>Hq-6*)HBg>GF&q=cV2(D95CXVh28Fh4Z6V;6%^%oICD|^=O=gq1Gq~9mcy3Tlb&SzAb0PVw zsO%7|5^wr1j{OOL34SU&TqZM>MZ4Jrp< z6RLo7Z$G%&c6lC;+Qb0Blye3GoyL!kiRihKwAh#Y#?d7^OxPuH;+o! zvPQ`*<4RJB+=3?btj-1}qn4?Xoio$D5TwmRH0;af1b-@<^AIp3yPN~H2(Okx+|62b zf+Z-4AVz-+r$;zqauH?+W64@NOHj{OGXzj#!fVS66j32Xa)Lfy{(+GL3Ap>r41a_k z-z3*K3o#PcVoPVO%QZ7D66ZL~t=Sj~IEhA)cfGL)G^kG4U;i0=-lIlPaDCZ*c?|#i ztwvt0WU;Est7|sdTjFd9RmN3c5P~BR`YANjqp3+}RhqhkI_4}Yd4vYS3xa16HNkxq zm@CCHgI~hAL+sVw{{$~Acgm)IEf)T~Ea>&Z!Hu<;UWn>Yqst|w2H)n}p_h`iEY~oJ zcvqcHi^laj(?;X^5p@CU4b%e{YMSPY5aR=}#O3(bRY_5U^x#Y86~e|@5BV2O{y?3ZT2W>v zBu>o+ytpKB?zaZyufz-~2Zpd@pzBPe0QoiJZV!c%BKq z-H(bC8PzRjpoAu_7MIwm90sIdBQ&{$J|efjYsTgsJ zGrc;*cf2x0egVx`W`oJ>{Wj_~$Ax#-nGJG5HXVL)GCo;c)R;3(^DwGl>5ZiH&~qDl zFOlbn-OhM7gq`3;DtmCRm`}jo65MAxcnT9v{TYXwV-@rH+O>0@Ee#Ebx#7ydeR zP7DYA&_PT3QgRAZ2SHpAs)VRmXn>H57zxYH13B|^c;4(8)d!;Y2~PH+vuLL+d26z> zLhEp!zpA3&CFXn?QW2)l(rDnCcZqC*LaAbq{p78NtZt)Xn=tC+(ynXh6X;6;OSV13 ze)Wk|s`vdxlRprQ*LLYBTyfkC9Rw!-;SlXa|06;FV3r33+qn4SBo6J%@RzWKT$5c) zpA7B>I=vjw_MmSzRm%j_Q zC-VHGSrPJ&$}YE_mv(?y4m=r!WV2nPRNA@NCJfIse#jC>Gi{kPeZIXu3+o=Ar*mSV zCwXVr(bE-|UuJ!HS+5}-%o83f+GzRx4}r}AukleQFwL1T_PsB`8n-Zf5eOmyX%~0A zdGi8h3Mqvv+0@n=aw*B@tQfhFVAAigTY>(mhCC38TT3`VO1Ixyc|r>o!pldike*U* zgS58ECbcD}x2t5K$Vm!3Db*Gr14}`M1Tc+Y!ruCS=CNy@98TYgDsyGgptAj_mlom& z8zK(caVIqa@lr>R#6Gu8=tF;wWdNk1 zKRi(?DQT{b3-pm%{+&!Ef#}3KldwT$WW*W6Lnh!AOmV>SJ3$DfuwYU1cD}@FH8{#} zu*B{NB^^4#Bc3VN$&ZC^ba?On%M;lj6Nt1iibXT7-y|LI2BD}L6~LK^GI`Y+jDd3-}h`(s^2l`w|(*U6xo89g48@0pL3LSJ;c0jQOqtV&<_2oh! zgMJzQ5)$xFrDX+M3xZGk0Q<(Pry)yX1@? zJfwr-OvUkcuN;Z#F!XxSysTASrpX;7*f3W-03^;D5>nCC`u|Q6T|FUbQCX)bZfjPY1$OD8Th{9%UPKA-;@m7J^ z9DpG{%NNub)URee$UR_>tYMZF|4|t5ur2z8zs0@~{gR4bA1KB?2;y2N(M-!N3OYI| z+dZu7HDx$(lOP}E!5T$vM5AZE?k&=#A}v(AIS(z4SQ?tk^aAwAaG{f0JoG?A9N4oO__qgE8hY-T|3g z!YZ;%C2?)oOLfyr$OH=h0(CDUcwY+mXISDdn*4z`9NxBeKpoYFj7!W_M3mj>D!_yH z?>n2)QO>tW0=F+;hQEaQm1%En${VyO7sY-G39zqJrFa^_z5*)h35?;K-&~gMCpYD? zEf;-qCi>lEK=jrj4f)=V?AR9CFJ8vzM~~wb(W%yx95aLM${z4Uq8UN0gP#muZee*Y z*22fw8@=xL%0cFIZig8oWQ=RGa4@?gP^@Bl+^x`ZDrC}aCY>(-63`ow0h8jLV>2qZ>OudTGBIg9(oPChQE~ZId4DS4@#i0 zEZ9-hoQf>D8}O$f*J#Uf3^T6xIGWCgtS{O9l!vK&cO8KW1i*1Qu=_O1NxYkMIE{3g zB8C=qB&b9CwdFz)ii##qn3EnS%f<&xKGcIj@Q7vjp0aAzEnN`o1$;&0+DgtAvU%EG zk?UmA1)Jh_o?Y7qDPLfL*%`D6Q1=Gkp3CPY-0(mI$1+#Yk9_TA3%NAJ$;Jx(bTFD} zD}VWR;Fnua#pxrcg4Vw40Z|WMmrB@#-z6=;K$jz53ixM?<}aH3fz>2R>iJGCAAp;s zKP1w``RKt30>okJSq8T>ictLFrk6z!6i~x2gZZf=LmT07jq$ zy7#;%} zSCPS3EC~sB>ef~%?T8tjvr$ip(b+}NGR0A%@dLirv*qE9A_dz3cir&h!0&+JxkXTG zHvspFti(5ndP-0LtJ?x0F^8QtH#UnDBOf>P_pE@1r@#(H3@5r% z>(cNY2A!bEwr6Xqlo%JNlk9>N3sc*t9@0z~U9%=i%gJ5=HVt|Ak@dN#>g8pYRRq_U zw%R3+2Kj-MY%Q>=TpZtuRJh8i{wRXeA1nLKv`dW_N4g&CR!U;ieH|d1S-#$;#qWDg z{6sd0)BtMAn_+57tP2EX7FcUqlM$7hph}b(TA>~KmNa6RGh|(2U;h&FXTa(&n*4!? zB9wJ>C7@C|z)eTntv0Sd}pAiwW#t82)c)pTe+}qrpT1iVO*|~>*-f;;QQIia@)FM^?Ok)esnl4{gZee z^(W~kMBHSFLj++{#W@z%qFjF7?YwY;$2=hAHenEJ8t_!#9K0{tX$)2k9}dqb<|fu|g!eLfqC%GYGr#=L4Q4j||9$2 zw><7$%!%0Zy@$>AT&(6b6rDpCo?V9NRpG3%{ITAKnRSY`IQd=_S0fHt9O0fD4{m-O zz{P)FDLC-kzCl1HO3VUDw#JSVC&=V=-9fP@4N|M<6KF64gbP$TBvUQF{vtllH|73F z-u@&kU2du%{)}n`gSi^KwWGSPv8Dst2VE2o`?*e8WsKc9gi*l(=_EUv%TQ4P{B3}3 zOP$Egnkm*mKCp!fb3yeAmE2zaKuXSwo?#b}iQB7ou&D!Y`&Dkuth84L zYG^WmjU-U^aw%$ZHcG%Umb#KI5C?oJPV6rdnGum;ypj|X3H01%{GDXHr0#JI+6M$< zkHd{GcfdcRcYo334>XE|*(m4I(H86gYKb78fZnZgd)P}C#@>CC7?TI~Rr@mhB}7Gr zI&T1|DnkL@^^;lN41^K5XPd z#2fc~bjzovXd|MF7t-vx+D%sw+rmZ&y~xOtr+rJ?3n3yKuBtPXW`-u!P&>)S%*4cb z5Edhcj-NG&ao;GecmPG-4=wYG8N97in!-4l%X3<3;Xwq$eVV)@aE(VDQ>E%2sbF(R z!2w`!nOF>(qf~FEmYj1ssGjP;hEuWlT&EUW`!{_w9=53~34J_4mD$2a|^ zcY|lTE^$f4gHP>#wv=d5(yx~^bDB9+WH_LI@RraZV4{8&<4GGb1FRqSK&Nnj5m>Iq zp0^`sJOx8hDYZ)>ca3?ExpF1jhdbQfGkOIc?MsY5S^E$;)m`Yf>J)D^izVw(oq@L* zi;Pf7%fs;A$0!!xy}pB_lZrhjfZd(RC{}hh9LEGoR|DJHFLI2pKY0z*#3()8pZuD z@B9}5*+K+!o|02p7&)4LbqT~ZWv25(3gAa9)WW3z3U+z#gJh3A>5Ylk=m=D(t8k8c>qmOaxr7O+7C|mZK>S|5F(&(ovT;N9=3OnQ`feI zd;mnI?4kg`uxwJJ-H_Da?Q2B98U>@GeNg+G3EnvY4H#BK5&)J_h$1yDQis&Z*0;Od z>Lp{Lex6?Cl<^sOwL{Ni_}=RaZwB)}wp+JU0ND@|?dONr5{4HxH#xDYDhcsB#V@t@%GO1V^S+f`BR(g;QeH8q29 zx2VygH5~3;7<7=n2rDuw2^kD(wzN$Ih>TJ9VNe6%9*V7pV6%XMWe%4z6y>q&gv@C+ zKX{_fbD>=me>-oVCU?suCnSSy!@~{c$@EZaqo-cVK1wZ-G6^M7c1gGYA;}`%A(Kvm ziPV}v({$B_siWmTH#kHJAWoe`-Z?rW4f9eP72nN|zp%KTjuDA#7)6Wn5G*REzjw`E~DH3n|z zHy9!xY_E=^f3~3Qw-{CRlAC9FP(cX$3gSVxaHM}jz^!z)%Ph`2hjwIXnbENr2Hj2r z%e;mY!^KUS(I|uY;D0J09c+bP8Bna3Xh`uD1m#aAa+dBYBDZ>C-U zi@rMpJ|PH#sC*CQrZyH{!p=~}9j~-}77-SR+SO>X>k&N+cd-@06u%Yq{6N(5gaI3P zY(4W2FLb97pDS{Dydcw&1fm;fXdqS~r&QaV+a8E%Nm3`3cZ@{cWA4peU$G4zBPA(6 zdVwIUvr$?C=CecXTRVMiQsMk{MgB{g0cL6D__HdXpeRoSVG!yABq12mwss`K|5W%Q z@9lc#>Xyp`*PuxQVJ<`il}V;8^8xbmcea>Z@^Sl{ch`ei1LLM-U8}I<$+QZ@X}DW$ zZ@DeT<^dXk5tDx4P?(a-dk?QXnX~Zgq+|Gaj1=P02w{nNqr-+AuUmoXnvYfXEq>M0 zPTW`JUUW*?FtBzWH-xj+}UDP8Y=)YQ8C4^l#olPk64^l+@yd zh~bf0JCNzS<#6+Z4hdF=q_)u^@Ds}qHYzv{Y2}%s10#2CfO2^acS-+-fLX4D6iCQ@ zxtUSPsB)nnWdO&Bqo^aFa^Ns5-Y~MKc>h$ursUjH96Uo{{I|n5u!D5m1-=&W7$k?> zc-F&f(d17Vk}|+0m&{fpL#;k%-w^<+s55!Irb&f{X2|y6iN_0V92+j2%tv-c{NlHt z>hC>lBg!sqAVbkEhPt(Y=VU-&2s}PC0LxzWou&!ZZfky33T`}I^|6`QPPoogIAfIK zhD`A^BYY|yR^L1#AQ@(A+h)t7uS5~$9wDGht!T)k-XGSo%k(wyvYcxjO!?Gzv5$y4 zeBn;K@$??-<-vY9e)i#BAY61XxScW!5IuC9)}DKZ%TE4u<;=s7E0%niQYbGaLjdem zfnH}DD11SYq#E+#z+BYqlfRgXCVkVE7f+;PpoT>pmA9WjbgByL_+0E0k2&>?h!Op+ zl;Ib{-6C+7^89qm7mcK5gT}C*eFhDVf1P16OfC`y>2D;TCWF+Qwkc(Pdk! zFEXBd1&d)2`#Gs*hcx-c7O)<+ZvfUvR0x{~hGuloe;IZ3ri=0reRtjFT8XcuR;e>p zw&~vxu(ts;sFF9uS5au}K@+?K{3j53gi>bwotkw|)ZR8j&p#C~TCbNe^?9c1%^yo4 zN_B#Vmzo=f{hK-ZX?6o4ujZ8t#nVy`X|+LMN#Z4IHX^mg5@@3ze%UGvxFlYs>sv?J zylC8s8V7WIe`beCAVV6D;uQ*}Z(vf}>hgZZd(QWg+M$oj`(VrtB-s-{f45FKbr;&O zr&2muQ${WrR|w_N*c#Bz7_s2Uu8t`NON$3x8@W5~*Dm)rB4geVW;O#M=1?9lelAbH z=d)nsGaSB+sp&{*B`@?V7&Js9{0ez!YZJ9uRVs(|@%eey=^FgaUBf<5C-B}A;AGfn z8-zL1!)1%p&Hc>CyRn7Xs1{rm`D)aX?0|^jNfMYW8Va=ULG?(SRtgw+4Qk?t18=PH;&Lm;e9kmY42#mKE zZ;v8lgOWJ;(69miuJvySxSBnc)+bs3QimlEpO9fZwv3c6s|~?4Z_wV+l_f%MSEbI#co4qbFY95LTho8l#(orh6HuaduS ze+Mm#4qT&jw0b=wkY3z4hQ0|X)J`Xn1P{MFRIN^&HfHAaCBh5Z! z{QLZpqB|?1c0xr;FzSo|>nl#@Y^SkDa%P#tanx|2RH#=3p}_gOEn}__q|Kv~VVPo-;uX&3AfHvk?kH}QT=2_15f0%DcP8jTsF19v^O9*5 ze2AdwA!^CCk$d81u&uxp@5d7Mm^V86(Y12**3vZuD`4@swMaqnw;QPm+AZz5iRjLI z@}91$^HQ)Nq5@rfHx!BnM)Z+IQVsJ)Kue1572R~{02KKi<)sTyOlKcVbAV5bh>Lwl z7Ruh#73>4)Jw@aO*7eJj+S~uq@jSDAmkW!X#m6jPMIG6f3d3_GWA7$yLIA0UdhZ#9 z0pMJ4*v|Y{$l(kTE6*klIxy|3h1_oNsmo{PT{zqUg>T`YaE1?Mj-b^pS1Q8NISy`E z7Pdi5L?07wGn8Og`rWF5aNuaP$~E0;mGAi3)QyV78x^I46$YA9g54Zs*d*@mHr??( zgER$7&n7%LYBH$h2k9)P!aAS^sf$J3C|0u4q0_1184v_^+P@(n4DQ}00pCPK90wGo zF~egsMZDIiSN@4B&??mNR_@xsKNXOTDA{inCK2~04pJ=O53U8TbP zTck_1QHG866hmEWX@X69aVM>#M?`ef1Ok>kEYuj-B>Fr^c#MQZdxAskMOmeiml+M} zD-kmlFjvDp)3#d81y&A3`VI=&tYl|~`v*Ad8DuFk0v+Kj)KOgugy9x>dFH2+`AOc1 zf#S8%%Qh`%K@iT*E^?adUuw32;_*O!F6M1MQ7mjZ#PQqq^FZG`@x!z_=Cc`1m_nzB zyUTz^5Xpm#zW;=;S-vqxigf}M-|ivN zU1?u6E5^&Cv6uj(mF!j#fZKBy4RP6E0@XUxT-!Ku+x^MZYf;SFDl~tXKaQN^1Lg0L z7<8#Y4mh3L#yZIew>0z(A$!Oei9w70f@YKqcNXb-FYbGdl|cpD3XYzP^NTP*Kj~39 zvxj4N7x9Iwp7;cN?6|JDf1t#WW#wlq2j7jZm?&EFLLs^!Hl5Wg55lMvhwpVt%Nt={ zz%FdGP!MLy{dn}Uklc!v>yvLy`kv^ocjxV2A!=1yXZEMC4mpi&++MP`mQuMKssDz6 zKSJm9LkAgXgC@s|@L8N4Jg$gIBMfhU@rX-93kZl@|5E{35a*w;^%uW?igIFg*#WfcV$~Ub9yQ1W1J*dS^TURwQIV{TeQVxDqZ^OVUIS=)9;ZXoN)yV@hf)Iab_7fmF-U*r`QR(l`=Ju*+ zaCDes&Lt*4Z~nh=tS>eB5~i)`Y6ZW8ymEl}O9^$dhBQt$lQ2e^{X{Q7qQo)sTKqEn zmvg`Zt5{XU1?_uC2t0qa&4G4~neDrn&sWD4xJy|SgE!9&Wy?0wkEieRok^T9P8e_G z$jF$&uS-=)g3-t;GGJz4a#Q@Z$BQPuhx+Hkz+AFE}CZ?l=CZMrsIGP=(M;; z_{E$0m*h96Z|f4g_g$zl8)kP6{sj!z5(zeb5gF{<+<3@kBb3KF{PDIF@lYc zpi6;$*5YrB2lu7Ni)AV&n~k(X+0`&lpq2#kq)QP4bk_ z4a4@|x6H+%+D>np?|O^aAPT$1?8E`8n<*6N?Wc!%=w;6j@_uW!;6D9mUc|#76fJq%oQc;IitN zvG%jO?21Snv&Qth$N}$x6$&kEku}C(0Q7I`8G5iGlX_O4F$j>lfM`?$t1uhPS%zgd zZU`W3PK_#zZ}KRGZRn+J~T) zX_mm=fBwED=RMwt`A-GR!ObRmC!d*;PK`}yw>gGnc==iAg3QGSPr&K9ccHugE`VB0 z8f<9=ogKB0YAq%xqZ*ejO1ZMYzE5*K9zcBhbchkAm2s&7?q-#NU3;)t>v*hhRm~cA zA8bwrbo|&hJhQ*Ds6h_`Fe<#RI~V0L$aw;WXu@|XCg8wmfCJ7 zc)=olW6bCUT0ViLh6j&^y!BfAsLfmZn^|P)%A zo%_#vzIk>|pfiWUssy154taURkV1zRG+a9FYb6;xY=31RG|>4(Et z4tBC4?IEoyN{Ea@$`lT&Rtsu!*E@wVnX}RCHDr5nX-5URJghSsnA87Exm=y4$zn ze2(Kp_DA8^`WR^eI(M)TbC8L^=&?{hQeG{MCl(WNIdXWX;KTv*(cTx?mL~+_6iqPC zEf4tB*oNqCD<4lyDq8cce-HlAQr4?33m8W@tE@(h-4G zf6v!Qx>9IP+9KwX=CZ0pfz))3(BQpR1OJr)#i{s;g4ldH=!bc}_-%kFiQZ7CqM|Su z+{;4@Sj9#n9sc(|(hW+A8v_`W=zR5fw)#Wg6q`FHop+e>-N!ZU-YjxJp8!;q-)7gh zKqA+*EGD#>bMoSY|I9#-Nn`5kOWFFq=gm1*xpVJ1zSH<)-Ex!eYW;%J3f2q#F_{3n zjfIr^OV5}ggYEvnK`{c3Mhq2(JI#=v;NrIFay*IW2MD;0d6@bZO(Dnpe983en%r$I z*&^;$>5rQr?_xeZeqMBq_6jW+59#`GPQwr;^3X>%Wj3`%vSBkPe=pm^(g_h}X*>e; zzBSDyL@JqC6OO^{-+iXrWv97R zpY1nU@je|(z<_pqOZ5-v>~z?I=OsKy+UE;c87NJ=E``f7p*$rbsQ1$7WXU#j(2Pe4 zt|_fSa4L#>v2nuJwh*+)EyhD6D44+?uZ>;x4r5ADW+)Y1mepA{(lI&E*gy1BCtJ^d zE&MCzfX(_5&bvKRoZAf1DM15_p*Mm(5&r$1#@jXE*!`LV8vj(lA~n%*cbDPIfF8Ay zDML6olJE*M6-q)T{4TYycDGNpixfv^GH!y3ApN0i;7_$etuUi_Op zJ_N*{{4T?@;GR)JOD!gAPb&D*`k%oIgO~E0#0}U<2}X=`AC)NfD%;wDH6b7aL!yza z5tgv4EHZ8e{Rj)6AQ>xKe5s65^M|Az1iU~S^pn=dC|4po`r*ZPtBe3o32L?h-41v& zmWqXxtltj7%&MM$ezK_X+O9Wy0jm~0Q`tQZ%pRxAiG#$}1DW?<;ZrUWvA9x^h65YVO~ANk*-u@lQHo z&U?J011~v-F_;fvKn4ukKC*Uh-=U9rfzj~7mnPtlVfEkG0byYGXB`GnTuOS4l17k0Ha|#%5>a&PM)->O98XpSUjM0p z!4{5-Bf@fE91XM2bIxm_-kIA}Mb_i{(3?MgW-929L-2L`O%GQOROV;3Fa5IoNQh|u z%>K`;vdO-gSmbmb)*Fgu1SsTLSDVR%Ku%cIXtao@%VKD)9ya5t0lrRPdX0qbQXh;A zGYqYQR3;IVIKZA)>J#M%Lmz=Os2uTCF)P<5fVqVj+hU3pqGu5AMia5Np_dsNDVptffsRcvhE-$T-Bs<(3IMisT1#s8{T@a*JR> z3VlD;z`9q_>yxpMP0sR>E0z~_vC=$WJ5cdrd!3JQr23DazQ-{Hn|t+=LPH5v20veb z&rk^x!^3{K&@~Zf89=-JBJ396x{SBS7J@@`*-2?1wxXBz`+)741Y@)B$J1R6b8I<$ zJ!)`;oYGHmb>2e$H6#JMl`~`*kkWtn;TA7?bYVMDr&9Z?t8W!B%~L)Bz%%Gd(=3?F z%q)0?M@=_^P4Y)4dEY|?^9tyDN2X#j>^Cl_c}t%;Nf^Z*|H@gm8Nb%D0|H?8LLGg^ z``7sDah~DWJ*L#WCMdd$N4p^6SF(vBX5?(u{eC4{G;_fz(vGrJrgs{U+x5C*=)pzW zSz56giNQ0BXMb4NNtj2@aV~dyh7}>75jx<1LqH~cmWyw4?BX+9fM6;!49j;~?=cY} zu~>@}!g?-4EV%zvz$9Lhtw2yOaQRP!)-`i94K#iN$dGd(Y^E#@q}fE!osCPzZWHKa zt~$k_e=D}yE;s;GK z&k<(ccCWW&+!A2Jm<+Ws(_#w&=hyC-@m0VBpwY2s+d>h>o*uQA!_bBS!zbO7_hwKd zVya&R8uKs`B_CCnRF7r1-U^zT`(3jyV)P8PZidinLLp5RMqCI(L=sY_UvcEm95Eck zg(zfD-O7R2uS>DMCn5P8gC3r@aJ_L7JkV6n?nX0S71E(kA6k37;waOCdFF)9fDPe%ic)~9T zB_Sg31f7-+0UXB`Gp4^zDDhJ0bFw7dg!=nLo%{MP0K$0D1LEcZ(whlqvLYSwUha1w z{E=2iua3ESdT+ zQ+(gs3l@NG1yYQmUKB$fFlTgfuM(#Am`pQp^9X9Ms~xjsV<`6?krHYggJL*kI3~CC zQLuPz9Q7LdVjV#CYfmAF%i+B&;=I8z;z{ExXVI5J3XXl%Qn+H|4o0zoNdLx4J>}MDMq|grDElI26l3wA)vPfg_;yEE{CwGd%#MFN0_921TRu7}l zgjt>&>STB4%uZY;bN+pK!I2`Ch+53M0d$Y^OlLPLRhZvlidi9kzCGcUj_x(oON{aa z-BRfX8oVJj9?~u6QOL%M|K(hVIQq$4YbXd)f8+F%#$x~wuUc>s)rlz=i~;S_kSf>d zpUtDByvC0%`8H;kO@@n|HmZgEiR-ss4SYS`E!gA&SAmPScxTa5hKX@3DR;UYa9EoZ ziR`)oza_Z=$FX6x7|)x$W3043Nex$AZV0mw&TTfT)o|b|mvkH9`6qWk*-&kavhAH) zn|fBe0|STXOjCZII~n*On8H8(e|z<5|0}@GV57UWp;+7!nZBK!e%C}M7?2^3)pb^+ zKtQ$d4bkS%ipE>V_~AY|Fr2b8pqD}5^UT+U!cS3nq2@7W9{-}jx`P0XihP?({RFLT z3gacQm&1h}dWIw;rWI#E#eR`|q;EvRUQd@RjLP4%KAgTGwSaFXj`~Xy6jXJB$Vc6& zO~ER=Wy`#CAhmJ`PVauUQ(YT5LagQFk6mw5)^He^bcoCX$@6KN#x9n1JM4)Iqi z+cb$aEV#3;p3xrbdbl8bt3^IR(TE8nVNzEOF|Unp&5G)4!zIb4mH1_(OU;qed@V`W zS48X9$7{O*^k`x~tWv=+hCnv>`syoRSkvx@Ic76GqY77zI(Jjb>(-ndvr6oMY2Fh9 z2Jq6Bqe}PKM798&*e4HLKfq-V&3WM_)!?B$(L%zN)>?mv5vGtQ>XdY&JtoHnxfFz! z;}Z|4lDmTyBd(F~_f=*1l-ut7Uk_x!B9-QwI!+*(h!mqXo}d@-w2-rR8{%4J6rtE% z87`C7L)dJ@;HR`xs21$5bhgxu@A{w2d2|veUkY4!WeEn9FJ}^Vu;v>~qm>Vj8WQn! z3_wSGVHd|1o^`K!2Cc!Q<%t_Z+=&u33pwgDb-F6!NZ@*#0m{Iw@z}WCIOdB zbOY_O|NmX4ybWASM?K0&?0Db1T^q~U?hM@k^}`jkjOGii$ZNXNVf4jcy-}`kamn!S zUef((iD4K3AqSz3{1*UaUep~7N@rNj9Jv3~#Yi3?Oo+xV(KhZ5a9~NP>DGAtSAaU1 zdpMZ3q%qZrfQ?;MiBqbR22qg>8aLxh*n4=TMJ#ELiAw&`K{2I+QoMUQ*zrw-yn7!ZHY8?h-fbtd+5 znf!>jxtexc>?+VteY@N3eD8P;OKE<^#wBnRZ)!I1O2nN6N255rNID!VYw>PD>GEe& zywtR{Qdb`G-^X{qDy|GDiwv0X7MAau&|AizybnOpl<$5sj?MY|{6(sm#6mkMPGc(@ z)zkupqgdsw{lWPjDn4i&-}6Pq+{BFuqG0yF+KVC+2LXiumtyM^=9ovRrD-D86QIt8 zT%u+zo#RPfb^@maf>CLqBHfsdu}n6$-rN&?@W_KKT)7kh?+k{x7g>zg1SBZfjX{`Y zv#_d`Jxz(DGUEBN-Q^FCiE!dvO!VBzR%W>;eM(v=63-CtZ_s;327BcBDUlKAxanSF zyeCR)F@jI)7TWa}t}J{H?>7dp*2b`Fv*(Aoa%8^*zr3Qa%P+gEpW_bk`A1C>&qVv# zR;A9zdarbpy!8;&CnonS0jtQukiVk-1way!^NQ6?B$7Od*y=70yB}&#c}D-?L0@(6 ze0RY9xTOCIkdf+Tv_9BNEwV6GM$fX`kLlAv9tPiaWLWN9RI)^!1vMV4T+Msl4kFf1 zhiXSd2T4n4n4!sx=Z)RxE$Qo;QtkPnB^9585IwyVOqBgrI!lAzR1<@`Fa)9Y0OQO& zCf^JbkQ=bR=_LT==|xC_T2gpaM@L#Fn+4ofpFA`Ai@CpCpW3^XwH3#Uu56H4zUGNI zz5}%;6ZHepn^%jFwcalitd8hGitMT4=el}w_j`R)ei3=6f|t?Xv8i=*&inuIZSkE8 z({milg-3TsRe~(_CSJ+2lnNUp9B+`I1hnQlnh5xP{4jt)vZCl8Hjt&=&u@u##~9rK zR2)mD0S6qYL$k0{Mu=GpEf8PdGOLL@k&DVLSUjqnCe~SlY zJaa(J6rYGmwdUf}g6_JfA1u6DdF*ZLC((hei*BWC`G3=>57B62D%tp9r<)HvPG6b5 z5@j)c#M;xYjx7gDFtsArxa-TU93aD!%P`?m+6JdG^v=BvA_U#C_+XKaKZu`SM`D+; zsO6N2-!@h@GCyY+QjG@ zCpmPwMS=1!6C-20cUk|&{^trGb-3)+zKbfjy72 zvwlE|y7slyyyzDwC6*u^!TM;$O*|}P=RzqF2QZDs%JqKjAcQ?p*0Fmruk<+$wFRM; zA4GlWi3x1(2+e?)bRPI+L8c6(6{Vb+kUxEhKsL$ETx@1Zr6YR6{Dugq1LTd(zw@$6 zadA9$b!@7(rKXL%yi|L$_=;W5EE2TjuWhac=PVq0{OMW8@RVc4LbrlN9W*kYC8(KaJ(!etgHXL4_V~-lST>@~$(T01%J)4_*M-1<*i*Tr-UIK%* zzPLxAX)HMH2b{3JPT=8?6tMf5se9smaq@8Wi=AdBw>$NJEX%rOd^Zhan37=FQ1Wxe zQE@5X1CuUfd~3R2d&1LhDySJLk%-H>>r?bB#_V#{Be})hogeF<#0kME_E^SoU3VPm zFPVadP!=!ZWVwY*nZV{AmXhRhPXbG=^~SOU;=1U~^NhSah1+^MZGew7X@-;cwo-7$84dNL;X5gt633IzdXEh>43zWP0>a zL|k_hcgrz}yiln+c1ABKegl7#HmGnf>p~7Od)SMn4G8b&e5lzn>gwlugdu{XVu8o6 z!&j$x2)?j^@Q2c8M5}yXI|Lpc=>@eeb}m>P^EqoPlB$XQ%g)gJ?N>M{gwlVN2c4m^ zV3nw#oYS%4iuFj(4BPV^vV#~8S42E8@P^vVsZd&n8Ze`z=#Fw9qTMR?v}CGCA;}{` zkS;$6Fm=2I)$Xvg;=;N|Tv<$Ce-lf!2+Bd3-!+L*GVJQ(={7A~&q&l8?EIsn0Kf7l z0@~UP#NZ@R=YFGQK_$S19MS8PFIPCo_k7^92JZA6n$buBU+ZnFY0v#@oG8PnbkbXJ z`uqOfO%?qI-Ct*Tdie{>7Nhghf|B9m2Ax{rn6;;LAw%&G$zv#-W3WMX5o7VJlh_{0 zGrGzWrIIk$SPy_I9s@jZvq(?M8Jo^bCLuv;-;SAi4N|rf*yEH2DI-1Vyh~eoG-udH z$}m&xzW~VSh0W&?_%-yD8_^ty7+^wrLIm@S9hAz5l!{vCJ&y8U0piz)yrn9_^1for zxcH^&=CF)uEg^A5PHn>q3KjqZRjiVdZpe(3@?R~H+q@@Q!D2Rq+a* zb-#hLMq!iQwq8#IB!B_$kI)H(ix9~X>b77gx4A7Pju>P7ytOAp=f9T~ zMU}cqbQx)wr?B7j@`NYbPcJm>mnhH9Sqn)raNd7etNn>GTCvORJssQ-Nx zIJA#!&h}uIM|>X-x74HXWVWYcbwozx$)?0|-Knya@Frt0c63_TEANs>tsUv@1#6Z$ z5ZE6vlx=G))J>*hiBkC&0H29V&90PPF02n-mVN0Y5?a7jHt*-FiBKcT(H7b!1^+8R z3v46Jav!!=I+S`c^6DWr^c#3}T|G@ zd#Jkf!5QI$d5{jK{uLnwvuNnS(QLP192zRVw)1UDB33}h5KkdPJ@6m~?3iG=K1709 z)`nC=r@uPIsEKt|D!>S5u#f{e+hp(2qzqMYl)GAP;Om59k9&6pDcmn&3m`88Da(50 z9*FezI15hgUs_;;e}MPS8fbS>^fI5rbRXKs^{MuSV@G-!$C~zc3gCGWd1W?Ia{I>( zrx?Z`MK~>KC~e;DHhw?bI-Qx5sBdJfM{%y6$N%LHbfKA&yL)21}F~JwL-}*qQ z`Hk-NeBS_CvkpAX1v9c6;)0)n$*MwJYGDm3ER9-M9X&;!ZL`21O&!e{@flO{-htr+ z5;(9{dLZy_kO@V})~x+qt|HES;P=)b4~IP=z6Q@ev;K6{lQdF9#zZrr6Fh-$ z@U_L6ws(Fc%%!SR(W9+89-;l2=Z<=8uswWp!|-(2rAZq%yfc%Z@1v|AwN8`9v5r3Z zuXK1ATnnCXs-)5%f*ez0%3^8Y++plmTpxpXzGbxw=L6ZQe*v(kwp?=E0v(2{nFrnl zffQ9O)vW2DpCy06q5N;Au1VN`1xS{wO;oPz=ZeEZDBU?&`4)q<<17#=7W4Z_8O4y2 zn7u6$EI>G~rka`cOcUPVhH6Z-bYtTBaw_SPwAGW^<}{zn%j%AVVwFjimp_vf$AOl8 zcyG;*C)u?KPwm1Qn?R3NN2-!aKW7n8w7PP#r`DsBtipJs1x1URk7B+* z@Ct;Bz`>4>2is)w(|x0^assI1KwH<|OgWtwcsjxyvkwItK^D$|@=en0d7hF`m>0l{ zFzGe$OEux)0C3fTQeA?`0OtX0#G^K;P0H0x{ni8!q?am7d)90kW?-lOT(R+_o^}>` zt8(vcJ!f_%NkK;Jv{su7kpTLaH%uW#skP2t5$9*0h0-NP8dv=rEuLz z^Za04EculzvjgQBu0j~fM3$%vs1k9Z=eGak`WrY*_%iR&j+nP~&khLx=!aOONGq@o zMGfB}{*<-qY{Il!g~{DYJFEg$9l$n|axrgF`E9_bZ!gG79ukx~wn=M>R`G-uD(M9m z($C2DDD12Fu=w@*4NfAD@+4BSPt*TDEbW@KNm?@I{bzq%_6&#Mf{{4Ii80JsXZw_v zI`_~uC6pxeOx^IU%aDrE>ap z2}JeoF4rmi>~`$lI00rip#Tl%+^AIAK#h$U!iVQ;^LyL3d|GL%3=VjX>c#N}6dalC z>9#36gXVb&YkltOmb-{GY(>)?+^rIrtPz{_%zbFmk)tZm$pA4iq4#*|pmQXB@I zXaCk}UF2T?wBsN(YqibbD!Gee>uFE`Ag~LX9nD5I&y*rF$RE?c`L6(hvP3+pR16=?yf)hl+NMrNW29U^c@5wrZ$n0-fN1?ZtySn)_{^NqXD#{% z3wadueh9Z87t0mnILT(TH|=SeD-2z{w9F1`9J$WmKvp(7@*81i&nOsFn*x+4n{a8< zk=Wnr3$#=}Vsa2=JD5XyIvhIV$+d7|c z*8wbYW#6@~k6rjFqFgl#L=LB(GCCDb>*cGQv+P>OTQ6PtZ9k4W=84fn*2%d2(_nTH zm2P%x;7wiePfYByRPO^W*&O)Zt3G0(SmuUW|JELT21=m-C4FyMr zGJ39THwRwGeW3TS(k3b8>%2;pbwtCuU@u?#W8VL`zl+;?G)rAVJIFV_Ij!*4TU`o*AbN zHXr2Hfzgl*yZ3$Kkn646lp>&H!pHtLTZ}l%t2L74Z{eH} zhfNwj9QuhgR}iDBeLdIYkX0HIh#oMo$xWb_qU-n^*h$}*>#1%6)YSrbpKA4;*g}F7 zm!6p8S!-ldb0Z}#(l>4A9*q-My#ih~L06#K{iU1=D|7*1Bg!}Alb#tN&DJKcX&EPO;0$`!)4Vs@|d zTkT*$zMg>a{FLLC7waF`4XKKVKN!1Ei}rdn9mL3 zkV!IA-4j69NeX+RT=O0X*EkpFsoL_UFFqtZmgoXha$*})vb|y>!Yd}_=E@#k@T*r9 zQ(<@I)3tU|jk6REzHg&KYJ0l&Q{-0UVu*Rbf7VH?H}W0OhDbjIO5@x* zD!|F0yaQ(OF)jQdSF$4?LVCw;A*mv!gZGN74$#svH|Je*>am%YH#lggjcFaUqhbDg z>L{ZvL~G+KA+#(GXc2+)$q}fpvbIC&0_^LC%MeVJVBTW=McC}SaEW!ge;>gFqO$#m zyD8A&Ym|UWG8q*9`{2|UY8SLJSURV&fYPCA>xF^g-u$sWtYerUGwx;A#O?pH1taKU2MrH}ql;EPqsow-fz2&TZHk5L-TLQN zwgc<46yH9ildKU@bi!B&zYS?>ip-fNf27^srlu{2GBkC;OOnvEOMNPS3`^eu1a@K} zNvcOb6z^kMx|-(HD)WJV@*ldJ5fD(c6o4uA!3mwe~xFnkS zLet719z9f6I4)kiopR>Sh2>6U^!+mVy^XxCKkM@DFFwXMX*CXTpCZ||j=LWCuT}mF zfCo;#!%94M$98siew6Ld$EdhFgkLj|5IAdnGOC3;O#c<2<2w?%mBwp@lwx4V#83eJ z*9ajJ+6Fl5ZB*RN>Lp+yiu_Lb#Pd3;P-*v@^pVE^^P;FHDKIFwm;rBTLbatShU(q8EM z&;qT6ZRy4bSso!_oSK)aRmJHf2_~JN;mB|6jDFh3zixpkZ7c;STF*j@ZzbTP)i4$5 zh?>X60bHcf6!4BSXQBY4S~_0@Ul$O`%vs9*(gys#x~7}NYV2PG4)H3(GAGX!1Cm2) zNGCXG#So2nLe{TaL~K0#3EbX)@^|c2%D6}+!yN!Sb9e5Pv zRk`HAe%j@;fQ8hOS-lx`ooDF=bp3_%Q`B=C4ciDMT-H7}WON@=*9<j!Ih0P;#aJ!#dX+87+|JqvR_4}`JqiMp7QlNGwCMk-Su1vDomvsJ4BPIDV zCQ41UBT6r?RClXFl)d_?*a5Lwu*qf+sBUpGk|vcbqH9;=Oh9q%6_i#pv@6uu%E4#% zv9!kW?_(}Znf?oanA>vQaI!s}J&P=EvO*P9J|jnp*$8gNo}K&h_&T4q{}rHj#g}Y5 z0kS?`ldOz3JD9-alLND-Qw^Ahu8NO`73=PDgUgD6(&bU-8m$7AceCrH&DD4cof3eO zYG6r%jX&n9N4k+6SrH}jml7k{O=MBm>~su*C5A$IPc$7HVW zXK*eye&Q^~!vnzEbAu7}gnQ&F-}8lm+|ajAjxg`X5cL%wLdmjW8Kh)#Ik=CcplqkU ziB}OLpbO@SED|}1kfL?Rj#do{0o?FNI(9orJ>?1*yIL4y+AMg?p{tfDj|Fm5CswG< zEF?u_M+`+$$o>);jb%u*5ch-o;AW-C+Z?KqNXS=ECgN+`&*Q_Xd974;gq|g3P=pVssV~N2{>S5u zZU+v^T$+HgW2)C4x3Y#Qd=nmu>wKGcFb*nZ0(l=#SoG3`{nLV~#i98drQtf3j(g8n zj|?!v*tu_)>3inD^*cc0G3t z9aucMtdl4j{GX_;zvbx7=5)cs{{rA2EMBDfygE%)W~2q1;%>5fM?JDZ40wVhXR$o9qf)JkNAXDws zRB8%~O>6k0kWerFS*c`gp}i67uFaAjbUT3b5cN6C%djIWh-6Ngj??t9=4wc)tk*gmTELC-`?`efQ8lBp6Utp$xsg>@~)kb zPb*R$XghE7mfm$Y|I%r6y=~8@EJVM7+#VXwI9qJFYS((uJceRbWoJl}arG(8{00r#y64`%07#Z(zh~&Xp@(oE?x=rdvtjh$ zb+;N-_ET+Uv&>#4!}VVQBJ;ax84#9C|DGYb1Q4$c6K(^q!JxdM;#71tj6NrxyBG&g zwM}2Lc#p2}_G%vs;ScQHGfD**3+nKqUyZkNp@I79suX_!l^|)iFkCk=%_wDD4Vjxn z%mMM#s@ah)8M}tbL6^QVI3Nq1(zFW}{Q=-n6M=!`x;2M3<<}Z?yZDGn6`>3?TB|k)0w~B!8j4?IUqAE~{3s}?F zpFqp2Ak6NzaA!4v!|}a1I-R8he*E7yvcxp*PWaHz$mn$6Z;F0p>6)UDPGDeYP?q-{ z?bT~=9Co}^)UY!clsf)}1U;J7Q_gx&Lyo$fcY=|k5Z`^~KK5Wg zH!6FR3p*Ks#(GV`wdqb8W#0j-kFm4yKSy&kkc!__PxUZLisRhSz2ix!mcBOH?VE`F zUEl@jXW&C1xiakmBaWRW^*KfNRdpi)1qcy@I;dL&?00aYJ}B zUNwiHdx<}#vRtf?<(52oIS?m`RabL?hXTFog)5b@U565k&w z;y7>esc$w35JsBh=P~mt?`#;3VkukivVM=30D67e?LSB2UnTRGMd0;ETUDQl`m-9% z&ar4PLC_e}qXes7*0^roTASqEZ6NLPIQLv7)hm0)vzi;P|M7+iO;;*W1ot18hUQ=*y-qK>bUFFS}I zXE4|@zRqP%OT!l679U}kTQp7c&O00VC!b=B<>mx#*i9`}3(I;0Xdq#2kA#!|Nsjv6 z_rCzRizDK1Ij9zgCD%iC|B-kI)<~)2cX?YiplipEEM2VsUn`(v1n7b2L=1d$*R+8! zNb!ttBGJBLE#HIpq8yUtYHs)wtSS^sk^mzWCA5Q#r_OC4jJC$y>d(+F$={$9%b
YE=pZf9ISrPT6?ngCwoUBS43zIJC^5uUQjSLMm#a>`R{;6l#*cXy+WMn=G?azFh_o`B4HP{apfoD zo|E(zf@=eMOnX-z4ijqSiuiv9CIQmtq~gSKx4zUU$0_=Z2-AR`f1|T(c*cHSA<+s$7e&`fVNU=TQT2 zB+X-STW*MKmB^4eGqi9Dhh%uO$j@fto3%n6`oEF(!F#FXf&(LdRq^2Sy2-{_+Y*y) z-A%8b{B8lO_c(5(JmIQTlQYk1zLb817Bb+8DG^++1wHphx&(^&J*7p5-7SxgoH&nIC6jxW*0w~fjmd5hgL4mI{x1N|=7M-)Dep-R9b9LqvBA;isQnF>rs%bC3YSAgBl zd5*W;==Vhxjh6u`d0ytOey1S8MN6CmCB-7;xTQ#!dUfMAJD1g5Yl-{g!>RUT2>XxT z%QMWm)Xi`4oEZ^B5%ajADNaSSJ-#+%y$`&0>Gzjs$-OwtC>4)+C|@b%11+qrQ20hZ zk}t#2$}5IJwf*g2Xx_$7nB^Jb)__qPNpWK(ON_1E5`eIK$n)!MT81501T#dQb=Rds zylAlLLMaKR=kktkphNkpmY8%;M4W1_rEX$ge+|tw7BLUX7<6>2;=F$H-6HVn{my*Azy!E>XlwOB(Vcr##ZeWIDaMiQu*R`kpd455$=mAOAszdzGe zpdxM-R{YuB0|?@vTmVr){atr00!{`5D^iT8Nkjt(O6Q4(T4}*4cMd{i3cFULNY&uM6F>di< zRY-&uK(E3NY@xS422+oj)^M_Ps>4AX@A%!bBfy}Edtuzmbrv#^Tdy6KJ85V^4rHA& z##3nf_DP!T)MQ{>rc`H$OU-PtjuE`IB*e!WgJh0qsH<<$YRs+>EwlQoHUAw1(dq~h zEA8cc4&XO$?4H{MeLuo3_jYe^MoYE%r@S zDjpJ>9Rc`sF^!IOsI~;Y{#mRlH^rh|&VF#`!O7ijRqJ5sQ09DQTzJEwsfrZnY?kr4 zT{PQqjPC9Qfur1zHStSJ6j;BfrI$ls4KCBP3KvJ+Kdewyi?pynE|zC~;;Fw;?euez zUgPHk0RqcFtLRUU)!zw zv?-ff3&&V3j`JMW-XUZNcUpRxY09q!`~VWLa356(C#r6TY62T@c}cewnHk$Hd32Yd zs_+naMnJ++Ld3?Gt|}yd9SXY+PTr6-PTw zQrF}jb5sioeCH)SBzN+-WW(*2TwOWuU_-4nK8>EYSgpQ3et-EUc@0xNnTSxfK^Mh; zvN_-l&Fcz*u)dU{RN@{D7MOzaB=jT&Gh7t4yCyugtkVAlKp%a%x78-D!!!i*Bad`5 z4|5`ov9aYA*IUZekg`Hx!v6~J;?$s10g1vv{LXxAb(nc(-Evq7*5DfY&cbO!%uE^U z+1VnAPWz1MQQ=hHCpwB7nbrC8Q#I!G$%fl}!|L-nS0Zav2gS1|0rt zrvsBwnK)R;EO?R=^bbp{%Wn1W2=MIi(wp_G4^u0@3`7h9uSJ)+BBOZ|iRq6-;MfYm z=qZ(s6E|z8-O7q^XapWc8BT+D&QtJEMKG*~Xsrz4%FmgxyLBG!%IqRh#_2HH`!1wS z23>+u&u5NlrY?pg1VF!c&=$qxW98~LQ*yWe3ya#LZkju7F9$7WUSrmj={Fc zTCnZ3_2m_oCF3&JN%yw`xt9(GFeL^_`MRnoX9TDhT?tg(6(TTTRo@-bm;VBw$d_sQZ zr*104!tTP+04rGE0#AVHSpNCzi>Rw{*2TFN%uJ3NG;BsbncnoE8J3UDRo+L1DSm>* zm18?nXphP3h!sRBSUwpyuARJnhLG5H9CDu*-T^!)KRW~ZU}Qv}c4*b=T*a`G=uMXQ zygUS}_kbP`Cprlc)v9RAHCx|ZI6D*Db1|hGu}W*@)8F`5 zJXx{cqJp4Bz9GbR-s}J^LJP}U#eT0`Oxm8)DxJO7RnYgqidjMxdr^nx*~W7aR=*() z&xx*n7@&%ID{rCbRmP(Ny%Lu2fdUf*iTM=OHs)r=L#Q}Fmz+U&Cz0wK1r*^s8_}2a zVsJdNT|dB78q&z6I#lPYD6@9aivC>`zSj(DQvmZBxl=|Ite982X^z2HA>NLQr(h5J zY$6h#%v*t7FVdmp7QDUothXNx*NKwNuO?9bQp3kj4u3~M_jWZtQaJJfy>RMg0moQS z>L?WPF93d~wQDSY7Yib!T(pjMX}xTP{DRReE=KhYUyMYq%H#i6fNk+Q_01-5S!s2$ zl02v8jeco47s?;4&4eG7CgTv_QfVce6e?<=(N;VNbTu z%;hDrO-i&fp+Yl)CigP5#x8N|xwtwu)lyZ_5yb2Tu?hixL}IT9xR4g?wCYzlL~TIa zFA|LE98=m_<`jSXl&>%|MeQVP!|pMlmQ9;6=|q_y6sZ6F46d-qI?p9dfHP6LKR8$Q z3ON3DgE$=zaWN>P(Fn%PN2EKgR?GHCV}4VwD*XCzQ|Qz7uI^TQ32s_IA;SZ;l_|_+ z8=Ke_-X1Cg-%dKwpw$Ebf}U*OH8{Ey`29 zhEZK}K%hky724In`c>Tl(>cpCFddw`F4xA6xf=c9x}GBlET6h~U{I4F#*gxU&@*|V zR?4AHq<{yN{FNp5F}H`MuJBZUn8|E923n{gDqm*W%Sg%@9>Is>>;?SL1l0a9^0n;2 z;p#zPgxI9p4jWu7cI6X63q0sZpH=<4G1MTcRMd-E!o^D!FWyKaI@%T_A#`^SN}E85 zXTI0G0*@pKKw*JSH{G^2Dlb6}W`b5qOqq7Aw$Q91@QvUjA2r z?#F_-9K$;9m&>e+h#yTrcJbZ$Z@DQP1c?nCBfj2dF?x4tBA3V(l3fm{-B;Q?Er@L! z*N*U@AVOEH#sL%{mhvJMSOi7qBLJt`)0sYUTVC7KIn0Jnm~Guzi+*#*RopXL{1P8RA{#2!=Xar+&U_5uO2> z5kxXJy|Wa-(kO8D522B3sPyA-zxs;lQ>3(j#Hf$h%s~7o${INCozUCqp9P;zdsyaFfLQU|IWpH>cx|hM&g#!L z98l~$Ri2rmeTIHzXNd{*+OrJCb?<4f!uoXH9PMZQD);L0p8TN_WeT98LQbo0I!{4e zp>5GGU5NQY{-IRPGM!ol@^r9T>sa!rY5o@g6@2JGSQdSm{0f4hDV?kOLhI74l0JVA z)P$?&C(s;(|0}@BZNoF&(AAgE&)Q}c*kDVte4sVx6Hla(dSgX}cO7L3%6QW9OB~P! zE3@^~6XYvMLR^F;0AK`WZJHl-+%qNroe>PSRvlEEIt^c|1skluFXTYLzXt$)zN241 z_RJl_y;Fu$n|+uUJMXas*GX;e%_=nfH6dUHFur`b4iyQhd><~ic_Ym}h0koN4dX~x z4O&QNzuO7bWSb_oWI*q?RM+hWrOxExpwAPE<41Wh$_ zWou17?XIh050K1he6#RW1tW)Kz zPXbT`96KXar?daW`XZFL*FRbg41lJa*8}CZqdG)h7XvXn%qJg&!SXhW(^z37N9=LO zxcz_1dd-35Qx@b_0_yBYB>|a9v@43M4A=fCHIWNH7Y-$V@1B7Ji2tq2G)nqDA;CD^ixwi? zOE`d4%Zbp9L6&B?OlPz$pKrbQUjZK5Niw57;*@!NoLLu|Q2I01Q3=sEr}L~EQetfzm+wTbyt9Tsk^a2Ot74Ot*drO8Ji|)%mb1(2m z?hVLPAd?!?zjsvuLKe#cYo;NdTOzB7+uai(kiq781*Cyh&FtMGVH347(4|QofN%7O z+MYd#I{AvvgiWO#S?LMLg7nG7Q8&wEOM63$nfr>jW$o{eRnB?;j%KC_;?T>@!!jk# zsqNo2X39faqz6R63~`>k3OYNLy%oPAM@ORrRob{x6&DMw4Nm$vt#_j-SIGID+kLWa zu{g%tts9waWK!W+;(<+3h9mi8k#5h|83R)EC2vAE0mxdG=3+6w>-*>0KMt+<;!O%I z_C-i#a8CV62buekT$wXMkO<4sBCmOy14{^_!i|7G=JP4{&_+A&qoO>TzU-eU z0f&o_C;ZZv|ZC zTSC@8Kk>3qf%uWM4Uly@Hqn^yLMQA|_VGik1EJVTC<56~aNLk7P3ta5OaDY@m7CHz zuxZzXnh@pnu@Avp<%>_DnY$Q+pt2i5r#fcfDPUYC; zysixKGPe8|01HHGtgI&b0SfGzm;oQQ$FeEW6qn-u*V}o=bM?M|+}=}wsq^xWqWXlMVh>WaA2-%_(LRLnyiQnhH^ZVmG?$bB#_vi2DVJI$zW-ucqO{(nPcfo3O`}yfK<4z!tCJ;dEVEa5^%RA- zAP%pfSAfo-W|I~6^`iI}*AE__^E}RXA|>?q=q+O+p7+F(Hb)`TEEHFBhobIl5u}c2YY{q+W9U$vqHu;tHaxB+dg_u@`R0iy3SDC8x7yK z(|pfao#m!5NKCHjYdn6J;>DK)t~W2(o^Tb*XD62INvCTS36z{uS9sYNG9S6v>n!`Iz6Kg4!cSg60`UMTdDY7 z=5Z65DF0i(ix_17-FTImM`dz*>hM*}>z7y^B<(Ly`aYq1G<_(#t@u<{uOCV%{R*?K z(KLeqbPPZ}_4F$?Y*`0JD3+&ZlDRmG{4QpP>(kjk=vrU7p=iPKzYak9ft!y5T5F8Y z_QkQ0Cu>e=b2|8?-NquJlq?@BIDvIXCpzAH@jgq<#s_WLSyOCE`SRF^1}jtlpWd_S z)c8y;iS2uWMH7wvS3)kA7vK->x8}S1Ys- zT79ACrHZeIH&b$)gaVgMT~;CS*YwbRt~uuC!3|r-^As~=edBr<0ww%Rj_ilP*UsEfb`zz2YX#lVjUR)N$PK&lUdO}u6+sa zqh9=!psb@4DWS|!a`imlVdk>U&(|FO=oo;y{M@aD^k16z&8i;=?BEe_;>de6ZCYtR z)H3sT;pY9HJERJl`&vZ%tqr&se~nEE{C4rse{j9wEBz(mHt%Uump5jL5tmQW(hV|a z`zGmk3=NmC^Ya#nm6~-^%Ac3eA$8u67wsil6%Fi`w8qlE+A{IhT>0#PCV?oqvChr2 z*F2wEu(EcW4k(%9i*XDHP|a_bKF%y7WP#9(;SQX ztKa*DlYGT?WXA-bI9Eus4Q6P%C(_cfpV2Aor|rN=p}p}sByYPh+3m(IZs%#v-YVQy zdSe@1K?=M>yZXNRTn$-{fx8rWRhD$5OE1Y&j`z)n{a7HwQ~PIhIVkiqN?s%^sx=@a z(^^smFJtb#ORt|vSwZl<3!~G5cI|c7rP{Rmq|YZiR=9Fl3uoWia>qM4?ryezc)ai1 zCQoCqpR9MUg2(q_<&u-{)#GM#>@RVB&&$(YD``G)LP$CX?z%ur| zWo)=}&G`Z!hGyW$OmnTmoaO-?dd@(}KwRo*aXx67_6ucQUD z4;L6?pl3}r3B?7jrXY>tnAP4oI=?&&@$~EzsV-24Cw-Ve*@uV z2g`QGc15%FGBe6$bq<4#Lu1_Sy-@?Vm0ijBq5eLgF%Ty1+1E(oT=LkTxt;Asw(L%L zdDgty%k*oei`O#GZWt}l7zqF2I@f%A?)QAy%Xo2N;({H!V@uAh*n?Aj=a%+|2r@*` zp-XU->`S~uE7IZ+3v4wK2NN$QA;)9y2#OhG%V#ffqSk-`0^v3eX*eUES$weRqLL1W zluFgb>n>*!-AEoY-kn+WEyx2J10lEQzn^UJp7=Z=IIfS(9fwpXdno>G)q*uj=TW4F zDmO-lE?R-t!$c@i$sPLn4-%;&ad{hG-nEQ>f$L-Dx7i_(0`4>njQ&Xd>wun-8k0e~ z0Dq1!aCPrGdt%Ft0E&O6H0dbFV!ghUw($B(KJR#KnpB^kZ7|hl`n#98Ozig;7-Q&% z4un+7+!X>v_?i~D)pL{G4?S+oAf zoZ2Y14`fp>Zf{PfD>6^*t9R+*-ZzAF0m=)6Rs9cjFL0l;-`IZrQg)q~;>`MnjS`nM zr)m9c5tipBcp|YNy$-p*`wyC}TImwi zGfoP>1_lU(M>-J;=o#sp1xOc|VMjeeH8CUA`)lzN4CGzrE}SGiK5*yX9eXI-nPV%) z12+9ShSNf_PhVx4uf)=ucb8;p@iga9!WaWRL(rW$;5_FIbKzf3&uOgXJxgN>v;Lv8 z{=qCy(du)>nUC; ze1B8gKq~QoUf@RH+Z-Yqd`K7Iumi%#D60*+#AXZe(B7x^O|CSIq*zxA1P+Meuf!O5 zY;Dy6je+n;Swo;_q;wEEbg@rhHpm{FKf!qtWxwCyW&X@Txy@g&ZQg|Rkrbh&!3$u3 zK!`cT$pX)Ref~|cICy9>+M7hnBisAC|AYCGc(wNZo=nbo7DyMEVTZY|b)aVmrhc>e zWRQ*Ou4OyPI~;NA)$Ro?CttCoNeb`JcIzp-8Za?KlL#REb^n-uYsTBwJyAK?%Ypu+ zilX7v0nZ;LRR$(;11i%h^>)}6~zYuB`P*;b@4;`|`j z|9}l>41{j1aTi>e8usky{SvlU`3(|s-z4g2ek0X*6{4!s?0kfg7%M$`kgTB56I`gM z=UCp7qEqnZ+{8$KfXYlniEJ}wvP7kL{V2pvDAy~tsQLiKOk6Y%qw(+KrCF?St z`4e^h%Np(l#j^yZr~6<$Ltz2onq+{-4V;ED8{rYr7p&ZG(y!vSZT$W=t`+0`i(2m2 zE;`0b5anC$dgrI_O$CR=l8Gs>S|Th1T@>{iyt$`*3|FbZg#GxeZSBcV1Ph)q!SK+o{{?IqC=OPuS#=& z8M?83=w#9jyhOqAcB771oI2^MXH>y+AGHUAzyN`e;E{u&h+ERq$@n5X6@D7eAGN%J z#H#U?-Bu>zUA2#_fyO}Sn;SM5t|2p7emh?~HfQ>~DP?yc9-(ah#!q4eZW(hbbm($@ zvC8>(EyeF^^5Gpmj}JKZXY!A&ooMbDl>BYul5!&j7$6WL&0>JYK#0`d0v)vhA@h80Gb={~cQ`E;FhC$YQaJ+9Gg6Wp9lG2;?|S~UQ&1Pa z+@OXI?u_kD;FkyD49^nBF~e=Y5I}U5alh-E7h1 zW)u_NT`FqdaQY88kMT?WT?u`0Tp(=|+34EtM% z3JJC&i$~ZvMfje#bw;&JECC-J5FTlRGte{Awi$Hj;_&uND*s33ll?VTdOmACYw825 zgF|x_ZbI+98hU2)h=BnDVUMNCBZ@SxVZ|rcBvbskk$(*OzIR;#>G+KD~ zuH@|AZT#MRVt?bw0XE|&@M*%p=xg*;2Uhj;EAF!!yA@y(-X!s z6c!Ll2kBI>j!xr@iOlozjLeH&$uteiA7ZbjDOn7&2!9Sbxf2u4tZ!3} zbM!w{VAp7!4+^GaAv+!zGgZ9B?g%sn!Xq8G4D^h-@e85 zDx(AgG7?&KZ^;uKyu*a+PaBw!%)$p2y_0`o6W%y9)VoROky%Tqto zw5rcK64~FCVCKK|uu4m^AxvN>QUn?=FaZK{$V2;f1$Uws@@{SRioI|AcG7%KP2dVG zr-}I46>n}Ko6E=Os?!_XRB6)S^4>ttkS@Qg_C)+F?SHJDD=8#qz9-+KcW#L9p!xoz zj;k$~H+xUO#GI=vKc_IjkRC(=1EH9$*YDt0VG-Qw!9IaPq0R`?k-CN_NW}QZH59jh z>eurF>LS>|Ca!Ypr!3QNm}@}V^Pl4Kt2nU~@A9#jqJ#TJF)in9X~M3}l;-f2n>-!* za9I+dX9(UERqW%iqjT!eP%`Az4q4k|b%+zxHl&*PWJK9j{~a3`77!-+#JV_&h&S5N z`V_FWZ!KO>+wSuHbfpPbWkZzV#k49=7br0ga}8+85UDl*#u(@sf`73d0EGo}SqLJn zWJ0 zI{m4%19>xtEV*d{r#YQ~?kYy0F%TjhtBnp_EPoLQX)$|U9>ja}1&fZ=@4jijfb(Sf zMho`pt&J?dQec2Uh`Br_(v~uG=<<#9q>r$UeS;fw$+57Ff;qJTmw!8Q#r-bw4%~I@ z#`6H)IuIgNV*rhT5b5v{bm%fKzUE7v;KyPdzIwL+XC+q$X9lk)kHNlQ(-fD3!72+F zAP^oqIX!+B^&u)jl9by{Xsd?nky+>6yaivJ&!qDcIb-jD#z3ePK4=x*Cd41w(_L^z z%-hp%m5n3H=9jyyniiRyobpdd7ns*K=Dsq3o*{^|x(*XF^zj3Pe?AX@aRXe7fog1a zRIXM#OqjCV9Jcj#)7RBpZt(3^?)S6Q($I+Sxn`*Lzj}tgzKU$S-?MoV6YI|^<9ud3 zw{Tog?deY~0^X$WG*#C`DF|RZLx06E5Q-V8mI*js(DdyH6Cltt1TEHHs)*jAmNI>H zIw4t2W<5?P{{us|#v|4d7y1W$@9khbL!ts<8N0qw-YuU4?D1zx>S<~y!S4J~qXCtR zwTkhWVPP8W&Emv7)=A>YZ z|Ijm11uXCt0)$p_u^-|%zt4KLSr#>pwlm70dh zfq6|qg^@52`nXAaS0vsr+BjA^c0L|;2Rx^8L4Im(gi3l5DMCh5tCWF zjlYw5RBKFtb@mq@{=M{EkK?5px!3(u`igpZWcvg91y3aDD=OG%zrF%%I!zWG zx-6emudC-ft}4_QtCmE*UGx317|WT5<7p!2EswYSQn3ZzIuMqfH%}&H6=cFrW>5B; zZ#KIT)@9CnkMoP`x=41`E)JLu4*yhbhL7PFalz+SsqTwLkL_{m$}o+yjU z$g`VhapH-L0|Nv?%qdQu>6R1zqI-F&>BCI+M(Dz1aBrSL6yNYiDXIQ1D;F-JLl?|F z)s|QIp`{m6?&~RZZJH7`5Srj{M(JlhS~tAj95X#B)C?3A2>(3)0(1j}584We?_HKC z51UZy5V1cmc~auu_motYu#LrNCd#?9MnGdA%ADguC_0wGc@Hlz#C4G`MU?5|}_6~!gae~2Gb zSGP~Gq7&*>_)K%wTbccINe?#A7zi=8esr#op-%7HchWF@LR**dq&D&07V7D zo#u&1fu0ZmmbzwrJYFpE6leP#wMZ&o=5u6v z<=I?Gdqs5_Q$$%JP+S#Hk7sD&Kb-+qM0j7dd8c`Yd8uix z3Uj3JmOh~M{F#asQVt9Y2+f9)CIYx+8%s$YxrT-O=*F@w;@(_wCuwS5nok>0tU`w_ zNdI}j0D%y5J13EPkf1{sq`iW`TL(gT4P`ID>YJ*|4l3<|AB%^YU*&+C%(d3!NQ(0ekxh!|DphFkT{c9lY{ss;< zAp9@Z?J>7&6?5OaNT)KPW4s{MWC0%>5Mpj66Dc=>4qcFHY=E~8gh*BVKw}_8I!qnX z1?FoVSTg}-?O6?ro{o~clW$Da+#t5-jHhI&&zk68T=p{1di_K)a_{dwBUSDJZygAc z_V5CAfl^N}w+7{R|Dwtk=ox}YU0Q%)0pXw5sRHE%LZr1`pfM0)uCfNHrvz}kplNOj z^WcDlrJyM@%w-{nv=a`}1^D2A5Oce~RljaV+SA><$@MDKXo+PvZ{N^KV*K9Z zrP}z`jm1859{fBruWbOVIhUwmqfPMUqwid#vUTB*dfi@nt?u^$v%;O=7BPH5)|GzLP< z&AgC$5u#(fAoUIe-Z~H>RR;o%f$+cR7lc$B01OZaF;|a?)S(F-;{~a98hGnKh`Eec zB+1uYH$tfbiA|10E_&&e0FUqN5|s+S#@4mPsM;?5zx_wR0DI%ULiZD4s9U0~-0+4!Qp<}!t)dm1X1w!18b`(>Wb!ZY{!WZ%V zj`B^M@V33#h4zD%C-)?}wvNzo2dT3RI&^8jM;X-1^r<8*Z^duOXOhBYmS{xlI(et} z@l=M-Md5_NTL;29y#0b5&&53!f)td5O7K`9VR)OzS_-&-Qh;=8MX zJXt5$MGMviwqI0s$Uhjcy|)Wv{D+>s)SUQT9}7=qCOt8FH=r^oQIS`-LqL<;cr5kb>V%w1hN#n>r!2mj z=Mw7G`#pMzrP@fM&4G^J5o5hlt#Okg`;&krZ~Qm*i-g!eN8IYBHV3?7qNviI{00UH zgh;1|0F8n0zvvf)GzS9)2!u$hfBK0N(1_*?hn;#){Qbfn^2y>^PVXiV5 zsaF^}beZottN)IN%l}Tnmu z8}k?T$almu79XTtvv;wY-m+Ib(0v0m212CTT6E}wRKo`h5D1ZW^8$^5@W1H0g}H11 z=FU|>>WG1k@q)S21Toi99jQknI&?YFmO?o>8N+`kW#Tp#b6sU&UG!C{)6_A#l;o%v zb@$Ey2Q(03F5`vtPDF<;NHyiaTL(hS?Z`mtxd-V2^R*785?~-S4?xP2z!(EPLsOi; z=wJ(#<-lAPiWzASFQg0b!2u!W{!J?xWvF^Ho*U{5Q}sFL)UlVkn+Y~Dc~Gu)I#;u# qiuVJ<0z#yd3DBVn=DK5HP8Xz#X>{m?H+HGUoPF8H&PF8HL*tTukSg~!}wr$(a-7hKop1+iH=1YBP?Pc~l`W&qA#Xa(^o}S`PlBY&=RUy56RD*ICFJ0dS5_<9r?6-VcAxJQVtU{}*%D zuR0Yh4nJd1yD5#u6oh}H{6EV53-_OJmmew17sF7}7L(CfdDl~nZ+8BA?lVnSMB|&p z!{X)KDX7wpGgn_JS!_E0QU3lvA4MSC)a`lV5>^eXH+)o^0?>>aG2Nt*<-o|$1>GK4 zsQ+0uk@0U5T}}KV58sXsIFjxyEp@SUR1rRn+d%*A?b((0Kg;!Di2JF)g_hd=y0AnyI}2%`p?c1mt|>7TG_^X-dNATdG^t zdf6Q-sRxg;sYF~|Lu=dF+cKMrMDr`xfh&X^1;YnDBd^aaR)#O-qDN8t{m z*|jMkuioejcvYANUW-XuRN0H!6HxK*WaI#$7YV71Nn4`4b^g}-x4rr-1x(o^)(Q8l zXCHQzW5%$n$M-F^P=Yd*fyA5Xe0{5GS8e@96sjKhq>9n2XVUelFZm)C0>W1Tae-`Y z-#*aTK7F4fEQF!pplK5>ZmxqnvA>~G6^PnN=wCIXA1ilh6mB=}hB8 zL#*T`077SV9F#ONCn?H|qjZ{)T!0GKf#`=D8x?ozuoWEunPG`NqoT)1F)7*J4*&$U zj7BvdwdChKVUTOipcJA5yA8-d6akqJ4%&uKmq^7f$QDkNx3KhAU+$olkl_Cs^z+F7 z&|rG+w+Y$=wMiE6(L)l$QYLTI_aAy=eqOFNV)IM58Vn%m3f+g!^&xrPmn5=cv?BRv`~R93K+JbecAX%bMlRCCjRFxT7e< zID*3G;LlE~^wg>wtz&hEgg3<|M^WzczI<1XvB#lQF2e1_$3v$w@G4v$o`*6ruka;~}xT65?R<7TqK0emiV3 zPQS`Cpw|e%9vWOFHoCuWQ<;X#&c6oz?E61FNt-8d`CX!PL@&bZtAQ9RK8B>}r4kL} zSVO#%C}idQllx(`>$dey`M@7O^Bi>&tV4PCHT{g{DHD8a06;8Fwno@r^xUU~zA9OI zV&ix=9D(wK>N&5tN<5R8h1KA>3oCul!)~*jbdZ*?!k-PR*NMuk2E+#V~Alg6z?c|*5N4;I5!W>ZR9Rx zuOl+h$1RgSibo$L+AyTcVbuna7jBl^0Pwq2dde!D0Q%lVGFVMnu)Op91|9&vC^d#C z4nV=w>D$FTwoB~Ysn1@oZ7eh{_6yp5MG2`+;P1@m1&Sj`lEQk;rxQ(yqeK{)_HVWp zZ%dye%5hgh8-kxu8m0aQXkWm8?2r?t~Nwf5Bx?V&lbbh830~*z?m5C2v zpG(j5alx*i@o`TblPB$^uW6{Iizcqhf$ZVCZPU6^QS=aOdxzsQB@Ec+YB}8Ms>H=E zBv7qqA2;p@g9o_@${90upQ45^IKo!k+OTObFvl0j({V-5K!q3|F%pEX_iK{?ZA(NA z3nrG=ZI7c+9?bR)#w;AO*NQ3Jp#RB$z=TAyh-&OAK1&S~k2Q6<-+54Vcr2Lw*Px#t z{~uZ>^+~(Z%ADUw(dgyUY=O3U7=pjY4)AK?EzOP1pGyAZekcSp%9qV`<3j|?uH2u? zBqx=u7Sp3+_})?Ko-F_SToB_tw?1g<%58JZm+;)UMb+2o&%IvK?K@vqQppVpf}#y_ z{RoJPn9E}0831Kgf$WBl$+zz&!U{M`6hFLB$Lrs3MbdimU`F4=@ajp|g|cd+!WDEo~RBoC$t-*RN)vwHpq3_o6sopl>?-+*KBE&)k-HoHt#9OPKZ;|;66JCn+U zI&37$)nhIc<`3!6J`JDQN4`6qZ-dNyXEB%Q8dXjL+S({JvtNB&gFUtf45m{lbLsN$ zv0QrDgVd*eoer9@4kwir7{61q{B4N7>(Ta^K0TQ%?S^^sdJCVIak&l~#MhbyUrVhx zg^rI>oE{m*Y!?tO6}p+}kSLr6!>XJduuMGM7L>u!rUyjCO2IS9k}4-PzPl2(ms(C5 zpN)^7HSz`F*m{CnqdvmGTL$mq(XsC`YESP$<}n48ltWyR`3=hl(-TMCjFb&oHy97< zgBzK=$3$_8ah0GiP{J=*Rs{M*Qct~VL{4K5*meIx%JyWJc5y&l;E8}9ppU2>cdN)2 z9M^&ma)}G=1SDFgW6ZUc;;LDZ+3h~I)9im!6EakuL*btGg&T4*bfaf(R+ATys%u`a zkLIH8Knq3bPDAkhT3=C_6TNu{mJRvWpr1b|KP&@Jlpt~oCX?HKxk2Zm*HR+Id=8#X z0!7^%Kxj|&3HizWux5_ocAZA3gP83c#$emhXNXtK0ZgPblI za+pj1xA0xJJ-7^ky_s6agZQ@}%v$60-@=PDB#@%MG&P9z^C3$9wGUWc+yQsju_j}1 zOSuWM45I8s@r8-6v@fd@3)`5B+LmBujOBz#Z$9GPY)Z@ah-mtoC~yjG_=!`8JQS73 z@l7xZ=+zl0NGo6pVquPWu&0m~dqV@WRA&R6-^NWmXz93Q!HF~=#*}+GcF;)dH=2O& zOm?@HdIGbG7RwM4ov4Mr29R)x3_DkoSk*;y{h13 zU#!F;IBx4+C+)3?0BT%JeI|XpXbmFJgl#r$ff1B!%MPSrjnu}bw(DXF{6l}S+qrm( zmun^>!hI>ybmEc*uK|k&udp4Rb&oMWEVRy7##QM;#bZ?Y6fdp517>|!vo|Q*Hej*{ zwf6yrXraRwUC(zv8^LAb)fAhjiC~(YgvUVm-DM*)5ii5TPVGE%MSC=Z+kGx~+eQqm z43EEgYI;5-;U|^y;(PqulQMj^Oknt*@8aO{8X=da**BV=U&yJg5dG;`-ng@Z{r@$H zA_kes+PDKt|fal=^2O;<)y0ke!SC1|oUF*e5&g)M?GQ6G-Es*LVgKkX%N%8c{$ z*UC?^HP!p*k0-^8<8QjhPro^;wH?`U%tNC41Hiha+~sLa@V(1%LauI=FbSUnCJVpl zITuat2^%OSE`7Ch6GXi5+tZB3uO6u^!$?UA z7Vje`3r1YiEVy>>7iVg8NDtb-v(%4-gCY&xUV=m|)bzB)aQB!7D|#ZNHH`plr38htr#|g8bG*^QSD}&|1+PD$kTV^{`r&pp`HB@@l6N}+#gHy_1}?1L&s1%6Yfp$^H5M@ zq~|(rzXet^K=Gr5PF|ZCvW*iVO6-ba2=u@UmX+S}?#f@nf6<2p_=F^1#u1I{r z!FSp*6R(32pG;iL_`E%DPPWkFUjRk=(S>7s$fgr9twP3k{y{^XjS? zlug^HE#KSxEUBXSrLLvZzgcB~^EyVdF~`k^Q)L=)TIo!qUr z*@tjwU(DUh%lWB)@>bw`{sKnqX>OdrE}+~2mw_}%SGW%4>B z#$1XaXx&UmTdK$WMgNbG(Yh8bDq6A&e2^`7Bhji~o2&ui<4Do3)m!T==M-2PPW6aA zUNbEZ>8vrtK^(WO5XS`QmzLJIf2n7o~- zd8hllXl42Y@XG$}aD;W-*G|5Zi5N1WvGI9XApa?W$Yqg2f9*{gACty0yp-) zj$t~96nh$9C0O`nswy?Hp^4nT8)C()A$gH5ZgrT@^!&URn<0(hxT4 z9#pT>#!Jiu)c5@Fk&mqdkw=T6TCSI&5r&5Gve|rA{XxD*FBGr+}bLX|<@A zl}Y50=ya&&WfN|XQi>PqGf;I2jsbO9w<5eDT{rM1yw%1bI{mN~@B2g8aI5Q*x(Yt;zJFMfjJpCgwFB~2gzDe*SkpvI!zS9ALm;GTb!RGwMP$3b!MQUV&NT$XHQ%8E)ofYz0&z=CkLksR|` zpjDxgZfBX!m`M0oTPNWl8R?gfQA;tg~QEw*3CA731l9}nO z=6<`w9&RAEjO`1=*W~TM;y*p_55(>TirIScL)J1Uh5zjn9&8qG5nRe33uvh0dDJZp z?l??UYjzpu68(Fj_c+~)8TDlsdyGz5$^~$iZ2TGQMA~BRP#%;yUW>mqpHJk6y`+56 z1ej=>b`m84(+otxu;PjsX67;CIM}3)#HWg>09Mr%%>UV7(37X{VxZ`Qh#Hs$YoS^t zVWxEm`@)0LETFZk_`Ik@`7+%Z3@AFL!L@1h8>S>Wq>33%FDE?g#shQ-UN$dohzGSc zJpTpLC?U6S;iXR;B6Po+8DS_uW00@f=H{{+Q;IgxnVC(k9*io+#bgF?m9=-_V5wFu z0ls|E$dM>CTe}d(TcZX8?ae9643zsP zZ-d(TY%ikC`Bn&`5qa?D4sZE~YWxFmSo{fqV$#2fV8)JBr1vVe^r1-I$SkYeU5})$ zyUffUO@zg}ZS)6S3j$RCezS?$4qplyj$W9mpFLs~F+Wdc>ZTM_L3%t4-q=y<=xw+< z{h4k0fXHkYfQSSt;bQ9norAAarjjfia7~@Qi@d~FSXT8)byOZ$0(y^IIbX09ODS5= znaWZClX^KXB$cdS)DGl)My4SqSOAw+o}vMGKCPi`dEr*a!>5zP!5tFlS_mFOQUd*Q zHE&%P9mfVE6nMicJ&<)m9ISYjN9;ylO1n@p(T4^#PoihRtvB!EFHIun#&o(b<_Qyv%Y$UI z&?uyuv{2_fbQvZkoEp2{2X|q283n$2ROEz^#fBE`Nw;EJ$xo#Gi9tD+qy;x2=i*-z z)*~T{&9bOhC3(xqCR#mTQJ&vdUc=?Gi(JQauYPD#=_hJ98CNA@)N?Pf-XZ!-plW1= z5O1J*`65b7HG>s57>jQS6WqbcorA%j1_a-g8ig9Xqz z!nx6uo$A04rC-m)LLD#yL(%&4uuZ>8eCQKSlCmZgC#cEhJWzY-Ky z30-beeX(iyIq6$M^st%t^|E2=bk)BG{oEA(aHTei*EkBepG0t__0-_F8GP&7q7Z4W z9w`NOv1clv;wSe*5?;S3uk||CMRrwh7|w3lBRb-qN$cHD=U~1%viNc+8dvRFCO-8y z+fZxe@1xSt)QFlQP8BJT)G-w~^QVeu8uMgq%JID0N;R zS+h<~0^C0e;%)TT7NO{&?!PQSGn>ig zWtz1M(_;y2mC-H|P$KtKVC4i!IJ^KpU0NJNAx0CFVkOMV_}$P0 zpw4u>1~$GBU4L7BG=C`j*vWt4MCpGUf$mi_7-ctQn92E~{}riIfd!oLRn z+i?7-lM}+LBHQ^LewEy#No~ns$_Se_VDGsP{1%wO6wk1^vioa0HC4&wLC)i!A zt8f97`oy@BiFNC;XpJRR;jD+kmuL*|^jmbKo3x#Y8p6&PqX{Y*X+T5%@_B%XwB6o0 zLlC%2jXtMSV~dFS>piY_Q)IvHNRv^X_d%(q^FR>^5A}vAlD1IWqb}cV43;jw}pXPBDvk`eo zW2AZu6xkP$ik9(xmhHV;cfgFq*6(Bzw)z!hyJ%b{E`_37s{tj7-*V|PZ;k%6^N{-@ zhXWw0I23AnhWmh$e~Ys=zRpX1d%7<}_X2M(xKYPl?F|7 z*<0563cnVE_^cE;M3kLw0?m#QqS{}tN;Oc^YJn(&6KNpfqgi#o=PCfO3Q_lXrd7@Z z@-$|!Q^th!-Waa`4Ng?4k~eV~Uj$=YMnWa!ScF{GNMhb9N(Q|U49HQq;8+d)#GdfQ zm?2dT)2eIVJ(AS060|c^GP4}BW5+1^`(J~8uE2k26Vbni5Cxsb?L6E%p*6YGggfiC zCVpG0ITxb+AzNuxITbe2n?CIQ!%g z@UBT?Zq-P4mslA}(r|$ACz3uBrooyC0^?93nc=3G7q^cxnf>>cR7!37F$9PX$eP-G z-9@38(pe^GzsmBfUzA=IM?3lE;+h;w9rM(t|XD%cg5V~gN4alt~F!Rx=MN!ePM z?`kdWBwKEYutZ=k9O=tLe!vV{VO@f)Rbva zVCHPTn+_V?=1tWvcgZzegQ?}6?Vns}AlzIpM;4O&Wg6A55XKR4%qm19-p&jzM(wLZ zGohk2Bx~w0;xnR}{F31&#!kkUJISseuxZ8dp6Kg}<2$`L~vz>d@1GfCx-bWTFU_b(xKNA}GR*A3G%X!|2lj65E{p#Lf!o zUxR+;Abu!Ok0WAw(HP4m*TSOAx+455q)DQXIj`K~Q3VWiaq-Vj?uSmEp(Gu0oGKTc zrMTVS6QyO-leg;xneD!e!>7c-^3-J8!@H7@;_8^MJ&PaNI`=7O<@E5^o?mx0ZhbuU zR1UD6(Ay!{@4&P9dS-G026zScRyJPepFukI{%%37=hb#{PV9IT7llxnt&m57XzjA8 z=^vDjunG~f`lJnr7Nf>Sx#}r+-Kt@8(Q|K^A=X8ZwX8~X2EJh{<=qrstU~a(N_9Vg5G0;M3kRK zTO;s*_Vyc-{p-07um|xCMKGCi`;ae*6}wBZ!b7JXEL^&IPdVzRewz&^GS-rjY$I$H4+ zHTy{_CEV6;%Rt5?uFzN8FaXyz z6GR>Q_kFGB>!s3iFf=*hLCeVb2)T!r z&Z2AT^A=&i+?-}5E0z`tH_cO3BWSlw6|Qx)f|SH6gaUrhddmZ-PCdc53Tfo)Tw+B%32tn?LD6b7^{WMR(?nXs{-gL6hreyfC zL9FShjW^cvPDM$MHrOqZdgjW3=}8qCr_AB)2Q!_wY%CiJ1Ow!FPhh?uC-r zvi}dkfuzt#vE9BbpHDi^M$V2M=>UBKeSU=&XlG}2ohvp&D_AfGmGgM;8HnU6Sx zy)V=|^?Y)PQ-~)23Tud@4E&LjkU4GItcxxRo}Y<#UOAvue^tk(-=^fI0aYGEj=h~TUH)kTtzTg}3j!=khnP$|NO~kYCo(p?&9)9PL>&();v)&>E z%6{pSE~Oomc?su^XHDg7o#&w!3<4!Xw1-7#&(zHp9x?@Xdc=OY$2o6e=!<=)$(Jpr zv2jZ2n0fK>JG`nX42;-vyLpYV3=0|K36~;2D|U)OXoNn>;Qm5dJ3D}8murf)6{Z+( zQstBPbF7CfwjX2=M0%9lfMFsJ>;mzPX|WoRk^p1No{aV;4MR_7!LT$9+7LkytN*lK ztNYm-UU6$iK1!)>Q`uBC0iL9wpgc>dX^ZP%Zv+hKj`gX}B^~3)5?-6opJeR&rh8CI zXZ9|ir$X;Bpg<%NfV5HR!qi-YBRlyR>%-BhW`;eg=Gu0&7ViU>3me03dS6it<0S-~ z9vEq1a0{U-{VsdYgKOlv5&)^fvpF`NbQFln4f)Lkp;hhlk!}9r?xTrHFWM7X&H~_9 znEaoQB^7kub>gcR14``jN=b8>5s9fVj{P>CXv$0gx5WQ$Q2rI~U?Ztg_>GP-OBmy{ z_d7!PBBft62<;V9Rt%?`>yJT1v{6JNyoXO=a)E3cMilqFcVvtw8PsN16oPeks1B&7 zflA)$??i2pJZS?Lbefa368-LIk!^PKo^HpBegkPF$IJ>T1H9*U^BXa%*jZ zv&rF#VTfy11{pI%l}}1*s^LdU1?~CY)TfnE6?(>2MfT=5RM2el9=CDB;P!rNx#wf1 z!-k(jJiLA?GX41t{sLw>V_YWE;l^hG;`sh;^Z?7sHL#9s0@#|!lfeolJ^MHGfQRy; z2o&{e)fI}{=8yUOalptyY<2x*Q3Lf$fFP-H0~Br_^|!H&^^2M(etiPiKU za?@K^XA`_ODBX$9mzo9Pzv1I~J9&|u8OL2$@VLxK{y@nX`bzQqEfL1E7P2=WpunS!ba`S0@SwkT%MU+>!I>JS11kY=o8CdazSr-lVM1WMFQ@s8qXo zo{biiScMQt-Gao<JlG#rMINnUv&p%TbKU`qXlFZdif4lEef-N+6ID55UMhM zb{^5b#G2x>MyvAhPISHYl;Movk(zU8S0ktrp7M8NW!ueXL&KeRIbX}&boc=TF=WLHHp`(a3Git?IA!2$z>g$sP z?G>J7qH*#qT16vGjo7A}rMFiiHek`NBxO=Vbb7Fh;d@cR(ayR8SPEf|bSUCvI1!9* zBfwVaMW_@o{2zR7#;wnE(GTYYceO4nAQQ<2zv1=T>PlQ#t1K@b@`mH&kx|98c=14C zrc*$^N^G-|jxD7~JP4NGQMCgv;xzW>{mKl9j{^x}Q{Sa7ypm;*+bf4tiD)e~3s$gUEre{gM_Yu05X3^kuq$}4HJXj!o}~BB6m~~b?*&aR z=Hx-(F#k3jG!G-&A0kK;<=(&dpm9Xvdl3oPYTK|Z$vX)#;OEdu*HAp4<2u-{A1#^I z%~3UT!!@trL?3`nOZ@{=s27<_6*9hjG+iR7!&RXqUtAcYmRw3YegEHd$Bz*|oXgTi zx9Ce7K2`=BguFX!wLuH@90Q4-N1fN*~+0sFQdX1av@z3 zP~{cXwp)Ge{hI6Q3$|d|>pqSxJL1v6KIhmY@u|_*NFutWkN{*lm4k@a^n-@K1g6(^ z=*9aagfNt_P+HtrH0x(A7qlEUDHy)oNK1a~=$3SRdI{Rfys2E{j)4QulKJSY8@+Xx z{sYofWVoq+x%*#Pe4Ta z_m_W2tg(}Yh;Zd+4wSVz7f~A;06c$jSbnUyrxYH*nAqohppRAXvS7H=v6Kyw&I0$< z4&cTqhuXAnqv)y87!v!hlNQn<7@)C4w6|mzK+To3xtm~4!cK2^S;J1H^R%f99ixLI z_E!Xv{qI^2Cqy3m0Q!1M-pewCsa9gT!=B;}DcV9`U^a;X5ONnV8CksS+!JC6CR2+u z-5!`oVBz7D9t;mws|mq6Wt-v>^pakQLQnx=tl*HN@LF6ifS3~>Z_|~2p~nQk7^+bq zFuBw`T2Cm5Fh;^{6EGzkuN;{D@->?Oz6T*!SA$MT1NHzY{LvF`-@qB9^ofB`A&UZF-8X*2j1UGuv?XQ7on)^` ze=5}T1L-N!X5@EbV!1Gr(O<)z(+zeprd(ng00)`cR9I+Tmf2OVfm8TS&1$8HhGOE; z)5+m zHEuyNUcRVx0E@*`V#gU%1&BAYR4{*#lH?`>cDeV1SRq+J#Qhy!H1X0tpb?qOwE~kq z%TnAYP0jd6z?AwMhM8|UwSn!trFb-a@Gq%|3Okk*M+TGVCJA`OJO2qZ z-3VMTi#@!FEB;-Up6RhFom#W3TQD@}DU5)vtg9Ip=raf} z+5sNNqPD@iY3DJJstNsZZye&ievs*m^GpKt64LSPoDP_?#b{_9_-8gU;lhchy|W~l z?&QB4#7U}QzKLU(S(TL2*QL8NX7jSBHW1WrE@ z_TN3sPFMDrGyR~G#$MK;c>?1A(_*kUQ6m<1zCngM-iLrtoM0qYIIou5OfSrRz9e!X zCx3VEJ}Sok45B<#bm6Kt&bmHO<;xYLH-=y^_HiDXK%MuIyC~zm`12R&l*m_XNX~KoI`;EQEqD+}6G8v`r4pn3;PEn0YKbVo#+4ln@BF*iOxoU=3 z&ZisB>8;S#^Vzgl0eZb8}|5Il^W7adl_qyu>PE}E0hWno-?A60}A zfMqPch5!VYwTBw`nJ_1t!u9lF83=3*_9k~e=|HpmcZ0lUpS*sD6-OD)cz$I2H`vyf z?WxT#urP5)TpM^<*!%w&B$fJ022*&3%q<9k3}2;z*a%Sp@q8> z4Gq#wRr>yp_NCBRCl-aIKVfg_+*A{3q$3}aDiInSIC;YxA^!fECc?&PrjfDnG<-}G zhdsDMg7iS!x=7@hnk-0(4fmN|O{VGq85;q=9}Vi$7b?7HY{+;i;pHx_m-ED54Z}9oew2D9 zvQ~YKZAQL4-?$L|V8@1dcv(%MmqZdvP1qRe0U@o|TxGt*3`W`TXHOd{Ac&fCQ~F|4 zsP}?1Q49Sd7^!)mi>_X{YF>pNPiNy@V6fwEJGr`G+GU6fQSuv$mo3Aq^gsLj}2NfVuGSk3lhG63`*p zDm3QMO@w6}k==_;M*`h|_TE(_7dp~YU$e39Oq2XVLwdK2L&T~}f8bFrSy0FeJuL*( zILv#+bOv>dlZ(UunP$J`%5iHA$qCh@mUHgbdo9Bra_?a?^lYnIA-4QJdSdEOsJx7FaxFp+*3$(d@D-T z4AcU-Q`ch0!IdACvu?D?kPJeOWt>VlO_fVa_33gL@#~sJ7@_ltLOrS51EnKjD0PC7 zx?q0Wok0kUat;wS(CdN$niw{q-O^zCqsX6CULYmmqqM?_QEwsANgG8<0HU@FHG&kf zLc|Mq_E2nmkS-jJ;s?ih6)mp8*G$0`5D-H}7ft4Y<(-WoEtWuRcd8PSXDVs`OYxhJ z{k(mEu|r>XXoGwJiEo4)?$oYn@TovL0fTh4C{SoOxcWLiY#Z*c$NVG*j&op-pU1+q zNp_er4mg&_h*xoH8-N@un$u<6$244-0)?3Ao`=L%E<~FCYtm_EiBI)&G|Po+GBUla zIoZCLSt#_dGzdqa$cO{^I`qfZ8b7NS+n-{ zuh5GfdEe@-!yEtBvHE@!GvP3kgNPMDav+u!q1DcnOo^)#V$vIdb}hf`K1c~8jUUC?E>POD_pYFH9}X3U+=12y3B;=#tmV#>*6SSxTtIR@SvkOktZWiwlz3$U?jeoE z32nro%rQMGwrt$ul=Sg7ggC)8{S%o;M}9pX9s!8n0R~ecFgkYLfU!?dsKKhld^AHI z`)?V}eK$m3%nQcAG2&|#^VYF))vd|62Tn|uPb^2jfROt_!Ji)P$kJsk;2Lws#wuq8 z`eJdOTrFF!>7EYPT5NVlVkVzGz&1Cs1C7FPG-$OQMD82>l_e3Mg5EYS228K^FjA(n zY%>Y`9Q@Uk2@}O&<2xbL$++7t3JBwMJX?G*9t56M?%J$irbV zF;t}~<2DI=LJ)kE8Wt=fhOMvM8LO?3^r<&V59wn>;+8kqp&_k(sI_~#h6aSvEU^je zS-wjhn*ZgQ?YRxorKMX-iupNd|HwmNpQ6Er46LBdn;lv?S5ItQDT33317ca3I%BLM z`j(OmQytX2)q9u#jxxpRu8w};5dvlSyYj{6H-su`b-sS;GBw#=aP3W{`yJtL4ICL$ z_pTPw3kpW!)L&9UHJ(v#b!nj<@{t^_J1Y2)8?0WkP4PwRAQo-5Tme~hbAXj^McVP3 z%09|dJbS;EWX zx6ZyM$mV3~9`uMV(#Jcf8q$ix7bK~!WKt&P38FXC#r8@=hw{~ss*OZ^=~TOawcroH~~hhM4JMO;82(kEWLu+&B>cDXZLDs*}!028XUSEW-l=w zg_yUr1Xrl{tgy1G<%Mnef`vd`H|F8;CyJz@%1c67w!SVDa98}Bc6(?#K{ zJ}%(1SE1CTCrNT}ZNjZ>Bu))|h_X}se=?9jI4xhn>tOKOfRXWZu2=qAV3Va@zr35F zU<9uvZ3O3|cFN*aT?2TeRQIfn)tKvQr4%#igAteX76MWq0OW-Zo1AH|8u)0rmdU!Q zh~B3bDMIAOLLrumlc4Qj*573gFDj!{GqHzDej%g=wb+j5>4mHgMgvu3I8JL3wGP-} zzAv7-@sRVA*>+&iCeZeZdUCiNe%jm-?N-O0S5`ojA=?S+x~;=CRWlrPt(m5^YWfdU z?boB$c7%mR-Vd0|b~}bs*AsEfWf^X^E+9K3%hM+AZb82=*@3Hgg~k5$b33iEwm!te zpwz?d6lkDEZV3%xe{`X9@Q6#ceP&fGKjy6oz`?8q68I=Zwy z-Lfo1!>yMN|6@>~L(F7OJ5iJ;w2M2{FezU46^oFW02}InoXtwRSp!h^o3h5kdtneI zuf9Xb9XK1&a-3zk!-I>YOUpG&pI}jIq)dO}eLl|c@a%%xHz!u0ZS{XR~U&?~q^37r|!`Qx++JcX6c}tkBp-uc*}o;G_s>qlHZrf~L&l*a7;d z{t1ETn-}z~@(Aja=nXYrO{~h|;#)#h*^Y|dbX_%&Y zB7x=3Fi{VrN`mj(*Lt{V_0Xf$QU_aREK;3rRNOUH#j8ku@fe0%29RC?CEe`Gd$Z3o zc}KU$BK0O}Mu%%&*1AG4yP%?*wPiXEHUuSQ7JZO5t+W&Op@sm1H+&-z zrQy2jZkHXcelZAw&onrD+H0sLbSh?&vntyB|aj%pud=s^sStY`yiG?kf4#`htx0z3F`^Jyim%R3}b@3#^oMDj93sOT0 z03-nrr2)CJl|K2K4+%cFMv{Ka1YSV=rpjI7{+>kCo0S3h1SHpH9fOM22Oh3%fUi7x zZ+q-g<U#I3oHf3N-${^)vA|BBl(nc#}>u*e(Bf`F|&uF(DYk#vrnuIx4!2^A_%{&JOA=&9`}H# z>A3h}s;9pNSicYMx~P7ev{!JL3sKZP5$>B4{yT$KRp5B*Psz{eyaHd0Fdfr#!#|r7 zK=tu^G~uZH@_#S?6~wg&E$c&yyb5Jxj^&<3PQJmYn_V9MaRhC)>bFoh{`Is^JQwH< z0a5S7*fi2#;E5}{E)i(XopfF+q%W4iGm;7*y72PA0XFpGE6BwOWSJ1PALsFDVbwS` z?<^h(`3E|p?@33)czCQES+gZpozu>L8v`3tzl_$~12MaUm8$ZqGlgQ7nlOd3P56|) zFM+KY*8qDGAzKn%q6TEo*4xmQJDzY?ypZFltFS_j}E_tu4 zvF)T68+HLT3b>itdoZEL$zh^(02Is0D-gags7ipTeCC@bcp31o>O#jt030W0|Osw zpW=+ra}}H|`22uTXsz#S+B$xCF(id*`zg6qPH7=c17^Gxg8B3u=K{~c<5Cb}HQlzL zTz5Zi6H@RzpZx*(Dw_9o46I8Q)`y5^{*mb{syp+qRIQKG{$ zTM-jd0VlB8-+ax5rRf)(g`ycxdKzWYbFY!Du1xnk;w>#B$#38TANb^ib5a#9p~=f( zVU)&3nY`mFnXdR~>nbN_DA!gD(47;t0{|)qzJQ#x%PLD&zg}hQ(}IHu@x_w7X)*(ZBg+E$w-GV`JvC|pj;*8>}BGq;q0+w zn(*;M+OQe%YP~M_wWb5x21k$9(qcVvY+rp-8aNGx!4qD5;gJ>>Mx4bw@kjytmyOr8 z6$+6RIOn|XP3^ly*Sd7~X+?{lSwuFLm}YCP1x{v1f(A2PWimcKRaU1lug#BkA)gV% z93Wukar1GbNCbMKck`{)i~D%`>3#1Dy}6t`?U7G6hCazI1t$l_isB=$G*r*r8g~PM zs+TGwEh*PwTdPo^4OH4D&k(pkf|O;p3yP278l-TU>38@3y8~HgO|tC-JOd%HVGZ;RisJbLbkaCr4))f=nlP zdftU_>GziOx;PB2hk6qDR2++N@n#}!K@2aYCz5{_F zC?0{%IY+r1tY0W#a6LMg{=_3A1aFdIS5-tviN?&1pK;{>7KG$DVsb@Nn#zHVaKumQ zi#S)QE1H-_T6t|P1)5Up^YB*?e{~i3w^!TO$v5t?&GJ1WTo|Y>3ql&EPTuw(E5lN= zVMJxm55W6^17(dNaU|7p{%PVdGpU{vCEcZMtc&~EP}b^nfV;}P>(y7#495vJ_q)5m zHnTpYp7=Yz71zpViKQ)A87IOH2;%5~i9$8M%Mc$D^_xy}2`85_2mbci^oyQ^pZF%I{(-LK(c3>4%(*VJdXbTnF~uj| zF1I8O@}u4t?t$Q3zUpDG+umG{2`L#9)Ek=?4m70iZX+Bt9|eP&hQ%1eaC0Adc_m+&p*3DD|RHU7-2%_^s0#U(B<5iDg9(^*)O5BY{t7rv2xNGzMr& zWY#J4S0nma&AHT9-THmrJx(&DyjX&(h`~O51S}3DO?~cp=N{hORDt+;t_6N?Qx&W) zu~bG--agkUdj&pxNiN03JLW{nQ9*h~^O*!8`%6pbuS9rIvQ>1JHDz7C=HL|PHKSo; z+eGkC_o6qyy`t}slS9$w{b4jx;8Ww5$f9S4?NEVFw>o5_V~Xm=R050+nC6354T2Fw zO?u9lkf9SJ6JeG@|1HQP?yNrrD5H>y?qUkS%UoxBV$`_&K{RC?I~ai(*dzO|APZN^ zt_$6|N168r4zBsIL*=`bU_{zFJC1j+)%}?5d^Q7&Yv~6N!Gh(TLMc1^!37Ai+ z|1A)soIqc;7y5%{P>%L?5_aU%Wa$_wCqz69Wg#2ny#g{9W~tP)lj5W&4X0M>Y?2Ci zTK@2zYg!9tCFl&Vi$p5XS!W6_zXG6O&WkI|+~gEHi?YIhBDdk}Ml*aepCqlPagB5UgSzvu`w+_^tId(a`9T(ZfzX9k!YJ_PTK`i9Q}fK1s$k_BrB1Mkt_|rKY=jIa*9d-?r{fBLT4~VUd8V zk(z{G&@Vp|#PoH*_ zUO~(pE7<8CjMYHYI?mU#$t`phij^)x}vip5oY-(mP$FOZM!-wuscke0)Fq$KdSzI zt_Bo`MBdwC+fl^3k(!$d(}8E##L@Z=jcgJHZyPH*^%(#Kd($lXt$q3dXqjuRwz<6S zI&FJOuu@S5Q)u_@s9vg*(PSg_YN;qF-1$(>Bj)^7b9F!T;-mF_{=CZqH!%Lgpx%#$ z+^Z*N6{qm6YOcc(veM9KDW>?VcwWTU*L zy7eO*cVgK#nTYB`LHTEHxqJ1XuAp+reVM7-+;WbgKP?9wDi)UW!D{58AnS6oIpO!R zzX6RRjAS#xM{+byfHv-rE#zYe(pn*M;)bqnCwOx>X)ln(g1MaL=RD=<+M>?6@i#4} zB`F`M8Y-f>V4_S*T7XcZf5&yxG_Qd0jR1*u^L!f$*gJ)^KimN?e3eqQV;estts|OV zo-sfnd5Q5U&5hbS2<`DQyiSp$Y+?7|FYE5Tv12isYb3T8@GNudw}pKT#8Xs^wLrs! z9m|SPM~!UJ$0XV(v#{;&9d`|ge+%l~pb82DGI~NNYfRJyOZaerF&uEKd?J3HKsc_^ zHN5#N$ZY@!QGzJz+7^T0GxAlYlAP_T8x8p9DS54Te)B6J9UF74o2~bLy5e)4$w_kS zN&_$_zb6&b+_HFZGx-BEq;wN5yA3tNFKvloRAgF`#^e3Bz;O&waq5}ui)NS6jaJDn z5Bpq=5Q&<&-OCz+)tMe;A;}#)SspbWlc}I-*Fm~Q3p~n9mFgn-1)F=7h8#~1y*pN- z3&k|0&!o{?DFXQbZvKi!?a17?eD_Rf2|) z4!O*&rI9}5fJY993>wW0P>IXEbfe1H5;fu}ANk)BIU--rV?Ks-zh~^qmhl#+N$5mM z2&Jf22iO-%M2GDd`~>Amw@^N9HLG(C$x|#IfnskPHbFZeu8&G`J(Jeg0*DNWFa*lY zqfwgtPTo7vCvJj1ir=|_g0&4}Bh1K-pp=yR((zbSS5jy=lD4yFVqJQd=y>jT+r0zM zYlp9Y70MPh_vSz}@>2+LjmQ0hGuUxi*w+n#z2733LFww?C^jbB(vmM-ovtzJ zfx1-$Yp%V#BFqjxYqubelb*RAzPjd(K&kB&B`JQIYEh~a19c14#-2qYx`o;4`~M~m z|BCqg>|i2YJXx@sK#jP4Wv9_q7~14Xo_4p7{g8@jJdF-E|NA*xvtyS52qX8HEGE~& zKLLMWf69GIG`5crV^)RP5Aa^s3!AOcGU@1#Q03@Gpu(;;?_aG+H9Ma34=e94xrP1g zJtk%L-4QkC!1LH&Q1~bUV~MGTiYY$m4VIA1f7 z4E-Xs_Avh@(avh62W97MyC&EEjz zv>%+V7TlUCHYe4-^lkIn_0|Och{4n9DD;Y_^{n4(_)~F2Ua*);)jri8k^Jsj{oic8 zGqK`f!Nq2VV}oD_K!OgH%npb#l=jeB<)K1m1uRwKXbcTT=Fg!VOT8mS#lEHuh%F|E zarE#hrkp(&r7^$e>HD8WbW_=&6hqyi4@%zt3M)@QayI8{tdtOPaY&fBGH|qgh}n#I z&+ATmOeK}fiza%kIr8_tAUfEPw5i6HcUtJI8Yh2+e`ioD(g|$T;SDn#0~8NZvkEi( ziL2&>Z)@@?t{Egme<|o+L2>EfRm#B_#d*xR!wRUCcONV;!KgW#nH=bVF~p z8s!5kJkbf}uEi5{nZ9gRRSsYm5Io^xdV|_&*~EOc4Q}-B%AxHt7Q0%mEI`D2RWVH;d8HLWHWFxO*AkTz>3gcRujxzBK z;E{5UU7g@gw<5a!<&Er4pd#4tZ%{1NvmO$b_TZa2;_I6q+P&O#Pn@9&QHgq$!iUQG z`d2CWz!4YXc>cEGMbm7n1eZU6&-W~FLe8=oEC5|^YG;9^g$u=~AsH!CkY(Wt)4K50 z$D3p5FEy6~V8!1?8=>0XR_!yM|22qViwo@cDz=Cb3)_W}AFg^lxsmQjx(L@=)$d;* zJF7Q)t+o5tX9vBg*YanlR%_xU@^Hq_*6k<2l_`aDe!s3NyQ8Q@V$g0v$lD83-8>u=C=G}p=tq;8$z%7b>Vky_zhlaHTMUXyvgD6m4f#bd_# z1!=tVma^+{p@_a(PL$n#o@r(x@)mVf`sg^;eMkc+ljM$&Bm&k@DlzWHTE{NVAL1qM z&E(f;H)&)kf&Onnf(P)CRGk&`Od%J#)in^IAZ!>{pHb6->~%ST2Uuh%e+4z+(|NlW zd++|7M;&LK1(EsaBx^c$T5gwi7NC0@?!Uo7{I_i&MN!es3YA9e`s8j>&?e;&#21lLXL5Uuvf52E32IVv`1*Hn z0*fAj4uI~&v%ryiHZ7YO+O3Q(BxyaMnme+8SV~zJmu(R;*L_Y_VM?XUG&*A^SYc*% zgk;S%g?-D_6Q-hYKKzq$H2}M_p3ZIH7`@Km-n6i)- z$xlc}W{$S%B5Fp&JV+QK;hx7eNCWGK1G<6E{bg~#d=epTT&hoo^zs*f1PkmE4DEn& z7PFaO#pdv#Zg>5&Y)4!$vyzwq6f-fPG#_iK)WQ3GHcL4RFR@~=>Mto*rsOVe2=`h7 zQqj1V(_#Af=4m$!mzG>-G>jz%^Hu$ObuKlL0M-k38qVk1jmd-^`QhyOnR*UAPRa5| zH5m_m`@u?XyT+}iz?Hdy(>#0Pvh5(Hd8)J1?ssGWeX8)Q4&@t_W0Nu__cM(NN)y~; zX7ju~$K@=Yt2$-}JPhYkibIALFtB*jIg}hN_ufLkLM;KuAAbFh?hcyG(H;RAJSMbl z9 zHK5OfVgf1$2?oNj@SG+OiHVV7!rVBKxg02iBX#43HE!u%4w3lWnxx-4>;y`3>8b$3 zH0OW-`r&VQabqPXOn_M($2Pl%KpN{6%0hSp6_|e8=A70`xkk|IQ)HdQq3!Aem@%?JO?G8bX1q~09m`1Ffqdh4_j90n8Mzy9Yk2)xR1mdRuZ>D!g8~bPflc54uNOhL}eBIjP zuOMo`gIZFZ!-8va+xWqtt&fs$`!W(R&bde|3?@pEmlZ4664R2Jh2BNR#}YGq5Q1?Z`qV^`z$nZ4?}vw}EJ*>T=vYUX7ICKL7(cuOq^&UH z(nj<<#4+>BU@i|josfB1Gn!0 z3$E(rm+=P=jN6H5>ZHM_2G=yjUB+v>fMausbf!7*FpXj7Hvxpv(sbYTl3sMTAi(M0(TMJj504%!?Z(1SqXy12C zli8S8-f}9hV3T^$bT=a$U5W*=8CL)C<}O2@s+vwyIg5ZSLYZLL{rGJ<;oW!c!-4^z z>6dWKg{{yzQfA1Mi|j#RMZN}c=DUBf8Y{)rJ5zCIda>rf}Czv zFFoRMcjI1mMOFObSmz5T2-4|a?PkCZR5;4G!~Y60xra)z{Q25g!6>L~Y=H>f=iqk{ zcFw4!|2BLkNZ&)X?lkCt(4TPvS{68jKYESX+?StFR~9(Znt{<_n+@2UEyXmbIGjWM=L0nUfl=jx(8R(Id%^v;qCCyAcmeCFG&IH~b71ui zKDiB`&Wwl;VG`y)xmOFBkO*-)4X9co2YY`?0|ge|l95A`(IK zXY}AsJ}jN#N&2q!2rn_TIFGukfWtVK1;)bP7n;4`pT(@T9f6>29%BN6kvj{xF>J^I z%GOHfgXn;|ze1NM(mZa2u{$dbn@Yt3xm2H@BWWe|G~DUf&w2-I3UAR$Z<5m7S;YbX2^3*7E;cZwlx_85AFy zfSY<=L+>3&Cv?n7NM<_Y=4ZMY#l0p=fy^zM;86iyRXz z6eoCSI=Lrs^8MUquUj3fk$rO{^~Sh^yJzEb_Tj!?3{_uwm~4Ji{a(^kSfU5*rMM!1 zc3EG%Q`KbXF-{O|1fZP7D&yfn+8#LjC1uRBC$?0%bb{vAw6Xp@z?G`^=e@izagA@6 zM^=vQMIzmXHPK}*s2Rv)p$XuaOE%2@f6R*+zi$hkj4jAnhXiAz-rZ{=pqFOM4*Qsz z8meOxdqV!-tcH?Bc^g4~F8hah)*(F`Kon>fD zwQ#nFbrkkg^tiT#Mn(O zUeUx-zwm#X(>G49Rd!KqESJsS)D*QPnBZYx6-loA0*kVZTd`|wlH-zfYEz)38NEUG z4&WJ_r*g|9bCjg8iw0=6CNVs19uI;e?!$Ld#dSSLHJiKR;$b9*5YM%u=o46tyz)xq zZ`7Y&EaAf?j#xZuw!m9sDQq51P<;I@D!-(LMJF07^4hs0@R6}JDwVmGfU^m8 zi>nAuwkIR__M;~9N+fI{uSuSA-NSzBN*RRyaQ%RRhdH4NA9kW1o#wJlV)4*p!%RIb zN=t!*UU#CjpZugb(aPRx{iIG@9>DI}c@3%Q>8ea zVjTD188iVx*nTnYi_$9>qncueO$?5Peh$zpo>;}pC5+t6q5L<4Tx7k6@J$HKzMJ>* zDn&!Y{mxc7#Bseb3V^B>*H?2I-0;B&a=sY!q>+xNMM+kc)29XrDi5;;Ty0Wo#vVUx zlb^%HLff}Hk}KeH6xHoa&9$y~HzOJlvk5MR9JH}jjSiE?6y=6lG*$cc$cgktuTXY` z$U%`$L{y+Lq8>W3n>}~lH;d_@`2mFQJYk{?8xU%Y zh1tM8J#5VPRcPANS&3c%Myy13L!G8q#_%W_QLil}S{gqr`hBL-JR#9cp?HIDML_FV zsx~dNZLG8-GG|SJx8Z1NPlQbF%I{VIycfYM{8zVn)H21mmU4&nqh(^uIh)jd6&O<&nao>)c2iIWJ%W=07wi2#1|R#Uqb;bR$mwAbU69$WjjbjzMzaEudZQm4{6=0miP9~?tkwx!}^~fJyh#1Rok2D48KNPCQYZ8 z$8LHnKtc>#2XMyVuYRh@ELYSSw>YqdKWD3jlctZu=w8&Si#-=7^Iif@t`cgK8c`Lv zAKJ3J_q#DCqvSvddI18J8KMlk&zyyB3TmP*(qVgpBxKSC5MZnW-v_UF9W{0&5Gl|F zI6AqxD7lp`-DJh0w6{4MYr3?stl?APPTC!#IJB+Y|c55oL z9ll@#m?Jtwa;3RBMz}LU`AXn3yf+XxSwW2@zc*p$eedt{zg2;E9y#(YY)lOKV)Gd6 zNV@MSe3K-8ouR`QEZe`v8zG7zc{K}>nz^l0`iL^`AGvR>q5EoeyUeYIGEF1V2_j0W zX6un6=E0wZFjohns^DvH*mc~aM(G`5dyJl-PtV%>D8yozA#=iU*4>``IX5gD;#L>L z=IxAqGUaf4X7u;Pm5xeS9XgStEDs|<(MdU~jX0VR?vf-S2VHck=*0Qyx=`aT&siz9 zz#IEhgNA*k_wHl>-%Y?=@sbo#MS*0_gxUljNQjMyGErISSko5SYfmE-GGzsOR!DUJ zc-X&47XpZK3K~Czu1Jg4q}%3N#AfN!%56R{x;fq@xHGtDBIZ%s2ST~-9E=rr7V>Np zEkvD$;8KMfzYd!kB9OPU&||WJx2&z0z$meM6D>DLpEuM0^aRu;nc#P%2SvBVr6rWJ z2nmlY;uO5&pu#2!UTY8UNeTV^YHOev+jJq;LtpI0^&q+0Y`d5hXZQB2fYSpHTKC#( z%^|qlnWOxn5_%5?NeZEM)ItFHReM#?N)rBij>C&MD+~*tX)u z^b`pYF+(J{2j#y7jho}>(hcX3zyeo;Q0%Df6(iyaGR$Fuh7=R)?_90x{mme(KMyv- z4}zKMKXMAXt$QB_0w3!%;VqGgjFFLy8><+3cORk2TI1mlCH}I8*mN^box69KI38q& zat@9|<|6;yk}uJ%YN=L{F^R*a-1~Z6_aWK%pw1l&J*WUt#mAMMbv_5|vhWqV5#)hpU+~lFP z7g(ki@A}_X2W=i#wAU}l8`#=Iwg%tLCDQBYMhPitqKU=#+yGT6xL|z{vXDPYNSL#` zCBIR?M388Ivgbv!7X4aO?V zhM-Mm#_Ueh*7f~88+RgKv3?|TZ!W#Q%KAs9pjZ2TtA}f8QxcQ((rbCabLr1XV6ksW z)OT#oSnqmJ0$yf#Li~_4d&6(=B&_uG;f+}awa<`zI)VbL%xq-s{aesZZXP5S@NfT} zvS^fW?=6>HW4o^LWq`H=smTrcH4farf=K)&^&YzrS#i}Hz&xHG)i=}of4KkKq@{YF z^D=Q9?3WJ9n<FW5xgotC}uXhU)VqN{Y6 ze-OD;*X2D)e64JVSL%9Qj?Jv%AHUa7Ly;82C7Lx!0>r4CBuL^VE`X@F_I5~530`NV z!=N7k)Fh#yRm( zs+XTs?A`(DD@~^X$DaRlXTl?qKb9PDho@XU+8RhW+H}Z}c;EdSbomd~EY90Hvk3r_ zUtTr}r9Zn52x@}IrU9~2aAVXRJ&a8N@3`S~kpP6jp@^4=mhO!=v~75=+=Igiq6yl( zIi(sF04F8RIIAaH#pKlrAGknTFELf>(n^zaq!6Gex|V_UCWFgJS(%~LZM5g-PU=X( zu~FGV7P3|fts5B^6jwErd#j)uq?%F=qH)-N4C3cP;{f+HBaw&bbZElgD0y4cfOxr*F{HS4Q?BW%> z`zr|YDU{IboxKOmG6geG)1WT)29EckBjVJnuz+s>F6U8@Kye?huW(ef#Buv_FdUXk z1jl|Nx*U=%G--lMj7vVw&6?-tg#X+c;_`y&Pyz+J)b-|0O9G{xz5)Vhb(16 zr#Zh5!wO{Is7F;l$#&5X75Lfwhc7@-;_@)lakV`fe()Ih7rgvrSKRObp_CCRB=oDK zT7c3u$W1WC#d=aiPv-@!dH=qV)q3l0XZ?m{ZMxUm?Pn{t5)>1qcSqXV7=77>xgv9m z+1L3jYHQT^ZNTw7Rd{CeCOAFU-5;^j+v ze#|>b=!GW99~s;Qd=r@_ZxZ1Ea;6YK#k@o}OS0QIv$#GfiCD@IJiashy|d$;M8doT zF%V?vSoR;MfxK9`!bBs@ER{3v=43bj7Q`@{ zJVTGV>SynhE)l(meqZu2yTbeY^mN!h#V}u(^M6$h?=JM}5g7YUU8kF|EXe0-0B>>p zAq>6HqXu7$5Uu;0I7L*2xalWQFKa@3D6V9}9)!|sgZt;9?n%E0zSZ1RVv2)mf zYgS?4awMz3E+=FI###O+JgT?j9=Ut-+t73q?H00ZDiavgAjuAnp3lGd)4B?8;R9>f zsjo2Mn|);7Qpm(NK59xW8)QhN3=ZUa&yaw7pR0sN#$FEDhWi6V8 zsl9jPf7))KLk|8P>B$XM@!Fa8+5~)MadQ@~S zx=7WHVyq?hi^!;+6Y02^a+-5tuGweE@x#ieF;`6%4SAvA8yd!X8odwHzb(QD?}O!@ zq{-}C7b$AB{Z@gBou*wM{Z2rv8DuH~JiZp%3U#vCC~OL@AAG5IAA6@HnQT=O=u^+H zNL0wRo;kZ}>btC=Apq!v*zjnE9T=$oM{R!2C~whO(8`_g+XE3V7Vp?vof8Ym1D3$D z*`K{h*og9P24yFD>>Tn#t@>TSP$JiButzKT*dESEmwMVd$_!u<0~c5)KLTCb#o(Cd zE~1Kt0pW^YBV<}c1OD!wNm_J923s>TVH$xNFN_?wQeNi{**kswvUW*WQkbYzPdecq zRQsAWJ616`7B$CdN!!bhtN=9tZ&Qc!mZ@r@YfKb68LgHZdoied82QcYj?cLs?nT&a z>b%|jV*Cj3%dP{*IezliRuPZBx5ooq#Tc*xRfq1@a*y@9xKaf#yOUzIf0y6Uf0~9> za)hF)nybzyUa4dP<%6TmrVK@cD3?b&sMG9m76dPWTdcj~^O*5DJpw(+4u+OFv+2k0 zXu)-JbgQ#!G8@bRyY>kqHM;_~H_>?^^EUa(AQ$0;cKx6XzWMf7djT)Z8U8e(K9>wcXlnn?Q14_ZosodSkjrU7T2^3HAI;THUB16vWe9Jq>EU~MNGGBp% zEy1L3vC9<6%id;4;~%UPBKx;MI}n0P4eVse@;r<=n-T z%>QvGowk5P*aazh9gZYbRZ*ds?t*SaM%MLEkUAv@y$gCb_g7HSHogQ->uP)vjK5)i z^T*f~8+T;rNc_O|aBN@lW@K8B+>F)%-x2_BXpO4cC@=s&MSm*gj?bQY^_beKUrloGI%WKwmU{Q!k&Q-q9O&6 z7*}l0jH>BxWu*&Qb9~$7PS$26o5Mc{2w$xX@nd%@_C$tGAiUiZ8bbJ7kG$i%TUb2~ z$%V|(K85~dBo7-rCcT}46NL60$~_uIc5MzfV02q5{m!GH)^QkrQ)*^luRY%C>sf8= zx>SkO_G)6iicH5g&r~ySNsCDxGqapc>ngkJu=rhQ&lO zEr)s=AdE5xm|F5TVhspEJ1znwU0D~~`U0kh5&*@RbB^$bwah7u77wvrIsjEP2de)= z?Z=USFn&1|0Q^q>$*OAKF9;25HXnFnPutfoUKHa{WQffmVlBl&i+ZNt3}YKYjNXgs z2LOYXvM3L&{q<7RwoQZY`av0&F!l-U5$JFyefb{mc<}jL!A9GvpKlN^-M%VYiG;0x z)z*lfa8?GW0j#Cp+X~7%Ya6vxxO$#l{YV0m|1$CaZ3F%*;_vg>qFtekh@P^2F~jPU zMn>@Np=}Ed+OOdB>Ln;?&ienlTm|!<1dhfI*u(5m2pb`^u}&y;6x#!u4uV-8f;KsK zkY{=@KM9TO9YFT(!Q?a8rB3)Z;gIa1JGpvX(l8wC#pWOs_vfDk--I3=}kZf3n zC4Z@$a*!lcicQQXB6C_|&9@#$W<2f}>I!_`{K*_l+X+G3Fl44OPpl`DVZiAcS2OVa zZ^}m+>lbbt^-qF8e36EAXNiBeurn(RcHWLxGCd$O~BKsA95u@XLnGlVzr^+=w5c=3wT zo#~*(iyK33pxLxg>hm4Csp+9LfNj2!~f zq}s8hnw9oOqzVcmU?P?6`GNs~a+4O++H;2v)Le&QY+ zv4ycxss34tDgCEt3|0#jmr{j3Pg-ZfJU}bxq&Cm14W^UsI2{N_OX;=WHtJ|ydwpIX z)=*y7|6Vvp{s`1j`%Q-XwGN(^jPu~W3!iK&RMAyu}f7BR0%1&@W(Oa28Do+Q0h(poS13cq*}0U z0?36{T6>7_7#v+aOj&AmeEP&wA%{=;E4^qgzuB#R@ z%l*=#CeYtwVhsxCcms4i1f)r`88w<_pqThA#yzJ5m0=k|11{U^MiN8S2UL`N{Fhir zx%zcoT^pSMtKTNX0yUzxG$d4<$j&Ua;0&*&!XTKiR50v9vHbx0jN#BWY~Qws27%RC z4b03S-keMHodN=SLut#E&W4~&^J;WPv_D0g5n_LBI(H#WED`_#Ra=?*zfaPdONZu% zmV45oZtcj>n>tc!I!xJ${yzC&3!%>FJ?%z+1xcwL-Pc?nt@QUz9&Ip$V5XEt&d|KZ z{b)W9EUxJfsnu%~(`z>e7-S|tW5}(C=p+!CJ?o;gKD5}ta3lL9T#^Jy&o03Mn`(xY zE`vSL7rK}jIE9>vHI6LMcF;sr^P)AG;{+3WWhDWYrEleeT@yJz0w`BL)}d>_%Afds zjo!ywm#O=&&>J#{qz5`(SI;NV0Ob%dFcTjhL32d-xJ_^s>${W~BZO*?pP5yhD}ERM z{uzLfAh(W%SLSg_!}oR6pw(!Hvm@!c|Iv3q7xM$wM{UB!AtQV~A!f!wrdJI3?jcnm zDX*>>Hc#ouAxuStSSebcVT+G*PNt+3sFYMghhRjYI`aa|JVlwiC@lqIW{%|1iayOS zxyH&N2?f3MIA<@&$E+r%hs+|@S*#Fq8T-YxQD8AHLK9z|nRW7Z45$vdB`h=8ICQ?R zXI9X7g*U0liao75pOS*QgI6L?dt^Z?`AX-aWDi zRq?GBT?%<1PZ?~#76F^O(vxN7q&NU0ZkaRIr1g6~tewebO!eu(u~{&wb2*|ay3N;h z?t)5E@*t9ImSb=)MFu6ETMkO^i1^=v7O|v&Oo-?O?x?N#3$tMdZ`8w(Rt>KMY8~}a zox*`+{|f4W4Efc$ck0j`&BC*Z&BAn>*hWVeN=EeDmRH=rlHc>^^opmId3y6CkD0Ea zt0kx+@K2QuKB{L-PD(LBRQ2_!2R_jguqjPR33*OXjCd?1~_Ea!5@B&d=69|ZEH>=Reane#(^~?=uyh`@_OsbmcNI3e@z+l z=fx}KeZiIot}Fw+Z`lnaV?GwQh6zZy<3W8RiY1M7Bl4G`2dhJyMf9Jdpb*s29{HUA z@hfIUog#;uPnvR;$tF&H4K%~4kFo)>PM7d*z74w~W}5L=5F7ZKis7H?<94Mn^n^x# z5eCjOC4P4-YJt$lb9#bI+{LrZ^)+)G9HituMZS+xdzW&5D<5G$%vK%$@fAbS&h!0& zeX68f%HJ3^(EwxL@IZCQIavo-9Y5{!kQkywv(6?VpJxL`$UWq<87RMmjr`(Y{4dhg zC^zS7XK9a?MYIe};O*5lRIRdZG(VQc*pHle9hewU10aAc$f%J#B=3wPTNvfx&LrXM z)A4#%o`@oR8l|8%xC>ER-B@P^a`#&9!k7rpbm(Tg?)n*L5Z43{FKjhuu=^TgfT4HCeF{@=5ljc)R*Ak$!17ooD|GtQj5LP%E1*IFC@W>88+aT zkJ66A!y(m<6`B=4?n1lwnC*jB{$ShI@{jf|y?WZoMH1qwrMZxyMzksp*U|7AYZmh_ ztjZMQEPcsKU?92pOC><_YfJIaf26kNo5q{!sD|#{&`V8~dD41WX9B>y45X8MS=>Gh z?BsRFf|apqP6i%<(I5gaTmQ+bBEiE{2ukkQjgokpb`+nNPYZb}08Rk>T zyD&6)#hI3)bzXRTjlKdVAV0Fr25li_l$1f+$^CZ*DGuQ)5!vWgK|8;_=U?%XLa7w< zm|@VL9J9r6BdJ*aUu9M1i@RIdQ1g1hu2Yd5Q~UjWW1*YCNfRcS1&U}qnBoMXp3>aI z_=XUUe~Er%7c*VUO}l^9uRw8}*fgQvtU;JFe$QFBAi>?Xy)=0=O|?d)Wa49f!T+fa zI`;0^4wBCd#M`SeTmw8qN2rM5ze-rj+HA1a5=k)~sJ^9qW3M=Xh0BvA?d0iGh`R+iuUd1 zW?c}<{f<$ACjn2#_O73JhBd5Z_Pi+pnLPNRcmon+d&sn3jvoVa#-93{R6~sqMX`zV z1;`1ENSqFN6*)TH-8Ek+$bh*0%$T^_j?g2%&v%j>!i z%ZjXQrJ<~qtYe7^q@=I=btl7^Xmv+Sr9I1v*m0@pI+_^u?z}*8q;qa!C75`NoKa3_ zTCTlYg9Q$hOfr1&rT2{g`%{#g$e&@kW>v+o-Mrq*bw>|Y(mnU~7S9L;bW0(D3B8;6 z8Tqn-C$bWlhJR#P2hPT~E@BRTi&uD`=qqx~+EO>CgaF3LBcaCfo@uHnL}?&{VJiHBX0j@(7nv8#%ro} z?gTZvYP>*$PR?ti+47PIbwn-ZnmiVm<=FE@5Irr$JQG6WB}#qgjZJSU``ZOD#N#Ih z1+zQxRaQjriCaNeQFV#WBvM@J1s+&4MVhDDX4~6%%_E`u6qE}!zcn3ivJo=GG^&e0 z*(B+`)aEuXA4fQZtA6F(fD1STpM$7Qn$}_cAbw^gj})$k^VC-*I$a~P045VHfw@!j z^oM1mC?dSw5&m2@sw3N6;+L78$Z+;z`n0lW#0cORR3VecOq-Lcl4`aztQ%o|@b~a~ z|0H&G`>=f3BUw@78>Xv$O0+oXRTB0C4j=2j;C24wZYspvtUnH}f?DFA(rl8WHrO)) z7j(<>k9z!=wyEK91FW7fXk-|bxJtDJW}FcW-Y?>lmOS;rN{1Gepc&)~Hs9&qy;ydb z+{t1U=#}GsnDe4O&w)tcR(ykeA|=+707I@+Fx^HY`3)u_q5N#u4d8OrR0Tg1(IQ1Y z7wx69z9vCB%x&_$K+n9fWg60@w0QNI<^4bD>`4-Kua)rh`hVDl znu~Rmt@{O-oPZ;#XFHIEgYeS88j(u*RbIGuym~hj<2F`$GL$e(J(bI|cS|hLly>TD zoOEarCK9YY%fd*~~0|4BaKk5Fvm!GY`_AP1)B9I?rYGESjP%rTM+vLJhq zBd*o}yUNdz07bD^kmM}Y22a*{FH9w7?_5m17czzR7G_8C^IgI$Kz^1scNS)>PXp z%n>h2T5`ZXRv4=l=>7rCc~8iZToSZWVipgSjrJaucD0>=PUMlKqUoI7JLWgVrT$j4 znh8JGab5K_%0`)so279QD#B;OHBtlxIG$A=t91wAlo`OMqVEHu?>@Wend&3r8V+Ql z8@e4W44|&Zruh}!pd{ZP4toQvlE&4NTuqc&R;=ik&OUvch=p2RI%O5U8Z)iG4CZ}f zUa@Aic;f26D9m@fw!#VjL~~K3J-Tksfj^>279z(LlA`lXWJns$(~KXgX-+O1$X@}G z3Qnoah}~#=?V=L8f5br(UK;J(WXpCv8p7u&08%Z@cs(2 z<#FgX9E&{7e(&EA0e%G_%EaA!RsVw5m=)f0bwJ1z+cR_~=(jY^_i|(I%2Ah$cas6u z1Rh`fL77l9~Ip$O(`nK7`{*o6bYmZ z1Xs}hdtYz0n9`~=86iis!L&4RP9$>}a3vzu366;2;*{;(;mPehAva}wqeKtvt8Px$ zm~OCDA|{3kw2JM%(uz7roMkU#d_{!a>-Ud1?aEs_34yam-$=#wCDl$KU#-=Ihg)B~ z_`KW;`%^`l5_J(YMd2@|S# ziL99QiPw6k$b^roxw1{ww>*SIsqIL4*G&HP&g&y#lP252K-$r-_F;ug&WS-$-$ya- zUk9jfDgCk?1hltgbMiLw9xt*w(pF}HJpGE-hC3zMU-v;MZial3q3idN_;7)_n!&_p z3ctKi>YAM*)-|=8#b&6T?C@&SAJ__RnaPqGXxAY$@Y_uay<)dG#}f|rv~_#BmqjzO zXlK%A&(~fs)udD|H9L&iPfJ6@PC|V!C*10JF|<`asurs&*k?pM zSDZ)r0Zg~QcYz&+D;R<^+zO#&oqf6(zYDkT=2%yA#)LId??o`*@MX?2^lw1}?7Sz` z=!#Df0Rx4I|F5(2j_30G|38xLZIdLcNZFClKv^X_WQ52b5i;u~+1bg=9--`+QMQng znXJgns_gmgpJ#rz_xtwxU0$ER?*BZla~{`uU9anUo^zh(9I(uAH%0ATxWzIMRCk zTlqqm^kBBk@1U1Ps?+KMR@rWLo8Q;Eb%L~2HGiD!UVm||ZdbE!S1jiEY}}VOa^c$CmJn)l=qi1o&We25?Z%^T%5GoYqlP~3PVjgCshmY>q}vq`IiOVqurWR;z)=QU@fE_(fJ?dfW|KBGHrW#$th z(^PN&Q#$-r*?g9O_MIZC@yM*7sSAp3C@f)OiZ)JSqAr{D)R>637VE4pY4U8Ny!C{h zVqNYI4O|{#>cSe#$>Q_waS^j_Tdv3V*Z1f#2|3mzmRDcKER2080>2>bCa)x8+4p!d zo=)bv=N|Z1NUcL2ZlOYUmQ)^#-MuF{rdMawYgH(0z%N#sXC5J2dp&;z%e|3klq+KS zXg<4{`b(j}RkyjIXIg{l6A!g<rY)ew9m3;h%HN>0at*0=NOW48o4p*r#-S05!c795eGx*{ghcPFmK5S}3aL`L6b z)rcCWUpbPUllU22lDFh`zGW&6`)A7BZdf9oZ7l8N*R5^(5G`nQX&7jw^0sH8tQE>3{Su#3k=^H|c4-*{r$zg>r(wQznSi4}iLEKYcH zY2UTDi`ezg(#|b&z7%!|A(0w`p}Xq^RRyVUwa%P~ey?}#_|rbG1yjes8#6)vroZKd z@EK3NmbmW35Y?JHq~j^XkDGTd;VcdoOG^32M?uvhL0dtt(j_T~Wk{Ubt>Lk!A}`;yE&0@R zEMKm7>o)dC386?OzOSEiEo33_oJ~(&WYL1FNtjRIK zg+euf)v(PIzuzpTT)L)uEH^9ot>5xiLBIHjPTz1RyKTs&`=0vzb7og0`c88xNOxD= zBM|@m6o*8F3Tx}jNP)6xc9Qy&`zfN?#JF#VoJ7T<-{5~yrraF3h-b|sWUIxtqo`qY zT)QYRDdZ7neW4Er#V(IUF6Gh0ZlBPI^bFo-vKIMoj!t0P^w;Oy6dWet@^r%4RZBMX zr6%Fq4Oe7I|erN1iQ1TfyExLH5H$WlqK^G=8T=C4|r0uCUC}*#K4EP`0#h zGh_aqQ>%fMObIVbCnDS~W5eqcD_8h%V96t8rYR%QUw`%mFctSYwq@wNL>9^ahQ)PI zd&)8G&#}4DTWcHPSU+Y^p1b24n-%=!IYBqfrbNGLBxFv0O$|o>1a&yiW=_99;iQ>5 zefeyA^A=vs%?T+MRN+LEOs&SJX?b{pjt35=)oz3e3f8$8wLY4a3B0#w`hu>HwkAMP zlz`O=rS4#N@8)$^)Pz&|&i7TmmMtIi5l70>=~xt_HPgcEq?t@c@mjFV#kCWABLl5! zTZSxm#Z9`j1sRI)BEt->XWr`+Yt5#bf4n+!9@kgGhCIsr9ZGgVt4}=C{KbX5#bYPG zV(FZws#+HGynWHqNuR!zzw)63wM(e8IPcuCmoFDJ5~4+_47G`C@8_SIw8<@elM~pj zTht__~{x zZ>vv9iDAjmnA=F~XaKbCJP+pSjr{nVS82a%Djx~pA|=25cYOKtj&hFR|wk)ib6pu?8slVjT1?w2DX5q>DWia4tI{H}B(N4_RR&c3R z?$4B$={ws7k@-%mO1d1!mpJ#1(TZ3PpX9Y-iYQ&RXP9u zOZpO%c*^G5kmB%*hPF6vRLIX<9J^XVt1$|!uS^aW&M`M;!=tvnN2{$ZZ~OcrXo=e9 z3sdxDv=hA;%vSEvc#_GA>7mv+RX-bV`>`tPOgkd;ae87$|R9QZlT$_8$NKyX`I@s_WyG#!WX8vA8u($RW3v0VJfqGwoW#KZ?Zk7 z>KFXSyUejK*1{9ijQ4QczT?j}4os_7ABl3p_S6VW;@@~P^*7q7J)32^+1UO4vh?gH zx#u6*)nf;_X97w@w9^x@H2B1(vkm7udG*goc6~6%8g$UtU*?YYR`-_Nm?NiV`j9n- zbHd=E$P~@ktMIaf-xY7y&Yk{Y@Zt4~Z#O3PXd22LaM#d#w*R1wEIuR*Y9)T65A!`DK1rx8@Csmjr)Bv;}Q;1eW#~dNK`6mu5iz< z!oa}Yv2!FmP@7OdZse*p+0T`VrT4*`Qn%+==|^+NR9!|pr4^YZjGviHdU0O7$i;g} zcY#IcA?^tulD+KTL(dsirK6X9D;jA;l?){0)8)w~VreygQ^O4;W*D~g?35o~p_Tzo(dslU=%DK-5 zwuhH`>dWc+dOO*)KX_SLvwh)Vfb^+^Da{WK8?WrP24{@t@m694sQP9b>k9}He?G5m zKBar&=c~AhF~jVM@AzGNm)NfplMJMVxinpmnOTzZq>;MvvwMBS)iAA1tAhCg1J)H| z+t90}1#aEHy=bMKb*>8q3mHkOO|d#snVnZ76s|mR+oEvh-SvdFfkv9OR{Oc8y2;nr z?^trUaXPL_8h>r5RK%K(!p24|VtXuN^T87V>~vPY`tG=GXiM(#vSOcCn( zyIZCUj)0uZ0%0TEPOsY}*M}G0lfesM6o(h4TY*`L)?k z5SY7DIQ>})%jjjr7$jitYWH4%x7U=V@?_IE=%J1MTl+6!J;Dxg3m>+f8L=Q+E%a`D zrp*!NM%5^jSlK`f=QBh3s>eHmHP*4d#+ZsqN?-oj7OuBm!tt)9TT-%+BZ{fx^no)> zuLM01=>%zT1VlP%@xU3J7z|f_Q$Fnw>}=od#b7S>jN<_t>Z7NSW3Qhxs@+p^npL$-<-#@2Vw*>k0+CKmUO<`2B%OLSy>RXWJ}Ru^K~9wB%a^e-X9fd_d3} zM6Hq`Xn%nhbii}B$B%Df?H48ggFDfSqK_mmB(AU))^q!HbhAS(CjM{6I_PIy3&raV zv%WEjtp+I>R!j+W*Va^-TM7vsnXw!;YS~8bZ0ggJ*e}x;+|E(6!Hmb<+4LJ;PV=mt zCt@EleomeCF%oJ3N)}ptZhU|GvLI1{!E3rOU1nCVR+_L<8~KvL{{B;svOx#re%7K* za(4Rk-kPB*R~D=L118Hl8{Fcj-)~=e7HNcqci;@uw*crlgQ?C~Uu}CP;w4Gd!SVFt z;*-xhw4^mT{=@tDW~37%L;o0(2TbF83P@gl{=2Mm68#wt6X7mg9-q$(90I3ufwXm^nyCWD#;!XDGeT#=Nv9HI(vdn&Eu#b2~dUej#k zJMiahJ_cQ8&;t!S#^p?C+qg{Pk4&vZ@b|@C3EbD=GZw!h_RjbkHzhkE`3-{%$t%J8Vr#3ql@eP*c~fSf2WJ!bt!^aiWvPtLv6kPHpAdP;;XLsBr9yG^sdmn$ z2K9FutN}rvH)<C*Pb7Dk^iCW7bLU;ePhD%ywUd6lwq-cFYs>NF!Va&Y-p7Nj>xD32q3)vQyQq zm{uR^YL>GJL1mECurBKgla{zBQh**f6Wz`HNba@T6rYqJN{b>ywuMsLT|J1zKJV1n zi)&VnP%f~nW1BScYN!vs8(Mx zah=}1XhvLyS6_}()b3;QnZ#dw<;cLI4(NcG{+dXq%%OKiySk#qUhMllXM6JEOmZ$N zG`e~ufqsl&MCojJQ*@RuI07E_R94UdZZmE4xGWiAHIXaS`S`Q222j|4w7-PUHmCArg2>L+EM4D zc!W#mm+=_Tvmeh zbsu}=j!PA}`Tsv(C%6Lm&xH8P=}W^3r#ORE3RLpcjoa;GhLqJPJ@-UI`TY5JK?gi+ z-XrLd@&gXq%=jaZ^w)O=zaZYa7J?Tv7{ux3pk7K zKo7stTPNo8tm87>PJTo~>xy*`uvkQ(q&ax%)W?Ua5^9?UizTslc^`At( z4N;$DDO@;mZ+>2)0B69Di7o9r|M-$>x8eCUC|DO$PH}VaUff8$@%YD)OCwQ5P%iKs z(T4##7JF1ZpWoGc0jhGRzlPF{(Jnb~o7Y*33>Rb{ID=LOR_dP<waULWhE8YuL zNc^}PTKhKQY!xZ5r1SK{p0&{@XJOx;8ox=qMN9^&MH8uA&^s&fW9e&NakA%Ed|}0P z;~1&K`?BX%(bUo7vTErcMDaJE2H;_{BSDXp3y0nr8wq_^-P^e{uX0((o+Q;r1_|+} zWOA^1-!hg5sL+?t4{88D(jQPbRqa7> zL%V^TAjM1Eyvu>rdbZgf<8jG~?NO4cQ+UmzQwoZQ8)36~Bb5{3EgbM!hu)C|m1xR~o z9yo(HAbfF=dbXekBE31GT%b)r-8gw5aq=a{rt&u3PkkkfUG9UjKQpyI``q18%r-l4 z2KNHgp&TLhNPQtKSlcNe!RMixbGN8*5wHeFz{8&32708o%z-mZ#~nZqyis^U%5=UsL`nuJcH_`-?vt3u! z@s9TP>cDsg@bDupa4*0&t2Ee8ECy~ACw58}Y~H?dEam0Uj{9jIKbjR~4$bjfpaULu zKN-*?jZq%78C(H$%jw)JV$;M6J#LcGP*sgmx^Yg; z_+x%0oZ)afAC|9TKYLAv;mO09ZLL!617|QA0B?cl-M!PlO)vFDXtl=IU!N9z{6?mL zNng;vBd+7*PL6IG5j2P2N@yvq*4G*hvK) z@Pr~86Ezt_alVGq36F~8M2}pq{6&+CCUtU~GNY)XAoR{cy>g@|imS%mrjA{6uyt=$ z`1mM{uIgUHsUC%_>E)Vgr~!D`UEM&Bw1*dZXD2oV#DA@Lh+2Nu%X*O6A+J-|r1joC z_BOGJNQ^04`e&#C7)soGEd5?f2` zjIEs9R^yHuveZUz0z5@5FRr|Y4=&$&y2osB#*5GFE$D!28Jev7ce6iJ){uqIyx>h3 z->IchqhG<+;n#|p-Edw*?+j^#7is_=_H0Pd|L-v^rX#|KU4;zJ{`VfrtJIil?lDCR z33Q^=)~fw3m)v$XT#f3sew}m{eH)ZQ3^f4XeoMCQUis#)e806FE1$GquDONGiueAJ z_@1CZeo?*#&;gNV+|d7;NY4S(0L1jWiPYCZ@9fG{l+ers*QoN*#Qd)G;KAX$s}{Pk zW;gPa7uHJ^q;0_w5YzECq>;k|XYjRwvF8wPGR4K4A1_4OWm=BrOsx zJf%VJ!$+yM~j&JUcy6~G>1y%YJ`-I~P>Dm`VJMw)Xyf(s(+A7X`QFNO4n z+yBiU0zIbR45TOvy)&elQE&w0l+|xwec;1$_1hC?J=?{`B)&^o7jRTZ7;^;2W&5oE z^F^oJ3}|)llgKU+zR-&QXW!9YA->s78&!@_^UT(F+m0m%&R`4?dJvuC#ZA9k*NMMG znrwWyJ*qPzN>clNzW>N|vf)z{cbv8Vtw$Q8ggXEtjT|00gDZf?nVhEDud`8l1k=7O zc2NJ3BHOuak<@QdaxE@?{#X+~=zxdamjm=j??3d;JV`dFIUb%}8)>tQEgY2Bnjo;i zS+`05^l*hyv9)g)2WkKw_Ea;_Bb{M&;0$IE4m+0ydLYuw1C$HArF={{>-{y4q?X6d zLjs?djEV3!S_H&bjd3qI{n&34IdBGV0f=dlVpPz>Y+?$LLXwO$w)}LHoaXQJ(wD|9 z$=q)y=)(&4r4=iVE3m-WB$SUdrg`8DS{Z1f@U%qw%Zt4;{-Z&jQ@3g|Gap&nt6!fD z9io$a>{gu&Iv}P!8l-*(dS{rv1*mf0_5$&V2jg`IRf;VlN3h=cOcE&K!R`>HT9;I=Oo9C3T0%41{@y=Jd-c9X#F@)JVGIxz=Q+3J{)OXYY?(ahfrL9NJB1DTYCHBK7Ifw;84}DyE}UNb^+aoy{0kc(GWc?kx`F z?}?!1O|}!e?S_tW&4o=eb>)W zj94RN7RezMZ*2%htz8gkueIv@2{iyQoq0ui%N(>Bywf3uK+19eJ+O9QN7u{P=>0kD zj3K6@BK*Y@5a??rqcl)r^ znRjiYA%&jH%o@s>m4A0406qP>>P`si-Qo+8UlkHewmp7lZNJNFs`h;33P^v#Lih!A zK%{XK^v*CHLBaIDiL`?Lz!`jT4?AB8dLYs)B9secu<-Tq>XGTNMU^}>bZ^s`q^&;Z znz7cgkQ@`u;W%L)6UCd^!Bb2)66=Pie+ZE9z>##XU0#H2^WqK19k1MDGkK+YcN854!>x^hhgp z&^yC)RuI!(9n!q}fiuX~g2*o3FtB#Vu0UQpCD6|)%8te5J~_`xyQ+ErZq!{Y8#N-( zL*EH0D-0q`C?6>@{jW24KOtrNfioad{uz`DM0)=nID_9GC=x&NVY=d!&r7kLKI0Kz zLaBkd-COD7!q@_8%6m-?!?n5(Ylzcevs1vLOM{hB>IQjivO zq|5%ZiEDLq`w>%Bk%#<&b=u!DoH*&3MUUkkID_Z}#?O#)>Oc=fifW)-;Qv0uPep|# zZ=EX{mzcV!MfdrE9O0UqVAhPr5|M)swk~H%A+!|`(|L5HnT>-sgBpM$&&E#up1bqv z#rXzNBZYdUD(zjfs{5?H1YfAnN;7?t1RW65JJA*JFb}t>o6N*1u%XuIj_N8*%2A`- zN;}%hmVLW`w;H`O?tvd`l=%0K&R8>Un>6PI;EHU_U#!sMoY8VSIvg1r3XXskvd(J} z=eo+8i;TD|NM>nw=W(kl*--T+TW+rx>>2;F)(0ua`M?=O&5-TqxD-@TN46x`=)3>> z75z7l+$g+Irr3tNCDB5rxZ*?E|JEaAAHp2~k$#f{XK)1&X`UN&K%^cA=zxdMz=SJ+ zNdIck0WqBuM(Wd}Z!=8uW0Agta0kG{uXlo&0iu+|vGgK_wrrB)#e0j_CtsB+R0MAL z&gPHp=iXON(IIC4w;m~K0!KiknO*d4hUtIpu=_HBGo%<8y)#U+P!GFP0yz8Mqixj< zCNar;eu^6|0(oo{7GuSvM+nn4PngjY444t+GW;iN71NAOq?wR|HiK9ba-Fyg&sL0bWl=9~|l!HffB)g!H60zEJb z|HTb9XKxOr(T!y~F+N>2cKUViu85;F!}a79FP`{7xxmA(DuR0fVj6E@n&pZVnI5zm z^ljklgp`X8dLYuMJCqAVTDy4Q4B85)io+N++I{Nv1ud&BCK{X6qp^81`Vr0j&z}4C z=zkz30v&L`n5HSKsQ+o4&S1Jm4yQVKk_2`6wLu0?x^3zb=PUnNQL|a=F{aEYnU+tX z*LLS^NUMzMLUTQSYjz=Jum*j_4QKSuFwLmMbX*rHj}pBzOjkW3t;K=11s;Ch2}Xk9 zzlJm`2znsW2r`rlMEWnIZ!@Il0PY=#>FhSr91VJBNYOAj0%AIThSWbr@2t>|Vj}8D znx#3xX$2SAmJi&`Z>L5U8GlB)%xGunDLw~BKuq&HkfM3?&So@CSRFo-xZ=IDpCW7H z>4})Q@|mj=??Ee_QE-DNDIqulVw&TJw9W;+Gfdx4n2yjO<L4coVvXa;OYH(%wom_T`&Y8a);fHuNjCdZB+V`R&c(gRcwxPN>yg&4gCihPl!d;{ dFr7!jboDOM*g1Mj} diff --git a/share/empty.go b/share/empty.go index 6b5d7bb1e8..b5a8b11041 100644 --- a/share/empty.go +++ b/share/empty.go @@ -4,7 +4,6 @@ import ( "bytes" "context" "fmt" - "math" "github.com/ipfs/go-blockservice" @@ -22,8 +21,7 @@ var ( func init() { // compute empty block EDS and DAH for it shares := emptyDataSquare() - squareSize := uint64(math.Sqrt(float64(appconsts.DefaultMinSquareSize))) - eds, err := da.ExtendShares(squareSize, shares) + eds, err := da.ExtendShares(shares) if err != nil { panic(fmt.Errorf("failed to create empty EDS: %w", err)) } @@ -51,7 +49,8 @@ func EmptyRoot() *Root { // redundant storing of empty block data so that it is only stored once and returned // upon request for a block with an empty data square. Ref: header/constructors.go#L56 func EnsureEmptySquareExists(ctx context.Context, bServ blockservice.BlockService) (*rsmt2d.ExtendedDataSquare, error) { - return AddShares(ctx, emptyDataSquare(), bServ) + shares := emptyDataSquare() + return AddShares(ctx, shares, bServ) } // EmptyExtendedDataSquare returns the EDS of the empty block data square. @@ -61,5 +60,6 @@ func EmptyExtendedDataSquare() *rsmt2d.ExtendedDataSquare { // emptyDataSquare returns the minimum size data square filled with tail padding. func emptyDataSquare() [][]byte { - return shares.ToBytes(shares.TailPaddingShares(appconsts.MinShareCount)) + result := shares.TailPaddingShares(appconsts.MinShareCount) + return shares.ToBytes(result) } diff --git a/share/get_test.go b/share/get_test.go index 0ea83fd178..8eafe84cd8 100644 --- a/share/get_test.go +++ b/share/get_test.go @@ -75,7 +75,7 @@ func TestBlockRecovery(t *testing.T) { t.Run(tc.name, func(t *testing.T) { squareSize := utils.SquareSize(len(tc.shares)) - eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, rsmt2d.NewRSGF8Codec(), wrapper.NewConstructor(squareSize)) + eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) require.NoError(t, err) // calculate roots using the first complete square @@ -88,7 +88,7 @@ func TestBlockRecovery(t *testing.T) { rdata := removeRandShares(flat, tc.d) eds, err = rsmt2d.ImportExtendedDataSquare( rdata, - rsmt2d.NewRSGF8Codec(), + DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize), ) require.NoError(t, err) @@ -101,7 +101,7 @@ func TestBlockRecovery(t *testing.T) { } assert.NoError(t, err) - reds, err := rsmt2d.ImportExtendedDataSquare(rdata, rsmt2d.NewRSGF8Codec(), wrapper.NewConstructor(squareSize)) + reds, err := rsmt2d.ImportExtendedDataSquare(rdata, DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) require.NoError(t, err) // check that the squares are equal assert.Equal(t, ExtractEDS(eds), ExtractEDS(reds)) @@ -116,7 +116,7 @@ func Test_ConvertEDStoShares(t *testing.T) { // compute extended square eds, err := rsmt2d.ComputeExtendedDataSquare( shares, - rsmt2d.NewRSGF8Codec(), + DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareWidth)), ) require.NoError(t, err) diff --git a/share/getter.go b/share/getter.go index 163741f925..fa2cf5f055 100644 --- a/share/getter.go +++ b/share/getter.go @@ -57,7 +57,7 @@ type NamespacedRow struct { // Verify validates NamespacedShares by checking every row with nmt inclusion proof. func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { originalRoots := make([][]byte, 0) - for _, row := range root.RowsRoots { + for _, row := range root.RowRoots { if !nID.Less(nmt.MinNamespace(row, nID.Size())) && nID.LessOrEqual(nmt.MaxNamespace(row, nID.Size())) { originalRoots = append(originalRoots, row) } diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 8751f49da0..c9bf82031a 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/rsmt2d" @@ -136,7 +137,7 @@ func TestStoreGetter(t *testing.T) { assert.Len(t, shares.Flatten(), 2) // nid not found - nID = make([]byte, 8) + nID = make([]byte, namespace.NamespaceSize) _, err = sg.GetSharesByNamespace(ctx, &dah, nID) require.ErrorIs(t, err, share.ErrNamespaceNotFound) @@ -213,7 +214,7 @@ func TestIPLDGetter(t *testing.T) { assert.Len(t, shares.Flatten(), 2) // nid not found - nID = make([]byte, 8) + nID = make([]byte, namespace.NamespaceSize) emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) require.ErrorIs(t, err, share.ErrNamespaceNotFound) require.Nil(t, emptyShares) @@ -239,8 +240,16 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData randShares := share.RandShares(t, n) idx1 := (n - 1) / 2 idx2 := n / 2 - // make it so that two rows have the same namespace ID - copy(randShares[idx2][:8], randShares[idx1][:8]) + + // Make it so that the two shares in two different rows have a common + // namespace. For example if size=4, the original data square looks like + // this: + // _ _ _ _ + // _ _ _ D + // D _ _ _ + // _ _ _ _ + // where the D shares have a common namespace. + copy(randShares[idx2][:share.NamespaceSize], randShares[idx1][:share.NamespaceSize]) eds, err := rsmt2d.ComputeExtendedDataSquare( randShares, @@ -250,5 +259,5 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData require.NoError(t, err, "failure to recompute the extended data square") dah := da.NewDataAvailabilityHeader(eds) - return eds, randShares[idx1][:8], dah + return eds, randShares[idx1][:share.NamespaceSize], dah } diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 5ad8c92fb2..c2f8f18f9d 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -54,7 +54,7 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in // wrap the blockservice in a session if it has been signaled in the context. blockGetter := getGetter(ctx, ig.bServ) - s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowsRoots)) + s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 9d727635a8..db60e0138a 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -15,8 +15,9 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/namespace" libhead "github.com/celestiaorg/go-header" - "github.com/celestiaorg/nmt/namespace" + nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" @@ -162,10 +163,10 @@ func newStore(t *testing.T) (*eds.Store, error) { return eds.NewStore(tmpDir, ds) } -func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, namespace.ID) { +func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, nmtnamespace.ID) { eds := share.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) - randNID := dah.RowsRoots[(len(dah.RowsRoots)-1)/2][:8] + randNID := dah.RowRoots[(len(dah.RowRoots)-1)/2][:namespace.NamespaceSize] return eds, dah, randNID } diff --git a/share/getters/store.go b/share/getters/store.go index 156c9cf2ee..48b08d9759 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -57,7 +57,7 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) - s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowsRoots)) + s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound diff --git a/share/getters/utils.go b/share/getters/utils.go index b80478af15..a2b172e51e 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -32,8 +32,8 @@ var ( // filterRootsByNamespace returns the row roots from the given share.Root that contain the passed // namespace ID. func filterRootsByNamespace(root *share.Root, nID namespace.ID) []cid.Cid { - rowRootCIDs := make([]cid.Cid, 0, len(root.RowsRoots)) - for _, row := range root.RowsRoots { + rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) + for _, row := range root.RowRoots { if !nID.Less(nmt.MinNamespace(row, nID.Size())) && nID.LessOrEqual(nmt.MaxNamespace(row, nID.Size())) { rowRootCIDs = append(rowRootCIDs, ipld.MustCidFromNamespacedSha256(row)) } @@ -68,7 +68,7 @@ func collectSharesByNamespace( // shadow loop variables, to ensure correct values are captured i, rootCID := i, rootCID errGroup.Go(func() error { - row, proof, err := share.GetSharesByNamespace(ctx, bg, rootCID, nID, len(root.RowsRoots)) + row, proof, err := share.GetSharesByNamespace(ctx, bg, rootCID, nID, len(root.RowRoots)) shares[i] = share.NamespacedRow{ Shares: row, Proof: proof, diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index 91c413e05b..e5e7d41cd1 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -34,13 +34,13 @@ const ( // nmtCodec is the codec used for leaf and inner nodes of a Namespaced Merkle Tree. nmtCodec = 0x7700 - // sha256Namespace8Flagged is the multihash code used to hash blocks + // sha256NamespaceFlagged is the multihash code used to hash blocks // that contain an NMT node (inner and leaf nodes). - sha256Namespace8Flagged = 0x7701 + sha256NamespaceFlagged = 0x7701 // MaxSquareSize is currently the maximum size supported for unerasured data in // rsmt2d.ExtendedDataSquare. - MaxSquareSize = appconsts.DefaultMaxSquareSize + MaxSquareSize = appconsts.MaxSquareSize // NamespaceSize is a system-wide size for NMT namespaces. NamespaceSize = appconsts.NamespaceSize @@ -67,7 +67,7 @@ const ( func init() { // required for Bitswap to hash and verify inbound data correctly - mhcore.Register(sha256Namespace8Flagged, func() hash.Hash { + mhcore.Register(sha256NamespaceFlagged, func() hash.Hash { nh := nmt.NewNmtHasher(sha256.New(), NamespaceSize, true) nh.Reset() return nh @@ -144,7 +144,7 @@ func CidFromNamespacedSha256(namespacedHash []byte) (cid.Cid, error) { if got, want := len(namespacedHash), NmtHashSize; got != want { return cid.Cid{}, fmt.Errorf("invalid namespaced hash length, got: %v, want: %v", got, want) } - buf, err := mh.Encode(namespacedHash, sha256Namespace8Flagged) + buf, err := mh.Encode(namespacedHash, sha256NamespaceFlagged) if err != nil { return cid.Undef, err } @@ -159,7 +159,7 @@ func MustCidFromNamespacedSha256(hash []byte) cid.Cid { panic( fmt.Sprintf("malformed hash: %s, codec: %v", err, - mh.Codes[sha256Namespace8Flagged]), + mh.Codes[sha256NamespaceFlagged]), ) } return cidFromHash @@ -172,7 +172,7 @@ func Translate(dah *da.DataAvailabilityHeader, row, col int) (cid.Cid, int) { return MustCidFromNamespacedSha256(dah.ColumnRoots[col]), row } - return MustCidFromNamespacedSha256(dah.RowsRoots[row]), col + return MustCidFromNamespacedSha256(dah.RowRoots[row]), col } // NamespacedSha256FromCID derives the Namespaced hash from the given CID. diff --git a/share/ipld/nmt_test.go b/share/ipld/nmt_test.go index cd2f22cb5b..b52a75c150 100644 --- a/share/ipld/nmt_test.go +++ b/share/ipld/nmt_test.go @@ -12,8 +12,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" - - "github.com/celestiaorg/celestia-node/libs/utils" ) // TestNamespaceFromCID checks that deriving the Namespaced hash from @@ -22,6 +20,7 @@ func TestNamespaceFromCID(t *testing.T) { var tests = []struct { randData [][]byte }{ + // note that the number of shares must be a power of two {randData: generateRandNamespacedRawData(4, appconsts.NamespaceSize, appconsts.ShareSize-appconsts.NamespaceSize)}, {randData: generateRandNamespacedRawData(16, appconsts.NamespaceSize, appconsts.ShareSize-appconsts.NamespaceSize)}, } @@ -29,12 +28,11 @@ func TestNamespaceFromCID(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { // create DAH from rand data - squareSize := utils.SquareSize(len(tt.randData)) - eds, err := da.ExtendShares(squareSize, tt.randData) + eds, err := da.ExtendShares(tt.randData) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(eds) // check to make sure NamespacedHash is correctly derived from CID - for _, row := range dah.RowsRoots { + for _, row := range dah.RowRoots { c, err := CidFromNamespacedSha256(row) require.NoError(t, err) @@ -45,7 +43,8 @@ func TestNamespaceFromCID(t *testing.T) { } } -// generateRandNamespacedRawData returns random namespaced raw data for testing purposes. +// generateRandNamespacedRawData returns random namespaced raw data for testing +// purposes. Note that this does not check that total is a power of two. func generateRandNamespacedRawData(total, nidSize, leafSize uint32) [][]byte { data := make([][]byte, total) for i := uint32(0); i < total; i++ { diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 944b763229..8c5a132fdc 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -14,7 +14,8 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-app/pkg/namespace" + nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -34,7 +35,7 @@ func TestExchange_RequestND_NotFound(t *testing.T) { t.Cleanup(cancel) root := share.Root{} - nID := make([]byte, 8) + nID := make([]byte, namespace.NamespaceSize) _, err := client.RequestND(ctx, &root, nID, server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) @@ -47,7 +48,7 @@ func TestExchange_RequestND_NotFound(t *testing.T) { dah := da.NewDataAvailabilityHeader(eds) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) - randNID := dah.RowsRoots[(len(dah.RowsRoots)-1)/2][:8] + randNID := dah.RowRoots[(len(dah.RowRoots)-1)/2][:namespace.NamespaceSize] _, err := client.RequestND(ctx, &dah, randNID, server.host.ID()) require.ErrorIs(t, err, share.ErrNamespaceNotFound) }) @@ -116,7 +117,7 @@ func (m notFoundGetter) GetEDS( } func (m notFoundGetter) GetSharesByNamespace( - _ context.Context, _ *share.Root, _ namespace.ID, + _ context.Context, _ *share.Root, _ nmtnamespace.ID, ) (share.NamespacedShares, error) { return nil, share.ErrNamespaceNotFound } diff --git a/share/share.go b/share/share.go index 643f0c477a..06f911636d 100644 --- a/share/share.go +++ b/share/share.go @@ -20,7 +20,7 @@ var ( const ( // MaxSquareSize is currently the maximum size supported for unerasured data in // rsmt2d.ExtendedDataSquare. - MaxSquareSize = appconsts.DefaultMaxSquareSize + MaxSquareSize = appconsts.MaxSquareSize // NamespaceSize is a system-wide size for NMT namespaces. NamespaceSize = appconsts.NamespaceSize // Size is a system-wide size of a share, including both data and namespace ID diff --git a/share/test_helpers.go b/share/test_helpers.go index f3cb6776f2..c02bfc55ac 100644 --- a/share/test_helpers.go +++ b/share/test_helpers.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/rsmt2d" ) @@ -46,23 +47,19 @@ func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { // to be able to take both a *testing.T and a *testing.B. func RandShares(t require.TestingT, total int) []Share { if total&(total-1) != 0 { - t.Errorf("Namespace total must be power of 2: %d", total) + t.Errorf("total must be power of 2: %d", total) t.FailNow() } shares := make([]Share, total) for i := range shares { - nid := make([]byte, Size) - _, err := rand.Read(nid[:NamespaceSize]) + share := make([]byte, Size) + copy(share[:NamespaceSize], namespace.RandomNamespace().Bytes()) + _, err := rand.Read(share[NamespaceSize:]) require.NoError(t, err) - shares[i] = nid + shares[i] = share } sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) - for i := range shares { - _, err := rand.Read(shares[i][NamespaceSize:]) - require.NoError(t, err) - } - return shares } diff --git a/state/integration_test.go b/state/integration_test.go index 25b8e00a2d..e7d2496397 100644 --- a/state/integration_test.go +++ b/state/integration_test.go @@ -17,8 +17,8 @@ import ( "google.golang.org/grpc" "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/testutil/testfactory" - "github.com/celestiaorg/celestia-app/testutil/testnode" + "github.com/celestiaorg/celestia-app/test/util/testfactory" + "github.com/celestiaorg/celestia-app/test/util/testnode" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/core" From 6e0f3b9467d77493c959f48e5c0f293d877a9400 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 29 May 2023 20:43:00 +0800 Subject: [PATCH 002/388] refactor(misc): pretty print namespace.iD and peer.ID (#2255) ## Overview Log `namespace.ID` and `peer.ID` in pretty format. --- das/daser.go | 2 +- share/empty.go | 2 +- share/getter.go | 2 +- share/getters/cascade.go | 3 ++- share/getters/ipld.go | 3 ++- share/getters/shrex.go | 3 +++ share/getters/store.go | 3 ++- share/getters/tee.go | 3 ++- share/getters/utils.go | 3 ++- share/p2p/discovery/discovery.go | 6 +++--- share/p2p/peers/manager.go | 23 ++++++++++++----------- share/p2p/shrexeds/client.go | 6 +++--- share/p2p/shrexeds/server.go | 4 ++-- share/p2p/shrexnd/server.go | 5 +++-- 14 files changed, 39 insertions(+), 29 deletions(-) diff --git a/das/daser.go b/das/daser.go index 8b7e2a145c..d4ad0ee641 100644 --- a/das/daser.go +++ b/das/daser.go @@ -137,7 +137,7 @@ func (d *DASer) Stop(ctx context.Context) error { // save updated checkpoint after sampler and all workers are shut down if err = d.store.store(ctx, newCheckpoint(d.sampler.state.unsafeStats())); err != nil { - log.Errorw("storing checkpoint to disk", "Err", err) + log.Errorw("storing checkpoint to disk", "err", err) } if err = d.store.wait(ctx); err != nil { diff --git a/share/empty.go b/share/empty.go index b5a8b11041..0b7ea2e775 100644 --- a/share/empty.go +++ b/share/empty.go @@ -31,7 +31,7 @@ func init() { minDAH := da.MinDataAvailabilityHeader() if !bytes.Equal(minDAH.Hash(), dah.Hash()) { panic(fmt.Sprintf("mismatch in calculated minimum DAH and minimum DAH from celestia-app, "+ - "expected %X, got %X", minDAH.Hash(), dah.Hash())) + "expected %s, got %s", minDAH.String(), dah.String())) } emptyRoot = &dah diff --git a/share/getter.go b/share/getter.go index fa2cf5f055..f7a7b9c129 100644 --- a/share/getter.go +++ b/share/getter.go @@ -71,7 +71,7 @@ func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { for i, row := range ns { // verify row data against row hash from original root if !row.verify(originalRoots[i], nID) { - return fmt.Errorf("row verification failed: row %d doesn't match original root: %s", i, root.Hash()) + return fmt.Errorf("row verification failed: row %d doesn't match original root: %s", i, root.String()) } } return nil diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 3dcb2a472b..d65b902e81 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -2,6 +2,7 @@ package getters import ( "context" + "encoding/hex" "errors" "go.opentelemetry.io/otel/attribute" @@ -70,7 +71,7 @@ func (cg *CascadeGetter) GetSharesByNamespace( ) (share.NamespacedShares, error) { ctx, span := tracer.Start(ctx, "cascade/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", id.String()), + attribute.String("nid", hex.EncodeToString(id)), )) defer span.End() diff --git a/share/getters/ipld.go b/share/getters/ipld.go index c2f8f18f9d..04a1f4f728 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -2,6 +2,7 @@ package getters import ( "context" + "encoding/hex" "errors" "fmt" "sync" @@ -93,7 +94,7 @@ func (ig *IPLDGetter) GetSharesByNamespace( ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nID", nID.String()), + attribute.String("nid", hex.EncodeToString(nID)), )) defer func() { utils.SetStatusAndEnd(span, err) diff --git a/share/getters/shrex.go b/share/getters/shrex.go index a8dda90045..58a85abc0c 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -2,6 +2,7 @@ package getters import ( "context" + "encoding/hex" "errors" "fmt" "time" @@ -204,6 +205,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( if getErr != nil { log.Debugw("nd: couldn't find peer", "hash", root.String(), + "nid", hex.EncodeToString(id), "err", getErr, "finished (s)", time.Since(start)) sg.metrics.recordNDAttempt(ctx, attempt, false) @@ -245,6 +247,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( } log.Debugw("nd: request failed", "hash", root.String(), + "nid", hex.EncodeToString(id), "peer", peer.String(), "attempt", attempt, "err", getErr, diff --git a/share/getters/store.go b/share/getters/store.go index 48b08d9759..91200b78f3 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -2,6 +2,7 @@ package getters import ( "context" + "encoding/hex" "errors" "fmt" @@ -98,7 +99,7 @@ func (sg *StoreGetter) GetSharesByNamespace( ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "store/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nID", nID.String()), + attribute.String("nid", hex.EncodeToString(nID)), )) defer func() { utils.SetStatusAndEnd(span, err) diff --git a/share/getters/tee.go b/share/getters/tee.go index 41e9073fd4..50e2e1b55d 100644 --- a/share/getters/tee.go +++ b/share/getters/tee.go @@ -2,6 +2,7 @@ package getters import ( "context" + "encoding/hex" "errors" "fmt" @@ -75,7 +76,7 @@ func (tg *TeeGetter) GetSharesByNamespace( ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "tee/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nID", id.String()), + attribute.String("nid", hex.EncodeToString(id)), )) defer func() { utils.SetStatusAndEnd(span, err) diff --git a/share/getters/utils.go b/share/getters/utils.go index a2b172e51e..c99a7689b8 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -2,6 +2,7 @@ package getters import ( "context" + "encoding/hex" "errors" "fmt" "time" @@ -51,7 +52,7 @@ func collectSharesByNamespace( ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", nID.String()), + attribute.String("nid", hex.EncodeToString(nID)), )) defer func() { utils.SetStatusAndEnd(span, err) diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index 23489c3147..01001376d5 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -134,7 +134,7 @@ func (d *Discovery) Discard(id peer.ID) bool { d.connector.Backoff(id) d.set.Remove(id) d.onUpdatedPeers(id, false) - log.Debugw("removed peer from the peer set", "peer", id) + log.Debugw("removed peer from the peer set", "peer", id.String()) if d.set.Size() < d.set.Limit() { // trigger discovery @@ -307,7 +307,7 @@ func (d *Discovery) discover(ctx context.Context) bool { } size := d.set.Size() - log.Debugw("found peer", "peer", peer.ID, "found_amount", size) + log.Debugw("found peer", "peer", peer.ID.String(), "found_amount", size) if size < d.set.Limit() { return nil } @@ -323,7 +323,7 @@ func (d *Discovery) discover(ctx context.Context) bool { // handleDiscoveredPeer adds peer to the internal if can connect or is connected. // Report whether it succeeded. func (d *Discovery) handleDiscoveredPeer(ctx context.Context, peer peer.AddrInfo) bool { - logger := log.With("peer", peer.ID) + logger := log.With("peer", peer.ID.String()) switch { case peer.ID == d.host.ID(): d.metrics.observeHandlePeer(ctx, handlePeerSkipSelf) diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index 313cca8db0..2a7c1fee18 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -128,7 +128,7 @@ func NewManager( func(peerID peer.ID, isAdded bool) { if isAdded { if s.isBlacklistedPeer(peerID) { - log.Debugw("got blacklisted peer from discovery", "peer", peerID) + log.Debugw("got blacklisted peer from discovery", "peer", peerID.String()) return } s.fullNodes.add(peerID) @@ -136,7 +136,7 @@ func NewManager( return } - log.Debugw("removing peer from discovered full nodes", "peer", peerID) + log.Debugw("removing peer from discovered full nodes", "peer", peerID.String()) s.fullNodes.remove(peerID) }) @@ -256,7 +256,7 @@ func (m *Manager) doneFunc(datahash share.DataHash, peerID peer.ID, source peerS return func(result result) { log.Debugw("set peer result", "hash", datahash.String(), - "peer", peerID, + "peer", peerID.String(), "source", source, "result", result) m.metrics.observeDoneResult(source, result) @@ -318,7 +318,7 @@ func (m *Manager) subscribeDisconnectedPeers(ctx context.Context, sub event.Subs if connStatus.Connectedness == network.NotConnected { peer := connStatus.Peer if m.fullNodes.has(peer) { - log.Debugw("peer disconnected, removing from full nodes", "peer", peer) + log.Debugw("peer disconnected, removing from full nodes", "peer", peer.String()) m.fullNodes.remove(peer) } } @@ -328,7 +328,7 @@ func (m *Manager) subscribeDisconnectedPeers(ctx context.Context, sub event.Subs // Validate will collect peer.ID into corresponding peer pool func (m *Manager) Validate(_ context.Context, peerID peer.ID, msg shrexsub.Notification) pubsub.ValidationResult { - logger := log.With("peer", peerID, "hash", msg.DataHash.String()) + logger := log.With("peer", peerID.String(), "hash", msg.DataHash.String()) // messages broadcast from self should bypass the validation with Accept if peerID == m.host.ID() { @@ -390,15 +390,16 @@ func (m *Manager) getOrCreatePool(datahash string) *syncPool { } func (m *Manager) blacklistPeers(reason blacklistPeerReason, peerIDs ...peer.ID) { - log.Debugw("blacklisting peers", - "peers", peerIDs, - "reason", reason) m.metrics.observeBlacklistPeers(reason, len(peerIDs)) - if !m.params.EnableBlackListing { - return - } for _, peerID := range peerIDs { + // blacklisted peers will be logged regardless of EnableBlackListing whether option being is + // enabled, until blacklisting is not properly tested and enabled by default. + log.Debugw("blacklisting peer", "peer", peerID.String(), "reason", reason) + if !m.params.EnableBlackListing { + continue + } + m.fullNodes.remove(peerID) // add peer to the blacklist, so we can't connect to it in the future. err := m.connGater.BlockPeer(peerID) diff --git a/share/p2p/shrexeds/client.go b/share/p2p/shrexeds/client.go index 7c2591bcbd..387594186e 100644 --- a/share/p2p/shrexeds/client.go +++ b/share/p2p/shrexeds/client.go @@ -54,7 +54,7 @@ func (c *Client) RequestEDS( if err == nil { return eds, nil } - log.Debugw("client: eds request to peer failed", "peer", peer, "hash", dataHash.String(), "error", err) + log.Debugw("client: eds request to peer failed", "peer", peer.String(), "hash", dataHash.String(), "error", err) if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) return nil, err @@ -70,7 +70,7 @@ func (c *Client) RequestEDS( } if err != p2p.ErrNotFound { log.Warnw("client: eds request to peer failed", - "peer", peer, + "peer", peer.String(), "hash", dataHash.String(), "err", err) } @@ -95,7 +95,7 @@ func (c *Client) doRequest( req := &pb.EDSRequest{Hash: dataHash} // request ODS - log.Debugf("client: requesting ods %s from peer %s", dataHash.String(), to) + log.Debugw("client: requesting ods", "hash", dataHash.String(), "peer", to.String()) _, err = serde.Write(stream, req) if err != nil { stream.Reset() //nolint:errcheck diff --git a/share/p2p/shrexeds/server.go b/share/p2p/shrexeds/server.go index 60e4198842..fffa0e8152 100644 --- a/share/p2p/shrexeds/server.go +++ b/share/p2p/shrexeds/server.go @@ -70,7 +70,7 @@ func (s *Server) observeRateLimitedRequests() { } func (s *Server) handleStream(stream network.Stream) { - logger := log.With("peer", stream.Conn().RemotePeer()) + logger := log.With("peer", stream.Conn().RemotePeer().String()) logger.Debug("server: handling eds request") s.observeRateLimitedRequests() @@ -91,7 +91,7 @@ func (s *Server) handleStream(stream network.Stream) { stream.Reset() //nolint:errcheck return } - logger = logger.With("hash", hash) + logger = logger.With("hash", hash.String()) ctx, cancel := context.WithTimeout(s.ctx, s.params.HandleRequestTimeout) defer cancel() diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 748d9fbd10..67f64b8393 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -2,6 +2,7 @@ package shrexnd import ( "context" + "encoding/hex" "errors" "fmt" "time" @@ -82,7 +83,7 @@ func (srv *Server) observeRateLimitedRequests() { } func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stream) { - logger := log.With("peer", stream.Conn().RemotePeer()) + logger := log.With("peer", stream.Conn().RemotePeer().String()) logger.Debug("server: handling nd request") srv.observeRateLimitedRequests() @@ -99,7 +100,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre stream.Reset() //nolint:errcheck return } - logger = logger.With("namespaceId", string(req.NamespaceId), "hash", string(req.RootHash)) + logger = logger.With("namespaceId", hex.EncodeToString(req.NamespaceId), "hash", share.DataHash(req.RootHash).String()) logger.Debugw("server: new request") err = stream.CloseRead() From 009d5a95ae89880df6c309e277ea7a700f13d78a Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 29 May 2023 11:43:49 -0400 Subject: [PATCH 003/388] test: delete `Test_dataFromSharesBSR` (#2271) Closes #2267 --- api/gateway/share_test.go | 37 -------------------------- api/gateway/testdata/sharesBase64.json | 8 ------ 2 files changed, 45 deletions(-) delete mode 100644 api/gateway/testdata/sharesBase64.json diff --git a/api/gateway/share_test.go b/api/gateway/share_test.go index 423d08682b..16cf606680 100644 --- a/api/gateway/share_test.go +++ b/api/gateway/share_test.go @@ -2,11 +2,8 @@ package gateway import ( _ "embed" - "encoding/base64" - "encoding/json" "testing" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" coretypes "github.com/tendermint/tendermint/types" @@ -48,37 +45,3 @@ func Test_dataFromShares(t *testing.T) { require.Equal(t, testData, parsedSSSShares) } - -// sharesBase64JSON is the base64 encoded share data from Blockspace Race -// block height 559108 and namespace e8e5f679bf7116cb. -// -//go:embed "testdata/sharesBase64.json" -var sharesBase64JSON string - -// Test_dataFromSharesBSR reproduces an error that occurred when parsing shares -// on Blockspace Race block height 559108 namespace e8e5f679bf7116cb. -// -// https://github.com/celestiaorg/celestia-app/issues/1816 -func Test_dataFromSharesBSR(t *testing.T) { - t.Skip("skip until sharesBase64JSON is regenerated with v1 compatibility") - - var sharesBase64 []string - err := json.Unmarshal([]byte(sharesBase64JSON), &sharesBase64) - assert.NoError(t, err) - input := decode(sharesBase64) - - _, err = dataFromShares(input) - assert.NoError(t, err) -} - -// decode returns the raw shares from base64Encoded. -func decode(base64Encoded []string) (rawShares [][]byte) { - for _, share := range base64Encoded { - rawShare, err := base64.StdEncoding.DecodeString(share) - if err != nil { - panic(err) - } - rawShares = append(rawShares, rawShare) - } - return rawShares -} diff --git a/api/gateway/testdata/sharesBase64.json b/api/gateway/testdata/sharesBase64.json deleted file mode 100644 index e73d69f420..0000000000 --- a/api/gateway/testdata/sharesBase64.json +++ /dev/null @@ -1,8 +0,0 @@ -[ - "6OX2eb9xFssBAAAEQwBIdZAtC1rH+hFrZ7F6azCBAAAAAAQreNqM02s323cAAGBVRFEjy9ENcVuibWqbtSpxGWWVuESNkhDZEYxlVZdqWd2bLhU1GkFHxzQRl6TVTpW4xQ5FXdJqyTnp1KXuSnpaC0GJbKdn/73/fYHn3dNG1Nj24dbtlBwl5gUX4fwF1xmJg2X25QTU1a3ApPnwUCaSjDxwhTuIScHdY0ipKG1D5JOi66xWtbDcgtinX1PTO1YnWtRkfh+Pt+r+TwvkrrldM1ffDDrrg3sRRhsNFCj98EYXCuMeuapztgzSwTQbSKPKlnK3O3yDjyPuyDoNx7F5kx6jLc18UZZPpBdjJVMLTENBmrYF5WeOjgmentI1pBO5ILuduFCxc0PBIorzxZqvO4Vgmi2kuWlLZc5we9RFzue3XIuteS5tB8iTjcnn43hTXxM0JqLBtCOQFnN5IcY6Ej3Yo+GJx2O5dITSJM8v6/UxV1fmUDH12TqYhoE0UrYG7RAimeQxUfLTwQBLA7b8L/um/ggX+WL/ZlhqhBxMs4M03Z5ShCJ7Zrde/eKZZC/mCL+I8ZX7LEwrqIo8M8DM2QHTvoA0ZIfXStvLhrtI80zlYX3vSyFa5yzYn9ClG5Fqn870iex9XHv42Kg08S75gTytyENSVZFXTh/Zh6SGiHM=", - "6OX2eb9xFssA0rF2e0KnD5o9pNk4qqwxTgH1FbeHqPQuwUvJSU27f1b2T519HL3DZ9lSwLTjkPbG7cf37HVVvaXYqd0lMXg5VPSb70CdMLQ1vHV5ulsvDUxzgDS8V3KWimzsUHi5LHBjweGbAiXFdthUl1peSsrSM5sPAtMcIU2yjHIswLYbCPiMj30cWbrnl725tuPTubW3Vpk80nw1mIaDNIXV7KE9dP7bRsuUORVsMeddZT9WFNID68HBK3v5ccZgmjOkoSitGRSj8BsXN/EwuLHcv3dpd9P0lzX+meGPnu9/+h0PTHOFNIyxC83lyjHh7vBUB/nPNPjbLF1CPC9Tf+zvExnNY5/qaHITwuo5KNOh2ldOc32xJu65QQokrMBuWPBtiWfA6dWT7A+aG6SZPM+Z/cPPxqzhAnrVrSO2ITpx8sRDkacyOM5PPllcfBRMOwVpucEoTKlSsHL/Uizd5ipHIduZdbb0Cqh413zwzpy6MQlM84Q0QkztzIj3Lq6FMXCzxQvVa06MFY4+fiJYIiA5NMziHph2+v9Z7cO8X839w++Nm1TTspMyzFYf3I9HM9fLih7JESMGNDAND2nVnEppVJOlSCTVH1yrx+zJtlJ5hlhPfnxVoQqbxnME07whracrjtJM9+3bFs8oFuGq+agEnFNS8kYCCV36mR4=", - "6OX2eb9xFssATGwIpvlCmojMCZnoPswyLpXMPWwValWdC3rzlGDUhOhApyZ8+XsUmEaEtOk6q/c895prbA8li9SZP3WKrTH7ymMgMmJB8oNyTo8Kpp3p/jcAAP//G9WQYQEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "6OX2eb9xFssBAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=", - "6OX2eb9xFssBAAAC7gBH3m9XunkTbdymgmQ9EGUAAAAAAALWeNra4cfww3vB69mdR5Z4p8T9O/XhRMXJL1Y3mXwXyZuY/cyXUVk5vXTt8iYVvQXH1vN+kTk1L/iF77IaW02r+b9ril+9vVWz49KhjQbHPiyob0nJddM7ADFvpvxmVyfvNdsq330QChUvZipu3Dqhskl/Qo+YzFX+9a/2NanoL1jyNeZ8n1tC5bU/3WfbtJqrrW90nmiZ+LvD8Dqn6I+IPa9A5hlAzTPc/3d2j8lDd/7JfpVtnZkHnH9OPuZ8smOP95/c6co3VKcTa54R1Ly6unfXHLac3aclWXv63/LJJ2riWVoLc5YmxSw/WJHQ7bCfWPNMoOZtOGxz7dKGu3sKROMqAjrMK3xnn3t3e3etf47iluULrmsxEGueGdS83vsqTC/4X57I17nRpxCi8X1lF9/Mk4V5txbyBq/R+mG2kVjzLKDm/e81kP/w7+n9uPsf2SKO37f31Gx72SoTFaGzdHF55Ksvi4k1zwpqngV3p2344yMvVj/TeC3lMnHprSdT1YJfGHjyz+LrnZ6q3EqseTZQ88S+W96xfj1F2f2eUU/y16CrHcfnvf3qdOtlkHXH86+pK5ybVAwW/Dr1pGCZzK5QtStHtb6sVFuQsig4ftZq1a46qUdT7m3hTwc=", - "6OX2eb9xFssAmWcHNW+S6eFdYXNeXPrbcmVS4Jo9zG/2vrsme05s1tMfW6ILLH3mEmueA9S8tka+O/zFxSXz6hqnZpm2qt393sGal79490QTjXrxzf8/EmueE9S8yDZJq+P6fi+8XnRW7n/yzPKX4gfLkLrsmOPuDYuKWus1iTXPBWrem89/37l9ejxT/s0cZYe5K+IN6h6qzF377/rp9dnvVBrmXSLWPDeoeclv12ZbWd3OC3QV35SZeZija9qGZcu/bmfJtjc4azH/YS+x5nlAzWO7/o/LxPc+X1DUE8299hUT5bYXWs6c+FBhWuPC6T5sPMuINc/rACAAAP//N89ueAEAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA=" -] From 77d511d66e1da7b758003e90af0de739fb07c891 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Tue, 30 May 2023 10:35:32 +0200 Subject: [PATCH 004/388] fix(share/discovery): memoize discovery trigger and limit FindPeers operation (#2263) Closes https://github.com/celestiaorg/celestia-node/issues/2258 Two cases were possible: * Sometimes, the discovery is not triggered, and memorizing triggers might help * It's an unconfirmed theory, and we are still determining if it fixes anything yet. * We considered a case with @walldiss, but it should not happen. * FindPeers can get stuck sometimes for an indefinite time and so forth stopping the whole discovery. We should stop and restart it. --- share/p2p/discovery/discovery.go | 47 ++++++++++++++------------------ share/p2p/discovery/metrics.go | 27 ++---------------- 2 files changed, 23 insertions(+), 51 deletions(-) diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index 01001376d5..96eda8bd78 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -28,16 +28,15 @@ const ( // (by default it is 16) eventbusBufSize = 64 - // findPeersStuckWarnDelay is the duration after which discover will log an error message to - // notify that it is stuck. - findPeersStuckWarnDelay = time.Minute + // findPeersTimeout limits the FindPeers operation in time + findPeersTimeout = time.Minute - // defaultRetryTimeout defines time interval between discovery attempts. - defaultRetryTimeout = time.Second + // retryTimeout defines time interval between discovery and advertise attempts. + retryTimeout = time.Second ) -// defaultRetryTimeout defines time interval between discovery attempts. -var discoveryRetryTimeout = defaultRetryTimeout +// discoveryRetryTimeout defines time interval between discovery attempts, needed for tests +var discoveryRetryTimeout = retryTimeout // Discovery combines advertise and discover services and allows to store discovered nodes. // TODO: The code here gets horribly hairy, so we should refactor this at some point @@ -166,7 +165,9 @@ func (d *Discovery) Advertise(ctx context.Context) { } log.Warnw("error advertising", "rendezvous", rendezvousPoint, "err", err) - errTimer := time.NewTimer(time.Minute) + // we don't want retry indefinitely in busy loop + // internal discovery mechanism may need some time before attempts + errTimer := time.NewTimer(retryTimeout) select { case <-errTimer.C: errTimer.Stop() @@ -257,8 +258,7 @@ func (d *Discovery) discover(ctx context.Context) bool { // limit to minimize chances of overreaching the limit wg.SetLimit(int(d.set.Limit())) - // stop discovery when we are done - findCtx, findCancel := context.WithCancel(ctx) + findCtx, findCancel := context.WithTimeout(ctx, findPeersTimeout) defer func() { // some workers could still be running, wait them to finish before canceling findCtx wg.Wait() //nolint:errcheck @@ -271,26 +271,11 @@ func (d *Discovery) discover(ctx context.Context) bool { return false } - ticker := time.NewTicker(findPeersStuckWarnDelay) - defer ticker.Stop() for { - ticker.Reset(findPeersStuckWarnDelay) - // drain all previous ticks from channel - drainChannel(ticker.C) select { - case <-findCtx.Done(): - d.metrics.observeFindPeers(ctx, true, true) - return true - case <-ticker.C: - d.metrics.observeDiscoveryStuck(ctx) - log.Warn("wasn't able to find new peers for long time") - continue case p, ok := <-peers: if !ok { - isEnoughPeers := d.set.Size() >= d.set.Limit() - d.metrics.observeFindPeers(ctx, ctx.Err() != nil, isEnoughPeers) - log.Debugw("discovery channel closed", "find_is_canceled", findCtx.Err() != nil) - return isEnoughPeers + break } peer := p @@ -313,10 +298,18 @@ func (d *Discovery) discover(ctx context.Context) bool { } log.Infow("discovered wanted peers", "amount", size) - findCancel() + findCancel() // stop discovery when we are done return nil }) + + continue + case <-findCtx.Done(): } + + isEnoughPeers := d.set.Size() >= d.set.Limit() + d.metrics.observeFindPeers(ctx, isEnoughPeers) + log.Debugw("discovery finished", "discovered_wanted", isEnoughPeers) + return isEnoughPeers } } diff --git a/share/p2p/discovery/metrics.go b/share/p2p/discovery/metrics.go index c147a2eeeb..b6adbb1984 100644 --- a/share/p2p/discovery/metrics.go +++ b/share/p2p/discovery/metrics.go @@ -13,8 +13,7 @@ import ( ) const ( - discoveryEnougPeersKey = "enough_peers" - discoveryFindCancledKey = "is_canceled" + discoveryEnoughPeersKey = "enough_peers" handlePeerResultKey = "result" handlePeerSkipSelf handlePeerResult = "skip_self" @@ -37,7 +36,6 @@ type handlePeerResult string type metrics struct { peersAmount asyncint64.Gauge discoveryResult syncint64.Counter // attributes: enough_peers[bool],is_canceled[bool] - discoveryStuck syncint64.Counter handlePeerResult syncint64.Counter // attributes: result[string] advertise syncint64.Counter // attributes: failed[bool] peerAdded syncint64.Counter @@ -68,12 +66,6 @@ func initMetrics(d *Discovery) (*metrics, error) { return nil, err } - discoveryStuck, err := meter.SyncInt64().Counter("discovery_lookup_is_stuck", - instrument.WithDescription("indicates discovery wasn't able to find peers for more than 1 min")) - if err != nil { - return nil, err - } - handlePeerResultCounter, err := meter.SyncInt64().Counter("discovery_handler_peer_result", instrument.WithDescription("result handling found peer")) if err != nil { @@ -107,7 +99,6 @@ func initMetrics(d *Discovery) (*metrics, error) { metrics := &metrics{ peersAmount: peersAmount, discoveryResult: discoveryResult, - discoveryStuck: discoveryStuck, handlePeerResult: handlePeerResultCounter, advertise: advertise, peerAdded: peerAdded, @@ -130,7 +121,7 @@ func initMetrics(d *Discovery) (*metrics, error) { return metrics, nil } -func (m *metrics) observeFindPeers(ctx context.Context, canceled, isEnoughPeers bool) { +func (m *metrics) observeFindPeers(ctx context.Context, isEnoughPeers bool) { if m == nil { return } @@ -139,8 +130,7 @@ func (m *metrics) observeFindPeers(ctx context.Context, canceled, isEnoughPeers } m.discoveryResult.Add(ctx, 1, - attribute.Bool(discoveryFindCancledKey, canceled), - attribute.Bool(discoveryEnougPeersKey, isEnoughPeers)) + attribute.Bool(discoveryEnoughPeersKey, isEnoughPeers)) } func (m *metrics) observeHandlePeer(ctx context.Context, result handlePeerResult) { @@ -179,14 +169,3 @@ func (m *metrics) observeOnPeersUpdate(_ peer.ID, isAdded bool) { } m.peerRemoved.Add(ctx, 1) } - -func (m *metrics) observeDiscoveryStuck(ctx context.Context) { - if m == nil { - return - } - if ctx.Err() != nil { - ctx = context.Background() - } - - m.discoveryStuck.Add(ctx, 1) -} From 2496013598373aef411bc0bc37e16a1d736fd2ac Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 30 May 2023 11:03:23 +0200 Subject: [PATCH 005/388] feat!(blob): blob module (#2273) Co-authored-by: Hlib Kanunnikov --- Makefile | 2 +- api/gateway/state.go | 12 +- api/rpc/client/client.go | 3 + api/rpc_test.go | 8 + blob/blob.go | 93 +++++++ blob/blob_test.go | 82 +++++++ blob/blobtest/testing.go | 36 +++ blob/helper.go | 109 +++++++++ blob/service.go | 292 ++++++++++++++++++++++ blob/service_test.go | 412 ++++++++++++++++++++++++++++++++ header/headertest/testing.go | 25 ++ nodebuilder/blob/blob.go | 76 ++++++ nodebuilder/blob/mocks/api.go | 122 ++++++++++ nodebuilder/blob/module.go | 28 +++ nodebuilder/default_services.go | 2 + nodebuilder/module.go | 2 + nodebuilder/node.go | 2 + nodebuilder/node/admin.go | 2 +- nodebuilder/rpc/constructors.go | 3 + nodebuilder/state/mocks/api.go | 24 +- nodebuilder/state/state.go | 13 +- nodebuilder/tests/blob_test.go | 131 ++++++++++ share/nid.go | 26 ++ state/core_access.go | 12 +- 24 files changed, 1490 insertions(+), 27 deletions(-) create mode 100644 blob/blob.go create mode 100644 blob/blob_test.go create mode 100644 blob/blobtest/testing.go create mode 100644 blob/helper.go create mode 100644 blob/service.go create mode 100644 blob/service_test.go create mode 100644 nodebuilder/blob/blob.go create mode 100644 nodebuilder/blob/mocks/api.go create mode 100644 nodebuilder/blob/module.go create mode 100644 nodebuilder/tests/blob_test.go create mode 100644 share/nid.go diff --git a/Makefile b/Makefile index 79c45d6597..8f4d1b42a9 100644 --- a/Makefile +++ b/Makefile @@ -152,7 +152,7 @@ pb-gen: ## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api openrpc-gen: @echo "--> Generating OpenRPC spec" - @go run ./cmd/docgen fraud header state share das p2p node + @go run ./cmd/docgen fraud header state share das p2p node blob .PHONY: openrpc-gen ## lint-imports: Lint only Go imports. diff --git a/api/gateway/state.go b/api/gateway/state.go index 00a318c5ae..8997f4ae21 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -9,6 +9,9 @@ import ( "github.com/cosmos/cosmos-sdk/types" "github.com/gorilla/mux" + "github.com/celestiaorg/celestia-app/pkg/appconsts" + apptypes "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/celestia-node/state" ) @@ -137,8 +140,15 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { return } fee := types.NewInt(req.Fee) + + blob := &apptypes.Blob{ + NamespaceId: nID, + Data: data, + ShareVersion: uint32(appconsts.DefaultShareVersion), + } + // perform request - txResp, err := h.state.SubmitPayForBlob(r.Context(), nID, data, fee, req.GasLimit) + txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, blob) if err != nil { writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) return diff --git a/api/rpc/client/client.go b/api/rpc/client/client.go index 783d3f53cd..9e7e577074 100644 --- a/api/rpc/client/client.go +++ b/api/rpc/client/client.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/go-jsonrpc" "github.com/celestiaorg/celestia-node/api/rpc/perms" + "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -27,6 +28,7 @@ var Modules = map[string]interface{}{ "das": &client.DAS.Internal, "p2p": &client.P2P.Internal, "node": &client.Node.Internal, + "blob": &client.Blob.Internal, } type Client struct { @@ -37,6 +39,7 @@ type Client struct { DAS das.API P2P p2p.API Node node.API + Blob blob.API closer multiClientCloser } diff --git a/api/rpc_test.go b/api/rpc_test.go index f2e088ff1f..898a307389 100644 --- a/api/rpc_test.go +++ b/api/rpc_test.go @@ -21,6 +21,8 @@ import ( daspkg "github.com/celestiaorg/celestia-node/das" headerpkg "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder" + "github.com/celestiaorg/celestia-node/nodebuilder/blob" + blobMock "github.com/celestiaorg/celestia-node/nodebuilder/blob/mocks" "github.com/celestiaorg/celestia-node/nodebuilder/das" dasMock "github.com/celestiaorg/celestia-node/nodebuilder/das/mocks" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" @@ -81,6 +83,7 @@ type api struct { DAS das.Module Node node.Module P2P p2p.Module + Blob blob.Module } func TestModulesImplementFullAPI(t *testing.T) { @@ -336,6 +339,7 @@ func setupNodeWithModifiedRPC(t *testing.T) (*nodebuilder.Node, *mockAPI) { dasMock.NewMockModule(ctrl), p2pMock.NewMockModule(ctrl), nodeMock.NewMockModule(ctrl), + blobMock.NewMockModule(ctrl), } // given the behavior of fx.Invoke, this invoke will be called last as it is added at the root @@ -348,6 +352,7 @@ func setupNodeWithModifiedRPC(t *testing.T) (*nodebuilder.Node, *mockAPI) { srv.RegisterService("das", mockAPI.Das) srv.RegisterService("p2p", mockAPI.P2P) srv.RegisterService("node", mockAPI.Node) + srv.RegisterService("blob", mockAPI.Blob) }) nd := nodebuilder.TestNode(t, node.Full, invokeRPC) // start node @@ -376,6 +381,7 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, * dasMock.NewMockModule(ctrl), p2pMock.NewMockModule(ctrl), nodeMock.NewMockModule(ctrl), + blobMock.NewMockModule(ctrl), } // given the behavior of fx.Invoke, this invoke will be called last as it is added at the root @@ -388,6 +394,7 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, * srv.RegisterAuthedService("das", mockAPI.Das, &das.API{}) srv.RegisterAuthedService("p2p", mockAPI.P2P, &p2p.API{}) srv.RegisterAuthedService("node", mockAPI.Node, &node.API{}) + srv.RegisterAuthedService("blob", mockAPI.Blob, &blob.API{}) }) // fx.Replace does not work here, but fx.Decorate does nd := nodebuilder.TestNode(t, node.Full, invokeRPC, fx.Decorate(func() (jwt.Signer, error) { @@ -411,4 +418,5 @@ type mockAPI struct { Das *dasMock.MockModule P2P *p2pMock.MockModule Node *nodeMock.MockModule + Blob *blobMock.MockModule } diff --git a/blob/blob.go b/blob/blob.go new file mode 100644 index 0000000000..434ab220c7 --- /dev/null +++ b/blob/blob.go @@ -0,0 +1,93 @@ +package blob + +import ( + "bytes" + "fmt" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/nmt/namespace" +) + +// Commitment is a Merkle Root of the subtree built from shares of the Blob. +// It is computed by splitting the blob into shares and building the Merkle subtree to be included +// after Submit. +type Commitment []byte + +func (com Commitment) String() string { + return string(com) +} + +// Equal ensures that commitments are the same +func (com Commitment) Equal(c Commitment) bool { + return bytes.Equal(com, c) +} + +// Proof is a collection of nmt.Proofs that verifies the inclusion of the data. +type Proof []*nmt.Proof + +func (p Proof) Len() int { return len(p) } + +// equal is a temporary method that compares two proofs. +// should be removed in BlobService V1. +func (p Proof) equal(input Proof) error { + if p.Len() != input.Len() { + return ErrInvalidProof + } + + for i, proof := range p { + pNodes := proof.Nodes() + inputNodes := input[i].Nodes() + for i, node := range pNodes { + if !bytes.Equal(node, inputNodes[i]) { + return ErrInvalidProof + } + } + + if proof.Start() != input[i].Start() || proof.End() != input[i].End() { + return ErrInvalidProof + } + + if !bytes.Equal(proof.LeafHash(), input[i].LeafHash()) { + return ErrInvalidProof + } + + } + return nil +} + +// Blob represents any application-specific binary data that anyone can submit to Celestia. +type Blob struct { + types.Blob + + Commitment Commitment +} + +// NewBlob constructs a new blob from the provided namespace.ID and data. +func NewBlob(shareVersion uint8, namespace namespace.ID, data []byte) (*Blob, error) { + if len(namespace) != appns.NamespaceSize { + return nil, fmt.Errorf("invalid size of the namespace id. got:%d, want:%d", len(namespace), appns.NamespaceSize) + } + + ns, err := appns.New(namespace[appns.NamespaceVersionSize-1], namespace[appns.NamespaceVersionSize:]) + if err != nil { + return nil, err + } + + blob, err := types.NewBlob(ns, data, shareVersion) + if err != nil { + return nil, err + } + + com, err := types.CreateCommitment(blob) + if err != nil { + return nil, err + } + return &Blob{Blob: *blob, Commitment: com}, nil +} + +// Namespace returns blob's namespace. +func (b *Blob) Namespace() namespace.ID { + return append([]byte{uint8(b.NamespaceVersion)}, b.NamespaceId...) +} diff --git a/blob/blob_test.go b/blob/blob_test.go new file mode 100644 index 0000000000..5b3920ae9e --- /dev/null +++ b/blob/blob_test.go @@ -0,0 +1,82 @@ +package blob + +import ( + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/types" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + apptypes "github.com/celestiaorg/celestia-app/x/blob/types" + + "github.com/celestiaorg/celestia-node/blob/blobtest" +) + +func TestBlob(t *testing.T) { + appBlobs, err := blobtest.GenerateBlobs([]int{1}, false) + require.NoError(t, err) + blob, err := convertBlobs(appBlobs...) + require.NoError(t, err) + + var test = []struct { + name string + expectedRes func(t *testing.T) + }{ + { + name: "new blob", + expectedRes: func(t *testing.T) { + require.NotEmpty(t, blob) + require.NotEmpty(t, blob[0].Namespace()) + require.NotEmpty(t, blob[0].Data) + require.NotEmpty(t, blob[0].Commitment) + }, + }, + { + name: "compare commitments", + expectedRes: func(t *testing.T) { + comm, err := apptypes.CreateCommitment(&blob[0].Blob) + require.NoError(t, err) + assert.Equal(t, blob[0].Commitment, Commitment(comm)) + }, + }, + { + name: "verify nID", + expectedRes: func(t *testing.T) { + ns, err := appns.New( + blob[0].Namespace()[appns.NamespaceVersionSize-1], + blob[0].Namespace()[appns.NamespaceVersionSize:], + ) + require.NoError(t, err) + require.NoError(t, apptypes.ValidateBlobNamespaceID(ns)) + }, + }, + { + name: "shares to blobs", + expectedRes: func(t *testing.T) { + sh, err := BlobsToShares(blob...) + require.NoError(t, err) + b, err := SharesToBlobs(sh) + require.NoError(t, err) + assert.Equal(t, len(b), 1) + assert.Equal(t, blob[0].Commitment, b[0].Commitment) + }, + }, + } + + for _, tt := range test { + t.Run(tt.name, tt.expectedRes) + } +} + +func convertBlobs(appBlobs ...types.Blob) ([]*Blob, error) { + blobs := make([]*Blob, 0, len(appBlobs)) + for _, b := range appBlobs { + blob, err := NewBlob(b.ShareVersion, append([]byte{b.NamespaceVersion}, b.NamespaceID...), b.Data) + if err != nil { + return nil, err + } + blobs = append(blobs, blob) + } + return blobs, nil +} diff --git a/blob/blobtest/testing.go b/blob/blobtest/testing.go new file mode 100644 index 0000000000..395ef4167a --- /dev/null +++ b/blob/blobtest/testing.go @@ -0,0 +1,36 @@ +package blobtest + +import ( + tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/tendermint/tendermint/types" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/test/util/testfactory" + + "github.com/celestiaorg/celestia-node/share" +) + +func GenerateBlobs(sizes []int, sameNID bool) ([]types.Blob, error) { + blobs := make([]types.Blob, 0, len(sizes)) + + for _, size := range sizes { + size := rawBlobSize(appconsts.FirstSparseShareContentSize * size) + appBlob := testfactory.GenerateRandomBlob(size) + if !sameNID { + nid, err := share.NewNamespaceV0(tmrand.Bytes(7)) + if err != nil { + return nil, err + } + appBlob.NamespaceVersion = nid[0] + appBlob.NamespaceID = nid[1:] + } + + blobs = append(blobs, appBlob) + } + return blobs, nil +} + +func rawBlobSize(totalSize int) int { + return totalSize - shares.DelimLen(uint64(totalSize)) +} diff --git a/blob/helper.go b/blob/helper.go new file mode 100644 index 0000000000..1fef41dc22 --- /dev/null +++ b/blob/helper.go @@ -0,0 +1,109 @@ +package blob + +import ( + "bytes" + "sort" + + "github.com/tendermint/tendermint/types" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/shares" + + "github.com/celestiaorg/celestia-node/share" +) + +// SharesToBlobs takes raw shares and converts them to the blobs. +func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { + if len(rawShares) == 0 { + return nil, ErrBlobNotFound + } + + appShares := make([]shares.Share, 0, len(rawShares)) + for _, sh := range rawShares { + bShare, err := shares.NewShare(sh) + if err != nil { + return nil, err + } + appShares = append(appShares, *bShare) + } + + shareSequences, err := shares.ParseShares(appShares, true) + if err != nil { + return nil, err + } + + blobs := make([]*Blob, len(shareSequences)) + for i, sequence := range shareSequences { + data, err := sequence.RawData() + if err != nil { + return nil, err + } + if len(data) == 0 { + continue + } + + shareVersion, err := sequence.Shares[0].Version() + if err != nil { + return nil, err + } + + blob, err := NewBlob(shareVersion, sequence.Namespace.Bytes(), data) + if err != nil { + return nil, err + } + blobs[i] = blob + } + return blobs, nil +} + +// BlobsToShares accepts blobs and convert them to the Shares. +func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { + b := make([]types.Blob, len(blobs)) + for i, blob := range blobs { + namespace := blob.Namespace() + b[i] = types.Blob{ + NamespaceVersion: namespace[0], + NamespaceID: namespace[1:], + Data: blob.Data, + ShareVersion: uint8(blob.ShareVersion), + } + } + + sort.Slice(b, func(i, j int) bool { + val := bytes.Compare(b[i].NamespaceID, b[j].NamespaceID) + return val <= 0 + }) + + rawShares, err := shares.SplitBlobs(0, nil, b, false) + if err != nil { + return nil, err + } + return shares.ToBytes(rawShares), nil +} + +const ( + perByteGasTolerance = 2 + pfbGasFixedCost = 80000 +) + +// estimateGas estimates the gas required to pay for a set of blobs in a PFB. +func estimateGas(blobs ...*Blob) uint64 { + totalByteCount := 0 + for _, blob := range blobs { + totalByteCount += len(blob.Data) + appconsts.NamespaceSize + } + variableGasAmount := (appconsts.DefaultGasPerBlobByte + perByteGasTolerance) * totalByteCount + + return uint64(variableGasAmount + pfbGasFixedCost) +} + +// constructAndVerifyBlob reconstruct a Blob from the passed shares and compares commitments. +func constructAndVerifyBlob(sh []share.Share, commitment Commitment) (*Blob, bool, error) { + blob, err := SharesToBlobs(sh) + if err != nil { + return nil, false, err + } + + equal := blob[0].Commitment.Equal(commitment) + return blob[0], equal, nil +} diff --git a/blob/service.go b/blob/service.go new file mode 100644 index 0000000000..0fcf47bffd --- /dev/null +++ b/blob/service.go @@ -0,0 +1,292 @@ +package blob + +import ( + "context" + "errors" + "fmt" + "sync" + + "github.com/cosmos/cosmos-sdk/types" + logging "github.com/ipfs/go-log/v2" + + "github.com/celestiaorg/celestia-app/pkg/shares" + apptypes "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/state" +) + +var ( + ErrBlobNotFound = errors.New("blob: not found") + ErrInvalidProof = errors.New("blob: invalid proof") + + log = logging.Logger("blob") +) + +// TODO(@vgonkivs): remove after bumping celestia-app +// defaultMinGasPrice is the default min gas price. +const defaultMinGasPrice = 0.001 + +type Service struct { + // accessor dials the given celestia-core endpoint to submit blobs. + accessor *state.CoreAccessor + // shareGetter retrieves the EDS to fetch all shares from the requested header. + shareGetter share.Getter + // headerGetter fetches header by the provided height + headerGetter func(context.Context, uint64) (*header.ExtendedHeader, error) +} + +func NewService( + state *state.CoreAccessor, + getter share.Getter, + headerGetter func(context.Context, uint64) (*header.ExtendedHeader, error), +) *Service { + return &Service{ + accessor: state, + shareGetter: getter, + headerGetter: headerGetter, + } +} + +// Submit sends PFB transaction and reports the height in which it was included. +// Allows sending multiple Blobs atomically synchronously. +// Uses default wallet registered on the Node. +func (s *Service) Submit(ctx context.Context, blobs ...*Blob) (uint64, error) { + log.Debugw("submitting blobs", "amount", len(blobs)) + + var ( + gasLimit = estimateGas(blobs...) + fee = int64(defaultMinGasPrice * float64(gasLimit)) + b = make([]*apptypes.Blob, len(blobs)) + ) + + for i, blob := range blobs { + b[i] = &blob.Blob + } + + resp, err := s.accessor.SubmitPayForBlob(ctx, types.NewInt(fee), gasLimit, b...) + if err != nil { + return 0, err + } + return uint64(resp.Height), nil +} + +// Get retrieves all the blobs for given namespaces at the given height by commitment. +func (s *Service) Get(ctx context.Context, height uint64, nID namespace.ID, commitment Commitment) (*Blob, error) { + blob, _, err := s.getByCommitment(ctx, height, nID, commitment) + if err != nil { + return nil, err + } + return blob, nil +} + +// GetProof retrieves all blobs in the given namespaces at the given height by commitment +// and returns their Proof. +func (s *Service) GetProof( + ctx context.Context, + height uint64, + nID namespace.ID, + commitment Commitment, +) (*Proof, error) { + _, proof, err := s.getByCommitment(ctx, height, nID, commitment) + if err != nil { + return nil, err + } + return proof, nil +} + +// GetAll returns all blobs under the given namespaces at the given height. +// GetAll can return blobs and an error in case if some requests failed. +func (s *Service) GetAll(ctx context.Context, height uint64, nIDs ...namespace.ID) ([]*Blob, error) { + header, err := s.headerGetter(ctx, height) + if err != nil { + return nil, err + } + + var ( + resultBlobs = make([][]*Blob, len(nIDs)) + resultErr = make([]error, len(nIDs)) + ) + + wg := sync.WaitGroup{} + for i, nID := range nIDs { + wg.Add(1) + go func(i int, nID namespace.ID) { + defer wg.Done() + blobs, err := s.getBlobs(ctx, nID, header.DAH) + if err != nil { + resultErr[i] = fmt.Errorf("getting blobs for nID(%s): %s", nID.String(), err) + return + } + resultBlobs[i] = blobs + }(i, nID) + } + wg.Wait() + + blobs := make([]*Blob, 0) + for _, resBlobs := range resultBlobs { + if len(resBlobs) > 0 { + blobs = append(blobs, resBlobs...) + } + } + + if len(blobs) == 0 { + resultErr = append(resultErr, ErrBlobNotFound) + } + return blobs, errors.Join(resultErr...) +} + +// Included verifies that the blob was included in a specific height. +// To ensure that blob was included in a specific height, we need: +// 1. verify the provided commitment by recomputing it; +// 2. verify the provided Proof against subtree roots that were used in 1.; +func (s *Service) Included( + ctx context.Context, + height uint64, + nID namespace.ID, + proof *Proof, + com Commitment, +) (bool, error) { + // In the current implementation, LNs will have to download all shares to recompute the commitment. + // To achieve 1. we need to modify Proof structure and to store all subtree roots, that were + // involved in commitment creation and then call `merkle.HashFromByteSlices`(tendermint package). + // nmt.Proof is verifying share inclusion by recomputing row roots, so, theoretically, we can do + // the same but using subtree roots. For this case, we need an extra method in nmt.Proof + // that will perform all reconstructions, + // but we have to guarantee that all our stored subtree roots will be on the same height(e.g. one + // level above shares). + // TODO(@vgonkivs): rework the implementation to perform all verification without network requests. + _, resProof, err := s.getByCommitment(ctx, height, nID, com) + switch err { + case nil: + case ErrBlobNotFound: + return false, nil + default: + return false, err + } + return true, resProof.equal(*proof) +} + +// getByCommitment retrieves the DAH row by row, fetching shares and constructing blobs in order to +// compare Commitments. Retrieving is stopped once the requested blob/proof is found. +func (s *Service) getByCommitment( + ctx context.Context, + height uint64, + nID namespace.ID, + commitment Commitment, +) (*Blob, *Proof, error) { + log.Infow("requesting blob", + "height", height, + "nID", nID.String()) + + header, err := s.headerGetter(ctx, height) + if err != nil { + return nil, nil, err + } + + var ( + rawShares = make([]share.Share, 0) + proofs = make(Proof, 0) + amount int + blobShare *shares.Share + ) + + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, nID) + if err != nil { + if errors.Is(err, share.ErrNamespaceNotFound) || + errors.Is(err, share.ErrNotFound) { + err = ErrBlobNotFound + } + return nil, nil, err + } + for _, row := range namespacedShares { + if len(row.Shares) == 0 { + break + } + + rawShares = append(rawShares, row.Shares...) + proofs = append(proofs, row.Proof) + + // reconstruct the `blobShare` from the first rawShare in range + // in order to get blob's length(first share will contain this info) + if blobShare == nil { + for i, sh := range rawShares { + bShare, err := shares.NewShare(sh) + if err != nil { + return nil, nil, err + } + + // ensure that the first share is not a NamespacePaddingShare + // these shares are used to satisfy the non-interactive default rules + // and are not the part of the blob, so should be removed. + isPadding, err := bShare.IsPadding() + if err != nil { + return nil, nil, err + } + if isPadding { + continue + } + + blobShare = bShare + // save the length. + length, err := blobShare.SequenceLen() + if err != nil { + return nil, nil, err + } + amount = shares.SparseSharesNeeded(length) + rawShares = rawShares[i:] + break + } + } + + // move to the next row if the blob is incomplete. + if amount > len(rawShares) { + continue + } + + blob, same, err := constructAndVerifyBlob(rawShares[:amount], commitment) + if err != nil { + return nil, nil, err + } + if same { + return blob, &proofs, nil + } + + // drop info of the checked blob + rawShares = rawShares[amount:] + if len(rawShares) > 0 { + // save proof for the last row in case we have rawShares + proofs = proofs[len(proofs)-1:] + } else { + // otherwise clear proofs + proofs = nil + } + blobShare = nil + } + + if len(rawShares) == 0 { + return nil, nil, ErrBlobNotFound + } + + blob, same, err := constructAndVerifyBlob(rawShares, commitment) + if err != nil { + return nil, nil, err + } + if same { + return blob, &proofs, nil + } + + return nil, nil, ErrBlobNotFound +} + +// getBlobs retrieves the DAH and fetches all shares from the requested namespace.ID and converts +// them to Blobs. +func (s *Service) getBlobs(ctx context.Context, nID namespace.ID, root *share.Root) ([]*Blob, error) { + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, root, nID) + if err != nil { + return nil, err + } + return SharesToBlobs(namespacedShares.Flatten()) +} diff --git a/blob/service_test.go b/blob/service_test.go new file mode 100644 index 0000000000..87554eb3d0 --- /dev/null +++ b/blob/service_test.go @@ -0,0 +1,412 @@ +package blob + +import ( + "bytes" + "context" + "crypto/sha256" + "testing" + "time" + + ds "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" + mdutils "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + tmrand "github.com/tendermint/tendermint/libs/rand" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/go-header/store" + "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/blob/blobtest" + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/getters" +) + +func TestBlobService_Get(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + var ( + blobSize0 = 18 + blobSize1 = 14 + blobSize2 = 20 + blobSize3 = 12 + ) + + appBlobs, err := blobtest.GenerateBlobs([]int{blobSize0, blobSize1}, false) + require.NoError(t, err) + blobs0, err := convertBlobs(appBlobs...) + require.NoError(t, err) + + appBlobs, err = blobtest.GenerateBlobs([]int{blobSize2, blobSize3}, true) + require.NoError(t, err) + blobs1, err := convertBlobs(appBlobs...) + require.NoError(t, err) + + service := createService(ctx, t, append(blobs0, blobs1...)) + var test = []struct { + name string + doFn func() (interface{}, error) + expectedResult func(interface{}, error) + }{ + { + name: "get single blob", + doFn: func() (interface{}, error) { + b, err := service.Get(ctx, 1, blobs0[0].Namespace(), blobs0[0].Commitment) + return []*Blob{b}, err + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + assert.NotEmpty(t, res) + + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.Len(t, blobs, 1) + + assert.Equal(t, blobs0[0].Commitment, blobs[0].Commitment) + }, + }, + { + name: "get all with the same nID", + doFn: func() (interface{}, error) { + b, err := service.GetAll(ctx, 1, blobs1[0].Namespace()) + return b, err + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.NotEmpty(t, blobs) + + assert.Len(t, blobs, 2) + + for i := range blobs1 { + bytes.Equal(blobs1[i].Commitment, blobs[i].Commitment) + } + }, + }, + { + name: "get all with different nIDs", + doFn: func() (interface{}, error) { + b, err := service.GetAll(ctx, 1, blobs0[0].Namespace(), blobs0[1].Namespace()) + return b, err + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.NotEmpty(t, blobs) + + assert.Len(t, blobs, 2) + // check the order + require.True(t, bytes.Equal(blobs[0].Namespace(), blobs0[0].Namespace())) + require.True(t, bytes.Equal(blobs[1].Namespace(), blobs0[1].Namespace())) + }, + }, + { + name: "get blob with incorrect commitment", + doFn: func() (interface{}, error) { + b, err := service.Get(ctx, 1, blobs0[0].Namespace(), blobs0[1].Commitment) + return []*Blob{b}, err + }, + expectedResult: func(res interface{}, err error) { + require.Error(t, err) + + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.Empty(t, blobs[0]) + }, + }, + { + name: "get invalid blob", + doFn: func() (interface{}, error) { + appBlob, err := blobtest.GenerateBlobs([]int{10}, false) + require.NoError(t, err) + blob, err := convertBlobs(appBlob...) + require.NoError(t, err) + + b, err := service.Get(ctx, 1, blob[0].Namespace(), blob[0].Commitment) + return []*Blob{b}, err + }, + expectedResult: func(res interface{}, err error) { + require.Error(t, err) + + blobs, ok := res.([]*Blob) + assert.True(t, ok) + assert.Empty(t, blobs[0]) + }, + }, + { + name: "get proof", + doFn: func() (interface{}, error) { + proof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + return proof, err + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + + header, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + + proof, ok := res.(*Proof) + assert.True(t, ok) + + verifyFn := func(t *testing.T, rawShares [][]byte, proof *Proof, nID namespace.ID) { + for _, row := range header.DAH.RowRoots { + to := 0 + for _, p := range *proof { + from := to + to = p.End() - p.Start() + from + eq := p.VerifyInclusion(sha256.New(), nID, rawShares[from:to], row) + if eq == true { + return + } + } + } + t.Fatal("could not prove the shares") + } + + rawShares, err := BlobsToShares(blobs0[1]) + require.NoError(t, err) + verifyFn(t, rawShares, proof, blobs0[1].Namespace()) + }, + }, + { + name: "verify inclusion", + doFn: func() (interface{}, error) { + proof, err := service.GetProof(ctx, 1, blobs0[0].Namespace(), blobs0[0].Commitment) + require.NoError(t, err) + return service.Included(ctx, 1, blobs0[0].Namespace(), proof, blobs0[0].Commitment) + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + included, ok := res.(bool) + require.True(t, ok) + require.True(t, included) + }, + }, + { + name: "verify inclusion fails with different proof", + doFn: func() (interface{}, error) { + proof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + require.NoError(t, err) + return service.Included(ctx, 1, blobs0[0].Namespace(), proof, blobs0[0].Commitment) + }, + expectedResult: func(res interface{}, err error) { + require.Error(t, err) + require.ErrorIs(t, err, ErrInvalidProof) + included, ok := res.(bool) + require.True(t, ok) + require.True(t, included) + }, + }, + { + name: "not included", + doFn: func() (interface{}, error) { + appBlob, err := blobtest.GenerateBlobs([]int{10}, false) + require.NoError(t, err) + blob, err := convertBlobs(appBlob...) + require.NoError(t, err) + + proof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + require.NoError(t, err) + return service.Included(ctx, 1, blob[0].Namespace(), proof, blob[0].Commitment) + }, + expectedResult: func(res interface{}, err error) { + require.NoError(t, err) + included, ok := res.(bool) + require.True(t, ok) + require.False(t, included) + }, + }, + { + name: "count proofs for the blob", + doFn: func() (interface{}, error) { + proof0, err := service.GetProof(ctx, 1, blobs0[0].Namespace(), blobs0[0].Commitment) + if err != nil { + return nil, err + } + proof1, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + if err != nil { + return nil, err + } + return []*Proof{proof0, proof1}, nil + }, + expectedResult: func(i interface{}, err error) { + require.NoError(t, err) + proofs, ok := i.([]*Proof) + require.True(t, ok) + + h, err := service.headerGetter(ctx, 1) + require.NoError(t, err) + + originalDataWidth := len(h.DAH.RowRoots) / 2 + sizes := []int{blobSize0, blobSize1} + for i, proof := range proofs { + require.True(t, sizes[i]/originalDataWidth+1 == proof.Len()) + } + }, + }, + { + name: "get all not found", + doFn: func() (interface{}, error) { + nID := tmrand.Bytes(appconsts.NamespaceSize) + return service.GetAll(ctx, 1, nID) + }, + expectedResult: func(i interface{}, err error) { + blobs, ok := i.([]*Blob) + require.True(t, ok) + assert.Empty(t, blobs) + require.Error(t, err) + require.ErrorIs(t, err, ErrBlobNotFound) + + }, + }, + } + + for _, tt := range test { + t.Run(tt.name, func(t *testing.T) { + blobs, err := tt.doFn() + tt.expectedResult(blobs, err) + }) + } +} + +// TestService_GetSingleBlobWithoutPadding creates two blobs with the same nID +// But to satisfy the rule of eds creating, padding namespace share is placed between +// blobs. Test ensures that blob service will skip padding share and return the correct blob. +func TestService_GetSingleBlobWithoutPadding(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + appBlob, err := blobtest.GenerateBlobs([]int{9, 5}, true) + require.NoError(t, err) + blobs, err := convertBlobs(appBlob...) + require.NoError(t, err) + + ns1, err := appns.New(blobs[0].Namespace()[0], blobs[0].Namespace()[appns.NamespaceVersionSize:]) + require.NoError(t, err) + + ns2, err := appns.New(blobs[1].Namespace()[0], blobs[1].Namespace()[appns.NamespaceVersionSize:]) + require.NoError(t, err) + + padding0, err := shares.NamespacePaddingShare(ns1) + require.NoError(t, err) + padding1, err := shares.NamespacePaddingShare(ns2) + require.NoError(t, err) + rawShares0, err := BlobsToShares(blobs[0]) + require.NoError(t, err) + rawShares1, err := BlobsToShares(blobs[1]) + require.NoError(t, err) + + rawShares := make([][]byte, 0) + rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) + rawShares = append(rawShares, append(rawShares1, padding1.ToBytes())...) + + bs := mdutils.Bserv() + batching := ds_sync.MutexWrap(ds.NewMapDatastore()) + headerStore, err := store.NewStore[*header.ExtendedHeader](batching) + require.NoError(t, err) + eds, err := share.AddShares(ctx, rawShares, bs) + require.NoError(t, err) + + h := headertest.ExtendedHeaderFromEDS(t, 1, eds) + err = headerStore.Init(ctx, h) + require.NoError(t, err) + + fn := func(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { + return headerStore.GetByHeight(ctx, height) + } + service := NewService(nil, getters.NewIPLDGetter(bs), fn) + + newBlob, err := service.Get(ctx, 1, blobs[1].Namespace(), blobs[1].Commitment) + require.NoError(t, err) + assert.Equal(t, newBlob.Commitment, blobs[1].Commitment) +} + +func TestService_GetAllWithoutPadding(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + appBlob, err := blobtest.GenerateBlobs([]int{9, 5}, true) + require.NoError(t, err) + blobs, err := convertBlobs(appBlob...) + require.NoError(t, err) + + ns1, err := appns.New( + blobs[0].Namespace()[appns.NamespaceVersionSize-1], + blobs[0].Namespace()[appns.NamespaceVersionSize:], + ) + require.NoError(t, err) + + ns2, err := appns.New( + blobs[1].Namespace()[appns.NamespaceVersionSize-1], + blobs[1].Namespace()[appns.NamespaceVersionSize:], + ) + require.NoError(t, err) + + padding0, err := shares.NamespacePaddingShare(ns1) + require.NoError(t, err) + padding1, err := shares.NamespacePaddingShare(ns2) + require.NoError(t, err) + rawShares0, err := BlobsToShares(blobs[0]) + require.NoError(t, err) + rawShares1, err := BlobsToShares(blobs[1]) + require.NoError(t, err) + rawShares := make([][]byte, 0) + + // create shares in correct order with padding shares + if bytes.Compare(blobs[0].Namespace(), blobs[1].Namespace()) <= 0 { + rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) + rawShares = append(rawShares, append(rawShares1, padding1.ToBytes())...) + } else { + rawShares = append(rawShares, append(rawShares1, padding1.ToBytes())...) + rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) + } + + bs := mdutils.Bserv() + batching := ds_sync.MutexWrap(ds.NewMapDatastore()) + headerStore, err := store.NewStore[*header.ExtendedHeader](batching) + require.NoError(t, err) + eds, err := share.AddShares(ctx, rawShares, bs) + require.NoError(t, err) + + h := headertest.ExtendedHeaderFromEDS(t, 1, eds) + err = headerStore.Init(ctx, h) + require.NoError(t, err) + + fn := func(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { + return headerStore.GetByHeight(ctx, height) + } + + service := NewService(nil, getters.NewIPLDGetter(bs), fn) + + _, err = service.GetAll(ctx, 1, blobs[0].Namespace(), blobs[1].Namespace()) + require.NoError(t, err) +} + +func createService(ctx context.Context, t *testing.T, blobs []*Blob) *Service { + bs := mdutils.Bserv() + batching := ds_sync.MutexWrap(ds.NewMapDatastore()) + headerStore, err := store.NewStore[*header.ExtendedHeader](batching) + require.NoError(t, err) + rawShares, err := BlobsToShares(blobs...) + require.NoError(t, err) + eds, err := share.AddShares(ctx, rawShares, bs) + require.NoError(t, err) + + h := headertest.ExtendedHeaderFromEDS(t, 1, eds) + err = headerStore.Init(ctx, h) + require.NoError(t, err) + + fn := func(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { + return headerStore.GetByHeight(ctx, height) + } + return NewService(nil, getters.NewIPLDGetter(bs), fn) +} diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 76bda50db5..3e0da71d69 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -321,6 +321,31 @@ func FraudMaker(t *testing.T, faultHeight int64, bServ blockservice.BlockService } } +func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedDataSquare) *header.ExtendedHeader { + valSet, vals := RandValidatorSet(10, 10) + gen := RandRawHeader(t) + dah := da.NewDataAvailabilityHeader(eds) + + gen.DataHash = dah.Hash() + gen.ValidatorsHash = valSet.Hash() + gen.NextValidatorsHash = valSet.Hash() + gen.Height = int64(height) + blockID := RandBlockID(t) + blockID.Hash = gen.Hash() + voteSet := types.NewVoteSet(gen.ChainID, gen.Height, 0, tmproto.PrecommitType, valSet) + commit, err := MakeCommit(blockID, gen.Height, 0, voteSet, vals, time.Now()) + require.NoError(t, err) + + eh := &header.ExtendedHeader{ + RawHeader: *gen, + Commit: commit, + ValidatorSet: valSet, + DAH: &dah, + } + require.NoError(t, eh.Validate()) + return eh +} + func CreateFraudExtHeader( t *testing.T, eh *header.ExtendedHeader, diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go new file mode 100644 index 0000000000..8c309ba5b2 --- /dev/null +++ b/nodebuilder/blob/blob.go @@ -0,0 +1,76 @@ +package blob + +import ( + "context" + + "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/blob" +) + +var _ Module = (*API)(nil) + +// Module defines the API related to interacting with the blobs +// +//go:generate mockgen -destination=mocks/api.go -package=mocks . Module +type Module interface { + // Submit sends Blobs and reports the height in which they were included. + // Allows sending multiple Blobs atomically synchronously. + // Uses default wallet registered on the Node. + Submit(_ context.Context, _ ...*blob.Blob) (height uint64, _ error) + // Get retrieves the blob by commitment under the given namespace and height. + Get(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Blob, error) + // GetAll returns all blobs under the given namespaces and height. + GetAll(_ context.Context, height uint64, _ ...namespace.ID) ([]*blob.Blob, error) + // GetProof retrieves proofs in the given namespaces at the given height by commitment. + GetProof(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Proof, error) + // Included checks whether a blob's given commitment(Merkle subtree root) is included at + // given height and under the namespace. + Included(_ context.Context, height uint64, _ namespace.ID, _ *blob.Proof, _ blob.Commitment) (bool, error) +} + +type API struct { + Internal struct { + Submit func(context.Context, ...*blob.Blob) (uint64, error) `perm:"write"` + Get func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Blob, error) `perm:"read"` + GetAll func(context.Context, uint64, ...namespace.ID) ([]*blob.Blob, error) `perm:"read"` + GetProof func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Proof, error) `perm:"read"` + Included func(context.Context, uint64, namespace.ID, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` + } +} + +func (api *API) Submit(ctx context.Context, blobs ...*blob.Blob) (uint64, error) { + return api.Internal.Submit(ctx, blobs...) +} + +func (api *API) Get( + ctx context.Context, + height uint64, + nID namespace.ID, + commitment blob.Commitment, +) (*blob.Blob, error) { + return api.Internal.Get(ctx, height, nID, commitment) +} + +func (api *API) GetAll(ctx context.Context, height uint64, nIDs ...namespace.ID) ([]*blob.Blob, error) { + return api.Internal.GetAll(ctx, height, nIDs...) +} + +func (api *API) GetProof( + ctx context.Context, + height uint64, + nID namespace.ID, + commitment blob.Commitment, +) (*blob.Proof, error) { + return api.Internal.GetProof(ctx, height, nID, commitment) +} + +func (api *API) Included( + ctx context.Context, + height uint64, + nID namespace.ID, + proof *blob.Proof, + commitment blob.Commitment, +) (bool, error) { + return api.Internal.Included(ctx, height, nID, proof, commitment) +} diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go new file mode 100644 index 0000000000..93824bc625 --- /dev/null +++ b/nodebuilder/blob/mocks/api.go @@ -0,0 +1,122 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/celestiaorg/celestia-node/nodebuilder/blob (interfaces: Module) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + blob "github.com/celestiaorg/celestia-node/blob" + namespace "github.com/celestiaorg/nmt/namespace" + gomock "github.com/golang/mock/gomock" +) + +// MockModule is a mock of Module interface. +type MockModule struct { + ctrl *gomock.Controller + recorder *MockModuleMockRecorder +} + +// MockModuleMockRecorder is the mock recorder for MockModule. +type MockModuleMockRecorder struct { + mock *MockModule +} + +// NewMockModule creates a new mock instance. +func NewMockModule(ctrl *gomock.Controller) *MockModule { + mock := &MockModule{ctrl: ctrl} + mock.recorder = &MockModuleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockModule) EXPECT() *MockModuleMockRecorder { + return m.recorder +} + +// Get mocks base method. +func (m *MockModule) Get(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 blob.Commitment) (*blob.Blob, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*blob.Blob) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockModuleMockRecorder) Get(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockModule)(nil).Get), arg0, arg1, arg2, arg3) +} + +// GetAll mocks base method. +func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 ...namespace.ID) ([]*blob.Blob, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0, arg1} + for _, a := range arg2 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "GetAll", varargs...) + ret0, _ := ret[0].([]*blob.Blob) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetAll indicates an expected call of GetAll. +func (mr *MockModuleMockRecorder) GetAll(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0, arg1}, arg2...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockModule)(nil).GetAll), varargs...) +} + +// GetProof mocks base method. +func (m *MockModule) GetProof(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 blob.Commitment) (*blob.Proof, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProof", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(*blob.Proof) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProof indicates an expected call of GetProof. +func (mr *MockModuleMockRecorder) GetProof(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProof", reflect.TypeOf((*MockModule)(nil).GetProof), arg0, arg1, arg2, arg3) +} + +// Included mocks base method. +func (m *MockModule) Included(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 *blob.Proof, arg4 blob.Commitment) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Included", arg0, arg1, arg2, arg3, arg4) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Included indicates an expected call of Included. +func (mr *MockModuleMockRecorder) Included(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Included", reflect.TypeOf((*MockModule)(nil).Included), arg0, arg1, arg2, arg3, arg4) +} + +// Submit mocks base method. +func (m *MockModule) Submit(arg0 context.Context, arg1 ...*blob.Blob) (uint64, error) { + m.ctrl.T.Helper() + varargs := []interface{}{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "Submit", varargs...) + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Submit indicates an expected call of Submit. +func (mr *MockModuleMockRecorder) Submit(arg0 interface{}, arg1 ...interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]interface{}{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockModule)(nil).Submit), varargs...) +} diff --git a/nodebuilder/blob/module.go b/nodebuilder/blob/module.go new file mode 100644 index 0000000000..76e7677725 --- /dev/null +++ b/nodebuilder/blob/module.go @@ -0,0 +1,28 @@ +package blob + +import ( + "context" + + "go.uber.org/fx" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/header" + headerService "github.com/celestiaorg/celestia-node/nodebuilder/header" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/state" +) + +func ConstructModule() fx.Option { + return fx.Module("blob", + fx.Provide( + func(service headerService.Module) func(context.Context, uint64) (*header.ExtendedHeader, error) { + return service.GetByHeight + }), + fx.Provide(func( + state *state.CoreAccessor, + sGetter share.Getter, + getByHeightFn func(context.Context, uint64) (*header.ExtendedHeader, error), + ) Module { + return blob.NewService(state, sGetter, getByHeightFn) + })) +} diff --git a/nodebuilder/default_services.go b/nodebuilder/default_services.go index 03954c726a..0373e44244 100644 --- a/nodebuilder/default_services.go +++ b/nodebuilder/default_services.go @@ -1,6 +1,7 @@ package nodebuilder import ( + "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -19,5 +20,6 @@ var PackageToAPI = map[string]interface{}{ "header": &header.API{}, "daser": &das.API{}, "p2p": &p2p.API{}, + "blob": &blob.API{}, "node": &node.API{}, } diff --git a/nodebuilder/module.go b/nodebuilder/module.go index 3f3b08e68b..51edc26c72 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "github.com/celestiaorg/celestia-node/libs/fxutil" + "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" @@ -52,6 +53,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store core.ConstructModule(tp, &cfg.Core), das.ConstructModule(tp, &cfg.DASer), fraud.ConstructModule(tp), + blob.ConstructModule(), node.ConstructModule(tp), ) diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 90caac5dc9..243ac47163 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -21,6 +21,7 @@ import ( "github.com/celestiaorg/celestia-node/api/gateway" "github.com/celestiaorg/celestia-node/api/rpc" + "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -68,6 +69,7 @@ type Node struct { HeaderServ header.Module // not optional StateServ state.Module // not optional FraudServ fraud.Module // not optional + BlobServ blob.Module // not optional DASer das.Module // not optional // start and stop control ref internal fx.App lifecycle funcs to be called from Start and Stop diff --git a/nodebuilder/node/admin.go b/nodebuilder/node/admin.go index 4607c4d33f..41f8400101 100644 --- a/nodebuilder/node/admin.go +++ b/nodebuilder/node/admin.go @@ -8,7 +8,7 @@ import ( logging "github.com/ipfs/go-log/v2" ) -const APIVersion = "v0.1.0" +const APIVersion = "v0.1.1" type module struct { tp Type diff --git a/nodebuilder/rpc/constructors.go b/nodebuilder/rpc/constructors.go index 3d5c368042..ca30af6305 100644 --- a/nodebuilder/rpc/constructors.go +++ b/nodebuilder/rpc/constructors.go @@ -4,6 +4,7 @@ import ( "github.com/cristalhq/jwt" "github.com/celestiaorg/celestia-node/api/rpc" + "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -22,6 +23,7 @@ func registerEndpoints( daserMod das.Module, p2pMod p2p.Module, nodeMod node.Module, + blobMod blob.Module, serv *rpc.Server, ) { serv.RegisterAuthedService("fraud", fraudMod, &fraud.API{}) @@ -31,6 +33,7 @@ func registerEndpoints( serv.RegisterAuthedService("share", shareMod, &share.API{}) serv.RegisterAuthedService("p2p", p2pMod, &p2p.API{}) serv.RegisterAuthedService("node", nodeMod, &node.API{}) + serv.RegisterAuthedService("blob", blobMod, &blob.API{}) } func server(cfg *Config, auth jwt.Signer) *rpc.Server { diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index 3658c7ebf8..33663d7ce8 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -12,9 +12,8 @@ import ( types "github.com/cosmos/cosmos-sdk/types" types0 "github.com/cosmos/cosmos-sdk/x/staking/types" gomock "github.com/golang/mock/gomock" - types1 "github.com/tendermint/tendermint/types" - - namespace "github.com/celestiaorg/nmt/namespace" + types1 "github.com/tendermint/tendermint/proto/tendermint/types" + types2 "github.com/tendermint/tendermint/types" ) // MockModule is a mock of Module interface. @@ -189,23 +188,28 @@ func (mr *MockModuleMockRecorder) QueryUnbonding(arg0, arg1 interface{}) *gomock return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "QueryUnbonding", reflect.TypeOf((*MockModule)(nil).QueryUnbonding), arg0, arg1) } -// SubmitPayForData mocks base method. -func (m *MockModule) SubmitPayForBlob(arg0 context.Context, arg1 namespace.ID, arg2 []byte, arg3 math.Int, arg4 uint64) (*types.TxResponse, error) { +// SubmitPayForBlob mocks base method. +func (m *MockModule) SubmitPayForBlob(arg0 context.Context, arg1 math.Int, arg2 uint64, arg3 ...*types1.Blob) (*types.TxResponse, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "SubmitPayForBlob", arg0, arg1, arg2, arg3, arg4) + varargs := []interface{}{arg0, arg1, arg2} + for _, a := range arg3 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "SubmitPayForBlob", varargs...) ret0, _ := ret[0].(*types.TxResponse) ret1, _ := ret[1].(error) return ret0, ret1 } -// SubmitPayForData indicates an expected call of SubmitPayForData. -func (mr *MockModuleMockRecorder) SubmitPayForData(arg0, arg1, arg2, arg3, arg4 interface{}) *gomock.Call { +// SubmitPayForBlob indicates an expected call of SubmitPayForBlob. +func (mr *MockModuleMockRecorder) SubmitPayForBlob(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubmitPayForBlob", reflect.TypeOf((*MockModule)(nil).SubmitPayForBlob), arg0, arg1, arg2, arg3, arg4) + varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubmitPayForBlob", reflect.TypeOf((*MockModule)(nil).SubmitPayForBlob), varargs...) } // SubmitTx mocks base method. -func (m *MockModule) SubmitTx(arg0 context.Context, arg1 types1.Tx) (*types.TxResponse, error) { +func (m *MockModule) SubmitTx(arg0 context.Context, arg1 types2.Tx) (*types.TxResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SubmitTx", arg0, arg1) ret0, _ := ret[0].(*types.TxResponse) diff --git a/nodebuilder/state/state.go b/nodebuilder/state/state.go index 07f61e5952..fc1932cfae 100644 --- a/nodebuilder/state/state.go +++ b/nodebuilder/state/state.go @@ -5,7 +5,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/staking/types" - "github.com/celestiaorg/nmt/namespace" + apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/state" ) @@ -44,10 +44,9 @@ type Module interface { // SubmitPayForBlob builds, signs and submits a PayForBlob transaction. SubmitPayForBlob( ctx context.Context, - nID namespace.ID, - data []byte, fee state.Int, gasLim uint64, + blobs ...*apptypes.Blob, ) (*state.TxResponse, error) // CancelUnbondingDelegation cancels a user's pending undelegation from a validator. @@ -113,10 +112,9 @@ type API struct { SubmitTx func(ctx context.Context, tx state.Tx) (*state.TxResponse, error) `perm:"write"` SubmitPayForBlob func( ctx context.Context, - nID namespace.ID, - data []byte, fee state.Int, gasLim uint64, + blobs ...*apptypes.Blob, ) (*state.TxResponse, error) `perm:"write"` CancelUnbondingDelegation func( ctx context.Context, @@ -192,12 +190,11 @@ func (api *API) SubmitTx(ctx context.Context, tx state.Tx) (*state.TxResponse, e func (api *API) SubmitPayForBlob( ctx context.Context, - nID namespace.ID, - data []byte, fee state.Int, gasLim uint64, + blobs ...*apptypes.Blob, ) (*state.TxResponse, error) { - return api.Internal.SubmitPayForBlob(ctx, nID, data, fee, gasLim) + return api.Internal.SubmitPayForBlob(ctx, fee, gasLim, blobs...) } func (api *API) CancelUnbondingDelegation( diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go new file mode 100644 index 0000000000..db56bad8c4 --- /dev/null +++ b/nodebuilder/tests/blob_test.go @@ -0,0 +1,131 @@ +package tests + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/blob/blobtest" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" +) + +func TestBlobModule(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + sw := swamp.NewSwamp(t) + + appBlobs0, err := blobtest.GenerateBlobs([]int{8, 4}, true) + require.NoError(t, err) + appBlobs1, err := blobtest.GenerateBlobs([]int{4}, false) + require.NoError(t, err) + blobs := make([]*blob.Blob, 0, len(appBlobs0)+len(appBlobs1)) + + for _, b := range append(appBlobs0, appBlobs1...) { + blob, err := blob.NewBlob(b.ShareVersion, append([]byte{b.NamespaceVersion}, b.NamespaceID...), b.Data) + require.NoError(t, err) + blobs = append(blobs, blob) + } + + require.NoError(t, err) + bridge := sw.NewBridgeNode() + require.NoError(t, bridge.Start(ctx)) + + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + require.NoError(t, err) + + fullCfg := sw.DefaultTestConfig(node.Full) + fullCfg.Header.TrustedPeers = append(fullCfg.Header.TrustedPeers, addrs[0].String()) + fullNode := sw.NewNodeWithConfig(node.Full, fullCfg) + require.NoError(t, fullNode.Start(ctx)) + + addrsFull, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(fullNode.Host)) + require.NoError(t, err) + + lightCfg := sw.DefaultTestConfig(node.Light) + lightCfg.Header.TrustedPeers = append(lightCfg.Header.TrustedPeers, addrsFull[0].String()) + lightNode := sw.NewNodeWithConfig(node.Light, lightCfg) + require.NoError(t, lightNode.Start(ctx)) + + height, err := fullNode.BlobServ.Submit(ctx, blobs...) + require.NoError(t, err) + + _, err = fullNode.HeaderServ.WaitForHeight(ctx, height) + require.NoError(t, err) + _, err = lightNode.HeaderServ.WaitForHeight(ctx, height) + require.NoError(t, err) + + var test = []struct { + name string + doFn func(t *testing.T) + }{ + { + name: "Get", + doFn: func(t *testing.T) { + blob1, err := fullNode.BlobServ.Get(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) + require.NoError(t, err) + require.Equal(t, blobs[0], blob1) + }, + }, + { + name: "GetAll", + doFn: func(t *testing.T) { + newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, blobs[0].Namespace()) + require.NoError(t, err) + require.Len(t, newBlobs, len(appBlobs0)) + require.True(t, bytes.Equal(blobs[0].Commitment, newBlobs[0].Commitment)) + require.True(t, bytes.Equal(blobs[1].Commitment, newBlobs[1].Commitment)) + }, + }, + { + name: "Included", + doFn: func(t *testing.T) { + proof, err := fullNode.BlobServ.GetProof(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) + require.NoError(t, err) + + included, err := lightNode.BlobServ.Included( + ctx, + height, + blobs[0].Namespace(), + proof, + blobs[0].Commitment, + ) + require.NoError(t, err) + require.True(t, included) + }, + }, + { + name: "Not Found", + doFn: func(t *testing.T) { + appBlob, err := blobtest.GenerateBlobs([]int{4}, false) + require.NoError(t, err) + newBlob, err := blob.NewBlob( + appBlob[0].ShareVersion, + append([]byte{appBlob[0].NamespaceVersion}, appBlob[0].NamespaceID...), + appBlob[0].Data, + ) + require.NoError(t, err) + + b, err := fullNode.BlobServ.Get(ctx, height, newBlob.Namespace(), newBlob.Commitment) + assert.Nil(t, b) + require.Error(t, err) + require.ErrorIs(t, err, blob.ErrBlobNotFound) + }, + }, + } + + for _, tt := range test { + tt := tt + t.Run(tt.name, func(t *testing.T) { + t.Parallel() + tt.doFn(t) + }) + } +} diff --git a/share/nid.go b/share/nid.go new file mode 100644 index 0000000000..dfd6a5de7e --- /dev/null +++ b/share/nid.go @@ -0,0 +1,26 @@ +package share + +import ( + "fmt" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/nmt/namespace" +) + +// NewNamespaceV0 takes variable size byte slice anc creates version 0 Namespace ID. +// The bytes slice must be <= 10 bytes. +func NewNamespaceV0(subNId []byte) (namespace.ID, error) { + if lnid := len(subNId); lnid > appns.NamespaceVersionZeroIDSize { + return nil, fmt.Errorf("namespace id must be <= %v, but it was %v bytes", appns.NamespaceVersionZeroIDSize, lnid) + } + + id := make([]byte, appns.NamespaceIDSize) + copy(id[appns.NamespaceVersionZeroPrefixSize:], subNId) + + appID, err := appns.New(appns.NamespaceVersionZero, id) + if err != nil { + return nil, err + } + + return appID.Bytes(), nil +} diff --git a/state/core_access.go b/state/core_access.go index aed7db3df3..1a0a5650ab 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -20,11 +20,9 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/x/blob" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" libhead "github.com/celestiaorg/go-header" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/header" ) @@ -158,17 +156,19 @@ func (ca *CoreAccessor) constructSignedTx( func (ca *CoreAccessor) SubmitPayForBlob( ctx context.Context, - nID namespace.ID, - data []byte, fee Int, gasLim uint64, + blobs ...*apptypes.Blob, ) (*TxResponse, error) { - b := &apptypes.Blob{NamespaceId: nID, Data: data, ShareVersion: uint32(appconsts.DefaultShareVersion)} + if len(blobs) == 0 { + return nil, errors.New("state: no blobs provided") + } + response, err := blob.SubmitPayForBlob( ctx, ca.signer, ca.coreConn, - []*apptypes.Blob{b}, + blobs, apptypes.SetGasLimit(gasLim), withFee(fee), ) From 64c18a5554cbd39772921c5dcb46b8de06b9e9f0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 11:22:23 +0200 Subject: [PATCH 006/388] chore(deps): bump go.uber.org/fx from 1.19.2 to 1.19.3 (#2199) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Hlib Kanunnikov --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index abdc7b6afc..2773458ec2 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v0.34.0 go.opentelemetry.io/otel/trace v1.13.0 go.opentelemetry.io/proto/otlp v0.19.0 - go.uber.org/fx v1.19.2 + go.uber.org/fx v1.19.3 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.7.0 golang.org/x/sync v0.1.0 diff --git a/go.sum b/go.sum index 9eca643fef..3162a6abab 100644 --- a/go.sum +++ b/go.sum @@ -2086,8 +2086,8 @@ go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8= go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk= -go.uber.org/fx v1.19.2 h1:SyFgYQFr1Wl0AYstE8vyYIzP4bFz2URrScjwC4cwUvY= -go.uber.org/fx v1.19.2/go.mod h1:43G1VcqSzbIv77y00p1DRAsyZS8WdzuYdhZXmEUkMyQ= +go.uber.org/fx v1.19.3 h1:YqMRE4+2IepTYCMOvXqQpRa+QAVdiSTnsHU4XNWBceA= +go.uber.org/fx v1.19.3/go.mod h1:w2HrQg26ql9fLK7hlBiZ6JsRUKV+Lj/atT1KCjT8YhM= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= From a438788f474f65aae3e33b3b8ee6db055f8d3607 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 09:31:41 +0000 Subject: [PATCH 007/388] chore(deps): bump golang.org/x/crypto from 0.7.0 to 0.9.0 (#2276) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 2773458ec2..fd0bf7eb4c 100644 --- a/go.mod +++ b/go.mod @@ -72,9 +72,9 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 go.uber.org/fx v1.19.3 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.7.0 + golang.org/x/crypto v0.9.0 golang.org/x/sync v0.1.0 - golang.org/x/text v0.8.0 + golang.org/x/text v0.9.0 google.golang.org/grpc v1.53.0 google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 ) @@ -311,10 +311,10 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect golang.org/x/mod v0.9.0 // indirect - golang.org/x/net v0.8.0 // indirect + golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect - golang.org/x/sys v0.6.0 // indirect - golang.org/x/term v0.6.0 // indirect + golang.org/x/sys v0.8.0 // indirect + golang.org/x/term v0.8.0 // indirect golang.org/x/tools v0.7.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.103.0 // indirect diff --git a/go.sum b/go.sum index 3162a6abab..92583018a9 100644 --- a/go.sum +++ b/go.sum @@ -2152,8 +2152,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.7.0 h1:AvwMYaRytfdeVt3u6mLaxYtErKYjxA2OXjJ1HHq6t3A= -golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= +golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= +golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2286,8 +2286,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.8.0 h1:Zrh2ngAOFYneWTAIAPethzeaQLuHwhuBkuV6ZiRnUaQ= -golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= +golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= +golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2462,14 +2462,14 @@ golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.6.0 h1:MVltZSvRTcU2ljQOhs94SXPftV6DCNnZViHeQps87pQ= -golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.6.0 h1:clScbb1cHjoCkyRbWwBEUZ5H/tIFu5TAXIqaZD0Gcjw= -golang.org/x/term v0.6.0/go.mod h1:m6U89DPEgQRMq3DNkDClhWw02AUbt2daBVO4cn4Hv9U= +golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= +golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2480,8 +2480,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.8.0 h1:57P1ETyNKtuIjB4SRd15iJxuhj8Gc416Y78H3qgMh68= -golang.org/x/text v0.8.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= +golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 29203b67df7abcc7ce08dcea592c32b4dba439b6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 09:54:55 +0000 Subject: [PATCH 008/388] chore(deps): bump codecov/codecov-action from 3.1.3 to 3.1.4 (#2235) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/go-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 8f4729c7c8..b519db7763 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -59,7 +59,7 @@ jobs: run: make test-unit - name: upload coverage - uses: codecov/codecov-action@v3.1.3 + uses: codecov/codecov-action@v3.1.4 with: token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.txt From 5d0424aa06895d3782de2f5181679929647e7609 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 30 May 2023 10:04:08 +0000 Subject: [PATCH 009/388] chore(deps): bump alpine from 3.17.3 to 3.18.0 (#2198) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 214003dc08..9346c559d1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . RUN make build && make cel-key -FROM docker.io/alpine:3.17.3 +FROM docker.io/alpine:3.18.0 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From b62452af1e7dd1cc1b9a5ca1d96d1545154b4d04 Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Viet Date: Wed, 31 May 2023 09:49:03 +0200 Subject: [PATCH 010/388] chore!: bump arabica testnet params (#2281) --- nodebuilder/p2p/bootstrap.go | 8 ++++---- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index 0ceefa0453..25f0a85120 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -38,10 +38,10 @@ func bootstrappersFor(net Network) ([]string, error) { // NOTE: Every time we add a new long-running network, its bootstrap peers have to be added here. var bootstrapList = map[Network][]string{ Arabica: { - "/dns4/limani.celestia-devops.dev/tcp/2121/p2p/12D3KooWDgG69kXfmSiHjUErN2ahpUC1SXpSfB2urrqMZ6aWC8NS", - "/dns4/marsellesa.celestia-devops.dev/tcp/2121/p2p/12D3KooWHr2wqFAsMXnPzpFsgxmePgXb8BqpkePebwUgLyZc95bd", - "/dns4/parainem.celestia-devops.dev/tcp/2121/p2p/12D3KooWHX8xpwg8qkP7kLKmKGtgZvmsopvgxc6Fwtu665QC7G8q", - "/dns4/kaarina.celestia-devops.dev/tcp/2121/p2p/12D3KooWN6fzdt4sG5QfWRPn4kwCQBdkt7TDNQkWsUymAwKrmvUs", + "/dns4/da-bridge-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWDXkXARv79Dtn5xrGBgJePtCzCsEwWR7eGWnx9ZCyUyD6", + "/dns4/da-bridge-arabica-8-2.celestia-arabica.com/tcp/2121/p2p/12D3KooWPu8qKmmNgYFMBsTkLBa1m3D9Cy9ReCAoQLqxEn9MHD1i", + "/dns4/da-full-1-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWEmeFodzypdTBTcw8Yub6WZRT4h1UgFtwCwwq6wS5Dtqm", + "/dns4/da-full-2-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWCs3wFmqwPn1u8pNU4BGsvLsob1ShTzvps8qEtTRuuuK5", }, Mocha: { "/dns4/andromeda.celestia-devops.dev/tcp/2121/p2p/12D3KooWKvPXtV1yaQ6e3BRNUHa5Phh8daBwBi3KkGaSSkUPys6D", diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index 670699b292..bc0fc3810d 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -23,7 +23,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ - Arabica: "EE310062CBB13CE98CBC7EAD3F6A827F0E4A86043FDEB2DA42048821877FE45C", + Arabica: "E5D620B5BE7873222DCD83464C285FD0F215C209393E7481F9A5979280AD6CA2", Mocha: "8038B21032C941372ED601699857043C12E5CC7D5945DCEEA4567D11B5712526", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index 025ba3bddf..ded3f0f1b1 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -12,7 +12,7 @@ const ( // DefaultNetwork is the default network of the current build. DefaultNetwork = Mocha // Arabica testnet. See: celestiaorg/networks. - Arabica Network = "arabica-6" + Arabica Network = "arabica-8" // Mocha testnet. See: celestiaorg/networks. Mocha Network = "mocha" // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. From 011765ac0fbf33eb7666467e145663dae2e60589 Mon Sep 17 00:00:00 2001 From: Alex Kiss Date: Wed, 31 May 2023 10:42:58 +0200 Subject: [PATCH 011/388] feat(nodebuilder): add ip resolving capabilities to core.ip (#2277) Co-authored-by: Hlib Kanunnikov --- nodebuilder/core/flags.go | 23 +++++++++++++++++------ 1 file changed, 17 insertions(+), 6 deletions(-) diff --git a/nodebuilder/core/flags.go b/nodebuilder/core/flags.go index b98c7acd07..901d910ea6 100644 --- a/nodebuilder/core/flags.go +++ b/nodebuilder/core/flags.go @@ -2,6 +2,7 @@ package core import ( "fmt" + "net" "github.com/spf13/cobra" flag "github.com/spf13/pflag" @@ -21,7 +22,8 @@ func Flags() *flag.FlagSet { coreFlag, "", "Indicates node to connect to the given core node. "+ - "Example: , 127.0.0.1. Assumes RPC port 26657 and gRPC port 9090 as default unless otherwise specified.", + "Example: , 127.0.0.1. , subdomain.domain.tld "+ + "Assumes RPC port 26657 and gRPC port 9090 as default unless otherwise specified.", ) flags.String( coreRPCFlag, @@ -37,10 +39,7 @@ func Flags() *flag.FlagSet { } // ParseFlags parses Core flags from the given cmd and saves them to the passed config. -func ParseFlags( - cmd *cobra.Command, - cfg *Config, -) error { +func ParseFlags(cmd *cobra.Command, cfg *Config) error { coreIP := cmd.Flag(coreFlag).Value.String() if coreIP == "" { if cmd.Flag(coreGRPCFlag).Changed || cmd.Flag(coreRPCFlag).Changed { @@ -49,10 +48,22 @@ func ParseFlags( return nil } + ip := net.ParseIP(coreIP) + if ip == nil { + ips, err := net.LookupIP(coreIP) + if err != nil { + return fmt.Errorf("failed to resolve DNS record: %v", err) + } + if len(ips) == 0 { + return fmt.Errorf("no IP addresses found for DNS record") + } + ip = ips[0] + } + rpc := cmd.Flag(coreRPCFlag).Value.String() grpc := cmd.Flag(coreGRPCFlag).Value.String() - cfg.IP = coreIP + cfg.IP = ip.String() cfg.RPCPort = rpc cfg.GRPCPort = grpc return nil From d53efbd03851c487bd96ff0c88fb92c4d1ed89d1 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 31 May 2023 14:30:10 +0200 Subject: [PATCH 012/388] chore(share/ipld): pretify error (#2284) --- share/ipld/namespace_data.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index 9698ef990d..13be26c06a 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -2,6 +2,7 @@ package ipld import ( "context" + "encoding/hex" "errors" "fmt" "sync" @@ -166,7 +167,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( defer span.End() span.SetAttributes( - attribute.String("namespace", n.nID.String()), + attribute.String("namespace", hex.EncodeToString(n.nID)), attribute.String("root", root.String()), ) @@ -213,8 +214,8 @@ func (n *NamespaceData) CollectLeavesByNamespace( singleErr.Do(func() { retrievalErr = err }) - log.Errorw("getLeavesWithProofsByNamespace:could not retrieve node", - "nID", n.nID, + log.Errorw("could not retrieve IPLD node", + "nID", hex.EncodeToString(n.nID), "pos", j.sharePos, "err", err, ) From 135622abe39e1e1d767861b6c3419a8393c6cbba Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 1 Jun 2023 11:41:35 +0200 Subject: [PATCH 013/388] feat(blob): add marshaling for blob and proof (#2286) --- api/docgen/examples.go | 6 ++++ blob/blob.go | 75 ++++++++++++++++++++++++++++++++++++++++-- blob/blob_test.go | 12 +++++++ blob/service_test.go | 19 +++++++++++ 4 files changed, 110 insertions(+), 2 deletions(-) diff --git a/api/docgen/examples.go b/api/docgen/examples.go index ae65501ba6..8293396c7b 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -18,8 +18,10 @@ import ( "golang.org/x/text/language" "github.com/celestiaorg/go-fraud" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/das" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" @@ -128,6 +130,10 @@ func init() { Addrs: []multiaddr.Multiaddr{ma}, } addToExampleValues(addrInfo) + + proof := nmt.NewInclusionProof(0, 4, [][]byte{[]byte("test")}, true) + blobProof := &blob.Proof{&proof} + addToExampleValues(blobProof) } func addToExampleValues(v interface{}) { diff --git a/blob/blob.go b/blob/blob.go index 434ab220c7..9771714cb9 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -2,12 +2,15 @@ package blob import ( "bytes" + "encoding/json" "fmt" appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt" "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/share/ipld" ) // Commitment is a Merkle Root of the subtree built from shares of the Blob. @@ -57,11 +60,47 @@ func (p Proof) equal(input Proof) error { return nil } +type jsonProof struct { + Start int `json:"start"` + End int `json:"end"` + Nodes [][]byte `json:"nodes"` +} + +func (p *Proof) MarshalJSON() ([]byte, error) { + proofs := make([]jsonProof, 0, p.Len()) + for _, pp := range *p { + proofs = append(proofs, jsonProof{ + Start: pp.Start(), + End: pp.End(), + Nodes: pp.Nodes(), + }) + } + + return json.Marshal(proofs) +} + +func (p *Proof) UnmarshalJSON(data []byte) error { + var proofs []jsonProof + err := json.Unmarshal(data, &proofs) + if err != nil { + return err + } + + nmtProofs := make([]*nmt.Proof, len(proofs)) + for i, jProof := range proofs { + nmtProof := nmt.NewInclusionProof(jProof.Start, jProof.End, jProof.Nodes, ipld.NMTIgnoreMaxNamespace) + nmtProofs[i] = &nmtProof + } + + *p = nmtProofs + return nil +} + // Blob represents any application-specific binary data that anyone can submit to Celestia. type Blob struct { - types.Blob + types.Blob `json:"blob"` - Commitment Commitment + Commitment Commitment `json:"commitment"` } // NewBlob constructs a new blob from the provided namespace.ID and data. @@ -91,3 +130,35 @@ func NewBlob(shareVersion uint8, namespace namespace.ID, data []byte) (*Blob, er func (b *Blob) Namespace() namespace.ID { return append([]byte{uint8(b.NamespaceVersion)}, b.NamespaceId...) } + +type jsonBlob struct { + Namespace namespace.ID `json:"namespace"` + Data []byte `json:"data"` + ShareVersion uint32 `json:"share_version"` + Commitment Commitment `json:"commitment"` +} + +func (b *Blob) MarshalJSON() ([]byte, error) { + blob := &jsonBlob{ + Namespace: b.Namespace(), + Data: b.Data, + ShareVersion: b.ShareVersion, + Commitment: b.Commitment, + } + return json.Marshal(blob) +} + +func (b *Blob) UnmarshalJSON(data []byte) error { + var blob jsonBlob + err := json.Unmarshal(data, &blob) + if err != nil { + return err + } + + b.Blob.NamespaceVersion = uint32(blob.Namespace[0]) + b.Blob.NamespaceId = blob.Namespace[1:] + b.Blob.Data = blob.Data + b.Blob.ShareVersion = blob.ShareVersion + b.Commitment = blob.Commitment + return nil +} diff --git a/blob/blob_test.go b/blob/blob_test.go index 5b3920ae9e..7f34fbfc84 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -1,6 +1,7 @@ package blob import ( + "reflect" "testing" "github.com/stretchr/testify/assert" @@ -62,6 +63,17 @@ func TestBlob(t *testing.T) { assert.Equal(t, blob[0].Commitment, b[0].Commitment) }, }, + { + name: "blob marshaling", + expectedRes: func(t *testing.T) { + data, err := blob[0].MarshalJSON() + require.NoError(t, err) + + newBlob := &Blob{} + require.NoError(t, newBlob.UnmarshalJSON(data)) + require.True(t, reflect.DeepEqual(blob[0], newBlob)) + }, + }, } for _, tt := range test { diff --git a/blob/service_test.go b/blob/service_test.go index 87554eb3d0..4160f36205 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -268,6 +268,25 @@ func TestBlobService_Get(t *testing.T) { }, }, + { + name: "marshal proof", + doFn: func() (interface{}, error) { + proof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + require.NoError(t, err) + return proof.MarshalJSON() + }, + expectedResult: func(i interface{}, err error) { + require.NoError(t, err) + jsonData, ok := i.([]byte) + require.True(t, ok) + var proof Proof + require.NoError(t, proof.UnmarshalJSON(jsonData)) + + newProof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) + require.NoError(t, err) + require.NoError(t, proof.equal(*newProof)) + }, + }, } for _, tt := range test { From 1c1327b5a47db4377eed1213e06bfa72c117154e Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 1 Jun 2023 13:28:46 +0200 Subject: [PATCH 014/388] feat(nodebuilder): Add jwt.Signer to Node struct for auth token access (#2295) This PR adds the jwt.Signer to the node struct to be able to generate auth tokens for integration tests that use celestia-node's RPC. --- cmd/auth.go | 7 +++---- libs/authtoken/authtoken.go | 18 ++++++++++++++++++ nodebuilder/node.go | 2 ++ nodebuilder/node_test.go | 1 + 4 files changed, 24 insertions(+), 4 deletions(-) create mode 100644 libs/authtoken/authtoken.go diff --git a/cmd/auth.go b/cmd/auth.go index c4eb057a29..eb2000675e 100644 --- a/cmd/auth.go +++ b/cmd/auth.go @@ -14,6 +14,7 @@ import ( flag "github.com/spf13/pflag" "github.com/celestiaorg/celestia-node/api/rpc/perms" + "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/libs/keystore" nodemod "github.com/celestiaorg/celestia-node/nodebuilder/node" ) @@ -70,14 +71,12 @@ func newToken(cmd *cobra.Command, args []string) error { return err } - token, err := jwt.NewTokenBuilder(signer).Build(&perms.JWTPayload{ - Allow: permissions, - }) + token, err := authtoken.NewSignedJWT(signer, permissions) if err != nil { return err } - fmt.Printf("%s", token.InsecureString()) + fmt.Printf("%s", token) return nil } diff --git a/libs/authtoken/authtoken.go b/libs/authtoken/authtoken.go new file mode 100644 index 0000000000..c86272790d --- /dev/null +++ b/libs/authtoken/authtoken.go @@ -0,0 +1,18 @@ +package authtoken + +import ( + "github.com/cristalhq/jwt" + "github.com/filecoin-project/go-jsonrpc/auth" + + "github.com/celestiaorg/celestia-node/api/rpc/perms" +) + +func NewSignedJWT(signer jwt.Signer, permissions []auth.Permission) (string, error) { + token, err := jwt.NewTokenBuilder(signer).Build(&perms.JWTPayload{ + Allow: permissions, + }) + if err != nil { + return "", err + } + return token.InsecureString(), nil +} diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 243ac47163..fd32a966f4 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -7,6 +7,7 @@ import ( "strings" "time" + "github.com/cristalhq/jwt" "github.com/ipfs/go-blockservice" exchange "github.com/ipfs/go-ipfs-exchange-interface" logging "github.com/ipfs/go-log/v2" @@ -51,6 +52,7 @@ type Node struct { Network p2p.Network Bootstrappers p2p.Bootstrappers Config *Config + AdminSigner jwt.Signer // rpc components RPCServer *rpc.Server // not optional diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index 11b27b076a..6aa9782a85 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -34,6 +34,7 @@ func TestLifecycle(t *testing.T) { require.NotNil(t, node.Host) require.NotNil(t, node.HeaderServ) require.NotNil(t, node.StateServ) + require.NotNil(t, node.AdminSigner) require.Equal(t, tt.tp, node.Type) ctx, cancel := context.WithCancel(context.Background()) From 218ab99c5455557cac41444160557cbef70a75ec Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 1 Jun 2023 14:32:29 +0200 Subject: [PATCH 015/388] chore(nodebuilder): decrease lifecycle timeout for LN (#2290) LNs don't need that much time to run Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- nodebuilder/node.go | 23 +++++++++++++++++------ nodebuilder/node_test.go | 7 +++---- 2 files changed, 20 insertions(+), 10 deletions(-) diff --git a/nodebuilder/node.go b/nodebuilder/node.go index fd32a966f4..a6e3a3c3cc 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -32,8 +32,6 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/state" ) -var Timeout = time.Minute * 2 - var ( log = logging.Logger("node") fxLog = logging.Logger("fx") @@ -97,14 +95,15 @@ func NewWithConfig(tp node.Type, network p2p.Network, store Store, cfg *Config, // Start launches the Node and all its components and services. func (n *Node) Start(ctx context.Context) error { - ctx, cancel := context.WithTimeout(ctx, Timeout) + to := timeout(n.Type) + ctx, cancel := context.WithTimeout(ctx, to) defer cancel() err := n.start(ctx) if err != nil { log.Debugf("error starting %s Node: %s", n.Type, err) if errors.Is(err, context.DeadlineExceeded) { - return fmt.Errorf("node: failed to start within timeout(%s): %w", Timeout, err) + return fmt.Errorf("node: failed to start within timeout(%s): %w", to, err) } return fmt.Errorf("node: failed to start: %w", err) } @@ -142,14 +141,15 @@ func (n *Node) Run(ctx context.Context) error { // Canceling the given context earlier 'ctx' unblocks the Stop and aborts graceful shutdown forcing // remaining Modules/Services to close immediately. func (n *Node) Stop(ctx context.Context) error { - ctx, cancel := context.WithTimeout(ctx, Timeout) + to := timeout(n.Type) + ctx, cancel := context.WithTimeout(ctx, to) defer cancel() err := n.stop(ctx) if err != nil { log.Debugf("error stopping %s Node: %s", n.Type, err) if errors.Is(err, context.DeadlineExceeded) { - return fmt.Errorf("node: failed to stop within timeout(%s): %w", Timeout, err) + return fmt.Errorf("node: failed to stop within timeout(%s): %w", to, err) } return fmt.Errorf("node: failed to stop: %w", err) } @@ -183,3 +183,14 @@ func newNode(opts ...fx.Option) (*Node, error) { // lifecycleFunc defines a type for common lifecycle funcs. type lifecycleFunc func(context.Context) error + +var DefaultLifecycleTimeout = time.Minute * 2 + +func timeout(tp node.Type) time.Duration { + switch tp { + case node.Light: + return time.Second * 20 + default: + return DefaultLifecycleTimeout + } +} diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index 6aa9782a85..bd5d1da811 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -137,6 +137,9 @@ func StartMockOtelCollectorHTTPServer(t *testing.T) (string, func()) { } func TestEmptyBlockExists(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + var test = []struct { tp node.Type }{ @@ -149,10 +152,6 @@ func TestEmptyBlockExists(t *testing.T) { for i, tt := range test { t.Run(strconv.Itoa(i), func(t *testing.T) { node := TestNode(t, tt.tp) - - ctx, cancel := context.WithTimeout(context.Background(), Timeout) - defer cancel() - err := node.Start(ctx) require.NoError(t, err) From c77f6db35fe57ccef4110d0c7815e53a24921948 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 1 Jun 2023 17:55:09 +0200 Subject: [PATCH 016/388] fix!(blob/module): fix api by changing variadic params to a slice (#2298) --- api/gateway/state.go | 2 +- blob/service.go | 6 ++--- blob/service_test.go | 8 +++---- nodebuilder/blob/blob.go | 16 ++++++------- nodebuilder/blob/mocks/api.go | 26 +++++++------------- nodebuilder/state/mocks/api.go | 13 ++++------ nodebuilder/state/state.go | 8 +++---- nodebuilder/tests/api_test.go | 43 ++++++++++++++++++++++++++++++++++ nodebuilder/tests/blob_test.go | 5 ++-- state/core_access.go | 2 +- 10 files changed, 79 insertions(+), 50 deletions(-) diff --git a/api/gateway/state.go b/api/gateway/state.go index 8997f4ae21..2576ad4848 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -148,7 +148,7 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } // perform request - txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, blob) + txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*apptypes.Blob{blob}) if err != nil { writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) return diff --git a/blob/service.go b/blob/service.go index 0fcf47bffd..53ada45450 100644 --- a/blob/service.go +++ b/blob/service.go @@ -53,7 +53,7 @@ func NewService( // Submit sends PFB transaction and reports the height in which it was included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. -func (s *Service) Submit(ctx context.Context, blobs ...*Blob) (uint64, error) { +func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { log.Debugw("submitting blobs", "amount", len(blobs)) var ( @@ -66,7 +66,7 @@ func (s *Service) Submit(ctx context.Context, blobs ...*Blob) (uint64, error) { b[i] = &blob.Blob } - resp, err := s.accessor.SubmitPayForBlob(ctx, types.NewInt(fee), gasLimit, b...) + resp, err := s.accessor.SubmitPayForBlob(ctx, types.NewInt(fee), gasLimit, b) if err != nil { return 0, err } @@ -99,7 +99,7 @@ func (s *Service) GetProof( // GetAll returns all blobs under the given namespaces at the given height. // GetAll can return blobs and an error in case if some requests failed. -func (s *Service) GetAll(ctx context.Context, height uint64, nIDs ...namespace.ID) ([]*Blob, error) { +func (s *Service) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID) ([]*Blob, error) { header, err := s.headerGetter(ctx, height) if err != nil { return nil, err diff --git a/blob/service_test.go b/blob/service_test.go index 4160f36205..ee6e982fe8 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -73,7 +73,7 @@ func TestBlobService_Get(t *testing.T) { { name: "get all with the same nID", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, blobs1[0].Namespace()) + b, err := service.GetAll(ctx, 1, []namespace.ID{blobs1[0].Namespace()}) return b, err }, expectedResult: func(res interface{}, err error) { @@ -93,7 +93,7 @@ func TestBlobService_Get(t *testing.T) { { name: "get all with different nIDs", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, blobs0[0].Namespace(), blobs0[1].Namespace()) + b, err := service.GetAll(ctx, 1, []namespace.ID{blobs0[0].Namespace(), blobs0[1].Namespace()}) return b, err }, expectedResult: func(res interface{}, err error) { @@ -257,7 +257,7 @@ func TestBlobService_Get(t *testing.T) { name: "get all not found", doFn: func() (interface{}, error) { nID := tmrand.Bytes(appconsts.NamespaceSize) - return service.GetAll(ctx, 1, nID) + return service.GetAll(ctx, 1, []namespace.ID{nID}) }, expectedResult: func(i interface{}, err error) { blobs, ok := i.([]*Blob) @@ -406,7 +406,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { service := NewService(nil, getters.NewIPLDGetter(bs), fn) - _, err = service.GetAll(ctx, 1, blobs[0].Namespace(), blobs[1].Namespace()) + _, err = service.GetAll(ctx, 1, []namespace.ID{blobs[0].Namespace(), blobs[1].Namespace()}) require.NoError(t, err) } diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index 8c309ba5b2..7cbe312856 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -17,11 +17,11 @@ type Module interface { // Submit sends Blobs and reports the height in which they were included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. - Submit(_ context.Context, _ ...*blob.Blob) (height uint64, _ error) + Submit(_ context.Context, _ []*blob.Blob) (height uint64, _ error) // Get retrieves the blob by commitment under the given namespace and height. Get(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Blob, error) // GetAll returns all blobs under the given namespaces and height. - GetAll(_ context.Context, height uint64, _ ...namespace.ID) ([]*blob.Blob, error) + GetAll(_ context.Context, height uint64, _ []namespace.ID) ([]*blob.Blob, error) // GetProof retrieves proofs in the given namespaces at the given height by commitment. GetProof(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Proof, error) // Included checks whether a blob's given commitment(Merkle subtree root) is included at @@ -31,16 +31,16 @@ type Module interface { type API struct { Internal struct { - Submit func(context.Context, ...*blob.Blob) (uint64, error) `perm:"write"` + Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` Get func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Blob, error) `perm:"read"` - GetAll func(context.Context, uint64, ...namespace.ID) ([]*blob.Blob, error) `perm:"read"` + GetAll func(context.Context, uint64, []namespace.ID) ([]*blob.Blob, error) `perm:"read"` GetProof func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Proof, error) `perm:"read"` Included func(context.Context, uint64, namespace.ID, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` } } -func (api *API) Submit(ctx context.Context, blobs ...*blob.Blob) (uint64, error) { - return api.Internal.Submit(ctx, blobs...) +func (api *API) Submit(ctx context.Context, blobs []*blob.Blob) (uint64, error) { + return api.Internal.Submit(ctx, blobs) } func (api *API) Get( @@ -52,8 +52,8 @@ func (api *API) Get( return api.Internal.Get(ctx, height, nID, commitment) } -func (api *API) GetAll(ctx context.Context, height uint64, nIDs ...namespace.ID) ([]*blob.Blob, error) { - return api.Internal.GetAll(ctx, height, nIDs...) +func (api *API) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID) ([]*blob.Blob, error) { + return api.Internal.GetAll(ctx, height, nIDs) } func (api *API) GetProof( diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index 93824bc625..f99d1d8168 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -52,23 +52,18 @@ func (mr *MockModuleMockRecorder) Get(arg0, arg1, arg2, arg3 interface{}) *gomoc } // GetAll mocks base method. -func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 ...namespace.ID) ([]*blob.Blob, error) { +func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 []namespace.ID) ([]*blob.Blob, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1} - for _, a := range arg2 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "GetAll", varargs...) + ret := m.ctrl.Call(m, "GetAll", arg0, arg1, arg2) ret0, _ := ret[0].([]*blob.Blob) ret1, _ := ret[1].(error) return ret0, ret1 } // GetAll indicates an expected call of GetAll. -func (mr *MockModuleMockRecorder) GetAll(arg0, arg1 interface{}, arg2 ...interface{}) *gomock.Call { +func (mr *MockModuleMockRecorder) GetAll(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1}, arg2...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockModule)(nil).GetAll), varargs...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetAll", reflect.TypeOf((*MockModule)(nil).GetAll), arg0, arg1, arg2) } // GetProof mocks base method. @@ -102,21 +97,16 @@ func (mr *MockModuleMockRecorder) Included(arg0, arg1, arg2, arg3, arg4 interfac } // Submit mocks base method. -func (m *MockModule) Submit(arg0 context.Context, arg1 ...*blob.Blob) (uint64, error) { +func (m *MockModule) Submit(arg0 context.Context, arg1 []*blob.Blob) (uint64, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0} - for _, a := range arg1 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "Submit", varargs...) + ret := m.ctrl.Call(m, "Submit", arg0, arg1) ret0, _ := ret[0].(uint64) ret1, _ := ret[1].(error) return ret0, ret1 } // Submit indicates an expected call of Submit. -func (mr *MockModuleMockRecorder) Submit(arg0 interface{}, arg1 ...interface{}) *gomock.Call { +func (mr *MockModuleMockRecorder) Submit(arg0, arg1 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0}, arg1...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockModule)(nil).Submit), varargs...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockModule)(nil).Submit), arg0, arg1) } diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index 33663d7ce8..8398e0fd7b 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -189,23 +189,18 @@ func (mr *MockModuleMockRecorder) QueryUnbonding(arg0, arg1 interface{}) *gomock } // SubmitPayForBlob mocks base method. -func (m *MockModule) SubmitPayForBlob(arg0 context.Context, arg1 math.Int, arg2 uint64, arg3 ...*types1.Blob) (*types.TxResponse, error) { +func (m *MockModule) SubmitPayForBlob(arg0 context.Context, arg1 math.Int, arg2 uint64, arg3 []*types1.Blob) (*types.TxResponse, error) { m.ctrl.T.Helper() - varargs := []interface{}{arg0, arg1, arg2} - for _, a := range arg3 { - varargs = append(varargs, a) - } - ret := m.ctrl.Call(m, "SubmitPayForBlob", varargs...) + ret := m.ctrl.Call(m, "SubmitPayForBlob", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*types.TxResponse) ret1, _ := ret[1].(error) return ret0, ret1 } // SubmitPayForBlob indicates an expected call of SubmitPayForBlob. -func (mr *MockModuleMockRecorder) SubmitPayForBlob(arg0, arg1, arg2 interface{}, arg3 ...interface{}) *gomock.Call { +func (mr *MockModuleMockRecorder) SubmitPayForBlob(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - varargs := append([]interface{}{arg0, arg1, arg2}, arg3...) - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubmitPayForBlob", reflect.TypeOf((*MockModule)(nil).SubmitPayForBlob), varargs...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SubmitPayForBlob", reflect.TypeOf((*MockModule)(nil).SubmitPayForBlob), arg0, arg1, arg2, arg3) } // SubmitTx mocks base method. diff --git a/nodebuilder/state/state.go b/nodebuilder/state/state.go index fc1932cfae..57f46a8fd6 100644 --- a/nodebuilder/state/state.go +++ b/nodebuilder/state/state.go @@ -46,7 +46,7 @@ type Module interface { ctx context.Context, fee state.Int, gasLim uint64, - blobs ...*apptypes.Blob, + blobs []*apptypes.Blob, ) (*state.TxResponse, error) // CancelUnbondingDelegation cancels a user's pending undelegation from a validator. @@ -114,7 +114,7 @@ type API struct { ctx context.Context, fee state.Int, gasLim uint64, - blobs ...*apptypes.Blob, + blobs []*apptypes.Blob, ) (*state.TxResponse, error) `perm:"write"` CancelUnbondingDelegation func( ctx context.Context, @@ -192,9 +192,9 @@ func (api *API) SubmitPayForBlob( ctx context.Context, fee state.Int, gasLim uint64, - blobs ...*apptypes.Blob, + blobs []*apptypes.Blob, ) (*state.TxResponse, error) { - return api.Internal.SubmitPayForBlob(ctx, fee, gasLim, blobs...) + return api.Internal.SubmitPayForBlob(ctx, fee, gasLim, blobs) } func (api *API) CancelUnbondingDelegation( diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 27a10a5592..f740dc12fa 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -2,17 +2,60 @@ package tests import ( "context" + "fmt" "testing" + "github.com/filecoin-project/go-jsonrpc/auth" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/blob/blobtest" + "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) +// TestBlobRPC ensures that blobs can be submited via rpc +func TestBlobRPC(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) + + sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + + // start a bridge node + bridge := sw.NewBridgeNode() + err := bridge.Start(ctx) + require.NoError(t, err) + + signer := bridge.AdminSigner + bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() + fmt.Println("bridgeAddr: ", bridgeAddr) + + jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + require.NoError(t, err) + + client, err := client.NewClient(ctx, bridgeAddr, jwt) + require.NoError(t, err) + + appBlobs, err := blobtest.GenerateBlobs([]int{8}, false) + require.NoError(t, err) + + newBlob, err := blob.NewBlob( + appBlobs[0].ShareVersion, + append([]byte{appBlobs[0].NamespaceVersion}, appBlobs[0].NamespaceID...), + appBlobs[0].Data, + ) + require.NoError(t, err) + + height, err := client.Blob.Submit(ctx, []*blob.Blob{newBlob}) + require.NoError(t, err) + require.True(t, height != 0) +} + // TestHeaderSubscription ensures that the header subscription over RPC works // as intended and gets canceled successfully after rpc context cancellation. func TestHeaderSubscription(t *testing.T) { diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index db56bad8c4..51c2ee9d62 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -6,6 +6,7 @@ import ( "testing" "time" + "github.com/celestiaorg/nmt/namespace" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" @@ -54,7 +55,7 @@ func TestBlobModule(t *testing.T) { lightNode := sw.NewNodeWithConfig(node.Light, lightCfg) require.NoError(t, lightNode.Start(ctx)) - height, err := fullNode.BlobServ.Submit(ctx, blobs...) + height, err := fullNode.BlobServ.Submit(ctx, blobs) require.NoError(t, err) _, err = fullNode.HeaderServ.WaitForHeight(ctx, height) @@ -77,7 +78,7 @@ func TestBlobModule(t *testing.T) { { name: "GetAll", doFn: func(t *testing.T) { - newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, blobs[0].Namespace()) + newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, []namespace.ID{blobs[0].Namespace()}) require.NoError(t, err) require.Len(t, newBlobs, len(appBlobs0)) require.True(t, bytes.Equal(blobs[0].Commitment, newBlobs[0].Commitment)) diff --git a/state/core_access.go b/state/core_access.go index 1a0a5650ab..bd27f479e9 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -158,7 +158,7 @@ func (ca *CoreAccessor) SubmitPayForBlob( ctx context.Context, fee Int, gasLim uint64, - blobs ...*apptypes.Blob, + blobs []*apptypes.Blob, ) (*TxResponse, error) { if len(blobs) == 0 { return nil, errors.New("state: no blobs provided") From af8460aaee220162bbd7292a8df04c898fb34618 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Fri, 2 Jun 2023 12:22:41 +0200 Subject: [PATCH 017/388] fix(gateway): remove the GetByHeight hussle and rely on Module's logic (#2302) --- api/gateway/availability.go | 15 -------------- api/gateway/header.go | 17 ++------------- api/gateway/share.go | 39 ++++++++++------------------------- nodebuilder/blob/mocks/api.go | 3 ++- 4 files changed, 15 insertions(+), 59 deletions(-) diff --git a/api/gateway/availability.go b/api/gateway/availability.go index b35593e24f..e5e53e0dba 100644 --- a/api/gateway/availability.go +++ b/api/gateway/availability.go @@ -2,7 +2,6 @@ package gateway import ( "encoding/json" - "fmt" "net/http" "strconv" @@ -28,20 +27,6 @@ func (h *Handler) handleHeightAvailabilityRequest(w http.ResponseWriter, r *http return } - //TODO: change this to NetworkHead once the adjacency in the store is fixed. - head, err := h.header.LocalHead(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err) - return - } - if headHeight := int(head.Height()); headHeight < height { - err = fmt.Errorf( - "current head local chain head: %d is lower than requested height: %d"+ - " give header sync some time and retry later", headHeight, height) - writeError(w, http.StatusServiceUnavailable, heightAvailabilityEndpoint, err) - return - } - header, err := h.header.GetByHeight(r.Context(), uint64(height)) if err != nil { writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err) diff --git a/api/gateway/header.go b/api/gateway/header.go index 88ac3b0923..5b8a82351c 100644 --- a/api/gateway/header.go +++ b/api/gateway/header.go @@ -2,7 +2,6 @@ package gateway import ( "encoding/json" - "fmt" "net/http" "strconv" @@ -70,24 +69,12 @@ func (h *Handler) performGetHeaderRequest( writeError(w, http.StatusBadRequest, endpoint, err) return nil, err } - //TODO: change this to NetworkHead once the adjacency in the store is fixed. - head, err := h.header.LocalHead(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err) - return nil, err - } - if headHeight := int(head.Height()); headHeight < height { - err = fmt.Errorf( - "current head local chain head: %d is lower than requested height: %d"+ - " give header sync some time and retry later", headHeight, height) - writeError(w, http.StatusServiceUnavailable, endpoint, err) - return nil, err - } - // perform request + header, err := h.header.GetByHeight(r.Context(), uint64(height)) if err != nil { writeError(w, http.StatusInternalServerError, endpoint, err) return nil, err } + return header, nil } diff --git a/api/gateway/share.go b/api/gateway/share.go index 36a9778f1a..db5ed37286 100644 --- a/api/gateway/share.go +++ b/api/gateway/share.go @@ -4,7 +4,6 @@ import ( "context" "encoding/hex" "encoding/json" - "fmt" "net/http" "strconv" @@ -13,7 +12,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" ) @@ -44,14 +42,14 @@ func (h *Handler) handleSharesByNamespaceRequest(w http.ResponseWriter, r *http. writeError(w, http.StatusBadRequest, namespacedSharesEndpoint, err) return } - shares, headerHeight, err := h.getShares(r.Context(), height, nID) + shares, err := h.getShares(r.Context(), height, nID) if err != nil { writeError(w, http.StatusInternalServerError, namespacedSharesEndpoint, err) return } resp, err := json.Marshal(&NamespacedSharesResponse{ Shares: shares, - Height: uint64(headerHeight), + Height: height, }) if err != nil { writeError(w, http.StatusInternalServerError, namespacedSharesEndpoint, err) @@ -69,7 +67,7 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re writeError(w, http.StatusBadRequest, namespacedDataEndpoint, err) return } - shares, headerHeight, err := h.getShares(r.Context(), height, nID) + shares, err := h.getShares(r.Context(), height, nID) if err != nil { writeError(w, http.StatusInternalServerError, namespacedDataEndpoint, err) return @@ -81,7 +79,7 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re } resp, err := json.Marshal(&NamespacedDataResponse{ Data: data, - Height: uint64(headerHeight), + Height: height, }) if err != nil { writeError(w, http.StatusInternalServerError, namespacedDataEndpoint, err) @@ -93,33 +91,18 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re } } -func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID) ([]share.Share, int64, error) { - // get header - var ( - err error - header *header.ExtendedHeader - ) - - //TODO: change this to NetworkHead once the adjacency in the store is fixed. - header, err = h.header.LocalHead(ctx) +func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID) ([]share.Share, error) { + header, err := h.header.GetByHeight(ctx, height) if err != nil { - return nil, 0, err + return nil, err } - if height > 0 { - if storeHeight := uint64(header.Height()); storeHeight < height { - return nil, 0, fmt.Errorf( - "current head local chain head: %d is lower than requested height: %d"+ - " give header sync some time and retry later", storeHeight, height) - } - header, err = h.header.GetByHeight(ctx, height) - } + shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, nID) if err != nil { - return nil, 0, err + return nil, err } - // perform request - shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, nID) - return shares.Flatten(), header.Height(), err + + return shares.Flatten(), nil } func dataFromShares(input []share.Share) (data [][]byte, err error) { diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index f99d1d8168..b7c61aa450 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + blob "github.com/celestiaorg/celestia-node/blob" namespace "github.com/celestiaorg/nmt/namespace" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. From c6f4d6896e0cfe847aa9ecff4add0be5631ba6f9 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 2 Jun 2023 17:12:34 +0200 Subject: [PATCH 018/388] fix(blob): use default min gas price from core (#2306) Gas estimation broke from an incorrect constant, that can now be replaced with an app import Co-authored-by: Hlib Kanunnikov --- blob/service.go | 7 ++----- nodebuilder/tests/blob_test.go | 3 ++- 2 files changed, 4 insertions(+), 6 deletions(-) diff --git a/blob/service.go b/blob/service.go index 53ada45450..0d1ccf0bf1 100644 --- a/blob/service.go +++ b/blob/service.go @@ -9,6 +9,7 @@ import ( "github.com/cosmos/cosmos-sdk/types" logging "github.com/ipfs/go-log/v2" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt/namespace" @@ -25,10 +26,6 @@ var ( log = logging.Logger("blob") ) -// TODO(@vgonkivs): remove after bumping celestia-app -// defaultMinGasPrice is the default min gas price. -const defaultMinGasPrice = 0.001 - type Service struct { // accessor dials the given celestia-core endpoint to submit blobs. accessor *state.CoreAccessor @@ -58,7 +55,7 @@ func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { var ( gasLimit = estimateGas(blobs...) - fee = int64(defaultMinGasPrice * float64(gasLimit)) + fee = int64(appconsts.DefaultMinGasPrice * float64(gasLimit)) b = make([]*apptypes.Blob, len(blobs)) ) diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 51c2ee9d62..537e320c5a 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -6,12 +6,13 @@ import ( "testing" "time" - "github.com/celestiaorg/nmt/namespace" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" "github.com/celestiaorg/celestia-node/nodebuilder/node" From 124722530a5175995944226c4a0048edbfbece59 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 2 Jun 2023 20:32:59 +0200 Subject: [PATCH 019/388] feat(cmd/rpc): adding blob module to rpc cli and fixing namespace parsing (#2308) This PR does the following: ### Adds blob module support `blob.Submit` has been added with custom parsing for posting a single blob. `blob.GetAll` also only takes a single namespace id for now. This will make devx easier ### Adds --print-request flag Now the request can also be displayed along with the json response. This will both help for debugging and building things that need to build the requests themselves ### Fixes namespace parsing Namespaces are now encoded correctly again. In addition, if you pass a namespace of 8 bytes, it will zero-pad and use share/nID version 0. ### Fixes `state.SubmitPayForBlob` Another issue was the change to an array. We now submit an array of a single blob, and changed the order of the arguments. The order is now: fee, gasLimit, namespaceID, blob data ### Adds formatted output JQ is no longer needed to have an indented output. But colors are missing --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- cmd/celestia/rpc.go | 188 +++++++++++++++++++++++++++++++++++++------- 1 file changed, 158 insertions(+), 30 deletions(-) diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index afbe13c3f8..7992e8f2a8 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -18,7 +18,12 @@ import ( "github.com/cosmos/cosmos-sdk/types" "github.com/spf13/cobra" + apptypes "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/state" ) @@ -28,6 +33,7 @@ const ( var requestURL string var authTokenFlag string +var printRequest bool type jsonRPCRequest struct { ID int64 `json:"id"` @@ -36,6 +42,11 @@ type jsonRPCRequest struct { Params []interface{} `json:"params"` } +type outputWithRequest struct { + Request jsonRPCRequest + Response json.RawMessage +} + func init() { rpcCmd.PersistentFlags().StringVar( &requestURL, @@ -49,6 +60,12 @@ func init() { "", "Authorization token (if not provided, the "+authEnvKey+" environment variable will be used)", ) + rpcCmd.PersistentFlags().BoolVar( + &printRequest, + "print-request", + false, + "Print JSON-RPC request along with the response", + ) rootCmd.AddCommand(rpcCmd) } @@ -98,52 +115,117 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[0] = root // 2. NamespaceID - if strings.HasPrefix(params[1], "0x") { - decoded, err := hex.DecodeString(params[1][2:]) - if err != nil { - panic("Error decoding namespace ID: hex string could not be decoded.") - } - parsedParams[1] = decoded - } else { - // otherwise, it's just a base64 string - parsedParams[1] = params[1] + nID, err := parseNamespace(params[1]) + if err != nil { + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - return parsedParams - case "SubmitPayForBlob": + parsedParams[1] = nID + case "Submit": // 1. NamespaceID - if strings.HasPrefix(params[0], "0x") { - decoded, err := hex.DecodeString(params[0][2:]) - if err != nil { - panic("Error decoding namespace ID: hex string could not be decoded.") - } - parsedParams[0] = decoded - } else { - // otherwise, it's just a base64 string - parsedParams[0] = params[0] + var err error + nID, err := parseNamespace(params[0]) + if err != nil { + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - // 2. Blob + // 2. Blob data + var blobData []byte switch { case strings.HasPrefix(params[1], "0x"): decoded, err := hex.DecodeString(params[1][2:]) if err != nil { panic("Error decoding blob: hex string could not be decoded.") } - parsedParams[0] = decoded + blobData = decoded case strings.HasPrefix(params[1], "\""): // user input an utf string that needs to be encoded to base64 - parsedParams[1] = base64.StdEncoding.EncodeToString([]byte(params[1])) + base64.StdEncoding.Encode(blobData, []byte(params[1])) default: // otherwise, we assume the user has already encoded their input to base64 - parsedParams[1] = params[1] + blobData, err = base64.StdEncoding.DecodeString(params[1]) + if err != nil { + panic("Error decoding blob data: base64 string could not be decoded.") + } } - // 3. Fee (state.Int is a string) - parsedParams[2] = params[2] - // 4. GasLimit (uint64) - num, err := strconv.ParseUint(params[3], 10, 64) + parsedBlob, err := blob.NewBlob(0, nID, blobData) + if err != nil { + panic(fmt.Sprintf("Error creating blob: %v", err)) + } + parsedParams[0] = []*blob.Blob{parsedBlob} + // param count doesn't match input length, so cut off nil values + return parsedParams[:1] + case "SubmitPayForBlob": + // 1. Fee (state.Int is a string) + parsedParams[0] = params[0] + // 2. GasLimit (uint64) + num, err := strconv.ParseUint(params[1], 10, 64) if err != nil { panic("Error parsing gas limit: uint64 could not be parsed.") } - parsedParams[3] = num + parsedParams[1] = num + // 3. NamespaceID + nID, err := parseNamespace(params[2]) + if err != nil { + panic(fmt.Sprintf("Error parsing namespace: %v", err)) + } + // 4. Blob data + var blobData []byte + switch { + case strings.HasPrefix(params[3], "0x"): + decoded, err := hex.DecodeString(params[3][2:]) + if err != nil { + panic("Error decoding blob: hex string could not be decoded.") + } + blobData = decoded + case strings.HasPrefix(params[3], "\""): + // user input an utf string that needs to be encoded to base64 + base64.StdEncoding.Encode(blobData, []byte(params[3])) + default: + // otherwise, we assume the user has already encoded their input to base64 + blobData, err = base64.StdEncoding.DecodeString(params[3]) + if err != nil { + panic("Error decoding blob: base64 string could not be decoded.") + } + } + parsedParams[2] = []*apptypes.Blob{{ + NamespaceId: nID[1:], + Data: blobData, + ShareVersion: 0, + NamespaceVersion: 0, + }} + return parsedParams[:3] + case "Get": + // 1. Height + num, err := strconv.ParseUint(params[0], 10, 64) + if err != nil { + panic("Error parsing gas limit: uint64 could not be parsed.") + } + parsedParams[0] = num + // 2. NamespaceID + nID, err := parseNamespace(params[1]) + if err != nil { + panic(fmt.Sprintf("Error parsing namespace: %v", err)) + } + parsedParams[1] = nID + // 3: Commitment + commitment, err := base64.StdEncoding.DecodeString(params[2]) + if err != nil { + panic("Error decoding commitment: base64 string could not be decoded.") + } + parsedParams[2] = commitment + return parsedParams + case "GetAll": // NOTE: Over the cli, you can only pass one namespace + // 1. Height + num, err := strconv.ParseUint(params[0], 10, 64) + if err != nil { + panic("Error parsing gas limit: uint64 could not be parsed.") + } + parsedParams[0] = num + // 2. NamespaceID + nID, err := parseNamespace(params[1]) + if err != nil { + panic(fmt.Sprintf("Error parsing namespace: %v", err)) + } + parsedParams[1] = []namespace.ID{nID} return parsedParams case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": var err error @@ -284,7 +366,27 @@ func sendJSONRPCRequest(namespace, method string, params []interface{}) { log.Fatalf("Error reading response body: %v", err) //nolint:gocritic } - fmt.Println(string(responseBody)) + rawResponseJSON, err := parseJSON(string(responseBody)) + if err != nil { + panic(err) + } + if printRequest { + output, err := json.MarshalIndent(outputWithRequest{ + Request: request, + Response: rawResponseJSON, + }, "", " ") + if err != nil { + panic(fmt.Sprintf("Error marshaling JSON-RPC response: %v", err)) + } + fmt.Println(string(output)) + return + } + + output, err := json.MarshalIndent(rawResponseJSON, "", " ") + if err != nil { + panic(fmt.Sprintf("Error marshaling JSON-RPC response: %v", err)) + } + fmt.Println(string(output)) } func parseAddressFromString(addrStr string) (state.Address, error) { @@ -322,6 +424,32 @@ func parseSignatureForHelpstring(methodSig reflect.StructField) string { return simplifiedSignature } +func parseNamespace(param string) (namespace.ID, error) { + var nID []byte + var err error + if strings.HasPrefix(param, "0x") { + decoded, err := hex.DecodeString(param[2:]) + if err != nil { + return nil, fmt.Errorf("error decoding namespace ID: %w", err) + } + nID = decoded + } else { + // otherwise, it's just a base64 string + nID, err = base64.StdEncoding.DecodeString(param) + if err != nil { + return nil, fmt.Errorf("error decoding namespace ID: %w", err) + } + } + // if the namespace ID is 8 bytes, add v0 share + namespace prefix and zero pad + if len(nID) == 8 { + nID, err = share.NewNamespaceV0(nID) + if err != nil { + return nil, err + } + } + return nID, nil +} + func parseJSON(param string) (json.RawMessage, error) { var raw json.RawMessage err := json.Unmarshal([]byte(param), &raw) From 504ff93444eb46d9c3adeedb2f58d289ec1ef746 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Sat, 3 Jun 2023 03:12:18 +0800 Subject: [PATCH 020/388] fix(getter/cascade): cascade getter should return fast on context cancel (#2310) ## Overview Also fixes incorrect errors.Is for `share.ErrNamespaceNotFound` Resolves https://github.com/celestiaorg/celestia-node/issues/2174 --- share/getters/cascade.go | 16 +++++++++------- share/getters/cascade_test.go | 14 ++++++++++++++ 2 files changed, 23 insertions(+), 7 deletions(-) diff --git a/share/getters/cascade.go b/share/getters/cascade.go index d65b902e81..1a0d8fb274 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -122,16 +122,18 @@ func cascadeGetters[V any]( getCtx, cancel := ctxWithSplitTimeout(ctx, len(getters)-i, 0) val, getErr := get(getCtx, getter) cancel() - if getErr == nil { - return val, nil + if getErr == nil || errors.Is(getErr, share.ErrNamespaceNotFound) { + return val, getErr } - if errors.Is(share.ErrNamespaceNotFound, getErr) { - return zero, getErr + + if errors.Is(getErr, errOperationNotSupported) { + continue } - if !errors.Is(getErr, errOperationNotSupported) { - err = errors.Join(err, getErr) - span.RecordError(getErr, trace.WithAttributes(attribute.Int("getter_idx", i))) + err = errors.Join(err, getErr) + span.RecordError(getErr, trace.WithAttributes(attribute.Int("getter_idx", i))) + if ctx.Err() != nil { + return zero, err } } return zero, err diff --git a/share/getters/cascade_test.go b/share/getters/cascade_test.go index e3a324c5e9..d955c50682 100644 --- a/share/getters/cascade_test.go +++ b/share/getters/cascade_test.go @@ -52,6 +52,7 @@ func TestCascade(t *testing.T) { timeoutGetter := mocks.NewMockGetter(ctrl) immediateFailGetter := mocks.NewMockGetter(ctrl) successGetter := mocks.NewMockGetter(ctrl) + ctxGetter := mocks.NewMockGetter(ctrl) timeoutGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). DoAndReturn(func(ctx context.Context, _ *share.Root) (*rsmt2d.ExtendedDataSquare, error) { return nil, context.DeadlineExceeded @@ -60,6 +61,10 @@ func TestCascade(t *testing.T) { Return(nil, errors.New("second getter fails immediately")).AnyTimes() successGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). Return(nil, nil).AnyTimes() + ctxGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). + DoAndReturn(func(ctx context.Context, _ *share.Root) (*rsmt2d.ExtendedDataSquare, error) { + return nil, ctx.Err() + }).AnyTimes() get := func(ctx context.Context, get share.Getter) (*rsmt2d.ExtendedDataSquare, error) { return get.GetEDS(ctx, nil) @@ -96,6 +101,15 @@ func TestCascade(t *testing.T) { assert.Equal(t, strings.Count(err.Error(), "\n"), 2) }) + t.Run("Context Canceled", func(t *testing.T) { + ctx, cancel := context.WithCancel(ctx) + cancel() + getters := []share.Getter{ctxGetter, ctxGetter, ctxGetter} + _, err := cascadeGetters(ctx, getters, get) + assert.Error(t, err) + assert.Equal(t, strings.Count(err.Error(), "\n"), 0) + }) + t.Run("Single", func(t *testing.T) { getters := []share.Getter{successGetter} _, err := cascadeGetters(ctx, getters, get) From 118c4a90ce90fdd062e2298beec5a4f68db88914 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 5 Jun 2023 18:24:29 +0800 Subject: [PATCH 021/388] fix(nodebuilder): fix parsing url for core.ip (#2313) ## Overview Resolves https://github.com/celestiaorg/celestia-node/issues/2311 --- nodebuilder/core/flags.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/nodebuilder/core/flags.go b/nodebuilder/core/flags.go index 901d910ea6..01ff031991 100644 --- a/nodebuilder/core/flags.go +++ b/nodebuilder/core/flags.go @@ -3,6 +3,7 @@ package core import ( "fmt" "net" + "net/url" "github.com/spf13/cobra" flag "github.com/spf13/pflag" @@ -50,7 +51,11 @@ func ParseFlags(cmd *cobra.Command, cfg *Config) error { ip := net.ParseIP(coreIP) if ip == nil { - ips, err := net.LookupIP(coreIP) + u, err := url.Parse(coreIP) + if err != nil { + return fmt.Errorf("failed to parse url: %w", err) + } + ips, err := net.LookupIP(u.Host) if err != nil { return fmt.Errorf("failed to resolve DNS record: %v", err) } From 59b9daf44644f3c60d8e545664d7adb60f8721bd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 5 Jun 2023 15:26:56 +0300 Subject: [PATCH 022/388] chore(deps): bump github.com/filecoin-project/go-jsonrpc from 0.1.9 to 0.3.1 (#2318) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 6 ++---- 2 files changed, 3 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index fd0bf7eb4c..474dc02b05 100644 --- a/go.mod +++ b/go.mod @@ -21,7 +21,7 @@ require ( github.com/dgraph-io/badger/v2 v2.2007.4 github.com/etclabscore/go-openrpc-reflect v0.0.37 github.com/filecoin-project/dagstore v0.5.6 - github.com/filecoin-project/go-jsonrpc v0.1.9 + github.com/filecoin-project/go-jsonrpc v0.3.1 github.com/gammazero/workerpool v1.1.3 github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 diff --git a/go.sum b/go.sum index 92583018a9..285f5c257d 100644 --- a/go.sum +++ b/go.sum @@ -574,8 +574,8 @@ github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= -github.com/filecoin-project/go-jsonrpc v0.1.9 h1:HRWLxo7HAWzI3xZGeFG4LZJoYpms+Q+8kwmMTLnyS3A= -github.com/filecoin-project/go-jsonrpc v0.1.9/go.mod h1:XBBpuKIMaXIIzeqzO1iucq4GvbF8CxmXRFoezRh+Cx4= +github.com/filecoin-project/go-jsonrpc v0.3.1 h1:qwvAUc5VwAkooquKJmfz9R2+F8znhiqcNHYjEp/NM10= +github.com/filecoin-project/go-jsonrpc v0.3.1/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8wnQnIjHbRPpROol6iQKM= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= @@ -830,7 +830,6 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.7.4/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= @@ -1052,7 +1051,6 @@ github.com/ipfs/go-log v1.0.5/go.mod h1:j0b8ZoR+7+R99LD9jZ6+AJsrzkPbSXbZfGakb5JP github.com/ipfs/go-log/v2 v2.0.2/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.3/go.mod h1:O7P1lJt27vWHhOwQmcFEvlmo49ry2VY2+JfBWFaa9+0= github.com/ipfs/go-log/v2 v2.0.5/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= -github.com/ipfs/go-log/v2 v2.0.8/go.mod h1:eZs4Xt4ZUJQFM3DlanGhy7TkwwawCZcSByscwkWG+dw= github.com/ipfs/go-log/v2 v2.1.1/go.mod h1:2v2nsGfZsvvAJz13SyFzf9ObaqwHiHxsPLEHntrv9KM= github.com/ipfs/go-log/v2 v2.1.3/go.mod h1:/8d0SH3Su5Ooc31QlL1WysJhvyOTDCjcCZ9Axpmri6g= github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72gynbe/g= From f05b71f39720a202d4043a43eabc2fec401ec735 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 5 Jun 2023 20:30:29 +0800 Subject: [PATCH 023/388] fix(das): limit amount of recent jobs (#2314) ## Overview Resolves https://github.com/celestiaorg/celestia-node/issues/2312 --- das/checkpoint.go | 16 ++++++++-------- das/coordinator.go | 9 ++++++++- 2 files changed, 16 insertions(+), 9 deletions(-) diff --git a/das/checkpoint.go b/das/checkpoint.go index a38eca828c..bb023a19da 100644 --- a/das/checkpoint.go +++ b/das/checkpoint.go @@ -23,15 +23,15 @@ type workerCheckpoint struct { func newCheckpoint(stats SamplingStats) checkpoint { workers := make([]workerCheckpoint, 0, len(stats.Workers)) for _, w := range stats.Workers { - // no need to store retry jobs, since they will resume from failed heights map - if w.JobType == retryJob { - continue + // no need to resume recent jobs after restart. On the other hand, retry jobs will resume from + // failed heights map. it leaves only catchup jobs to be stored and resumed + if w.JobType == catchupJob { + workers = append(workers, workerCheckpoint{ + From: w.Curr, + To: w.To, + JobType: w.JobType, + }) } - workers = append(workers, workerCheckpoint{ - From: w.Curr, - To: w.To, - JobType: w.JobType, - }) } return checkpoint{ SampleFrom: stats.CatchupHead + 1, diff --git a/das/coordinator.go b/das/coordinator.go index 2184dce2c8..852a40d24d 100644 --- a/das/coordinator.go +++ b/das/coordinator.go @@ -81,7 +81,9 @@ func (sc *samplingCoordinator) run(ctx context.Context, cp checkpoint) { select { case head := <-sc.updHeadCh: if sc.state.isNewHead(head.Height()) { - sc.runWorker(ctx, sc.state.recentJob(head)) + if !sc.recentJobsLimitReached() { + sc.runWorker(ctx, sc.state.recentJob(head)) + } sc.state.updateHead(head.Height()) // run worker without concurrency limit restrictions to reduced delay sc.metrics.observeNewHead(ctx) @@ -146,3 +148,8 @@ func (sc *samplingCoordinator) getCheckpoint(ctx context.Context) (checkpoint, e func (sc *samplingCoordinator) concurrencyLimitReached() bool { return len(sc.state.inProgress) >= sc.concurrencyLimit } + +// recentJobsLimitReached indicates whether concurrency limit for recent jobs has been reached +func (sc *samplingCoordinator) recentJobsLimitReached() bool { + return len(sc.state.inProgress) >= 2*sc.concurrencyLimit +} From e5fcbe2a829b30c12df5c5ebfae12cb790a10239 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 5 Jun 2023 14:39:49 +0200 Subject: [PATCH 024/388] chore(state): extend error handling in core accessor (#2307) Co-authored-by: Hlib Kanunnikov --- go.mod | 2 +- state/core_access.go | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/go.mod b/go.mod index 474dc02b05..052eb5bcf5 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.20 replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1-lazypatch require ( + cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.0-beta.3 github.com/BurntSushi/toml v1.2.1 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 @@ -85,7 +86,6 @@ require ( cloud.google.com/go/compute/metadata v0.2.3 // indirect cloud.google.com/go/iam v0.8.0 // indirect cloud.google.com/go/storage v1.27.0 // indirect - cosmossdk.io/errors v1.0.0-beta.7 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect diff --git a/state/core_access.go b/state/core_access.go index bd27f479e9..0a265c2d08 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -6,6 +6,7 @@ import ( "fmt" "time" + sdkErrors "cosmossdk.io/errors" "github.com/cosmos/cosmos-sdk/api/tendermint/abci" storetypes "github.com/cosmos/cosmos-sdk/store/types" sdktypes "github.com/cosmos/cosmos-sdk/types" @@ -177,6 +178,10 @@ func (ca *CoreAccessor) SubmitPayForBlob( ca.lastPayForBlob = time.Now().UnixMilli() ca.payForBlobCount++ } + + if response.Code != 0 { + err = errors.Join(err, sdkErrors.ABCIError(response.Codespace, response.Code, response.Logs.String())) + } return response, err } From dd1a019349ff14b8b3883d2bb7899636bc318be0 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Tue, 6 Jun 2023 04:07:25 -0500 Subject: [PATCH 025/388] chore: bump celestia-app to v1.0.0-rc2 (#2305) Co-authored-by: Hlib Kanunnikov --- blob/blob_test.go | 2 +- core/eds.go | 2 +- go.mod | 12 ++++++------ go.sum | 25 ++++++++++++------------- share/eds/eds_test.go | 2 +- share/ipld/nmt.go | 10 ++++++---- share/share.go | 11 ++++++++--- 7 files changed, 35 insertions(+), 29 deletions(-) diff --git a/blob/blob_test.go b/blob/blob_test.go index 7f34fbfc84..3aabd6559b 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -49,7 +49,7 @@ func TestBlob(t *testing.T) { blob[0].Namespace()[appns.NamespaceVersionSize:], ) require.NoError(t, err) - require.NoError(t, apptypes.ValidateBlobNamespaceID(ns)) + require.NoError(t, apptypes.ValidateBlobNamespace(ns)) }, }, { diff --git a/core/eds.go b/core/eds.go index c435f0e649..dc9b2b4a9e 100644 --- a/core/eds.go +++ b/core/eds.go @@ -24,7 +24,7 @@ func extendBlock(data types.Data) (*rsmt2d.ExtendedDataSquare, error) { return nil, nil } - sqr, err := square.Construct(data.Txs.ToSliceOfBytes(), appconsts.MaxSquareSize) + sqr, err := square.Construct(data.Txs.ToSliceOfBytes(), appconsts.LatestVersion, share.MaxSquareSize) if err != nil { return nil, err } diff --git a/go.mod b/go.mod index 052eb5bcf5..5656aac5b3 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1 require ( cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.0-beta.3 - github.com/BurntSushi/toml v1.2.1 + github.com/BurntSushi/toml v1.3.0 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.0 - github.com/celestiaorg/celestia-app v1.0.0-rc0 + github.com/celestiaorg/celestia-app v1.0.0-rc2 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 github.com/celestiaorg/go-libp2p-messenger v0.2.0 @@ -61,7 +61,7 @@ require ( github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 - github.com/stretchr/testify v1.8.3 + github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.24 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.34.0 @@ -118,7 +118,7 @@ require ( github.com/cosmos/gogoproto v1.4.2 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.5 // indirect - github.com/cosmos/ibc-go/v6 v6.1.0 // indirect + github.com/cosmos/ibc-go/v6 v6.1.1 // indirect github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect @@ -136,7 +136,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/go-ethereum v1.11.6 // indirect + github.com/ethereum/go-ethereum v1.12.0 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -333,5 +333,5 @@ replace ( github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.21.0-tm-v0.34.27 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 ) diff --git a/go.sum b/go.sum index 285f5c257d..128a3b4b5e 100644 --- a/go.sum +++ b/go.sum @@ -220,8 +220,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.2.1 h1:9F2/+DoOYIOksmaJFPw1tGFy1eDnIJXg+UHjuD8lTak= -github.com/BurntSushi/toml v1.2.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.0 h1:Ws8e5YmnrGEHzZEzg0YvK/7COGYtTC5PbaH9oSSbgfA= +github.com/BurntSushi/toml v1.3.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= @@ -341,10 +341,10 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc0 h1:wpuP5fTIEbLCP+U5pGwKfSzXUTE/bE8oqKECFN5yoO0= -github.com/celestiaorg/celestia-app v1.0.0-rc0/go.mod h1:C8pNwFQWBLYIGpdrFesO1uezthrKjv0H5meecYQc1ek= -github.com/celestiaorg/celestia-core v1.21.0-tm-v0.34.27 h1:EdkqFRBypVEq/nX2ZE7KQ6dTlN8j3rEYe+WGahWuSUk= -github.com/celestiaorg/celestia-core v1.21.0-tm-v0.34.27/go.mod h1:GVo91Wifg9KL/nFx9nPkpl0UIFdvvs4fhnly9GhGxZU= +github.com/celestiaorg/celestia-app v1.0.0-rc2 h1:/u7eespYtBpQtBSz3P8/rKfz9rW7QOxkH8ebh8T4VxI= +github.com/celestiaorg/celestia-app v1.0.0-rc2/go.mod h1:uiTWKTtRpVwvSiFDl2zausrU1ZBHBWgk7z52pfzJqJU= +github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 h1:nmr9O5BflgNR1aWehs1ZFw4obA//M/+g+SrSMK9sOBA= +github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27/go.mod h1:GVo91Wifg9KL/nFx9nPkpl0UIFdvvs4fhnly9GhGxZU= github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 h1:Rd5EvJx1nG3KurBspVN51RVmvif0Lp2UVURbG2ad3Cs= github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11/go.mod h1:xCG6OUkJy5KUMEg20Zk010lra9XjkmKS3+bk0wp7bd8= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= @@ -460,8 +460,8 @@ github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4 github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.5 h1:rGA3hOrgNxgRM5wYcSCxgQBap7fW82WZgY78V9po/iY= github.com/cosmos/iavl v0.19.5/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= -github.com/cosmos/ibc-go/v6 v6.1.0 h1:o7oXws2vKkKfOFzJI+oNylRn44PCNt5wzHd/zKQKbvQ= -github.com/cosmos/ibc-go/v6 v6.1.0/go.mod h1:CY3zh2HLfetRiW8LY6kVHMATe90Wj/UOoY8T6cuB0is= +github.com/cosmos/ibc-go/v6 v6.1.1 h1:oqqMNyjj6SLQF8rvgCaDGwfdITEIsbhs8F77/8xvRIo= +github.com/cosmos/ibc-go/v6 v6.1.1/go.mod h1:NL17FpFAaWjRFVb1T7LUKuOoMSsATPpu+Icc4zL5/Ik= github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA= github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= @@ -543,7 +543,6 @@ github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5m github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/eclipse/paho.mqtt.golang v1.2.0/go.mod h1:H9keYFcgq3Qr5OUJm/JZI/i6U7joQ8SYLhZwfeOo6Ts= -github.com/edsrzf/mmap-go v1.0.0 h1:CEBF7HpRnUCSJgGUb5h1Gm7e3VkmVDrR8lvWVLtrOFw= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= @@ -564,8 +563,8 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWS github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= -github.com/ethereum/go-ethereum v1.11.6 h1:2VF8Mf7XiSUfmoNOy3D+ocfl9Qu8baQBrCNbo2CXQ8E= -github.com/ethereum/go-ethereum v1.11.6/go.mod h1:+a8pUj1tOyJ2RinsNQD4326YS+leSoKGiG/uVVb0x6Y= +github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= +github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= @@ -1924,8 +1923,8 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= -github.com/stretchr/testify v1.8.3 h1:RP3t2pwF7cMEbC1dqtB6poj3niw/9gnV4Cjg5oW5gtY= -github.com/stretchr/testify v1.8.3/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= +github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index 8df05d7d53..ea0f06c138 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -39,7 +39,7 @@ func TestQuadrantOrder(t *testing.T) { {"smol", 2}, {"still smol", 8}, {"default mainnet", appconsts.DefaultGovMaxSquareSize}, - {"max", appconsts.MaxSquareSize}, + {"max", share.MaxSquareSize}, } testShareSize := 64 diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index e5e7d41cd1..3923260555 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -38,10 +38,6 @@ const ( // that contain an NMT node (inner and leaf nodes). sha256NamespaceFlagged = 0x7701 - // MaxSquareSize is currently the maximum size supported for unerasured data in - // rsmt2d.ExtendedDataSquare. - MaxSquareSize = appconsts.MaxSquareSize - // NamespaceSize is a system-wide size for NMT namespaces. NamespaceSize = appconsts.NamespaceSize @@ -65,6 +61,12 @@ const ( NMTIgnoreMaxNamespace = true ) +var ( + // MaxSquareSize is currently the maximum size supported for unerasured data in + // rsmt2d.ExtendedDataSquare. + MaxSquareSize = appconsts.SquareSizeUpperBound(appconsts.LatestVersion) +) + func init() { // required for Bitswap to hash and verify inbound data correctly mhcore.Register(sha256NamespaceFlagged, func() hash.Hash { diff --git a/share/share.go b/share/share.go index 06f911636d..0178054a9f 100644 --- a/share/share.go +++ b/share/share.go @@ -8,6 +8,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/share/ipld" ) var ( @@ -18,15 +20,18 @@ var ( ) const ( - // MaxSquareSize is currently the maximum size supported for unerasured data in - // rsmt2d.ExtendedDataSquare. - MaxSquareSize = appconsts.MaxSquareSize // NamespaceSize is a system-wide size for NMT namespaces. NamespaceSize = appconsts.NamespaceSize // Size is a system-wide size of a share, including both data and namespace ID Size = appconsts.ShareSize ) +var ( + // MaxSquareSize is currently the maximum size supported for unerasured data in + // rsmt2d.ExtendedDataSquare. + MaxSquareSize = ipld.MaxSquareSize +) + // Share contains the raw share data without the corresponding namespace. // NOTE: Alias for the byte is chosen to keep maximal compatibility, especially with rsmt2d. // Ideally, we should define reusable type elsewhere and make everyone(Core, rsmt2d, ipld) to rely From 72757e32ae3a76a25b6669b9cd54652b9070610d Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Viet Date: Tue, 6 Jun 2023 14:46:42 +0200 Subject: [PATCH 026/388] chore: bump common docker ci version pipeline (#2330) --- .github/workflows/docker-build-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index ce629145b9..0ddebbd750 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -17,6 +17,6 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.1.1 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.0 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile From a2685d47a2ca1164cb1cd6d579b849a4f8372a80 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 6 Jun 2023 21:34:13 +0800 Subject: [PATCH 027/388] !feat(api/gateway): return txdata from submitPFB if it is present (#2329) ## Overview Closes https://github.com/celestiaorg/celestia-node/issues/2328 --- api/gateway/state.go | 16 +++++++++---- api/gateway/state_test.go | 48 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 59 insertions(+), 5 deletions(-) create mode 100644 api/gateway/state_test.go diff --git a/api/gateway/state.go b/api/gateway/state.go index 2576ad4848..fdd627b232 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -148,17 +148,23 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } // perform request - txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*apptypes.Blob{blob}) - if err != nil { + txResp, txerr := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*apptypes.Blob{blob}) + if txerr != nil && txResp == nil { + // no tx data to return writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) - return } - resp, err := json.Marshal(txResp) + + bs, err := json.Marshal(&txResp) if err != nil { writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) return } - _, err = w.Write(resp) + + // if error returned, change status from 200 to 206 + if txerr != nil { + w.WriteHeader(http.StatusPartialContent) + } + _, err = w.Write(bs) if err != nil { log.Errorw("writing response", "endpoint", submitPFBEndpoint, "err", err) } diff --git a/api/gateway/state_test.go b/api/gateway/state_test.go new file mode 100644 index 0000000000..3c01c9f0f8 --- /dev/null +++ b/api/gateway/state_test.go @@ -0,0 +1,48 @@ +package gateway + +import ( + "bytes" + "encoding/json" + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/require" + + stateMock "github.com/celestiaorg/celestia-node/nodebuilder/state/mocks" + "github.com/celestiaorg/celestia-node/state" +) + +func TestHandleSubmitPFB(t *testing.T) { + ctrl := gomock.NewController(t) + mock := stateMock.NewMockModule(ctrl) + handler := NewHandler(mock, nil, nil, nil) + + t.Run("partial response", func(t *testing.T) { + txResponse := state.TxResponse{ + Height: 1, + TxHash: "hash", + Codespace: "codespace", + Code: 1, + } + // simulate core-app err, since it is not exported + timedErr := errors.New("timed out waiting for tx to be included in a block") + mock.EXPECT().SubmitPayForBlob(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). + Return(&txResponse, timedErr) + + bs, err := json.Marshal(submitPFBRequest{}) + require.NoError(t, err) + httpreq := httptest.NewRequest("GET", "/", bytes.NewReader(bs)) + respRec := httptest.NewRecorder() + handler.handleSubmitPFB(respRec, httpreq) + + var resp state.TxResponse + err = json.NewDecoder(respRec.Body).Decode(&resp) + require.NoError(t, err) + + require.Equal(t, http.StatusPartialContent, respRec.Code) + require.Equal(t, resp, txResponse) + }) +} From ae8db72b5c956362a07bc5dd0c8d10e553b50aa0 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 8 Jun 2023 08:58:20 +0200 Subject: [PATCH 028/388] fix(cmd/rpc): plaintext parsing of data for blob.Submit and state.SubmitPayForBlob (#2340) Closes #2323 --- cmd/celestia/rpc.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 7992e8f2a8..4139e0f709 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -138,6 +138,8 @@ func parseParams(method string, params []string) []interface{} { blobData = decoded case strings.HasPrefix(params[1], "\""): // user input an utf string that needs to be encoded to base64 + src := []byte(params[1]) + blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) base64.StdEncoding.Encode(blobData, []byte(params[1])) default: // otherwise, we assume the user has already encoded their input to base64 @@ -178,6 +180,8 @@ func parseParams(method string, params []string) []interface{} { blobData = decoded case strings.HasPrefix(params[3], "\""): // user input an utf string that needs to be encoded to base64 + src := []byte(params[1]) + blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) base64.StdEncoding.Encode(blobData, []byte(params[3])) default: // otherwise, we assume the user has already encoded their input to base64 From a65280181615110e96908254336bf8d701098b66 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 8 Jun 2023 10:01:45 +0200 Subject: [PATCH 029/388] fix(dagstore): changing log level to WARN to shorten startup (#2336) Dagstore's debug logs increase startup time by 3-4x, and dagstore in general produces a lot of debug logs that are not very important for collecting. --- logs/logs.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/logs/logs.go b/logs/logs.go index 4ae1cddd3f..15c26888c5 100644 --- a/logs/logs.go +++ b/logs/logs.go @@ -21,6 +21,8 @@ func SetAllLoggers(level logging.LogLevel) { _ = logging.SetLogLevel("net/identify", "ERROR") _ = logging.SetLogLevel("shrex/nd", "WARN") _ = logging.SetLogLevel("shrex/eds", "WARN") + _ = logging.SetLogLevel("dagstore", "WARN") + _ = logging.SetLogLevel("dagstore/upgrader", "WARN") _ = logging.SetLogLevel("fx", "FATAL") } From 1cab8ebaf82da2562d3b9e1478dfeb2c6e2ead81 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 8 Jun 2023 12:19:22 +0200 Subject: [PATCH 030/388] fix(share): left pad nid in NewNamespaceV0 instead of right padding (#2341) --- share/nid.go | 8 ++++--- share/nid_test.go | 56 +++++++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 61 insertions(+), 3 deletions(-) create mode 100644 share/nid_test.go diff --git a/share/nid.go b/share/nid.go index dfd6a5de7e..b7fd4e5836 100644 --- a/share/nid.go +++ b/share/nid.go @@ -7,15 +7,17 @@ import ( "github.com/celestiaorg/nmt/namespace" ) -// NewNamespaceV0 takes variable size byte slice anc creates version 0 Namespace ID. -// The bytes slice must be <= 10 bytes. +// NewNamespaceV0 takes a variable size byte slice and creates a version 0 Namespace ID. +// The byte slice must be <= 10 bytes. +// If it is less than 10 bytes, it will be left padded to size 10 with 0s. func NewNamespaceV0(subNId []byte) (namespace.ID, error) { if lnid := len(subNId); lnid > appns.NamespaceVersionZeroIDSize { return nil, fmt.Errorf("namespace id must be <= %v, but it was %v bytes", appns.NamespaceVersionZeroIDSize, lnid) } id := make([]byte, appns.NamespaceIDSize) - copy(id[appns.NamespaceVersionZeroPrefixSize:], subNId) + leftPaddingOffset := appns.NamespaceVersionZeroIDSize - len(subNId) + copy(id[appns.NamespaceVersionZeroPrefixSize+leftPaddingOffset:], subNId) appID, err := appns.New(appns.NamespaceVersionZero, id) if err != nil { diff --git a/share/nid_test.go b/share/nid_test.go new file mode 100644 index 0000000000..8f83d430e3 --- /dev/null +++ b/share/nid_test.go @@ -0,0 +1,56 @@ +package share + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/celestiaorg/nmt/namespace" +) + +func TestNewNamespaceV0(t *testing.T) { + type testCase struct { + name string + subNid []byte + expected namespace.ID + wantErr bool + } + testCases := []testCase{ + { + name: "8 byte subNid, gets left padded", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + expected: namespace.ID{ + 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros + 0x0, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}, // id with left padding + wantErr: false, + }, + { + name: "10 byte subNid, no padding", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10}, + expected: namespace.ID{ + 0x0, // version + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros + 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}, // id + wantErr: false, + }, + { + name: "11 byte subNid", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10, 0x11}, + expected: []byte{}, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := NewNamespaceV0(tc.subNid) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, got) + }) + } +} From b046184fc15745913eb82238cddf0ec7d95f2dc9 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 8 Jun 2023 18:30:15 +0800 Subject: [PATCH 031/388] fix(daser) don't count recent jobs in total sampled stats (#2342) ## Overview Small fix for total sampled amount calculations --- das/stats.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/das/stats.go b/das/stats.go index 5799370f91..dda6be6cc0 100644 --- a/das/stats.go +++ b/das/stats.go @@ -34,7 +34,10 @@ type WorkerStats struct { func (s SamplingStats) totalSampled() uint64 { var inProgress uint64 for _, w := range s.Workers { - inProgress += w.To - w.Curr + 1 + // don't count recent jobs, since heights they are working on are after catchup head + if w.JobType != recentJob { + inProgress += w.To - w.Curr + 1 + } } return s.CatchupHead - inProgress - uint64(len(s.Failed)) } From f632908ebff8a349874d0bca3150ec7ca1ed3da1 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 8 Jun 2023 13:38:39 +0200 Subject: [PATCH 032/388] fix(state): fixing panic when err != nil but response is nil (#2339) There was a panic when err != nil but response == nil, for example in the case of: `rpc error: code = NotFound desc = account celestia1gchva5av4v798xmqttln9qw9smzrjm2cv2q8e7 not found` ``` 2023-06-07T14:20:38.731+0200 ERROR rpc go-jsonrpc@v0.3.1/handler.go:276 panic in rpc method 'blob.Submit': runtime error: invalid memory address or nil pointer dereference ``` --------- Co-authored-by: Vlad <13818348+walldiss@users.noreply.github.com> --- state/core_access.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/state/core_access.go b/state/core_access.go index 0a265c2d08..712100d921 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -179,7 +179,7 @@ func (ca *CoreAccessor) SubmitPayForBlob( ca.payForBlobCount++ } - if response.Code != 0 { + if response != nil && response.Code != 0 { err = errors.Join(err, sdkErrors.ABCIError(response.Codespace, response.Code, response.Logs.String())) } return response, err From 0823f6b644eb6ba176ef5ae3c008c65018ed5f46 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 8 Jun 2023 13:53:31 +0200 Subject: [PATCH 033/388] fix!(state): use node blob type instead of app type to fix inconsistent unmarshalling (#2338) Breaks API to v0.2.0 Because state.SubmitPayForBlob was using apptypes.Blob, which cannot be unmarshalled from a node blob.Blob JSON object, it was requiring two separate types downstream. Now both blob Service and state Service use blob.Blob type. This means introducing an interface in blob.Service called blob.Submitter to avoid that state module import. --- api/docgen/examples.go | 12 ++++++++++++ api/gateway/state.go | 12 ++++++------ api/gateway/state_test.go | 11 ++++++++++- blob/service.go | 23 ++++++++++++----------- cmd/celestia/rpc.go | 12 +++++------- nodebuilder/blob/mocks/api.go | 3 +-- nodebuilder/das/mocks/api.go | 3 +-- nodebuilder/fraud/mocks/api.go | 13 ++++++------- nodebuilder/header/mocks/api.go | 18 ++++++++++++++++-- nodebuilder/node/admin.go | 2 +- nodebuilder/node/mocks/api.go | 3 +-- nodebuilder/share/mocks/api.go | 3 +-- nodebuilder/state/mocks/api.go | 8 ++++---- nodebuilder/state/state.go | 13 +++++++------ share/mocks/getter.go | 3 +-- state/core_access.go | 14 ++++++++++---- 16 files changed, 94 insertions(+), 59 deletions(-) diff --git a/api/docgen/examples.go b/api/docgen/examples.go index 8293396c7b..b19b06a7f8 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -131,6 +131,18 @@ func init() { } addToExampleValues(addrInfo) + namespace, err := share.NewNamespaceV0([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}) + if err != nil { + panic(err) + } + addToExampleValues(namespace) + + generatedBlob, err := blob.NewBlob(0, namespace, []byte("This is an example of some blob data")) + if err != nil { + panic(err) + } + addToExampleValues(generatedBlob) + proof := nmt.NewInclusionProof(0, 4, [][]byte{[]byte("test")}, true) blobProof := &blob.Proof{&proof} addToExampleValues(blobProof) diff --git a/api/gateway/state.go b/api/gateway/state.go index fdd627b232..63ada8ebb2 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -10,8 +10,8 @@ import ( "github.com/gorilla/mux" "github.com/celestiaorg/celestia-app/pkg/appconsts" - apptypes "github.com/celestiaorg/celestia-app/x/blob/types" + "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/state" ) @@ -141,14 +141,14 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } fee := types.NewInt(req.Fee) - blob := &apptypes.Blob{ - NamespaceId: nID, - Data: data, - ShareVersion: uint32(appconsts.DefaultShareVersion), + constructedBlob, err := blob.NewBlob(appconsts.DefaultShareVersion, nID, data) + if err != nil { + writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) + return } // perform request - txResp, txerr := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*apptypes.Blob{blob}) + txResp, txerr := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*blob.Blob{constructedBlob}) if txerr != nil && txResp == nil { // no tx data to return writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) diff --git a/api/gateway/state_test.go b/api/gateway/state_test.go index 3c01c9f0f8..a613471a04 100644 --- a/api/gateway/state_test.go +++ b/api/gateway/state_test.go @@ -2,6 +2,7 @@ package gateway import ( "bytes" + "encoding/hex" "encoding/json" "errors" "net/http" @@ -12,6 +13,7 @@ import ( "github.com/stretchr/testify/require" stateMock "github.com/celestiaorg/celestia-node/nodebuilder/state/mocks" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/state" ) @@ -32,7 +34,14 @@ func TestHandleSubmitPFB(t *testing.T) { mock.EXPECT().SubmitPayForBlob(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&txResponse, timedErr) - bs, err := json.Marshal(submitPFBRequest{}) + ns, err := share.NewNamespaceV0([]byte("abc")) + require.NoError(t, err) + hexNs := hex.EncodeToString(ns[:]) + + bs, err := json.Marshal(submitPFBRequest{ + NamespaceID: hexNs, + Data: "DEADBEEF", + }) require.NoError(t, err) httpreq := httptest.NewRequest("GET", "/", bytes.NewReader(bs)) respRec := httptest.NewRecorder() diff --git a/blob/service.go b/blob/service.go index 0d1ccf0bf1..0b43493e27 100644 --- a/blob/service.go +++ b/blob/service.go @@ -6,17 +6,16 @@ import ( "fmt" "sync" + "cosmossdk.io/math" "github.com/cosmos/cosmos-sdk/types" logging "github.com/ipfs/go-log/v2" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" - apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/state" ) var ( @@ -26,9 +25,16 @@ var ( log = logging.Logger("blob") ) +// Submitter is an interface that allows submitting blobs to the celestia-core. It is used to +// avoid a circular dependency between the blob and the state package, since the state package needs +// the blob.Blob type for this signature. +type Submitter interface { + SubmitPayForBlob(ctx context.Context, fee math.Int, gasLim uint64, blobs []*Blob) (*types.TxResponse, error) +} + type Service struct { // accessor dials the given celestia-core endpoint to submit blobs. - accessor *state.CoreAccessor + blobSumitter Submitter // shareGetter retrieves the EDS to fetch all shares from the requested header. shareGetter share.Getter // headerGetter fetches header by the provided height @@ -36,12 +42,12 @@ type Service struct { } func NewService( - state *state.CoreAccessor, + submitter Submitter, getter share.Getter, headerGetter func(context.Context, uint64) (*header.ExtendedHeader, error), ) *Service { return &Service{ - accessor: state, + blobSumitter: submitter, shareGetter: getter, headerGetter: headerGetter, } @@ -56,14 +62,9 @@ func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { var ( gasLimit = estimateGas(blobs...) fee = int64(appconsts.DefaultMinGasPrice * float64(gasLimit)) - b = make([]*apptypes.Blob, len(blobs)) ) - for i, blob := range blobs { - b[i] = &blob.Blob - } - - resp, err := s.accessor.SubmitPayForBlob(ctx, types.NewInt(fee), gasLimit, b) + resp, err := s.blobSumitter.SubmitPayForBlob(ctx, types.NewInt(fee), gasLimit, blobs) if err != nil { return 0, err } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 4139e0f709..de8c55e3f4 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -18,7 +18,6 @@ import ( "github.com/cosmos/cosmos-sdk/types" "github.com/spf13/cobra" - apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/api/rpc/client" @@ -190,12 +189,11 @@ func parseParams(method string, params []string) []interface{} { panic("Error decoding blob: base64 string could not be decoded.") } } - parsedParams[2] = []*apptypes.Blob{{ - NamespaceId: nID[1:], - Data: blobData, - ShareVersion: 0, - NamespaceVersion: 0, - }} + parsedBlob, err := blob.NewBlob(0, nID, blobData) + if err != nil { + panic(fmt.Sprintf("Error creating blob: %v", err)) + } + parsedParams[2] = []*blob.Blob{parsedBlob} return parsedParams[:3] case "Get": // 1. Height diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index b7c61aa450..f99d1d8168 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - blob "github.com/celestiaorg/celestia-node/blob" namespace "github.com/celestiaorg/nmt/namespace" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/das/mocks/api.go b/nodebuilder/das/mocks/api.go index 68ffaf3c8c..c4046e90e8 100644 --- a/nodebuilder/das/mocks/api.go +++ b/nodebuilder/das/mocks/api.go @@ -8,9 +8,8 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - das "github.com/celestiaorg/celestia-node/das" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index 5ede6f27c5..ba88131695 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" + fraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" + fraud0 "github.com/celestiaorg/go-fraud" gomock "github.com/golang/mock/gomock" - - fraud0 "github.com/celestiaorg/celestia-node/nodebuilder/fraud" - "github.com/celestiaorg/go-fraud" ) // MockModule is a mock of Module interface. @@ -38,10 +37,10 @@ func (m *MockModule) EXPECT() *MockModuleMockRecorder { } // Get mocks base method. -func (m *MockModule) Get(arg0 context.Context, arg1 fraud.ProofType) ([]fraud0.Proof, error) { +func (m *MockModule) Get(arg0 context.Context, arg1 fraud0.ProofType) ([]fraud.Proof, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0, arg1) - ret0, _ := ret[0].([]fraud0.Proof) + ret0, _ := ret[0].([]fraud.Proof) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -53,10 +52,10 @@ func (mr *MockModuleMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { } // Subscribe mocks base method. -func (m *MockModule) Subscribe(arg0 context.Context, arg1 fraud.ProofType) (<-chan fraud0.Proof, error) { +func (m *MockModule) Subscribe(arg0 context.Context, arg1 fraud0.ProofType) (<-chan fraud.Proof, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Subscribe", arg0, arg1) - ret0, _ := ret[0].(<-chan fraud0.Proof) + ret0, _ := ret[0].(<-chan fraud.Proof) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/nodebuilder/header/mocks/api.go b/nodebuilder/header/mocks/api.go index 538169c6be..02529a8ef9 100644 --- a/nodebuilder/header/mocks/api.go +++ b/nodebuilder/header/mocks/api.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - header "github.com/celestiaorg/celestia-node/header" header0 "github.com/celestiaorg/go-header" sync "github.com/celestiaorg/go-header/sync" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -156,3 +155,18 @@ func (mr *MockModuleMockRecorder) SyncWait(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "SyncWait", reflect.TypeOf((*MockModule)(nil).SyncWait), arg0) } + +// WaitForHeight mocks base method. +func (m *MockModule) WaitForHeight(arg0 context.Context, arg1 uint64) (*header.ExtendedHeader, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "WaitForHeight", arg0, arg1) + ret0, _ := ret[0].(*header.ExtendedHeader) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// WaitForHeight indicates an expected call of WaitForHeight. +func (mr *MockModuleMockRecorder) WaitForHeight(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "WaitForHeight", reflect.TypeOf((*MockModule)(nil).WaitForHeight), arg0, arg1) +} diff --git a/nodebuilder/node/admin.go b/nodebuilder/node/admin.go index 41f8400101..cad8a51361 100644 --- a/nodebuilder/node/admin.go +++ b/nodebuilder/node/admin.go @@ -8,7 +8,7 @@ import ( logging "github.com/ipfs/go-log/v2" ) -const APIVersion = "v0.1.1" +const APIVersion = "v0.2.0" type module struct { tp Type diff --git a/nodebuilder/node/mocks/api.go b/nodebuilder/node/mocks/api.go index 39d500bc04..98df713429 100644 --- a/nodebuilder/node/mocks/api.go +++ b/nodebuilder/node/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" + node "github.com/celestiaorg/celestia-node/nodebuilder/node" auth "github.com/filecoin-project/go-jsonrpc/auth" gomock "github.com/golang/mock/gomock" - - node "github.com/celestiaorg/celestia-node/nodebuilder/node" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 1b26273c0f..586c6dab4b 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -8,12 +8,11 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index 8398e0fd7b..814b9a0113 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -9,11 +9,11 @@ import ( reflect "reflect" math "cosmossdk.io/math" + blob "github.com/celestiaorg/celestia-node/blob" types "github.com/cosmos/cosmos-sdk/types" types0 "github.com/cosmos/cosmos-sdk/x/staking/types" gomock "github.com/golang/mock/gomock" - types1 "github.com/tendermint/tendermint/proto/tendermint/types" - types2 "github.com/tendermint/tendermint/types" + types1 "github.com/tendermint/tendermint/types" ) // MockModule is a mock of Module interface. @@ -189,7 +189,7 @@ func (mr *MockModuleMockRecorder) QueryUnbonding(arg0, arg1 interface{}) *gomock } // SubmitPayForBlob mocks base method. -func (m *MockModule) SubmitPayForBlob(arg0 context.Context, arg1 math.Int, arg2 uint64, arg3 []*types1.Blob) (*types.TxResponse, error) { +func (m *MockModule) SubmitPayForBlob(arg0 context.Context, arg1 math.Int, arg2 uint64, arg3 []*blob.Blob) (*types.TxResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SubmitPayForBlob", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*types.TxResponse) @@ -204,7 +204,7 @@ func (mr *MockModuleMockRecorder) SubmitPayForBlob(arg0, arg1, arg2, arg3 interf } // SubmitTx mocks base method. -func (m *MockModule) SubmitTx(arg0 context.Context, arg1 types2.Tx) (*types.TxResponse, error) { +func (m *MockModule) SubmitTx(arg0 context.Context, arg1 types1.Tx) (*types.TxResponse, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SubmitTx", arg0, arg1) ret0, _ := ret[0].(*types.TxResponse) diff --git a/nodebuilder/state/state.go b/nodebuilder/state/state.go index 57f46a8fd6..c66205d594 100644 --- a/nodebuilder/state/state.go +++ b/nodebuilder/state/state.go @@ -5,8 +5,7 @@ import ( "github.com/cosmos/cosmos-sdk/x/staking/types" - apptypes "github.com/celestiaorg/celestia-app/x/blob/types" - + "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/state" ) @@ -36,7 +35,9 @@ type Module interface { // Transfer sends the given amount of coins from default wallet of the node to the given account // address. - Transfer(ctx context.Context, to state.AccAddress, amount, fee state.Int, gasLimit uint64) (*state.TxResponse, error) + Transfer( + ctx context.Context, to state.AccAddress, amount, fee state.Int, gasLimit uint64, + ) (*state.TxResponse, error) // SubmitTx submits the given transaction/message to the // Celestia network and blocks until the tx is included in // a block. @@ -46,7 +47,7 @@ type Module interface { ctx context.Context, fee state.Int, gasLim uint64, - blobs []*apptypes.Blob, + blobs []*blob.Blob, ) (*state.TxResponse, error) // CancelUnbondingDelegation cancels a user's pending undelegation from a validator. @@ -114,7 +115,7 @@ type API struct { ctx context.Context, fee state.Int, gasLim uint64, - blobs []*apptypes.Blob, + blobs []*blob.Blob, ) (*state.TxResponse, error) `perm:"write"` CancelUnbondingDelegation func( ctx context.Context, @@ -192,7 +193,7 @@ func (api *API) SubmitPayForBlob( ctx context.Context, fee state.Int, gasLim uint64, - blobs []*apptypes.Blob, + blobs []*blob.Blob, ) (*state.TxResponse, error) { return api.Internal.SubmitPayForBlob(ctx, fee, gasLim, blobs) } diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 12c36cb015..1c73c9170d 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -8,12 +8,11 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" + gomock "github.com/golang/mock/gomock" ) // MockGetter is a mock of Getter interface. diff --git a/state/core_access.go b/state/core_access.go index 712100d921..ca0947166d 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -21,10 +21,11 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/celestiaorg/celestia-app/app" - "github.com/celestiaorg/celestia-app/x/blob" + appblob "github.com/celestiaorg/celestia-app/x/blob" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" libhead "github.com/celestiaorg/go-header" + "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/header" ) @@ -159,17 +160,22 @@ func (ca *CoreAccessor) SubmitPayForBlob( ctx context.Context, fee Int, gasLim uint64, - blobs []*apptypes.Blob, + blobs []*blob.Blob, ) (*TxResponse, error) { if len(blobs) == 0 { return nil, errors.New("state: no blobs provided") } - response, err := blob.SubmitPayForBlob( + appblobs := make([]*apptypes.Blob, len(blobs)) + for i, blob := range blobs { + appblobs[i] = &blob.Blob + } + + response, err := appblob.SubmitPayForBlob( ctx, ca.signer, ca.coreConn, - blobs, + appblobs, apptypes.SetGasLimit(gasLim), withFee(fee), ) From c9443c6859808d1096f484eeb505a8ff9c3e12f9 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Tue, 13 Jun 2023 05:13:02 -0500 Subject: [PATCH 034/388] chore: mocha-2 support (#2326) Note that we're skipping mocha-1 because we reset the network after finding a state-breaking bug Friday Co-authored-by: Hlib Kanunnikov --- nodebuilder/p2p/bootstrap.go | 6 +++--- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index 25f0a85120..c534aa4fba 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -44,9 +44,9 @@ var bootstrapList = map[Network][]string{ "/dns4/da-full-2-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWCs3wFmqwPn1u8pNU4BGsvLsob1ShTzvps8qEtTRuuuK5", }, Mocha: { - "/dns4/andromeda.celestia-devops.dev/tcp/2121/p2p/12D3KooWKvPXtV1yaQ6e3BRNUHa5Phh8daBwBi3KkGaSSkUPys6D", - "/dns4/libra.celestia-devops.dev/tcp/2121/p2p/12D3KooWK5aDotDcLsabBmWDazehQLMsDkRyARm1k7f1zGAXqbt4", - "/dns4/norma.celestia-devops.dev/tcp/2121/p2p/12D3KooWHYczJDVNfYVkLcNHPTDKCeiVvRhg8Q9JU3bE3m9eEVyY", + "/dns4/bootstr-mocha-1.celestia-mocha.com/tcp/2121/p2p/12D3KooWDRSJMbH3PS4dRDa11H7Tk615aqTUgkeEKz4pwd4sS6fN", + "/dns4/bootstr-mocha-2.celestia-mocha.com/tcp/2121/p2p/12D3KooWEk7cxtjQCC7kC84Uhs2j6dAHjdbwYnPcvUAqmj6Zsry2", + "/dns4/bootstr-mocha-3.celestia-mocha.com/tcp/2121/p2p/12D3KooWBE4QcFXZzENf2VRo6Y5LBvp9gzmpYRHKCvgGzEYj7Hdn", }, BlockspaceRace: { "/dns4/bootstr-incent-3.celestia.tools/tcp/2121/p2p/12D3KooWNzdKcHagtvvr6qtjcPTAdCN6ZBiBLH8FBHbihxqu4GZx", diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index bc0fc3810d..0a36dc54cc 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -24,7 +24,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ Arabica: "E5D620B5BE7873222DCD83464C285FD0F215C209393E7481F9A5979280AD6CA2", - Mocha: "8038B21032C941372ED601699857043C12E5CC7D5945DCEEA4567D11B5712526", + Mocha: "1181AF8EAE5DDF3CBBFF3BF3CC44C5B795DF5094F5A0CC0AE52921ECCA0AF3C8", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", } diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index ded3f0f1b1..dd04cd377c 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -14,7 +14,7 @@ const ( // Arabica testnet. See: celestiaorg/networks. Arabica Network = "arabica-8" // Mocha testnet. See: celestiaorg/networks. - Mocha Network = "mocha" + Mocha Network = "mocha-2" // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. BlockspaceRace Network = "blockspacerace-0" // Private can be used to set up any private network, including local testing setups. From c41738a3f3d3743233e8b81f1c015f81dbd5eec3 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 13 Jun 2023 19:18:31 +0800 Subject: [PATCH 035/388] fix(nodebuilder): Fix dns lookup (#2332) ## Overview Cleanup core ip parsing logic Closes #2333 --------- Co-authored-by: Hlib Kanunnikov --- libs/utils/address.go | 9 ++++++--- libs/utils/address_test.go | 35 +++++++++++++++++++++++++++++------ nodebuilder/core/flags.go | 27 ++++++--------------------- 3 files changed, 41 insertions(+), 30 deletions(-) diff --git a/libs/utils/address.go b/libs/utils/address.go index 5971bf5a98..a8170e44b9 100644 --- a/libs/utils/address.go +++ b/libs/utils/address.go @@ -30,11 +30,14 @@ func ValidateAddr(addr string) (string, error) { } if ip := net.ParseIP(addr); ip == nil { - _, err = net.LookupHost(addr) + addrs, err := net.LookupHost(addr) if err != nil { - return addr, fmt.Errorf("could not resolve hostname or ip: %w", err) + return addr, fmt.Errorf("could not resolve %v: %w", addr, err) } + if len(addrs) == 0 { + return addr, fmt.Errorf("no IP addresses found for DNS record: %v", addr) + } + addr = addrs[0] } - return addr, nil } diff --git a/libs/utils/address_test.go b/libs/utils/address_test.go index c914a7d853..15452f4d1b 100644 --- a/libs/utils/address_test.go +++ b/libs/utils/address_test.go @@ -1,6 +1,7 @@ package utils import ( + "net" "testing" "github.com/stretchr/testify/require" @@ -13,8 +14,12 @@ func TestSanitizeAddr(t *testing.T) { }{ // Testcase: trims protocol prefix {addr: "http://celestia.org", want: "celestia.org"}, + // Testcase: protocol prefix trimmed already + {addr: "celestia.org", want: "celestia.org"}, // Testcase: trims protocol prefix, and trims port and trailing slash suffix {addr: "tcp://192.168.42.42:5050/", want: "192.168.42.42"}, + // Testcase: invariant ip + {addr: "192.168.42.42", want: "192.168.42.42"}, } for _, tt := range tests { @@ -27,23 +32,41 @@ func TestSanitizeAddr(t *testing.T) { } func TestValidateAddr(t *testing.T) { + type want struct { + addr string + unresolved bool + } var tests = []struct { addr string - want string + want want }{ // Testcase: ip is valid - {addr: "192.168.42.42:5050", want: "192.168.42.42"}, - // Testcase: hostname is valid - {addr: "https://celestia.org", want: "celestia.org"}, + {addr: "192.168.42.42:5050", want: want{addr: "192.168.42.42"}}, + // Testcase: ip is valid, no port + {addr: "192.168.42.42", want: want{addr: "192.168.42.42"}}, // Testcase: resolves localhost - {addr: "http://localhost:8080/", want: "localhost"}, + {addr: "http://localhost:8080/", want: want{unresolved: true}}, + // Testcase: hostname is valid + {addr: "https://celestia.org", want: want{unresolved: true}}, + // Testcase: hostname is valid, but no schema + {addr: "celestia.org", want: want{unresolved: true}}, } for _, tt := range tests { t.Run(tt.addr, func(t *testing.T) { got, err := ValidateAddr(tt.addr) require.NoError(t, err) - require.Equal(t, tt.want, got) + + // validate that returned value is ip + if ip := net.ParseIP(got); ip == nil { + t.Fatalf("empty ip") + } + + if tt.want.unresolved { + // unresolved addr has no addr to compare with + return + } + require.Equal(t, tt.want.addr, got) }) } } diff --git a/nodebuilder/core/flags.go b/nodebuilder/core/flags.go index 01ff031991..9cbed9b277 100644 --- a/nodebuilder/core/flags.go +++ b/nodebuilder/core/flags.go @@ -2,8 +2,6 @@ package core import ( "fmt" - "net" - "net/url" "github.com/spf13/cobra" flag "github.com/spf13/pflag" @@ -40,7 +38,10 @@ func Flags() *flag.FlagSet { } // ParseFlags parses Core flags from the given cmd and saves them to the passed config. -func ParseFlags(cmd *cobra.Command, cfg *Config) error { +func ParseFlags( + cmd *cobra.Command, + cfg *Config, +) error { coreIP := cmd.Flag(coreFlag).Value.String() if coreIP == "" { if cmd.Flag(coreGRPCFlag).Changed || cmd.Flag(coreRPCFlag).Changed { @@ -49,27 +50,11 @@ func ParseFlags(cmd *cobra.Command, cfg *Config) error { return nil } - ip := net.ParseIP(coreIP) - if ip == nil { - u, err := url.Parse(coreIP) - if err != nil { - return fmt.Errorf("failed to parse url: %w", err) - } - ips, err := net.LookupIP(u.Host) - if err != nil { - return fmt.Errorf("failed to resolve DNS record: %v", err) - } - if len(ips) == 0 { - return fmt.Errorf("no IP addresses found for DNS record") - } - ip = ips[0] - } - rpc := cmd.Flag(coreRPCFlag).Value.String() grpc := cmd.Flag(coreGRPCFlag).Value.String() - cfg.IP = ip.String() + cfg.IP = coreIP cfg.RPCPort = rpc cfg.GRPCPort = grpc - return nil + return cfg.Validate() } From c157db61736592213108d226df2810eae9aca12d Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 14 Jun 2023 10:05:09 +0200 Subject: [PATCH 036/388] fix(rpc/client): stop overwriting existing clients (#2361) While working on #2356 I uncovered a bug: When using the `client.NewClient` constructor, it uses a global static struct to set the modules. This led to clients overwriting each other when multiple are initialized. --- api/rpc/client/client.go | 36 ++++++++++++++++++++++-------------- 1 file changed, 22 insertions(+), 14 deletions(-) diff --git a/api/rpc/client/client.go b/api/rpc/client/client.go index 9e7e577074..7ac8a55b3d 100644 --- a/api/rpc/client/client.go +++ b/api/rpc/client/client.go @@ -18,18 +18,11 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/state" ) -// TODO: this duplication of strings many times across the codebase can be avoided with issue #1176 -var client Client -var Modules = map[string]interface{}{ - "share": &client.Share.Internal, - "state": &client.State.Internal, - "header": &client.Header.Internal, - "fraud": &client.Fraud.Internal, - "das": &client.DAS.Internal, - "p2p": &client.P2P.Internal, - "node": &client.Node.Internal, - "blob": &client.Blob.Internal, -} +var ( + // staticClient is used for generating the OpenRPC spec. + staticClient Client + Modules = moduleMap(&staticClient) +) type Client struct { Fraud fraud.API @@ -61,7 +54,7 @@ func (m *multiClientCloser) closeAll() { } } -// Close closes the connections to all namespaces registered on the client. +// Close closes the connections to all namespaces registered on the staticClient. func (c *Client) Close() { c.closer.closeAll() } @@ -80,7 +73,8 @@ func NewClient(ctx context.Context, addr string, token string) (*Client, error) func newClient(ctx context.Context, addr string, authHeader http.Header) (*Client, error) { var multiCloser multiClientCloser - for name, module := range Modules { + var client Client + for name, module := range moduleMap(&client) { closer, err := jsonrpc.NewClient(ctx, addr, name, module, authHeader) if err != nil { return nil, err @@ -90,3 +84,17 @@ func newClient(ctx context.Context, addr string, authHeader http.Header) (*Clien return &client, nil } + +func moduleMap(client *Client) map[string]interface{} { + // TODO: this duplication of strings many times across the codebase can be avoided with issue #1176 + return map[string]interface{}{ + "share": &client.Share.Internal, + "state": &client.State.Internal, + "header": &client.Header.Internal, + "fraud": &client.Fraud.Internal, + "das": &client.DAS.Internal, + "p2p": &client.P2P.Internal, + "node": &client.Node.Internal, + "blob": &client.Blob.Internal, + } +} From 55db897aab533d2f00f444685aad6ccb15674f76 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 14 Jun 2023 10:10:55 +0200 Subject: [PATCH 037/388] fix(state): JSON marshaling for sdk Address wrapper (#2348) While debugging #2322 , I noticed that it is not only an RPC issue, but also a broken API one. All RPC methods that used the type `state.Address` were unusable, both via the client and via raw JSON calls. This is because the server was unable to marshal the address string value back into the interface type. To circumvent this, I have embedded the sdk.Address type in the same way that we embed the fraud proof type, to allow us to unmarshal it back into a concrete type. I have also added unit tests to fix this. In addition, it fixes the RPC parsing - so it closes #2322 . --- api/docgen/examples.go | 2 ++ api/gateway/state.go | 2 +- cmd/celestia/rpc.go | 16 +++------- nodebuilder/state/mocks/api.go | 7 +++-- state/address_test.go | 57 ++++++++++++++++++++++++++++++++++ state/core_access.go | 6 ++-- state/integration_test.go | 2 +- state/state.go | 34 ++++++++++++++++++-- 8 files changed, 104 insertions(+), 22 deletions(-) create mode 100644 state/address_test.go diff --git a/api/docgen/examples.go b/api/docgen/examples.go index b19b06a7f8..80a8c64d93 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -83,6 +83,8 @@ func init() { } addToExampleValues(valAddr) + addToExampleValues(state.Address{Address: addr}) + var txResponse *state.TxResponse err = json.Unmarshal([]byte(exampleTxResponse), &txResponse) if err != nil { diff --git a/api/gateway/state.go b/api/gateway/state.go index 63ada8ebb2..c895e35bde 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -72,7 +72,7 @@ func (h *Handler) handleBalanceRequest(w http.ResponseWriter, r *http.Request) { } addr = valAddr.Bytes() } - bal, err = h.state.BalanceForAddress(r.Context(), addr) + bal, err = h.state.BalanceForAddress(r.Context(), state.Address{Address: addr}) } else { bal, err = h.state.Balance(r.Context()) } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index de8c55e3f4..767fca872c 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -5,7 +5,6 @@ import ( "encoding/base64" "encoding/hex" "encoding/json" - "errors" "fmt" "io" "log" @@ -15,7 +14,6 @@ import ( "strconv" "strings" - "github.com/cosmos/cosmos-sdk/types" "github.com/spf13/cobra" "github.com/celestiaorg/nmt/namespace" @@ -392,18 +390,12 @@ func sendJSONRPCRequest(namespace, method string, params []interface{}) { } func parseAddressFromString(addrStr string) (state.Address, error) { - var addr state.AccAddress - addr, err := types.AccAddressFromBech32(addrStr) + var address state.Address + err := address.UnmarshalJSON([]byte(addrStr)) if err != nil { - // first check if it is a validator address and can be converted - valAddr, err := types.ValAddressFromBech32(addrStr) - if err != nil { - return nil, errors.New("address must be a valid account or validator address ") - } - return valAddr, nil + return address, err } - - return addr, nil + return address, nil } func parseSignatureForHelpstring(methodSig reflect.StructField) string { diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index 814b9a0113..dbd1d5dabe 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -10,6 +10,7 @@ import ( math "cosmossdk.io/math" blob "github.com/celestiaorg/celestia-node/blob" + state "github.com/celestiaorg/celestia-node/state" types "github.com/cosmos/cosmos-sdk/types" types0 "github.com/cosmos/cosmos-sdk/x/staking/types" gomock "github.com/golang/mock/gomock" @@ -40,10 +41,10 @@ func (m *MockModule) EXPECT() *MockModuleMockRecorder { } // AccountAddress mocks base method. -func (m *MockModule) AccountAddress(arg0 context.Context) (types.Address, error) { +func (m *MockModule) AccountAddress(arg0 context.Context) (state.Address, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AccountAddress", arg0) - ret0, _ := ret[0].(types.Address) + ret0, _ := ret[0].(state.Address) ret1, _ := ret[1].(error) return ret0, ret1 } @@ -70,7 +71,7 @@ func (mr *MockModuleMockRecorder) Balance(arg0 interface{}) *gomock.Call { } // BalanceForAddress mocks base method. -func (m *MockModule) BalanceForAddress(arg0 context.Context, arg1 types.Address) (*types.Coin, error) { +func (m *MockModule) BalanceForAddress(arg0 context.Context, arg1 state.Address) (*types.Coin, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "BalanceForAddress", arg0, arg1) ret0, _ := ret[0].(*types.Coin) diff --git a/state/address_test.go b/state/address_test.go new file mode 100644 index 0000000000..d701b38aa8 --- /dev/null +++ b/state/address_test.go @@ -0,0 +1,57 @@ +package state + +import ( + "testing" + + sdk "github.com/cosmos/cosmos-sdk/types" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestAddressMarshalling(t *testing.T) { + testCases := []struct { + name string + addressString string + addressFromStr func(string) (interface{}, error) + marshalJSON func(interface{}) ([]byte, error) + unmarshalJSON func([]byte) (interface{}, error) + }{ + { + name: "Account Address", + addressString: "celestia1377k5an3f94v6wyaceu0cf4nq6gk2jtpc46g7h", + addressFromStr: func(s string) (interface{}, error) { return sdk.AccAddressFromBech32(s) }, + marshalJSON: func(addr interface{}) ([]byte, error) { return addr.(sdk.AccAddress).MarshalJSON() }, + unmarshalJSON: func(b []byte) (interface{}, error) { + var addr sdk.AccAddress + err := addr.UnmarshalJSON(b) + return addr, err + }, + }, + { + name: "Validator Address", + addressString: "celestiavaloper1q3v5cugc8cdpud87u4zwy0a74uxkk6u4q4gx4p", + addressFromStr: func(s string) (interface{}, error) { return sdk.ValAddressFromBech32(s) }, + marshalJSON: func(addr interface{}) ([]byte, error) { return addr.(sdk.ValAddress).MarshalJSON() }, + unmarshalJSON: func(b []byte) (interface{}, error) { + var addr sdk.ValAddress + err := addr.UnmarshalJSON(b) + return addr, err + }, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + addr, err := tc.addressFromStr(tc.addressString) + require.NoError(t, err) + + addrBytes, err := tc.marshalJSON(addr) + assert.NoError(t, err) + assert.Equal(t, []byte("\""+tc.addressString+"\""), addrBytes) + + addrUnmarshalled, err := tc.unmarshalJSON(addrBytes) + assert.NoError(t, err) + assert.Equal(t, addr, addrUnmarshalled) + }) + } +} diff --git a/state/core_access.go b/state/core_access.go index ca0947166d..7b59f3e714 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -194,9 +194,9 @@ func (ca *CoreAccessor) SubmitPayForBlob( func (ca *CoreAccessor) AccountAddress(context.Context) (Address, error) { addr, err := ca.signer.GetSignerInfo().GetAddress() if err != nil { - return nil, err + return Address{nil}, err } - return addr, nil + return Address{addr}, nil } func (ca *CoreAccessor) Balance(ctx context.Context) (*Balance, error) { @@ -204,7 +204,7 @@ func (ca *CoreAccessor) Balance(ctx context.Context) (*Balance, error) { if err != nil { return nil, err } - return ca.BalanceForAddress(ctx, addr) + return ca.BalanceForAddress(ctx, Address{addr}) } func (ca *CoreAccessor) BalanceForAddress(ctx context.Context, addr Address) (*Balance, error) { diff --git a/state/integration_test.go b/state/integration_test.go index e7d2496397..8862de1bf8 100644 --- a/state/integration_test.go +++ b/state/integration_test.go @@ -110,7 +110,7 @@ func (s *IntegrationTestSuite) TestGetBalance() { require := s.Require() expectedBal := sdk.NewCoin(app.BondDenom, sdk.NewInt(int64(99999999999999999))) for _, acc := range s.accounts { - bal, err := s.accessor.BalanceForAddress(context.Background(), s.getAddress(acc)) + bal, err := s.accessor.BalanceForAddress(context.Background(), Address{s.getAddress(acc)}) require.NoError(err) require.Equal(&expectedBal, bal) } diff --git a/state/state.go b/state/state.go index 987a783239..d55bb6901c 100644 --- a/state/state.go +++ b/state/state.go @@ -1,6 +1,9 @@ package state import ( + "fmt" + "strings" + "cosmossdk.io/math" sdk "github.com/cosmos/cosmos-sdk/types" coretypes "github.com/tendermint/tendermint/types" @@ -15,8 +18,11 @@ type Tx = coretypes.Tx // TxResponse is an alias to the TxResponse type from Cosmos-SDK. type TxResponse = sdk.TxResponse -// Address is an alias to the Address type from Cosmos-SDK. -type Address = sdk.Address +// Address is an alias to the Address type from Cosmos-SDK. It is embedded into a struct to provide +// a non-interface type for JSON serialization. +type Address struct { + sdk.Address +} // ValAddress is an alias to the ValAddress type from Cosmos-SDK. type ValAddress = sdk.ValAddress @@ -26,3 +32,27 @@ type AccAddress = sdk.AccAddress // Int is an alias to the Int type from Cosmos-SDK. type Int = math.Int + +func (a *Address) UnmarshalJSON(data []byte) error { + // To convert the string back to a concrete type, we have to determine the correct implementation + var addr AccAddress + addrString := strings.Trim(string(data), "\"") + addr, err := sdk.AccAddressFromBech32(addrString) + if err != nil { + // first check if it is a validator address and can be converted + valAddr, err := sdk.ValAddressFromBech32(addrString) + if err != nil { + return fmt.Errorf("address must be a valid account or validator address: %w", err) + } + a.Address = valAddr + return nil + } + + a.Address = addr + return nil +} + +func (a Address) MarshalJSON() ([]byte, error) { + // The address is marshaled into a simple string value + return []byte("\"" + a.Address.String() + "\""), nil +} From d77d0f2f22b107abb58585b3c02c47d7c1508092 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 14 Jun 2023 10:18:17 +0200 Subject: [PATCH 038/388] chore!(gateway): add deprecation notices (#2360) - Adds a warn log on startup if --gateway is passed - Deprecated endpoints are now disabled by default. This includes: - /balance - /submit_pfd - /query_delegation - /query_unbonding - /query_redelegations - Deprecated endpoints can be activated by passing `--gateway.deprecated-endpoints` - When using deprecated endpoints, the node will log a warning, giving the alternative RPC method to call in the future. The ugly config change/flag will be removed when fully deprecating the endpoints --- api/gateway/das.go | 1 + api/gateway/endpoints.go | 39 ++++++++++++++++------------- api/gateway/state.go | 10 ++++++++ nodebuilder/gateway/config.go | 7 +++--- nodebuilder/gateway/constructors.go | 3 ++- nodebuilder/gateway/flags.go | 16 +++++++++--- nodebuilder/gateway/module.go | 5 ++-- 7 files changed, 54 insertions(+), 27 deletions(-) diff --git a/api/gateway/das.go b/api/gateway/das.go index 565cbd7460..88dc97927c 100644 --- a/api/gateway/das.go +++ b/api/gateway/das.go @@ -10,6 +10,7 @@ const ( ) func (h *Handler) handleDASStateRequest(w http.ResponseWriter, r *http.Request) { + logDeprecation(dasStateEndpoint, "das.SamplingStats") stats, err := h.das.SamplingStats(r.Context()) if err != nil { writeError(w, http.StatusInternalServerError, dasStateEndpoint, err) diff --git a/api/gateway/endpoints.go b/api/gateway/endpoints.go index dfcb96bd06..0ae93b112c 100644 --- a/api/gateway/endpoints.go +++ b/api/gateway/endpoints.go @@ -5,21 +5,32 @@ import ( "net/http" ) -func (h *Handler) RegisterEndpoints(rpc *Server) { +func (h *Handler) RegisterEndpoints(rpc *Server, deprecatedEndpointsEnabled bool) { + if deprecatedEndpointsEnabled { + log.Warn("Deprecated endpoints will be removed from the gateway in the next release. Use the RPC instead.") + // state endpoints + rpc.RegisterHandlerFunc(balanceEndpoint, h.handleBalanceRequest, http.MethodGet) + rpc.RegisterHandlerFunc(submitPFBEndpoint, h.handleSubmitPFB, http.MethodPost) + + // staking queries + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryDelegationEndpoint, addrKey), h.handleQueryDelegation, + http.MethodGet) + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryUnbondingEndpoint, addrKey), h.handleQueryUnbonding, + http.MethodGet) + rpc.RegisterHandlerFunc(queryRedelegationsEndpoint, h.handleQueryRedelegations, + http.MethodPost) + + // DASer endpoints + // only register if DASer service is available + if h.das != nil { + rpc.RegisterHandlerFunc(dasStateEndpoint, h.handleDASStateRequest, http.MethodGet) + } + } + // state endpoints - rpc.RegisterHandlerFunc(balanceEndpoint, h.handleBalanceRequest, http.MethodGet) rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", balanceEndpoint, addrKey), h.handleBalanceRequest, http.MethodGet) rpc.RegisterHandlerFunc(submitTxEndpoint, h.handleSubmitTx, http.MethodPost) - rpc.RegisterHandlerFunc(submitPFBEndpoint, h.handleSubmitPFB, http.MethodPost) - - // staking queries - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryDelegationEndpoint, addrKey), h.handleQueryDelegation, - http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryUnbondingEndpoint, addrKey), h.handleQueryUnbonding, - http.MethodGet) - rpc.RegisterHandlerFunc(queryRedelegationsEndpoint, h.handleQueryRedelegations, - http.MethodPost) // share endpoints rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, nIDKey, heightKey), @@ -39,10 +50,4 @@ func (h *Handler) RegisterEndpoints(rpc *Server) { rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", headerByHeightEndpoint, heightKey), h.handleHeaderRequest, http.MethodGet) rpc.RegisterHandlerFunc(headEndpoint, h.handleHeadRequest, http.MethodGet) - - // DASer endpoints - // only register if DASer service is available - if h.das != nil { - rpc.RegisterHandlerFunc(dasStateEndpoint, h.handleDASStateRequest, http.MethodGet) - } } diff --git a/api/gateway/state.go b/api/gateway/state.go index c895e35bde..b584b00d36 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -74,6 +74,7 @@ func (h *Handler) handleBalanceRequest(w http.ResponseWriter, r *http.Request) { } bal, err = h.state.BalanceForAddress(r.Context(), state.Address{Address: addr}) } else { + logDeprecation(balanceEndpoint, "state.Balance") bal, err = h.state.Balance(r.Context()) } if err != nil { @@ -122,6 +123,7 @@ func (h *Handler) handleSubmitTx(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { + logDeprecation(submitPFBEndpoint, "blob.Submit or state.SubmitPayForBlob") // decode request var req submitPFBRequest err := json.NewDecoder(r.Body).Decode(&req) @@ -171,6 +173,7 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleQueryDelegation(w http.ResponseWriter, r *http.Request) { + logDeprecation(queryDelegationEndpoint, "state.QueryDelegation") // read and parse request vars := mux.Vars(r) addrStr, exists := vars[addrKey] @@ -202,6 +205,7 @@ func (h *Handler) handleQueryDelegation(w http.ResponseWriter, r *http.Request) } func (h *Handler) handleQueryUnbonding(w http.ResponseWriter, r *http.Request) { + logDeprecation(queryUnbondingEndpoint, "state.QueryUnbonding") // read and parse request vars := mux.Vars(r) addrStr, exists := vars[addrKey] @@ -233,6 +237,7 @@ func (h *Handler) handleQueryUnbonding(w http.ResponseWriter, r *http.Request) { } func (h *Handler) handleQueryRedelegations(w http.ResponseWriter, r *http.Request) { + logDeprecation(queryRedelegationsEndpoint, "state.QueryRedelegations") var req queryRedelegationsRequest err := json.NewDecoder(r.Body).Decode(&req) if err != nil { @@ -264,3 +269,8 @@ func (h *Handler) handleQueryRedelegations(w http.ResponseWriter, r *http.Reques log.Errorw("writing response", "endpoint", queryRedelegationsEndpoint, "err", err) } } + +func logDeprecation(endpoint string, alternative string) { + log.Warn("The " + endpoint + " endpoint is deprecated and will be removed in the next release. Please " + + "use " + alternative + " from the RPC instead.") +} diff --git a/nodebuilder/gateway/config.go b/nodebuilder/gateway/config.go index 903a27489a..f85f207ceb 100644 --- a/nodebuilder/gateway/config.go +++ b/nodebuilder/gateway/config.go @@ -8,9 +8,10 @@ import ( ) type Config struct { - Address string - Port string - Enabled bool + Address string + Port string + Enabled bool + deprecatedEndpoints bool } func DefaultConfig() Config { diff --git a/nodebuilder/gateway/constructors.go b/nodebuilder/gateway/constructors.go index c28153b0a5..c771c12023 100644 --- a/nodebuilder/gateway/constructors.go +++ b/nodebuilder/gateway/constructors.go @@ -10,6 +10,7 @@ import ( // Handler constructs a new RPC Handler from the given services. func Handler( + cfg *Config, state state.Module, share share.Module, header header.Module, @@ -17,7 +18,7 @@ func Handler( serv *gateway.Server, ) { handler := gateway.NewHandler(state, share, header, daser) - handler.RegisterEndpoints(serv) + handler.RegisterEndpoints(serv, cfg.deprecatedEndpoints) handler.RegisterMiddleware(serv) } diff --git a/nodebuilder/gateway/flags.go b/nodebuilder/gateway/flags.go index cd13e47162..4d72a278e5 100644 --- a/nodebuilder/gateway/flags.go +++ b/nodebuilder/gateway/flags.go @@ -6,9 +6,10 @@ import ( ) var ( - enabledFlag = "gateway" - addrFlag = "gateway.addr" - portFlag = "gateway.port" + enabledFlag = "gateway" + addrFlag = "gateway.addr" + portFlag = "gateway.port" + deprecatedEndpoints = "gateway.deprecated-endpoints" ) // Flags gives a set of hardcoded node/gateway package flags. @@ -20,6 +21,11 @@ func Flags() *flag.FlagSet { false, "Enables the REST gateway", ) + flags.Bool( + deprecatedEndpoints, + false, + "Enables deprecated endpoints on the gateway. These will be removed in the next release.", + ) flags.String( addrFlag, "", @@ -40,6 +46,10 @@ func ParseFlags(cmd *cobra.Command, cfg *Config) { if cmd.Flags().Changed(enabledFlag) && err == nil { cfg.Enabled = enabled } + deprecatedEndpointsEnabled, err := cmd.Flags().GetBool(deprecatedEndpoints) + if cmd.Flags().Changed(deprecatedEndpoints) && err == nil { + cfg.deprecatedEndpoints = deprecatedEndpointsEnabled + } addr, port := cmd.Flag(addrFlag), cmd.Flag(portFlag) if !cfg.Enabled && (addr.Changed || port.Changed) { log.Warn("custom address or port provided without enabling gateway, setting config values") diff --git a/nodebuilder/gateway/module.go b/nodebuilder/gateway/module.go index b3070e01a6..b727f4c04d 100644 --- a/nodebuilder/gateway/module.go +++ b/nodebuilder/gateway/module.go @@ -21,8 +21,6 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { if !cfg.Enabled { return fx.Options() } - // NOTE @distractedm1nd @renaynay: Remove whenever/if we decide to add auth to gateway - log.Warn("Gateway is enabled, however gateway endpoints are not authenticated. Use with caution!") baseComponents := fx.Options( fx.Supply(cfg), @@ -50,12 +48,13 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { "gateway", baseComponents, fx.Invoke(func( + cfg *Config, state stateServ.Module, share shareServ.Module, header headerServ.Module, serv *gateway.Server, ) { - Handler(state, share, header, nil, serv) + Handler(cfg, state, share, header, nil, serv) }), ) default: From 162e43899704956fa5e778e08d49f729d6f5485b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 10:11:36 +0000 Subject: [PATCH 039/388] chore(deps): bump github.com/pyroscope-io/client from 0.7.0 to 0.7.1 (#2319) Bumps [github.com/pyroscope-io/client](https://github.com/pyroscope-io/client) from 0.7.0 to 0.7.1.

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/pyroscope-io/client&package-manager=go_modules&previous-version=0.7.0&new-version=0.7.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5656aac5b3..bc783271b6 100644 --- a/go.mod +++ b/go.mod @@ -57,7 +57,7 @@ require ( github.com/multiformats/go-multihash v0.2.2-0.20221030163302-608669da49b6 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.14.0 - github.com/pyroscope-io/client v0.7.0 + github.com/pyroscope-io/client v0.7.1 github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 diff --git a/go.sum b/go.sum index 128a3b4b5e..ee2c193791 100644 --- a/go.sum +++ b/go.sum @@ -1772,8 +1772,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/pyroscope-io/client v0.7.0 h1:LWuuqPQ1oa6x7BnmUOuo/aGwdX85QGhWZUBYWWW3zdk= -github.com/pyroscope-io/client v0.7.0/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= +github.com/pyroscope-io/client v0.7.1 h1:yFRhj3vbgjBxehvxQmedmUWJQ4CAfCHhn+itPsuWsHw= +github.com/pyroscope-io/client v0.7.1/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE= github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic= From f291cf308352b69467e00362e14c3afad8b35983 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 14 Jun 2023 12:16:52 +0200 Subject: [PATCH 040/388] fix(header): GetByHeight requests for syncer head +1 shouldnt error (#2362) --- nodebuilder/header/service.go | 5 +++-- nodebuilder/tests/api_test.go | 38 +++++++++++++++++++++++++++++++++-- 2 files changed, 39 insertions(+), 4 deletions(-) diff --git a/nodebuilder/header/service.go b/nodebuilder/header/service.go index e6d7d46b8f..f410c04f04 100644 --- a/nodebuilder/header/service.go +++ b/nodebuilder/header/service.go @@ -37,7 +37,8 @@ func newHeaderService( sub libhead.Subscriber[*header.ExtendedHeader], p2pServer *p2p.ExchangeServer[*header.ExtendedHeader], ex libhead.Exchange[*header.ExtendedHeader], - store libhead.Store[*header.ExtendedHeader]) Module { + store libhead.Store[*header.ExtendedHeader], +) Module { return &Service{ syncer: syncer, sub: sub, @@ -66,7 +67,7 @@ func (s *Service) GetByHeight(ctx context.Context, height uint64) (*header.Exten return nil, err case uint64(head.Height()) == height: return head, nil - case uint64(head.Height()) < height: + case uint64(head.Height())+1 < height: return nil, fmt.Errorf("header: given height is from the future: "+ "networkHeight: %d, requestedHeight: %d", head.Height(), height) } diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index f740dc12fa..13cf083eee 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -2,8 +2,8 @@ package tests import ( "context" - "fmt" "testing" + "time" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/libp2p/go-libp2p/core/host" @@ -19,6 +19,41 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) +func TestGetByHeight(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) + + sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second)) + + // start a bridge node + bridge := sw.NewBridgeNode() + err := bridge.Start(ctx) + require.NoError(t, err) + + signer := bridge.AdminSigner + bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() + + jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + require.NoError(t, err) + + client, err := client.NewClient(ctx, bridgeAddr, jwt) + require.NoError(t, err) + + // let a few blocks be produced + _, err = client.Header.WaitForHeight(ctx, 3) + require.NoError(t, err) + + networkHead, err := client.Header.NetworkHead(ctx) + require.NoError(t, err) + _, err = client.Header.GetByHeight(ctx, uint64(networkHead.Height()+1)) + require.Nil(t, err, "Requesting syncer.Head()+1 shouldn't return an error") + + networkHead, err = client.Header.NetworkHead(ctx) + require.NoError(t, err) + _, err = client.Header.GetByHeight(ctx, uint64(networkHead.Height()+2)) + require.ErrorContains(t, err, "given height is from the future") +} + // TestBlobRPC ensures that blobs can be submited via rpc func TestBlobRPC(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) @@ -33,7 +68,6 @@ func TestBlobRPC(t *testing.T) { signer := bridge.AdminSigner bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() - fmt.Println("bridgeAddr: ", bridgeAddr) jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) require.NoError(t, err) From 2b8e151a3085fc5a4c81d162ef1f00697dbad34e Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 14 Jun 2023 13:13:05 +0200 Subject: [PATCH 041/388] fix(share/eds): fix test flake (#2364) --- share/eds/store.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/share/eds/store.go b/share/eds/store.go index f01e96a24b..f0d02a1141 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -119,6 +119,10 @@ func (s *Store) Start(ctx context.Context) error { // start Store only if DagStore succeeds ctx, cancel := context.WithCancel(context.Background()) s.cancel = cancel + // initialize empty gc result to avoid panic on access + s.lastGCResult.Store(&dagstore.GCResult{ + Shards: make(map[shard.Key]error), + }) go s.gc(ctx) return nil } @@ -132,10 +136,6 @@ func (s *Store) Stop(context.Context) error { // gc periodically removes all inactive or errored shards. func (s *Store) gc(ctx context.Context) { ticker := time.NewTicker(s.gcInterval) - // initialize empty gc result to avoid panic on access - s.lastGCResult.Store(&dagstore.GCResult{ - Shards: make(map[shard.Key]error), - }) for { select { case <-ctx.Done(): From 4c87c691ca1868f983bb45964b51c492db896ad8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:23:33 +0000 Subject: [PATCH 042/388] chore(deps): bump mheap/github-action-required-labels from 4 to 5 (#2354) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [mheap/github-action-required-labels](https://github.com/mheap/github-action-required-labels) from 4 to 5.
Release notes

Sourced from mheap/github-action-required-labels's releases.

v5

Tag that always points to the latest commit in the v5.x.x series of releases

v5.0.0

What's Changed

Full Changelog: https://github.com/mheap/github-action-required-labels/compare/v4.1.0...v5.0.0

v4.1.1

  • Fix build step by switching to ubuntu-latest

Full Changelog: https://github.com/mheap/github-action-required-labels/compare/v4.1.0...v4.1.1

v4.1.0

What's Changed

New Contributors

Full Changelog: https://github.com/mheap/github-action-required-labels/compare/v4.0.0...v4.1.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=mheap/github-action-required-labels&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index 314f2fcb14..bed2b3352c 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -12,7 +12,7 @@ jobs: label: runs-on: ubuntu-latest steps: - - uses: mheap/github-action-required-labels@v4 + - uses: mheap/github-action-required-labels@v5 with: mode: minimum count: 1 From 400517e0e32700fe7bc90e1de6644631c51b4996 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 14 Jun 2023 13:31:48 +0200 Subject: [PATCH 043/388] feat!(nodbuilder): making lifecycle timeouts configurable (#2347) Closes #2315 Doing this proactively even though it is a good-first issue in case other dagstore startup issues are not fixed in time. Breaks the config. --- nodebuilder/config.go | 2 ++ nodebuilder/node.go | 16 ++-------------- nodebuilder/node/config.go | 38 ++++++++++++++++++++++++++++++++++++++ 3 files changed, 42 insertions(+), 14 deletions(-) create mode 100644 nodebuilder/node/config.go diff --git a/nodebuilder/config.go b/nodebuilder/config.go index 3607aa593e..670bbf9bbd 100644 --- a/nodebuilder/config.go +++ b/nodebuilder/config.go @@ -25,6 +25,7 @@ type ConfigLoader func() (*Config, error) // Config is main configuration structure for a Node. // It combines configuration units for all Node subsystems. type Config struct { + Node node.Config Core core.Config State state.Config P2P p2p.Config @@ -39,6 +40,7 @@ type Config struct { // NOTE: Currently, configs are identical, but this will change. func DefaultConfig(tp node.Type) *Config { commonConfig := &Config{ + Node: node.DefaultConfig(tp), Core: core.DefaultConfig(), State: state.DefaultConfig(), P2P: p2p.DefaultConfig(tp), diff --git a/nodebuilder/node.go b/nodebuilder/node.go index a6e3a3c3cc..19760831cf 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -5,7 +5,6 @@ import ( "errors" "fmt" "strings" - "time" "github.com/cristalhq/jwt" "github.com/ipfs/go-blockservice" @@ -95,7 +94,7 @@ func NewWithConfig(tp node.Type, network p2p.Network, store Store, cfg *Config, // Start launches the Node and all its components and services. func (n *Node) Start(ctx context.Context) error { - to := timeout(n.Type) + to := n.Config.Node.StartupTimeout ctx, cancel := context.WithTimeout(ctx, to) defer cancel() @@ -141,7 +140,7 @@ func (n *Node) Run(ctx context.Context) error { // Canceling the given context earlier 'ctx' unblocks the Stop and aborts graceful shutdown forcing // remaining Modules/Services to close immediately. func (n *Node) Stop(ctx context.Context) error { - to := timeout(n.Type) + to := n.Config.Node.ShutdownTimeout ctx, cancel := context.WithTimeout(ctx, to) defer cancel() @@ -183,14 +182,3 @@ func newNode(opts ...fx.Option) (*Node, error) { // lifecycleFunc defines a type for common lifecycle funcs. type lifecycleFunc func(context.Context) error - -var DefaultLifecycleTimeout = time.Minute * 2 - -func timeout(tp node.Type) time.Duration { - switch tp { - case node.Light: - return time.Second * 20 - default: - return DefaultLifecycleTimeout - } -} diff --git a/nodebuilder/node/config.go b/nodebuilder/node/config.go new file mode 100644 index 0000000000..e44fe2f014 --- /dev/null +++ b/nodebuilder/node/config.go @@ -0,0 +1,38 @@ +package node + +import ( + "fmt" + "time" +) + +var defaultLifecycleTimeout = time.Minute * 2 + +type Config struct { + StartupTimeout time.Duration + ShutdownTimeout time.Duration +} + +// DefaultConfig returns the default node configuration for a given node type. +func DefaultConfig(tp Type) Config { + var timeout time.Duration + switch tp { + case Light: + timeout = time.Second * 20 + default: + timeout = defaultLifecycleTimeout + } + return Config{ + StartupTimeout: timeout, + ShutdownTimeout: timeout, + } +} + +func (c *Config) Validate() error { + if c.StartupTimeout == 0 { + return fmt.Errorf("invalid startup timeout: %v", c.StartupTimeout) + } + if c.ShutdownTimeout == 0 { + return fmt.Errorf("invalid shutdown timeout: %v", c.ShutdownTimeout) + } + return nil +} From 37d0af9141d2357349a463ecbb7459639cc8b8cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 11:41:58 +0000 Subject: [PATCH 044/388] chore(deps): bump golangci/golangci-lint-action from 3.4.0 to 3.6.0 (#2353) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.4.0 to 3.6.0.
Release notes

Sourced from golangci/golangci-lint-action's releases.

v3.6.0

What's Changed

New Contributors

Full Changelog: https://github.com/golangci/golangci-lint-action/compare/v3.5.0...v3.6.0

v3.5.0

What's Changed

... (truncated)

Commits
  • 639cd34 tests: increase timeout
  • 569abaa fix: out-format (#770)
  • c57cc43 build(deps-dev): bump typescript from 5.0.4 to 5.1.3 (#764)
  • 322510a feat: support out-format as args (#769)
  • 185e7a2 feat: add install-mode (#768)
  • 5be60c7 docs: improve args examples
  • 825a50d chore: update workflow and doc
  • 8c13ec4 doc: Add custom configuration file path to args (#767)
  • 416b5d0 build(deps-dev): bump @​typescript-eslint/parser from 5.59.7 to 5.59.8 (#765)
  • 66a6080 build(deps-dev): bump @​typescript-eslint/eslint-plugin from 5.59.7 to 5.59.8 ...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golangci/golangci-lint-action&package-manager=github_actions&previous-version=3.4.0&new-version=3.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- .github/workflows/go-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index b519db7763..e6c4e5881b 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -23,7 +23,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: golangci-lint - uses: golangci/golangci-lint-action@v3.4.0 + uses: golangci/golangci-lint-action@v3.6.0 with: version: v1.52.2 From d0e4353db6ffa9e28e97a8df867298e417e42edf Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 14 Jun 2023 13:56:45 +0200 Subject: [PATCH 045/388] fix(swamp): use synchronized map as container for nodes (#2358) --- go.mod | 2 +- nodebuilder/tests/fraud_test.go | 3 +- nodebuilder/tests/swamp/swamp.go | 123 ++++++++++++------------------- nodebuilder/tests/sync_test.go | 3 +- 4 files changed, 49 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index bc783271b6..524d1943ba 100644 --- a/go.mod +++ b/go.mod @@ -74,6 +74,7 @@ require ( go.uber.org/fx v1.19.3 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.9.0 + golang.org/x/exp v0.0.0-20230206171751-46f607a40771 golang.org/x/sync v0.1.0 golang.org/x/text v0.9.0 google.golang.org/grpc v1.53.0 @@ -309,7 +310,6 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/dig v1.16.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/exp v0.0.0-20230206171751-46f607a40771 // indirect golang.org/x/mod v0.9.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index 2022107cb7..c6876cb624 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -84,8 +84,7 @@ func TestFraudProofBroadcasting(t *testing.T) { require.ErrorIs(t, err, context.DeadlineExceeded) syncCancel() - require.NoError(t, full.Stop(ctx)) - require.NoError(t, sw.RemoveNode(full, node.Full)) + sw.StopNode(ctx, full) full = sw.NewNodeWithStore(node.Full, store) diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 5b99b577b1..1f21ac3bfa 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -5,6 +5,7 @@ import ( "crypto/rand" "fmt" "net" + "sync" "testing" "time" @@ -16,6 +17,7 @@ import ( ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" "go.uber.org/fx" + "golang.org/x/exp/maps" "github.com/celestiaorg/celestia-app/test/util/testnode" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" @@ -45,16 +47,16 @@ const DefaultTestTimeout = time.Minute * 5 // - Slices of created Bridge/Full/Light Nodes // - trustedHash taken from the CoreClient and shared between nodes type Swamp struct { - t *testing.T - Network mocknet.Mocknet - BridgeNodes []*nodebuilder.Node - FullNodes []*nodebuilder.Node - LightNodes []*nodebuilder.Node - comps *Config + t *testing.T + cfg *Config + Network mocknet.Mocknet ClientContext testnode.Context Accounts []string + nodesMu sync.Mutex + nodes map[*nodebuilder.Node]struct{} + genesis *header.ExtendedHeader } @@ -69,37 +71,38 @@ func NewSwamp(t *testing.T, options ...Option) *Swamp { option(ic) } - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - // Now, we are making an assumption that consensus mechanism is already tested out // so, we are not creating bridge nodes with each one containing its own core client // instead we are assigning all created BNs to 1 Core from the swamp cctx := core.StartTestNodeWithConfig(t, ic.TestConfig) swp := &Swamp{ t: t, + cfg: ic, Network: mocknet.New(), ClientContext: cctx, - comps: ic, Accounts: ic.Accounts, + nodes: map[*nodebuilder.Node]struct{}{}, } - swp.t.Cleanup(func() { - swp.stopAllNodes(ctx, swp.BridgeNodes, swp.FullNodes, swp.LightNodes) - }) - - swp.setupGenesis(ctx) + swp.t.Cleanup(swp.cleanup) + swp.setupGenesis() return swp } -// stopAllNodes goes through all received slices of Nodes and stops one-by-one -// this eliminates a manual clean-up in the test-cases itself in the end -func (s *Swamp) stopAllNodes(ctx context.Context, allNodes ...[]*nodebuilder.Node) { - for _, nodes := range allNodes { - for _, node := range nodes { - require.NoError(s.t, node.Stop(ctx)) - } - } +// cleanup frees up all the resources +// including stop of all created nodes +func (s *Swamp) cleanup() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + + require.NoError(s.t, s.Network.Close()) + + s.nodesMu.Lock() + defer s.nodesMu.Unlock() + maps.DeleteFunc(s.nodes, func(nd *nodebuilder.Node, _ struct{}) bool { + require.NoError(s.t, nd.Stop(ctx)) + return true + }) } // GetCoreBlockHashByHeight returns a tendermint block's hash by provided height @@ -158,7 +161,10 @@ func (s *Swamp) createPeer(ks keystore.Keystore) host.Host { // setupGenesis sets up genesis Header. // This is required to initialize and start correctly. -func (s *Swamp) setupGenesis(ctx context.Context) { +func (s *Swamp) setupGenesis() { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + // ensure core has surpassed genesis block s.WaitTillHeight(ctx, 2) @@ -180,7 +186,7 @@ func (s *Swamp) setupGenesis(ctx context.Context) { func (s *Swamp) DefaultTestConfig(tp node.Type) *nodebuilder.Config { cfg := nodebuilder.DefaultConfig(tp) - ip, port, err := net.SplitHostPort(s.comps.App.GRPC.Address) + ip, port, err := net.SplitHostPort(s.cfg.App.GRPC.Address) require.NoError(s.t, err) cfg.Core.IP = ip @@ -228,36 +234,29 @@ func (s *Swamp) NewNodeWithConfig(nodeType node.Type, cfg *nodebuilder.Config, o } // NewNodeWithStore creates a new instance of Node with predefined Store. -// Afterwards, the instance is stored in the swamp's Nodes' slice according to the -// node's type provided from the user. func (s *Swamp) NewNodeWithStore( - t node.Type, + tp node.Type, store nodebuilder.Store, options ...fx.Option, ) *nodebuilder.Node { - var n *nodebuilder.Node - signer := apptypes.NewKeyringSigner(s.ClientContext.Keyring, s.Accounts[0], s.ClientContext.ChainID) options = append(options, state.WithKeyringSigner(signer), ) - switch t { + switch tp { case node.Bridge: options = append(options, coremodule.WithClient(s.ClientContext.Client), ) - n = s.newNode(node.Bridge, store, options...) - s.BridgeNodes = append(s.BridgeNodes, n) - case node.Full: - n = s.newNode(node.Full, store, options...) - s.FullNodes = append(s.FullNodes, n) - case node.Light: - n = s.newNode(node.Light, store, options...) - s.LightNodes = append(s.LightNodes, n) + default: } - return n + nd := s.newNode(tp, store, options...) + s.nodesMu.Lock() + s.nodes[nd] = struct{}{} + s.nodesMu.Unlock() + return nd } func (s *Swamp) newNode(t node.Type, store nodebuilder.Store, options ...fx.Option) *nodebuilder.Node { @@ -284,43 +283,13 @@ func (s *Swamp) newNode(t node.Type, store nodebuilder.Store, options ...fx.Opti return node } -// RemoveNode removes a node from the swamp's node slice -// this allows reusage of the same var in the test scenario -// if the user needs to stop and start the same node -func (s *Swamp) RemoveNode(n *nodebuilder.Node, t node.Type) error { - var err error - switch t { - case node.Light: - s.LightNodes, err = s.remove(n, s.LightNodes) - return err - case node.Bridge: - s.BridgeNodes, err = s.remove(n, s.BridgeNodes) - return err - case node.Full: - s.FullNodes, err = s.remove(n, s.FullNodes) - return err - default: - return fmt.Errorf("no such type or node") - } -} - -func (s *Swamp) remove(rn *nodebuilder.Node, sn []*nodebuilder.Node) ([]*nodebuilder.Node, error) { - if len(sn) == 1 { - return nil, nil - } - - initSize := len(sn) - for i := 0; i < len(sn); i++ { - if sn[i] == rn { - sn = append(sn[:i], sn[i+1:]...) - i-- - } - } - - if initSize <= len(sn) { - return sn, fmt.Errorf("cannot delete the node") - } - return sn, nil +// StopNode stops the node and removes from Swamp. +// TODO(@Wondertan): For clean and symmetrical API, we may want to add StartNode. +func (s *Swamp) StopNode(ctx context.Context, nd *nodebuilder.Node) { + s.nodesMu.Lock() + delete(s.nodes, nd) + s.nodesMu.Unlock() + require.NoError(s.t, nd.Stop(ctx)) } // Connect allows to connect peers after hard disconnection. diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index 68f33bc61b..af311ba45c 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -121,8 +121,7 @@ func TestSyncStartStopLightWithBridge(t *testing.T) { require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) - require.NoError(t, light.Stop(ctx)) - require.NoError(t, sw.RemoveNode(light, node.Light)) + sw.StopNode(ctx, light) cfg = nodebuilder.DefaultConfig(node.Light) cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) From 7ef26dce43779715a17c14e0e0581bd8e350cea4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 12:04:05 +0000 Subject: [PATCH 046/388] chore(deps): bump github.com/imdario/mergo from 0.3.15 to 0.3.16 (#2355) Bumps [github.com/imdario/mergo](https://github.com/imdario/mergo) from 0.3.15 to 0.3.16.
Release notes

Sourced from github.com/imdario/mergo's releases.

Announcement: v1.0.0 will be released on June 18th

This release doesn't contain code changes.

After 10 years, with many corner cases covered, very few issues pending (at least, comparing them with the usage of the library as part of Docker, Kubernetes, Datadog's agent, etc.), and a very stable API, I think it's time to release a 1.0.0 version.

This version will be released under a vanity URL: dario.cat/mergo

PS: although I'll make sure that github.com/imdario/mergo will be available, I'm going to also change my GitHub handle, so expect for a few minutes to not be able to pull from github.com/imdario as I fork it from the new handle to the old one.

PS2: I'm creating a discussion for this release to make sure we can have a conversation around the topic, and anything else about Mergo that you care about.

Commits
  • 14fe2b1 fix: OpenSSF scorecard generation
  • 4cde94b fix: remove Travis link and fix tests actions
  • df62a52 chore: README and GitHub actions
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/imdario/mergo&package-manager=go_modules&previous-version=0.3.15&new-version=0.3.16)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 524d1943ba..9628a0c205 100644 --- a/go.mod +++ b/go.mod @@ -29,7 +29,7 @@ require ( github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-retryablehttp v0.7.2 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d - github.com/imdario/mergo v0.3.15 + github.com/imdario/mergo v0.3.16 github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.3.2 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index ee2c193791..bf659652b4 100644 --- a/go.sum +++ b/go.sum @@ -929,8 +929,8 @@ github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo= -github.com/imdario/mergo v0.3.15 h1:M8XP7IuFNsqUx6VPK2P9OSmsYsI/YFaGil0uD21V3dM= -github.com/imdario/mergo v0.3.15/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= +github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= +github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= From 168ff5f0e330ad46d37c1fdbd003b8ddd405d1d5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 14 Jun 2023 15:10:10 +0200 Subject: [PATCH 047/388] chore(deps): bump celestiaorg/.github from 0.1.1 to 0.2.0 (#2317) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [celestiaorg/.github](https://github.com/celestiaorg/.github) from 0.1.1 to 0.2.0.
Release notes

Sourced from celestiaorg/.github's releases.

v0.2.0

What's Changed

Full Changelog: https://github.com/celestiaorg/.github/compare/v0.1.1...v0.2.0

Commits
  • cb19bc0 feat: speed up amd64 availability & push amd64 on each commit (#60)
  • 5ff9239 SECURITY: minor update to scope language (#48)
  • 09f990f feat: TestGround GA (#57)
  • d51dd57 Add repository dispatch reusable workflow (#46)
  • fa412a8 chore(deps): bump actions/add-to-project from 0.4.1 to 0.5.0 (#55)
  • e3225bc chore(deps): bump actions/stale from 7 to 8 (#42)
  • 0a71110 copy over adr template (#40)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=celestiaorg/.github&package-manager=github_actions&previous-version=0.1.1&new-version=0.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> Co-authored-by: Ryan --- .github/workflows/ci_release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 574f425c7b..654c3c0a34 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -24,7 +24,7 @@ on: jobs: # Dockerfile Linting hadolint: - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.1.1 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.0 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: celestiaorg/.github/.github/actions/yamllint@v0.1.1 + - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.0 markdown-lint: name: Markdown Lint @@ -58,7 +58,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Version Release - uses: celestiaorg/.github/.github/actions/version-release@v0.1.1 + uses: celestiaorg/.github/.github/actions/version-release@v0.2.0 with: github-token: ${{secrets.GITHUB_TOKEN}} version-bump: ${{inputs.version}} From a4c66f49fafb6e69e943926a86d4413d67fd5027 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 15 Jun 2023 12:18:49 +0200 Subject: [PATCH 048/388] chore(nodebuilder/tests): Clean up sync_test tests (#2162) This PR is a part of the swamp cleanussy effort to clean up our integration tests and make them more deterministic + readable. This PR focuses on `sync_test.go` file. It also contains a change to fraud proof broadcasting test but I can happily split out into separate PR. --- nodebuilder/tests/fraud_test.go | 29 ++- nodebuilder/tests/p2p_test.go | 4 +- nodebuilder/tests/swamp/swamp.go | 39 ++- nodebuilder/tests/sync_test.go | 419 ++++++++++++++++++------------- 4 files changed, 299 insertions(+), 192 deletions(-) diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index c6876cb624..f652724d55 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -34,17 +34,18 @@ Another note: this test disables share exchange to speed up test results. */ func TestFraudProofBroadcasting(t *testing.T) { t.Skip("requires BEFP generation on app side to work") + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) const ( - blocks = 15 - bsize = 2 - btime = time.Millisecond * 300 + blocks = 15 + blockSize = 2 + blockTime = time.Millisecond * 300 ) - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) - t.Cleanup(cancel) - fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(blockTime)) + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, blockSize, blocks) + cfg := nodebuilder.DefaultConfig(node.Bridge) cfg.Share.UseShareExchange = false bridge := sw.NewNodeWithConfig( @@ -55,12 +56,13 @@ func TestFraudProofBroadcasting(t *testing.T) { err := bridge.Start(ctx) require.NoError(t, err) - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) - require.NoError(t, err) cfg = nodebuilder.DefaultConfig(node.Full) cfg.Share.UseShareExchange = false + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + require.NoError(t, err) cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) + store := nodebuilder.MockStore(t, cfg) full := sw.NewNodeWithStore(node.Full, store) @@ -72,9 +74,12 @@ func TestFraudProofBroadcasting(t *testing.T) { subscr, err := full.FraudServ.Subscribe(ctx, byzantine.BadEncoding) require.NoError(t, err) - p := <-subscr - require.Equal(t, 10, int(p.Height())) - + select { + case p := <-subscr: + require.Equal(t, 10, int(p.Height())) + case <-ctx.Done(): + t.Fatal("fraud proof was not received in time") + } // This is an obscure way to check if the Syncer was stopped. // If we cannot get a height header within a timeframe it means the syncer was stopped // FIXME: Eventually, this should be a check on service registry managing and keeping diff --git a/nodebuilder/tests/p2p_test.go b/nodebuilder/tests/p2p_test.go index 39c3c985a2..613dface94 100644 --- a/nodebuilder/tests/p2p_test.go +++ b/nodebuilder/tests/p2p_test.go @@ -142,7 +142,7 @@ func TestBootstrapNodesFromBridgeNode(t *testing.T) { // ensure that the light node is connected to the full node assert.True(t, light.Host.Network().Connectedness(addrFull.ID) == network.Connected) - sw.Disconnect(t, light.Host.ID(), full.Host.ID()) + sw.Disconnect(t, light, full) require.NoError(t, full.Stop(ctx)) select { case <-ctx.Done(): @@ -206,7 +206,7 @@ func TestRestartNodeDiscovery(t *testing.T) { connectSub, err := nodes[0].Host.EventBus().Subscribe(&event.EvtPeerConnectednessChanged{}) require.NoError(t, err) defer connectSub.Close() - sw.Disconnect(t, nodes[0].Host.ID(), nodes[1].Host.ID()) + sw.Disconnect(t, nodes[0], nodes[1]) require.NoError(t, node.Start(ctx)) // ensure that the last node is connected to one of the nodes diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 1f21ac3bfa..eabe33dc9c 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -51,6 +51,8 @@ type Swamp struct { cfg *Config Network mocknet.Mocknet + Bootstrappers []ma.Multiaddr + ClientContext testnode.Context Accounts []string @@ -210,6 +212,10 @@ func (s *Swamp) NewFullNode(options ...fx.Option) *nodebuilder.Node { cfg.Header.TrustedPeers = []string{ "/ip4/1.2.3.4/tcp/12345/p2p/12D3KooWNaJ1y1Yio3fFJEXCZyd1Cat3jmrPdgkYCrHfKD3Ce21p", } + // add all bootstrappers in suite as trusted peers + for _, bootstrapper := range s.Bootstrappers { + cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, bootstrapper.String()) + } store := nodebuilder.MockStore(s.t, cfg) return s.NewNodeWithStore(node.Full, store, options...) @@ -222,6 +228,10 @@ func (s *Swamp) NewLightNode(options ...fx.Option) *nodebuilder.Node { cfg.Header.TrustedPeers = []string{ "/ip4/1.2.3.4/tcp/12345/p2p/12D3KooWNaJ1y1Yio3fFJEXCZyd1Cat3jmrPdgkYCrHfKD3Ce21p", } + // add all bootstrappers in suite as trusted peers + for _, bootstrapper := range s.Bootstrappers { + cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, bootstrapper.String()) + } store := nodebuilder.MockStore(s.t, cfg) @@ -230,6 +240,10 @@ func (s *Swamp) NewLightNode(options ...fx.Option) *nodebuilder.Node { func (s *Swamp) NewNodeWithConfig(nodeType node.Type, cfg *nodebuilder.Config, options ...fx.Option) *nodebuilder.Node { store := nodebuilder.MockStore(s.t, cfg) + // add all bootstrappers in suite as trusted peers + for _, bootstrapper := range s.Bootstrappers { + cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, bootstrapper.String()) + } return s.NewNodeWithStore(nodeType, store, options...) } @@ -293,10 +307,10 @@ func (s *Swamp) StopNode(ctx context.Context, nd *nodebuilder.Node) { } // Connect allows to connect peers after hard disconnection. -func (s *Swamp) Connect(t *testing.T, peerA, peerB peer.ID) { - _, err := s.Network.LinkPeers(peerA, peerB) +func (s *Swamp) Connect(t *testing.T, peerA, peerB *nodebuilder.Node) { + _, err := s.Network.LinkPeers(peerA.Host.ID(), peerB.Host.ID()) require.NoError(t, err) - _, err = s.Network.ConnectPeers(peerA, peerB) + _, err = s.Network.ConnectPeers(peerA.Host.ID(), peerB.Host.ID()) require.NoError(t, err) } @@ -304,7 +318,20 @@ func (s *Swamp) Connect(t *testing.T, peerA, peerB peer.ID) { // re-establish it. Order is very important here. We have to unlink peers first, and only after // that call disconnect. This is hard disconnect and peers will not be able to reconnect. // In order to reconnect peers again, please use swamp.Connect -func (s *Swamp) Disconnect(t *testing.T, peerA, peerB peer.ID) { - require.NoError(t, s.Network.UnlinkPeers(peerA, peerB)) - require.NoError(t, s.Network.DisconnectPeers(peerA, peerB)) +func (s *Swamp) Disconnect(t *testing.T, peerA, peerB *nodebuilder.Node) { + require.NoError(t, s.Network.UnlinkPeers(peerA.Host.ID(), peerB.Host.ID())) + require.NoError(t, s.Network.DisconnectPeers(peerA.Host.ID(), peerB.Host.ID())) +} + +// SetBootstrapper sets the given bootstrappers as the "bootstrappers" for the +// Swamp test suite. Every new full or light node created on the suite afterwards +// will automatically add the suite's bootstrappers as trusted peers to their config. +// NOTE: Bridge nodes do not automaatically add the bootstrappers as trusted peers. +// NOTE: Use `NewNodeWithStore` to avoid this automatic configuration. +func (s *Swamp) SetBootstrapper(t *testing.T, bootstrappers ...*nodebuilder.Node) { + for _, trusted := range bootstrappers { + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(trusted.Host)) + require.NoError(t, err) + s.Bootstrappers = append(s.Bootstrappers, addrs[0]) + } } diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index af311ba45c..dfa3577599 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -5,73 +5,187 @@ import ( "testing" "time" - "github.com/libp2p/go-libp2p/core/host" - "github.com/libp2p/go-libp2p/core/peer" + ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-node/nodebuilder" - "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) // Common consts for tests producing filled blocks const ( - blocks = 20 - bsize = 16 - btime = time.Millisecond * 300 + numBlocks = 20 + bsize = 16 + btime = time.Millisecond * 300 ) /* -Test-Case: Sync a Light Node with a Bridge Node(includes DASing of non-empty blocks) +Test-Case: Header and block/sample sync against a Bridge Node of non-empty blocks. + Steps: 1. Create a Bridge Node(BN) 2. Start a BN 3. Check BN is synced to height 20 -4. Create a Light Node(LN) with a trusted peer + +Light node: +4. Create a Light Node (LN) with bridge as a trusted peer 5. Start a LN with a defined connection to the BN -6. Check LN is synced to height 30 +6. Check LN is header-synced to height 20 +7. Wait until LN has sampled height 20 +8. Wait for LN DASer to catch up to network head + +Full node: +4. Create a Full Node (FN) with bridge as a trusted peer +5. Start a FN with a defined connection to the BN +6. Check FN is header-synced to height 20 +7. Wait until FN has synced block at height 20 +8. Wait for FN DASer to catch up to network head */ -func TestSyncLightWithBridge(t *testing.T) { +func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) - fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) + // wait for core network to fill 20 blocks + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, numBlocks) + sw.WaitTillHeight(ctx, numBlocks) + // create a bridge node and set it as the bootstrapper for the suite bridge := sw.NewBridgeNode() - - sw.WaitTillHeight(ctx, 20) - + sw.SetBootstrapper(t, bridge) + // start bridge and wait for it to sync to 20 err := bridge.Start(ctx) require.NoError(t, err) - h, err := bridge.HeaderServ.WaitForHeight(ctx, 20) + h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) require.NoError(t, err) + require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) + + t.Run("light sync against bridge", func(t *testing.T) { + // create a light node that is connected to the bridge node as + // a bootstrapper + light := sw.NewLightNode() + // start light node and wait for it to sync 20 blocks + err = light.Start(ctx) + require.NoError(t, err) + h, err = light.HeaderServ.WaitForHeight(ctx, numBlocks) + require.NoError(t, err) + assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) + + // check that the light node has also sampled over the block at height 20 + err = light.ShareServ.SharesAvailable(ctx, h.DAH) + assert.NoError(t, err) + + // wait until the entire chain (up to network head) has been sampled + err = light.DASer.WaitCatchUp(ctx) + require.NoError(t, err) + }) + + t.Run("full sync against bridge", func(t *testing.T) { + // create a full node with bridge node as its bootstrapper + full := sw.NewFullNode() + // let full node sync 20 blocks + err = full.Start(ctx) + require.NoError(t, err) + h, err = full.HeaderServ.WaitForHeight(ctx, numBlocks) + require.NoError(t, err) + assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) + + // check to ensure the full node can sync the 20th block's data + err = full.ShareServ.SharesAvailable(ctx, h.DAH) + assert.NoError(t, err) + + // wait for full node to sync up the blocks from genesis -> network head. + err = full.DASer.WaitCatchUp(ctx) + require.NoError(t, err) + }) + + // wait for the core block filling process to exit + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case err := <-fillDn: + require.NoError(t, err) + } +} - require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 20)) +/* +Test-Case: Header and block/sample sync against a Bridge Node of empty blocks. - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) - require.NoError(t, err) +Steps: +1. Create a Bridge Node(BN) +2. Start a BN +3. Check BN is synced to height 20 - cfg := nodebuilder.DefaultConfig(node.Light) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - light := sw.NewNodeWithConfig(node.Light, cfg) +Light node: +4. Create a Light Node (LN) with bridge as a trusted peer +5. Start a LN with a defined connection to the BN +6. Check LN is header-synced to height 20 +7. Wait until LN has sampled height 20 +8. Wait for LN DASer to catch up to network head + +Full node: +4. Create a Full Node (FN) with bridge as a trusted peer +5. Start a FN with a defined connection to the BN +6. Check FN is header-synced to height 20 +7. Wait until FN has synced block at height 20 +8. Wait for FN DASer to catch up to network head +*/ +func TestSyncAgainstBridge_EmptyChain(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) - err = light.Start(ctx) - require.NoError(t, err) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + sw.WaitTillHeight(ctx, numBlocks) - h, err = light.HeaderServ.WaitForHeight(ctx, 30) + // create bridge node and set it as the bootstrapper for the suite + bridge := sw.NewBridgeNode() + sw.SetBootstrapper(t, bridge) + // start bridge and wait for it to sync to 20 + err := bridge.Start(ctx) require.NoError(t, err) - - err = light.ShareServ.SharesAvailable(ctx, h.DAH) - assert.NoError(t, err) - - err = light.DASer.WaitCatchUp(ctx) + h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) require.NoError(t, err) - - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) - require.NoError(t, <-fillDn) + require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) + + t.Run("light sync against bridge", func(t *testing.T) { + // create a light node that is connected to the bridge node as + // a bootstrapper + light := sw.NewLightNode() + // start light node and wait for it to sync 20 blocks + err = light.Start(ctx) + require.NoError(t, err) + h, err = light.HeaderServ.WaitForHeight(ctx, numBlocks) + require.NoError(t, err) + assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) + + // check that the light node has also sampled over the block at height 20 + err = light.ShareServ.SharesAvailable(ctx, h.DAH) + assert.NoError(t, err) + + // wait until the entire chain (up to network head) has been sampled + err = light.DASer.WaitCatchUp(ctx) + require.NoError(t, err) + }) + + t.Run("full sync against bridge", func(t *testing.T) { + // create a full node with bridge node as its bootstrapper + full := sw.NewFullNode() + // let full node sync 20 blocks + err = full.Start(ctx) + require.NoError(t, err) + h, err = full.HeaderServ.WaitForHeight(ctx, numBlocks) + require.NoError(t, err) + assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) + + // check to ensure the full node can sync the 20th block's data + err = full.ShareServ.SharesAvailable(ctx, h.DAH) + assert.NoError(t, err) + + // wait for full node to sync up the blocks from genesis -> network head. + err = full.DASer.WaitCatchUp(ctx) + require.NoError(t, err) + }) } /* @@ -85,106 +199,57 @@ Steps: 3. Check BN is synced to height 20 4. Create a Light Node(LN) with a trusted peer 5. Start a LN with a defined connection to the BN -6. Check LN is synced to height 30 -7. Stop LN -8. Start LN +6. Check LN is synced to height 20 +7. Disconnect LN from BN for 3 seconds while BN continues broadcasting new blocks from core +8. Re-connect LN and let it sync up again 9. Check LN is synced to height 40 */ func TestSyncStartStopLightWithBridge(t *testing.T) { - sw := swamp.NewSwamp(t) - - bridge := sw.NewBridgeNode() - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) - t.Cleanup(cancel) + defer cancel() - sw.WaitTillHeight(ctx, 50) + sw := swamp.NewSwamp(t) + // wait for core network to fill 20 blocks + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, numBlocks) + sw.WaitTillHeight(ctx, numBlocks) + // create bridge and set it as a bootstrapper + bridge := sw.NewBridgeNode() + sw.SetBootstrapper(t, bridge) + // and let bridge node sync up 20 blocks err := bridge.Start(ctx) require.NoError(t, err) - - h, err := bridge.HeaderServ.WaitForHeight(ctx, 20) + h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) require.NoError(t, err) + require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) - require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 20)) - - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + // create a light node and connect it to the bridge node as a bootstrapper + light := sw.NewLightNode() + // start light node and let it sync to 20 + err = light.Start(ctx) require.NoError(t, err) - - cfg := nodebuilder.DefaultConfig(node.Light) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - light := sw.NewNodeWithConfig(node.Light, cfg) - require.NoError(t, light.Start(ctx)) - - h, err = light.HeaderServ.WaitForHeight(ctx, 30) + h, err = light.HeaderServ.WaitForHeight(ctx, numBlocks) require.NoError(t, err) - - require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) + require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) sw.StopNode(ctx, light) - cfg = nodebuilder.DefaultConfig(node.Light) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - light = sw.NewNodeWithConfig(node.Light, cfg) + light = sw.NewLightNode() require.NoError(t, light.Start(ctx)) + // ensure when light node comes back up, it can sync the remainder of the chain it + // missed while sleeping h, err = light.HeaderServ.WaitForHeight(ctx, 40) require.NoError(t, err) - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 40)) -} - -/* -Test-Case: Sync a Full Node with a Bridge Node(includes DASing of non-empty blocks) -Steps: -1. Create a Bridge Node(BN) -2. Start a BN -3. Check BN is synced to height 20 -4. Create a Full Node(FN) with a connection to BN as a trusted peer -5. Start a FN -6. Check FN is synced to height 30 -*/ -func TestSyncFullWithBridge(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) - t.Cleanup(cancel) - - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) - fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) - - bridge := sw.NewBridgeNode() - - sw.WaitTillHeight(ctx, 20) - - err := bridge.Start(ctx) - require.NoError(t, err) - - h, err := bridge.HeaderServ.WaitForHeight(ctx, 20) - require.NoError(t, err) - - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 20)) - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) - require.NoError(t, err) - - cfg := nodebuilder.DefaultConfig(node.Full) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - cfg.Share.UseShareExchange = false - full := sw.NewNodeWithConfig(node.Full, cfg) - require.NoError(t, full.Start(ctx)) - - h, err = full.HeaderServ.WaitForHeight(ctx, 30) - require.NoError(t, err) - - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) - - err = full.ShareServ.SharesAvailable(ctx, h.DAH) - assert.NoError(t, err) - - err = full.DASer.WaitCatchUp(ctx) - require.NoError(t, err) - - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) - require.NoError(t, <-fillDn) + // wait for the core block filling process to exit + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case err := <-fillDn: + require.NoError(t, err) + } } /* @@ -198,59 +263,69 @@ Steps: 3. Check BN is synced to height 20 4. Create a Full Node(FN) with a connection to BN as a trusted peer 5. Start a FN -6. Check FN is synced to height 30 +6. Check FN is synced to network head 7. Create a Light Node(LN) with a connection to FN as a trusted peer -8. Start LN -9. Check LN is synced to height 50 +8. Ensure LN is NOT connected to BN and only connected to FN +9. Start LN +10. Check LN is synced to network head */ -func TestSyncLightWithFull(t *testing.T) { - sw := swamp.NewSwamp(t) - - bridge := sw.NewBridgeNode() - +func TestSyncLightAgainstFull(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) - sw.WaitTillHeight(ctx, 20) + sw := swamp.NewSwamp(t) + // wait for the core network to fill up 20 blocks + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, numBlocks) + sw.WaitTillHeight(ctx, numBlocks) + // create bridge and set it as a bootstrapper + bridge := sw.NewBridgeNode() + sw.SetBootstrapper(t, bridge) + // start a bridge node and wait for it to sync up 20 blocks err := bridge.Start(ctx) require.NoError(t, err) - - h, err := bridge.HeaderServ.WaitForHeight(ctx, 20) + h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) require.NoError(t, err) + assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 20)) - - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + // create a FN with BN as a trusted peer + full := sw.NewFullNode() + // start FN and wait for it to sync up to head of BN + err = full.Start(ctx) require.NoError(t, err) - - cfg := nodebuilder.DefaultConfig(node.Full) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - full := sw.NewNodeWithConfig(node.Full, cfg) - require.NoError(t, full.Start(ctx)) - - h, err = full.HeaderServ.WaitForHeight(ctx, 30) + bridgeHead, err := bridge.HeaderServ.LocalHead(ctx) require.NoError(t, err) - - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) - - addrs, err = peer.AddrInfoToP2pAddrs(host.InfoFromHost(full.Host)) + _, err = full.HeaderServ.WaitForHeight(ctx, uint64(bridgeHead.Height())) require.NoError(t, err) - cfg = nodebuilder.DefaultConfig(node.Light) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - light := sw.NewNodeWithConfig(node.Light, cfg) + // reset suite bootstrapper list and set full node as a bootstrapper for + // LN to connect to + sw.Bootstrappers = make([]ma.Multiaddr, 0) + sw.SetBootstrapper(t, full) + // create an LN with FN as a trusted peer + light := sw.NewLightNode() + + // ensure there is no direct connection between LN and BN so that + // LN relies only on FN for syncing err = sw.Network.UnlinkPeers(bridge.Host.ID(), light.Host.ID()) require.NoError(t, err) + // start LN and wait for it to sync up to network head against the head of the FN err = light.Start(ctx) require.NoError(t, err) - - h, err = light.HeaderServ.WaitForHeight(ctx, 50) + fullHead, err := full.HeaderServ.LocalHead(ctx) + require.NoError(t, err) + _, err = light.HeaderServ.WaitForHeight(ctx, uint64(fullHead.Height())) require.NoError(t, err) - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 50)) + // wait for the core block filling process to exit + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case err := <-fillDn: + require.NoError(t, err) + } } /* @@ -264,54 +339,54 @@ Steps: 3. Check BN is synced to height 20 4. Create a Full Node(FN) with a connection to BN as a trusted peer 5. Start a FN -6. Check FN is synced to height 30 -7. Create a Light Node(LN) with a connection to BN, FN as trusted peers +6. Check FN is synced to network head +7. Create a Light Node(LN) with a connection to BN and FN as trusted peers 8. Start LN -9. Check LN is synced to height 50 +9. Check LN is synced to network head. */ func TestSyncLightWithTrustedPeers(t *testing.T) { - sw := swamp.NewSwamp(t) - - bridge := sw.NewBridgeNode() - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) - sw.WaitTillHeight(ctx, 20) + sw := swamp.NewSwamp(t) + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, numBlocks) + sw.WaitTillHeight(ctx, numBlocks) + // create a BN and set as a bootstrapper + bridge := sw.NewBridgeNode() + sw.SetBootstrapper(t, bridge) + // let it sync to network head err := bridge.Start(ctx) require.NoError(t, err) - - h, err := bridge.HeaderServ.WaitForHeight(ctx, 20) + _, err = bridge.HeaderServ.WaitForHeight(ctx, numBlocks) require.NoError(t, err) - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 20)) + // create a FN with BN as trusted peer + full := sw.NewFullNode() - addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + // let FN sync to network head + err = full.Start(ctx) require.NoError(t, err) - - cfg := nodebuilder.DefaultConfig(node.Full) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - full := sw.NewNodeWithConfig(node.Full, cfg) - require.NoError(t, full.Start(ctx)) - - h, err = full.HeaderServ.WaitForHeight(ctx, 30) + err = full.HeaderServ.SyncWait(ctx) require.NoError(t, err) - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 30)) + // add full node as a bootstrapper for the suite + sw.SetBootstrapper(t, full) - addrs, err = peer.AddrInfoToP2pAddrs(host.InfoFromHost(full.Host)) - require.NoError(t, err) - - cfg = nodebuilder.DefaultConfig(node.Light) - cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - light := sw.NewNodeWithConfig(node.Light, cfg) + // create a LN with both FN and BN as trusted peers + light := sw.NewLightNode() + // let LN sync to network head err = light.Start(ctx) require.NoError(t, err) - - h, err = light.HeaderServ.WaitForHeight(ctx, 50) + err = light.HeaderServ.SyncWait(ctx) require.NoError(t, err) - assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 50)) + // wait for the core block filling process to exit + select { + case <-ctx.Done(): + t.Fatal(ctx.Err()) + case err := <-fillDn: + require.NoError(t, err) + } } From 9f304b42ea039a56bedee7f1833b9714b746c952 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 15 Jun 2023 19:52:59 +0800 Subject: [PATCH 049/388] feat(share): collect absence proof if namespace is not present in merkle tree (#2242) ## Overview Implements non-inclusion proof collector. Higher level integration (e.g. shrex) is coming as separate PR. --- share/get_test.go | 136 +++++++++++++++++++++++------------ share/getter.go | 4 +- share/getters/utils.go | 3 +- share/ipld/get.go | 37 +++++++--- share/ipld/namespace_data.go | 132 ++++++++++++++++++++++++---------- share/ipld/nmt.go | 19 +++++ 6 files changed, 234 insertions(+), 97 deletions(-) diff --git a/share/get_test.go b/share/get_test.go index 8eafe84cd8..25711e5a65 100644 --- a/share/get_test.go +++ b/share/get_test.go @@ -1,8 +1,12 @@ package share import ( + "bytes" "context" + "crypto/rand" + "errors" mrand "math/rand" + "sort" "strconv" "testing" "time" @@ -141,8 +145,8 @@ func removeRandShares(data [][]byte, d int) [][]byte { } func TestGetSharesByNamespace(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) bServ := mdutils.Bserv() var tests = []struct { @@ -168,6 +172,9 @@ func TestGetSharesByNamespace(t *testing.T) { for _, row := range eds.RowRoots() { rcid := ipld.MustCidFromNamespacedSha256(row) rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots())) + if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + continue + } require.NoError(t, err) shares = append(shares, rowShares...) @@ -182,8 +189,8 @@ func TestGetSharesByNamespace(t *testing.T) { } func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) bServ := mdutils.Bserv() shares := RandShares(t, 16) @@ -191,11 +198,7 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { // set all shares to the same namespace id nid := shares[0][:NamespaceSize] - for i, nspace := range shares { - if i == len(shares) { - break - } - + for _, nspace := range shares { copy(nspace[:NamespaceSize], nid) } @@ -230,59 +233,53 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { } func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) bServ := mdutils.Bserv() shares := RandShares(t, 16) - minNid := make([]byte, NamespaceSize) - midNid := make([]byte, NamespaceSize) - maxNid := make([]byte, NamespaceSize) - - numberOfShares := len(shares) - - copy(minNid, shares[0][:NamespaceSize]) - copy(maxNid, shares[numberOfShares-1][:NamespaceSize]) - copy(midNid, shares[numberOfShares/2][:NamespaceSize]) - - // create min nid missing data by replacing first namespace id with second - minNidMissingData := make([]Share, len(shares)) - copy(minNidMissingData, shares) - copy(minNidMissingData[0][:NamespaceSize], shares[1][:NamespaceSize]) - - // create max nid missing data by replacing last namespace id with second last - maxNidMissingData := make([]Share, len(shares)) - copy(maxNidMissingData, shares) - copy(maxNidMissingData[numberOfShares-1][:NamespaceSize], shares[numberOfShares-2][:NamespaceSize]) + // set all shares to the same namespace id + nids, err := randomNids(5) + require.NoError(t, err) + minNid := nids[0] + minIncluded := nids[1] + midNid := nids[2] + maxIncluded := nids[3] + maxNid := nids[4] - // create mid nid missing data by replacing middle namespace id with the one after - midNidMissingData := make([]Share, len(shares)) - copy(midNidMissingData, shares) - copy(midNidMissingData[numberOfShares/2][:NamespaceSize], shares[(numberOfShares/2)+1][:NamespaceSize]) + secondNamespaceFrom := mrand.Intn(len(shares)-2) + 1 + for i, nspace := range shares { + if i < secondNamespaceFrom { + copy(nspace[:NamespaceSize], minIncluded) + continue + } + copy(nspace[:NamespaceSize], maxIncluded) + } var tests = []struct { name string data []Share missingNid []byte + isAbsence bool }{ - {name: "Namespace id less than the minimum namespace in data", data: minNidMissingData, missingNid: minNid}, - {name: "Namespace id greater than the maximum namespace in data", data: maxNidMissingData, missingNid: maxNid}, - {name: "Namespace id in range but still missing", data: midNidMissingData, missingNid: midNid}, + {name: "Namespace id less than the minimum namespace in data", data: shares, missingNid: minNid}, + {name: "Namespace id greater than the maximum namespace in data", data: shares, missingNid: maxNid}, + {name: "Namespace id in range but still missing", data: shares, missingNid: midNid, isAbsence: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) - assertNoRowContainsNID(t, bServ, eds, tt.missingNid) + assertNoRowContainsNID(ctx, t, bServ, eds, tt.missingNid, tt.isAbsence) }) } } func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) + t.Cleanup(cancel) bServ := mdutils.Bserv() shares := RandShares(t, 16) @@ -306,6 +303,9 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi rcid := ipld.MustCidFromNamespacedSha256(row) data := ipld.NewNamespaceData(len(shares), nid, ipld.WithLeaves()) err := data.CollectLeavesByNamespace(ctx, bServ, rcid) + if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + continue + } assert.Nil(t, err) leaves := data.Leaves() for _, node := range leaves { @@ -317,7 +317,7 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi } func TestGetSharesWithProofsByNamespace(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) + ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) bServ := mdutils.Bserv() @@ -341,7 +341,7 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { } expected := tt.rawData[from] - nID := expected[:NamespaceSize] + nID := namespace.ID(expected[:NamespaceSize]) // change rawData to contain several shares with same nID for i := from; i <= to; i++ { @@ -356,6 +356,10 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { for _, row := range eds.RowRoots() { rcid := ipld.MustCidFromNamespacedSha256(row) rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots())) + if ipld.NamespaceIsOutsideRange(row, row, nID) { + require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange) + continue + } require.NoError(t, err) if len(rowShares) > 0 { require.NotNil(t, proof) @@ -432,10 +436,12 @@ func TestBatchSize(t *testing.T) { } func assertNoRowContainsNID( + ctx context.Context, t *testing.T, bServ blockservice.BlockService, eds *rsmt2d.ExtendedDataSquare, nID namespace.ID, + isAbsent bool, ) { rowRootCount := len(eds.RowRoots()) // get all row root cids @@ -445,11 +451,47 @@ func assertNoRowContainsNID( } // for each row root cid check if the minNID exists - for _, rowCID := range rowRootCIDs { + var absentCount, foundAbsenceRows int + for _, rowRoot := range eds.RowRoots() { + var outsideRange bool + if !ipld.NamespaceIsOutsideRange(rowRoot, rowRoot, nID) { + // nID does belong to namespace range of the row + absentCount++ + } else { + outsideRange = true + } data := ipld.NewNamespaceData(rowRootCount, nID, ipld.WithProofs()) - err := data.CollectLeavesByNamespace(context.Background(), bServ, rowCID) - leaves := data.Leaves() - assert.Nil(t, leaves) - assert.Nil(t, err) + rootCID := ipld.MustCidFromNamespacedSha256(rowRoot) + err := data.CollectLeavesByNamespace(ctx, bServ, rootCID) + if outsideRange { + require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange) + continue + } + require.NoError(t, err) + + // if no error returned, check absence proof + foundAbsenceRows++ + verified := data.Proof().VerifyNamespace(sha256.New(), nID, nil, rowRoot) + require.True(t, verified) + } + + if isAbsent { + require.Equal(t, foundAbsenceRows, absentCount) + // there should be max 1 row that has namespace range containing nID + require.LessOrEqual(t, absentCount, 1) + } +} + +func randomNids(total int) ([]namespace.ID, error) { + namespaces := make([]namespace.ID, total) + for i := range namespaces { + nid := make([]byte, NamespaceSize) + _, err := rand.Read(nid) + if err != nil { + return nil, err + } + namespaces[i] = nid } + sort.Slice(namespaces, func(i, j int) bool { return bytes.Compare(namespaces[i], namespaces[j]) < 0 }) + return namespaces, nil } diff --git a/share/getter.go b/share/getter.go index f7a7b9c129..6ba729c71a 100644 --- a/share/getter.go +++ b/share/getter.go @@ -10,6 +10,8 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share/ipld" ) var ( @@ -58,7 +60,7 @@ type NamespacedRow struct { func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { originalRoots := make([][]byte, 0) for _, row := range root.RowRoots { - if !nID.Less(nmt.MinNamespace(row, nID.Size())) && nID.LessOrEqual(nmt.MaxNamespace(row, nID.Size())) { + if !ipld.NamespaceIsOutsideRange(row, row, nID) { originalRoots = append(originalRoots, row) } } diff --git a/share/getters/utils.go b/share/getters/utils.go index c99a7689b8..c0b882f330 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -15,7 +15,6 @@ import ( "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/libs/utils" @@ -35,7 +34,7 @@ var ( func filterRootsByNamespace(root *share.Root, nID namespace.ID) []cid.Cid { rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) for _, row := range root.RowRoots { - if !nID.Less(nmt.MinNamespace(row, nID.Size())) && nID.LessOrEqual(nmt.MaxNamespace(row, nID.Size())) { + if !ipld.NamespaceIsOutsideRange(row, row, nID) { rowRootCIDs = append(rowRootCIDs, ipld.MustCidFromNamespacedSha256(row)) } } diff --git a/share/ipld/get.go b/share/ipld/get.go index 70385f73f7..6d7de35c27 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -103,7 +103,7 @@ func GetLeaves(ctx context.Context, // this buffer ensures writes to 'jobs' are never blocking (bin-tree-feat) jobs := make(chan *job, (maxShares+1)/2) // +1 for the case where 'maxShares' is 1 - jobs <- &job{id: root, ctx: ctx} + jobs <- &job{cid: root, ctx: ctx} // total is an amount of routines spawned and total amount of nodes we process (bin-tree-feat) // so we can specify exact amount of loops we do, and wait for this amount // of routines to finish processing @@ -123,11 +123,11 @@ func GetLeaves(ctx context.Context, defer wg.Done() span.SetAttributes( - attribute.String("cid", j.id.String()), + attribute.String("cid", j.cid.String()), attribute.Int("pos", j.sharePos), ) - nd, err := GetNode(ctx, bGetter, j.id) + nd, err := GetNode(ctx, bGetter, j.cid) if err != nil { // we don't really care about errors here // just fetch as much as possible @@ -149,7 +149,7 @@ func GetLeaves(ctx context.Context, // send those to be processed select { case jobs <- &job{ - id: lnk.Cid, + cid: lnk.Cid, // calc position for children nodes (bin-tree-feat), // s.t. 'if' above knows where to put a share sharePos: j.sharePos*2 + i, @@ -213,7 +213,7 @@ func GetProof( // chanGroup implements an atomic wait group, closing a jobs chan // when fully done. type chanGroup struct { - jobs chan *job + jobs chan job counter int64 } @@ -233,8 +233,29 @@ func (w *chanGroup) done() { // job represents an encountered node to investigate during the `GetLeaves` // and `CollectLeavesByNamespace` routines. type job struct { - id cid.Cid + // we pass the context to job so that spans are tracked in a tree + // structure + ctx context.Context + // cid of the node that will be handled + cid cid.Cid + // sharePos represents potential share position in share slice sharePos int - depth int - ctx context.Context + // depth represents the number of edges present in path from the root node of a tree to that node + depth int + // isAbsent indicates if target namespaceID is not included, only collect absence proofs + isAbsent bool +} + +func (j job) next(direction direction, cid cid.Cid, isAbsent bool) job { + var i int + if direction == right { + i++ + } + return job{ + ctx: j.ctx, + cid: cid, + sharePos: j.sharePos*2 + i, + depth: j.depth + 1, + isAbsent: isAbsent, + } } diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index 13be26c06a..b38e5884b5 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -18,6 +18,9 @@ import ( "github.com/celestiaorg/nmt/namespace" ) +var ErrNamespaceOutsideRange = errors.New("share/ipld: " + + "target namespace id is outside of namespace range for the given root") + // Option is the functional option that is applied to the NamespaceData instance // to configure data that needs to be stored. type Option func(*NamespaceData) @@ -40,11 +43,15 @@ func WithProofs() Option { // NamespaceData stores all leaves under the given namespace with their corresponding proofs. type NamespaceData struct { - leaves []ipld.Node - proofs *proofCollector + leaves []ipld.Node + proofs *proofCollector + bounds fetchedBounds maxShares int nID namespace.ID + + isAbsentNamespace atomic.Bool + absenceProofLeaf ipld.Node } func NewNamespaceData(maxShares int, nID namespace.ID, options ...Option) *NamespaceData { @@ -63,7 +70,7 @@ func NewNamespaceData(maxShares int, nID namespace.ID, options ...Option) *Names return data } -func (n *NamespaceData) validate() error { +func (n *NamespaceData) validate(rootCid cid.Cid) error { if len(n.nID) != NamespaceSize { return fmt.Errorf("expected namespace ID of size %d, got %d", NamespaceSize, len(n.nID)) } @@ -71,6 +78,11 @@ func (n *NamespaceData) validate() error { if n.leaves == nil && n.proofs == nil { return errors.New("share/ipld: empty NamespaceData, nothing specified to retrieve") } + + root := NamespacedSha256FromCID(rootCid) + if NamespaceIsOutsideRange(root, root, n.nID) { + return ErrNamespaceOutsideRange + } return nil } @@ -78,6 +90,14 @@ func (n *NamespaceData) addLeaf(pos int, nd ipld.Node) { // bounds will be needed in `Proof` method n.bounds.update(int64(pos)) + if n.isAbsentNamespace.Load() { + if n.absenceProofLeaf != nil { + log.Fatal("there should be only one absence leaf") + } + n.absenceProofLeaf = nd + return + } + if n.leaves == nil { return } @@ -140,6 +160,16 @@ func (n *NamespaceData) Proof() *nmt.Proof { nodes[i] = NamespacedSha256FromCID(node) } + if n.isAbsentNamespace.Load() { + proof := nmt.NewAbsenceProof( + int(n.bounds.lowest), + int(n.bounds.highest)+1, + nodes, + NamespacedSha256FromCID(n.absenceProofLeaf.Cid()), + NMTIgnoreMaxNamespace, + ) + return &proof + } proof := nmt.NewInclusionProof( int(n.bounds.lowest), int(n.bounds.highest)+1, @@ -159,7 +189,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( bGetter blockservice.BlockGetter, root cid.Cid, ) error { - if err := n.validate(); err != nil { + if err := n.validate(root); err != nil { return err } @@ -173,8 +203,8 @@ func (n *NamespaceData) CollectLeavesByNamespace( // buffer the jobs to avoid blocking, we only need as many // queued as the number of shares in the second-to-last layer - jobs := make(chan *job, (n.maxShares+1)/2) - jobs <- &job{id: root, ctx: ctx} + jobs := make(chan job, (n.maxShares+1)/2) + jobs <- job{cid: root, ctx: ctx} var wg chanGroup wg.jobs = jobs @@ -186,7 +216,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( ) for { - var j *job + var j job var ok bool select { case j, ok = <-jobs: @@ -203,13 +233,13 @@ func (n *NamespaceData) CollectLeavesByNamespace( defer wg.done() span.SetAttributes( - attribute.String("cid", j.id.String()), + attribute.String("cid", j.cid.String()), attribute.Int("pos", j.sharePos), ) // if an error is likely to be returned or not depends on // the underlying impl of the blockservice, currently it is not a realistic probability - nd, err := GetNode(ctx, bGetter, j.id) + nd, err := GetNode(ctx, bGetter, j.cid) if err != nil { singleErr.Do(func() { retrievalErr = err @@ -235,38 +265,11 @@ func (n *NamespaceData) CollectLeavesByNamespace( } // this node has links in the namespace, so keep walking - for i, lnk := range links { - newJob := &job{ - id: lnk.Cid, - // sharePos represents potential share position in share slice - sharePos: j.sharePos*2 + i, - // depth represents the number of edges present in path from the root node of a tree to that node - depth: j.depth + 1, - // we pass the context to job so that spans are tracked in a tree - // structure - ctx: ctx, - } - // if the link's nID isn't in range we don't need to create a new job for it, - // but need to collect a proof - jobNid := NamespacedSha256FromCID(newJob.id) - - // proof is on the right side, if the nID is less than min namespace of jobNid - if n.nID.Less(nmt.MinNamespace(jobNid, n.nID.Size())) { - n.addProof(right, lnk.Cid, newJob.depth) - continue - } - - // proof is on the left side, if the nID is bigger than max namespace of jobNid - if !n.nID.LessOrEqual(nmt.MaxNamespace(jobNid, n.nID.Size())) { - n.addProof(left, lnk.Cid, newJob.depth) - continue - } - - // by passing the previous check, we know we will have one more node to process - // note: it is important to increase the counter before sending to the channel + newJobs := n.traverseLinks(j, links) + for _, j := range newJobs { wg.add(1) select { - case jobs <- newJob: + case jobs <- j: case <-ctx.Done(): return } @@ -275,6 +278,57 @@ func (n *NamespaceData) CollectLeavesByNamespace( } } +func (n *NamespaceData) traverseLinks(j job, links []*ipld.Link) []job { + if j.isAbsent { + return n.collectAbsenceProofs(j, links) + } + return n.collectNDWithProofs(j, links) +} + +func (n *NamespaceData) collectAbsenceProofs(j job, links []*ipld.Link) []job { + leftLink := links[0].Cid + rightLink := links[1].Cid + // traverse to the left node, while collecting right node as proof + n.addProof(right, rightLink, j.depth) + return []job{j.next(left, leftLink, j.isAbsent)} +} + +func (n *NamespaceData) collectNDWithProofs(j job, links []*ipld.Link) []job { + leftCid := links[0].Cid + rightCid := links[1].Cid + leftLink := NamespacedSha256FromCID(leftCid) + rightLink := NamespacedSha256FromCID(rightCid) + + var nextJobs []job + // check if target namespace is outside of boundaries of both links + if NamespaceIsOutsideRange(leftLink, rightLink, n.nID) { + log.Fatalf("target namespace outside of boundaries of links at depth: %v", j.depth) + } + + if !NamespaceIsAboveMax(leftLink, n.nID) { + // namespace is within the range of left link + nextJobs = append(nextJobs, j.next(left, leftCid, false)) + } else { + // proof is on the left side, if the nID is on the right side of the range of left link + n.addProof(left, leftCid, j.depth) + if NamespaceIsBelowMin(rightLink, n.nID) { + // namespace is not included in either links, convert to absence collector + n.isAbsentNamespace.Store(true) + nextJobs = append(nextJobs, j.next(right, rightCid, true)) + return nextJobs + } + } + + if !NamespaceIsBelowMin(rightLink, n.nID) { + // namespace is within the range of right link + nextJobs = append(nextJobs, j.next(right, rightCid, false)) + } else { + // proof is on the right side, if the nID is on the left side of the range of right link + n.addProof(right, rightCid, j.depth) + } + return nextJobs +} + type fetchedBounds struct { lowest int64 highest int64 diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index 3923260555..df140ef8c7 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -20,6 +20,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/nmt" + "github.com/celestiaorg/nmt/namespace" ) var ( @@ -181,3 +182,21 @@ func Translate(dah *da.DataAvailabilityHeader, row, col int) (cid.Cid, int) { func NamespacedSha256FromCID(cid cid.Cid) []byte { return cid.Hash()[cidPrefixSize:] } + +// NamespaceIsAboveMax checks if the target namespace is above the maximum namespace for a given +// node hash. +func NamespaceIsAboveMax(nodeHash []byte, target namespace.ID) bool { + return !target.LessOrEqual(nmt.MaxNamespace(nodeHash, target.Size())) +} + +// NamespaceIsBelowMin checks if the target namespace is below the minimum namespace for a given +// node hash. +func NamespaceIsBelowMin(nodeHash []byte, target namespace.ID) bool { + return target.Less(nmt.MinNamespace(nodeHash, target.Size())) +} + +// NamespaceIsOutsideRange checks if the target namespace is outside the range defined by the left +// and right nodes +func NamespaceIsOutsideRange(leftNodeHash, rightNodeHash []byte, target namespace.ID) bool { + return NamespaceIsBelowMin(leftNodeHash, target) || NamespaceIsAboveMax(rightNodeHash, target) +} From 403eb61123d7d0659d38f0224dda060b9c3dd8da Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 15 Jun 2023 16:51:05 +0200 Subject: [PATCH 050/388] deps: bump libp2p (#2359) Closes #2211 This release brings a whole set of important fixes. --- go.mod | 78 ++++++------ go.sum | 164 ++++++++++++++------------ share/p2p/discovery/discovery_test.go | 13 +- 3 files changed, 133 insertions(+), 122 deletions(-) diff --git a/go.mod b/go.mod index 9628a0c205..5f5fe104a0 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( cosmossdk.io/math v1.0.0-beta.3 github.com/BurntSushi/toml v1.3.0 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 - github.com/benbjohnson/clock v1.3.0 + github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc2 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 @@ -31,7 +31,7 @@ require ( github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/imdario/mergo v0.3.16 github.com/ipfs/go-blockservice v0.5.0 - github.com/ipfs/go-cid v0.3.2 + github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.2.0 @@ -44,17 +44,17 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.10.0 github.com/ipld/go-car v0.6.0 - github.com/libp2p/go-libp2p v0.26.3 - github.com/libp2p/go-libp2p-kad-dht v0.21.0 + github.com/libp2p/go-libp2p v0.28.0 + github.com/libp2p/go-libp2p-kad-dht v0.21.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-routing-helpers v0.6.1 - github.com/minio/sha256-simd v1.0.0 + github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.8.0 + github.com/multiformats/go-multiaddr v0.9.0 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/multiformats/go-multihash v0.2.2-0.20221030163302-608669da49b6 + github.com/multiformats/go-multihash v0.2.2 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.14.0 github.com/pyroscope-io/client v0.7.1 @@ -74,11 +74,11 @@ require ( go.uber.org/fx v1.19.3 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.9.0 - golang.org/x/exp v0.0.0-20230206171751-46f607a40771 - golang.org/x/sync v0.1.0 + golang.org/x/exp v0.0.0-20230321023759-10a507213a29 + golang.org/x/sync v0.2.0 golang.org/x/text v0.9.0 google.golang.org/grpc v1.53.0 - google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 + google.golang.org/protobuf v1.30.0 ) require ( @@ -106,12 +106,12 @@ require ( github.com/cenkalti/backoff/v4 v4.2.0 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/chzyer/readline v1.5.0 // indirect + github.com/chzyer/readline v1.5.1 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect github.com/cometbft/cometbft-db v0.7.0 // indirect github.com/confio/ics23/go v0.9.0 // indirect - github.com/containerd/cgroups v1.0.4 // indirect + github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/cosmos-proto v1.0.0-alpha8 // indirect @@ -127,7 +127,7 @@ require ( github.com/davecgh/go-spew v1.1.1 // indirect github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c // indirect github.com/deckarep/golang-set/v2 v2.1.0 // indirect - github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 // indirect + github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect @@ -146,7 +146,7 @@ require ( github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/logr v1.2.4 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -154,19 +154,19 @@ require ( github.com/go-openapi/spec v0.19.11 // indirect github.com/go-openapi/swag v0.19.11 // indirect github.com/go-stack/stack v1.8.1 // indirect - github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 // indirect + github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/gateway v1.1.0 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect - github.com/golang/protobuf v1.5.2 // indirect + github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/btree v1.1.2 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect - github.com/google/pprof v0.0.0-20221203041831-ce31453925ec // indirect + github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect @@ -185,11 +185,11 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.1 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect - github.com/huin/goupnp v1.0.3 // indirect + github.com/huin/goupnp v1.2.0 // indirect github.com/iancoleman/orderedmap v0.1.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/inconshreveable/mousetrap v1.0.1 // indirect @@ -217,29 +217,29 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.15.15 // indirect - github.com/klauspost/cpuid/v2 v2.2.3 // indirect + github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/reedsolomon v1.11.1 // indirect - github.com/koron/go-ssdp v0.0.3 // indirect + github.com/koron/go-ssdp v0.0.4 // indirect github.com/lib/pq v1.10.6 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect - github.com/libp2p/go-libp2p-asn-util v0.2.0 // indirect + github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect - github.com/libp2p/go-nat v0.1.0 // indirect + github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.2.0 // indirect + github.com/libp2p/go-reuseport v0.3.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.0 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.17 // indirect + github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.50 // indirect + github.com/miekg/dns v1.1.54 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect @@ -250,11 +250,11 @@ require ( github.com/mtibben/percent v0.2.1 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect - github.com/multiformats/go-multibase v0.1.1 // indirect - github.com/multiformats/go-multicodec v0.8.0 // indirect + github.com/multiformats/go-multibase v0.2.0 // indirect + github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.5.1 // indirect + github.com/onsi/ginkgo/v2 v2.9.7 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -265,15 +265,15 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_model v0.3.0 // indirect + github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.39.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.2.1 // indirect - github.com/quic-go/qtls-go1-20 v0.1.1 // indirect + github.com/quic-go/qtls-go1-19 v0.3.2 // indirect + github.com/quic-go/qtls-go1-20 v0.2.2 // indirect github.com/quic-go/quic-go v0.33.0 // indirect - github.com/quic-go/webtransport-go v0.5.2 // indirect + github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -307,15 +307,15 @@ require ( go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.34.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect - go.uber.org/atomic v1.10.0 // indirect - go.uber.org/dig v1.16.1 // indirect + go.uber.org/atomic v1.11.0 // indirect + go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.9.0 // indirect + golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect golang.org/x/oauth2 v0.4.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect - golang.org/x/tools v0.7.0 // indirect + golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.103.0 // indirect google.golang.org/appengine v1.6.7 // indirect @@ -324,7 +324,7 @@ require ( gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect - lukechampine.com/blake3 v1.1.7 // indirect + lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) diff --git a/go.sum b/go.sum index bf659652b4..e548acff6a 100644 --- a/go.sum +++ b/go.sum @@ -295,8 +295,9 @@ github.com/aws/aws-sdk-go-v2/service/sso v1.1.1/go.mod h1:SuZJxklHxLAXgLTc1iFXbE github.com/aws/aws-sdk-go-v2/service/sts v1.1.1/go.mod h1:Wi0EBZwiz/K44YliU0EKxqTCJGUfYTWXrrBwkq736bM= github.com/aws/smithy-go v1.1.0/go.mod h1:EzMw8dbp/YJL4A5/sbhGddag+NPT7q084agLbB9LgIw= github.com/benbjohnson/clock v1.1.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= -github.com/benbjohnson/clock v1.3.0 h1:ip6w0uFQkncKQ979AypyG0ER7mqUSBdKLOgAle/AT8A= github.com/benbjohnson/clock v1.3.0/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= +github.com/benbjohnson/clock v1.3.5 h1:VvXlSJBzZpA/zum6Sj74hxwYI2DIxRWuNIoXAzHZz5o= +github.com/benbjohnson/clock v1.3.5/go.mod h1:J11/hYXuz8f4ySSvYwY0FKfm+ezbsZBKZxNJlLklBHA= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -382,14 +383,14 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= -github.com/chzyer/logex v1.2.0 h1:+eqR0HfOetur4tgnC8ftU5imRnhi4te+BadWS95c5AM= -github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= +github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= +github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= -github.com/chzyer/readline v1.5.0 h1:lSwwFrbNviGePhkewF1az4oLmcwqCZijQ2/Wi3BGHAI= -github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= +github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= +github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23 h1:dZ0/VyGgQdVGAss6Ju0dt5P0QltE0SFY5Woh6hbIfiQ= -github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= +github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= github.com/cilium/ebpf v0.4.0/go.mod h1:4tRaxcgiL706VnOzHOdBlY8IEAIdxINsQBcU4xJJXRs= github.com/circonus-labs/circonus-gometrics v2.3.1+incompatible/go.mod h1:nmEj6Dob7S7YxXgwXpfOuvO54S+tGdZdw9fuRZt25Ag= @@ -428,8 +429,8 @@ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1 github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= -github.com/containerd/cgroups v1.0.4 h1:jN/mbWBEaz+T1pi5OFtnkQ+8qnmEbAr1Oo1FRm5B0dA= -github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= +github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= +github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= @@ -492,11 +493,11 @@ github.com/davidlazar/go-crypto v0.0.0-20200604182044-b73af7476f6c/go.mod h1:6Uh github.com/deckarep/golang-set v1.8.0/go.mod h1:5nI87KwE7wgsBU1F4GKAw2Qod7p5kyS383rP6+o6qqo= github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6dtGktsI= github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= -github.com/decred/dcrd/crypto/blake256 v1.0.0 h1:/8DMNYp9SGi5f0w7uCm6d6M4OU2rGFK09Y2A4Xv7EE0= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= +github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0 h1:HbphB4TFFXpv7MNrT52FGrrgVXF1owhMVTHFZIlnvd4= -github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= github.com/deepmap/oapi-codegen v1.6.0/go.mod h1:ryDa9AgbELGeB+YEXE1dR53yAjHwFvE9iAUlWl9Al3M= github.com/deepmap/oapi-codegen v1.8.2 h1:SegyeYGcdi0jLLrpbCMoJxnUUn8GBXHsvr4rbzjuhfU= @@ -633,8 +634,8 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= -github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= +github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -670,8 +671,9 @@ github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= github.com/go-stack/stack v1.8.1 h1:ntEHSVwIt7PNXNpgPmVfMrNhLtgjlmnZha2kOpuRiDw= github.com/go-stack/stack v1.8.1/go.mod h1:dcoOX6HbPZSZptuspn9bctJ+N/CnF5gGygcUP3XYfe4= -github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0 h1:p104kn46Q8WdvHunIJ9dAyjPVtrBPhSr3KT2yUst43I= github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg7847qk6SyHyPtNmDHnmrv/HOrqktSC+C9fM+CJOE= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= +github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= @@ -732,8 +734,9 @@ github.com/golang/protobuf v1.4.2/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw github.com/golang/protobuf v1.4.3/go.mod h1:oDoupMAO8OvCJWAcko0GGGIgR6R6ocIYbsSw735rRwI= github.com/golang/protobuf v1.5.0/go.mod h1:FsONVRAS9T7sI+LIUmWTfcYkHO4aIWwzhcaSAoJOfIk= github.com/golang/protobuf v1.5.1/go.mod h1:DopwsBzvsk0Fs44TXzsVbJyPhcCPeIwnvohx4u74HPM= -github.com/golang/protobuf v1.5.2 h1:ROPKBNFfQgOUMifHyP+KYbvpjbdoFNs+aK7DXlji0Tw= github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= +github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= +github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= @@ -794,8 +797,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20221203041831-ce31453925ec h1:fR20TYVVwhK4O7r7y+McjRYyaTH6/vjwJOajE+XhlzM= -github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= +github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -901,8 +904,8 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.1 h1:5pv5N1lT1fjLg2VQ5KWc7kmucp2x/kvFOnxuVTqZ6x4= -github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= +github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -920,8 +923,9 @@ github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpO github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= -github.com/huin/goupnp v1.0.3 h1:N8No57ls+MnjlB+JPiCVSOyy/ot7MJTqlo7rn+NYSqQ= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= +github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= +github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3F6cCkg= @@ -972,8 +976,8 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= -github.com/ipfs/go-cid v0.3.2 h1:OGgOd+JCFM+y1DjWPmVH+2/4POtpDzwcr7VgnB7mZXc= -github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= +github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= +github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -1140,15 +1144,15 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.15.15 h1:EF27CXIuDsYJ6mmvtBRlEuB2UVOqHG1tAXgZ7yIO+lw= -github.com/klauspost/compress v1.15.15/go.mod h1:ZcK2JAFqKOpnBlxcLsJzYfrS9X1akm9fHZNnD9+Vo/4= +github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= +github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= -github.com/klauspost/cpuid/v2 v2.2.3 h1:sxCkb+qR91z4vsqw4vGGZlDgPz3G7gjaLyK3V8y70BU= -github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= +github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.11.1 h1:0gCWQXOB8pVe1Y5SGozDA5t2qoVxX3prsV+qHgI/Fik= @@ -1157,8 +1161,8 @@ github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxv github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.2/go.mod h1:XoLfkAiA2KeZsYh4DbHxD7h3nR2AZNqVQOa+LJuqPYs= -github.com/koron/go-ssdp v0.0.3 h1:JivLMY45N76b4p/vsWGOKewBQu6uf39y8l+AQ7sDKx8= -github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= +github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= +github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= @@ -1206,11 +1210,11 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= -github.com/libp2p/go-libp2p v0.26.3 h1:6g/psubqwdaBqNNoidbRKSTBEYgaOuKBhHl8Q5tO+PM= -github.com/libp2p/go-libp2p v0.26.3/go.mod h1:x75BN32YbwuY0Awm2Uix4d4KOz+/4piInkp4Wr3yOo8= +github.com/libp2p/go-libp2p v0.28.0 h1:zO8cY98nJiPzZpFv5w5gqqb8aVzt4ukQ0nVOSaaKhJ8= +github.com/libp2p/go-libp2p v0.28.0/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= -github.com/libp2p/go-libp2p-asn-util v0.2.0 h1:rg3+Os8jbnO5DxkC7K/Utdi+DkY3q/d1/1q+8WeNAsw= -github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= +github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= +github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= @@ -1255,8 +1259,8 @@ github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxn github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.21.0 h1:J0Yd22VA+sk0CJRGMgtfHvLVIkZDyJ3AJGiljywIw5U= -github.com/libp2p/go-libp2p-kad-dht v0.21.0/go.mod h1:Bhm9diAFmc6qcWAr084bHNL159srVZRKADdp96Qqd1I= +github.com/libp2p/go-libp2p-kad-dht v0.21.1 h1:xpfp8/t9+X2ip1l8Umap1/UGNnJ3RHJgKGAEsnRAlTo= +github.com/libp2p/go-libp2p-kad-dht v0.21.1/go.mod h1:Oy8wvbdjpB70eS5AaFaI68tOtrdo3KylTvXDjikxqFo= github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= @@ -1361,8 +1365,9 @@ github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0 github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= -github.com/libp2p/go-nat v0.1.0 h1:MfVsH6DLcpa04Xr+p8hmVRG4juse0s3J8HyNWYHffXg= github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= +github.com/libp2p/go-nat v0.2.0 h1:Tyz+bUFAYqGyJ/ppPPymMGbIgNRH+WqC5QrT5fKrrGk= +github.com/libp2p/go-nat v0.2.0/go.mod h1:3MJr+GRpRkyT65EpVPBstXLvOlAPzUVlG6Pwg9ohLJk= github.com/libp2p/go-netroute v0.1.2/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.3/go.mod h1:jZLDV+1PE8y5XxBySEBgbuVAXbhtuHSdmLPL2n9MKbk= github.com/libp2p/go-netroute v0.1.5/go.mod h1:V1SR3AaECRkEQCoFFzYwVYWvYIEtlxx89+O3qcpCl4A= @@ -1378,8 +1383,8 @@ github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/libp2p/go-reuseport v0.1.0/go.mod h1:bQVn9hmfcTaoo0c9v5pBhOarsU1eNOBZdaAd2hzXRKU= -github.com/libp2p/go-reuseport v0.2.0 h1:18PRvIMlpY6ZK85nIAicSBuXXvrYoSw3dsBAR7zc560= -github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= +github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= +github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= @@ -1468,8 +1473,8 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.17 h1:BTarxUcIeDqL27Mc+vyvdWYSL28zpIhv3RoTdsLMPng= -github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= +github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -1488,8 +1493,8 @@ github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.50 h1:DQUfb9uc6smULcREF09Uc+/Gd46YWqJd5DbpPE9xkcA= -github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= +github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1507,8 +1512,9 @@ github.com/minio/sha256-simd v0.0.0-20190328051042-05b4dd3047e5/go.mod h1:2FMWW+ github.com/minio/sha256-simd v0.1.0/go.mod h1:2FMWW+8GMoPweT6+pI63m9YE3Lmw4J71hV56Chs1E/U= github.com/minio/sha256-simd v0.1.1-0.20190913151208-6de447530771/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= github.com/minio/sha256-simd v0.1.1/go.mod h1:B5e1o+1/KgNmWrSQK08Y6Z1Vb5pwIktudl0J58iy0KM= -github.com/minio/sha256-simd v1.0.0 h1:v1ta+49hkWZyvaKwrQB8elexRqm6Y0aMLjCNsrYxo6g= github.com/minio/sha256-simd v1.0.0/go.mod h1:OuYzVNI5vcoYIAmbIvHPl3N3jUzVedXbKy5RFepssQM= +github.com/minio/sha256-simd v1.0.1 h1:6kaan5IFmwTNynnKKpDHe6FWHohJOHhCPchzK49dzMM= +github.com/minio/sha256-simd v1.0.1/go.mod h1:Pz6AKMiUdngCLpeTL/RJY1M9rUuPMYujV5xJjtbRSN8= github.com/mitchellh/cli v1.0.0/go.mod h1:hNIlj7HEI86fIcpObd7a0FcrxTWetlwJDGcceTlRvqc= github.com/mitchellh/go-homedir v1.0.0/go.mod h1:SfyaCUpYCn1Vlf4IUYiD9fPX4A5wJrkLzIz1N1q0pr0= github.com/mitchellh/go-homedir v1.1.0 h1:lukF9ziXFxDFPkA1vsr5zpc1XuPDn/wFntq5mG+4E0Y= @@ -1564,8 +1570,8 @@ github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9x github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= -github.com/multiformats/go-multiaddr v0.8.0 h1:aqjksEcqK+iD/Foe1RRFsGZh8+XFiGo7FgUCZlpv3LU= -github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= +github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1584,13 +1590,13 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= -github.com/multiformats/go-multibase v0.1.1 h1:3ASCDsuLX8+j4kx58qnJ4YFq/JWTJpCyDW27ztsVTOI= -github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= +github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= +github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= -github.com/multiformats/go-multicodec v0.8.0 h1:evBmgkbSQux+Ds2IgfhkO38Dl2GDtRW8/Rp6YiSHX/Q= -github.com/multiformats/go-multicodec v0.8.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= +github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= github.com/multiformats/go-multihash v0.0.5/go.mod h1:lt/HCbqlQwlPBz7lv0sQCdtfcMtlJvakRUn/0Ual8po= github.com/multiformats/go-multihash v0.0.8/go.mod h1:YSLudS+Pi8NHE7o6tb3D8vrpKa63epEDmG8nTduyAew= @@ -1599,8 +1605,8 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= -github.com/multiformats/go-multihash v0.2.2-0.20221030163302-608669da49b6 h1:qLF997Rz0X1WvdcZ2r5CUkLZ2rvdiXwG1JRSrJZEAuE= -github.com/multiformats/go-multihash v0.2.2-0.20221030163302-608669da49b6/go.mod h1:kaHxr8TfO1cxIR/tYxgZ7e59HraJq8arEQQR8E/YNvI= +github.com/multiformats/go-multihash v0.2.2 h1:Uu7LWs/PmWby1gkj1S1DXx3zyd3aVabA4FiMKn/2tAc= +github.com/multiformats/go-multihash v0.2.2/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= @@ -1652,8 +1658,8 @@ github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvw github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.5.1 h1:auzK7OI497k6x4OvWq+TKAcpcSAlod0doAH72oIN0Jw= -github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= +github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= +github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1661,7 +1667,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.24.0 h1:+0glovB9Jd6z3VR+ScSwQqXVTIfJcGA9UBM8yzQxhqg= +github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= @@ -1742,8 +1748,8 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.3.0 h1:UBgGFHqYdG/TPFD1B1ogZywDqEkwp3fBMvqdiQ7Xew4= -github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= +github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= +github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -1780,14 +1786,14 @@ github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKA github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/qtls-go1-19 v0.2.1 h1:aJcKNMkH5ASEJB9FXNeZCyTEIHU1J7MmHyz1Q1TSG1A= -github.com/quic-go/qtls-go1-19 v0.2.1/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.1.1 h1:KbChDlg82d3IHqaj2bn6GfKRj84Per2VGf5XV3wSwQk= -github.com/quic-go/qtls-go1-20 v0.1.1/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= +github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= +github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= +github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= -github.com/quic-go/webtransport-go v0.5.2 h1:GA6Bl6oZY+g/flt00Pnu0XtivSD8vukOu3lYhJjnGEk= -github.com/quic-go/webtransport-go v0.5.2/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= +github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= +github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= @@ -2079,10 +2085,10 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= -go.uber.org/atomic v1.10.0 h1:9qC72Qh0+3MqyJbAn8YU5xVq1frD8bn3JtD2oXtafVQ= -go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= -go.uber.org/dig v1.16.1 h1:+alNIBsl0qfY0j6epRubp/9obgtrObRAc5aD+6jbWY8= -go.uber.org/dig v1.16.1/go.mod h1:557JTAUZT5bUK0SvCwikmLPPtdQhfvLYtO5tJgQSbnk= +go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= +go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= +go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.19.3 h1:YqMRE4+2IepTYCMOvXqQpRa+QAVdiSTnsHU4XNWBceA= go.uber.org/fx v1.19.3/go.mod h1:w2HrQg26ql9fLK7hlBiZ6JsRUKV+Lj/atT1KCjT8YhM= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -2168,8 +2174,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771 h1:xP7rWLUr1e1n2xkK5YB4LI0hPEy3LJC6Wk+D4pGlOJg= -golang.org/x/exp v0.0.0-20230206171751-46f607a40771/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= +golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2201,8 +2207,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.9.0 h1:KENHtAZL2y3NLMYZeHY9DW8HW8V+kQyJsY/V9JlKvCs= -golang.org/x/mod v0.9.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= +golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2329,8 +2335,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.1.0 h1:wsuoTGHzEhffawBOhz5CYhcrV4IdKZbEyZjBMuTp12o= -golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= +golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2453,12 +2459,13 @@ golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2558,8 +2565,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.7.0 h1:W4OVu8VVOaIO0yzWMNdepAulS7YfoS3Zabrm8DOXXU4= -golang.org/x/tools v0.7.0/go.mod h1:4pg6aUX35JBAogB10C9AtvVL+qowtN4pT3CGSQex14s= +golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= +golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2818,8 +2825,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8 h1:KR8+MyP7/qOlV+8Af01LtjL04bu7on42eVsxT4EyBQk= -google.golang.org/protobuf v1.28.2-0.20220831092852-f930b1dc76e8/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= +google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -2872,8 +2879,9 @@ honnef.co/go/tools v0.0.1-2020.1.3/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9 honnef.co/go/tools v0.0.1-2020.1.4/go.mod h1:X/FiERA/W4tHapMX5mGpAtMSVEeEUOyHaw9vFzvIQ3k= honnef.co/go/tools v0.1.3/go.mod h1:NgwopIslSNH47DimFoV78dnkksY2EFtX0ajyb3K/las= lukechampine.com/blake3 v1.1.6/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= -lukechampine.com/blake3 v1.1.7 h1:GgRMhmdsuK8+ii6UZFDL8Nb+VyMwadAgcJyfYHxG6n0= lukechampine.com/blake3 v1.1.7/go.mod h1:tkKEOtDkNtklkXtLNEOGNq5tcV90tJiA1vAA12R78LA= +lukechampine.com/blake3 v1.2.1 h1:YuqqRuaqsGV71BV/nm9xlI0MKUv4QC54jQnBChWbGnI= +lukechampine.com/blake3 v1.2.1/go.mod h1:0OFRp7fBtAylGVCO40o87sbupkyIGgbpv1+M1k1LM6k= nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= diff --git a/share/p2p/discovery/discovery_test.go b/share/p2p/discovery/discovery_test.go index f0935086ef..06d88a9079 100644 --- a/share/p2p/discovery/discovery_test.go +++ b/share/p2p/discovery/discovery_test.go @@ -11,6 +11,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/p2p/discovery/routing" basic "github.com/libp2p/go-libp2p/p2p/host/basic" + "github.com/libp2p/go-libp2p/p2p/host/eventbus" swarmt "github.com/libp2p/go-libp2p/p2p/net/swarm/testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -21,7 +22,7 @@ func TestDiscovery(t *testing.T) { discoveryRetryTimeout = time.Millisecond * 100 // defined in discovery.go - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) t.Cleanup(cancel) tn := newTestnet(ctx, t) @@ -80,8 +81,9 @@ type testnet struct { } func newTestnet(ctx context.Context, t *testing.T) *testnet { - swarm := swarmt.GenSwarm(t, swarmt.OptDisableTCP) - hst, err := basic.NewHost(swarm, &basic.HostOpts{}) + bus := eventbus.NewBus() + swarm := swarmt.GenSwarm(t, swarmt.OptDisableTCP, swarmt.EventBus(bus)) + hst, err := basic.NewHost(swarm, &basic.HostOpts{EventBus: bus}) require.NoError(t, err) hst.Start() @@ -110,8 +112,9 @@ func (t *testnet) discovery(opts ...Option) *Discovery { } func (t *testnet) peer() (host.Host, discovery.Discovery) { - swarm := swarmt.GenSwarm(t.T, swarmt.OptDisableTCP) - hst, err := basic.NewHost(swarm, &basic.HostOpts{}) + bus := eventbus.NewBus() + swarm := swarmt.GenSwarm(t.T, swarmt.OptDisableTCP, swarmt.EventBus(bus)) + hst, err := basic.NewHost(swarm, &basic.HostOpts{EventBus: bus}) require.NoError(t.T, err) hst.Start() From d5b571de330e13f118507dde3fe8b36c864c9f2e Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 15 Jun 2023 17:55:54 +0200 Subject: [PATCH 051/388] feat(node): implement node.AuthVerify and node.AuthNew (#2370) --- api/rpc/server.go | 14 ++-------- libs/authtoken/authtoken.go | 18 ++++++++++++ nodebuilder/node.go | 3 +- nodebuilder/node/admin.go | 22 +++++++++------ nodebuilder/node/mocks/api.go | 4 +-- nodebuilder/node/module.go | 2 +- nodebuilder/node/node.go | 6 ++-- nodebuilder/node_test.go | 2 +- nodebuilder/tests/api_test.go | 52 +++++++++++++++++++++++++++++------ 9 files changed, 84 insertions(+), 39 deletions(-) diff --git a/api/rpc/server.go b/api/rpc/server.go index a4c8c21ce7..3357140e68 100644 --- a/api/rpc/server.go +++ b/api/rpc/server.go @@ -2,7 +2,6 @@ package rpc import ( "context" - "encoding/json" "net" "net/http" "reflect" @@ -15,6 +14,7 @@ import ( logging "github.com/ipfs/go-log/v2" "github.com/celestiaorg/celestia-node/api/rpc/perms" + "github.com/celestiaorg/celestia-node/libs/authtoken" ) var log = logging.Logger("rpc") @@ -51,17 +51,7 @@ func NewServer(address, port string, secret jwt.Signer) *Server { // reached if a token is provided in the header of the request, otherwise only // methods with `read` permissions are accessible. func (s *Server) verifyAuth(_ context.Context, token string) ([]auth.Permission, error) { - tk, err := jwt.ParseAndVerifyString(token, s.auth) - if err != nil { - return nil, err - } - p := new(perms.JWTPayload) - err = json.Unmarshal(tk.RawClaims(), p) - if err != nil { - return nil, err - } - // check permissions - return p.Allow, nil + return authtoken.ExtractSignedPermissions(s.auth, token) } // RegisterService registers a service onto the RPC server. All methods on the service will then be diff --git a/libs/authtoken/authtoken.go b/libs/authtoken/authtoken.go index c86272790d..3d6645c972 100644 --- a/libs/authtoken/authtoken.go +++ b/libs/authtoken/authtoken.go @@ -1,12 +1,30 @@ package authtoken import ( + "encoding/json" + "github.com/cristalhq/jwt" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/celestiaorg/celestia-node/api/rpc/perms" ) +// ExtractSignedPermissions returns the permissions granted to the token by the passed signer. +// If the token isn't signed by the signer, it will not pass verification. +func ExtractSignedPermissions(signer jwt.Signer, token string) ([]auth.Permission, error) { + tk, err := jwt.ParseAndVerifyString(token, signer) + if err != nil { + return nil, err + } + p := new(perms.JWTPayload) + err = json.Unmarshal(tk.RawClaims(), p) + if err != nil { + return nil, err + } + return p.Allow, nil +} + +// NewSignedJWT returns a signed JWT token with the passed permissions and signer. func NewSignedJWT(signer jwt.Signer, permissions []auth.Permission) (string, error) { token, err := jwt.NewTokenBuilder(signer).Build(&perms.JWTPayload{ Allow: permissions, diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 19760831cf..33132dc5e4 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -6,7 +6,6 @@ import ( "fmt" "strings" - "github.com/cristalhq/jwt" "github.com/ipfs/go-blockservice" exchange "github.com/ipfs/go-ipfs-exchange-interface" logging "github.com/ipfs/go-log/v2" @@ -49,7 +48,6 @@ type Node struct { Network p2p.Network Bootstrappers p2p.Bootstrappers Config *Config - AdminSigner jwt.Signer // rpc components RPCServer *rpc.Server // not optional @@ -70,6 +68,7 @@ type Node struct { FraudServ fraud.Module // not optional BlobServ blob.Module // not optional DASer das.Module // not optional + AdminServ node.Module // not optional // start and stop control ref internal fx.App lifecycle funcs to be called from Start and Stop start, stop lifecycleFunc diff --git a/nodebuilder/node/admin.go b/nodebuilder/node/admin.go index cad8a51361..c6c97625ef 100644 --- a/nodebuilder/node/admin.go +++ b/nodebuilder/node/admin.go @@ -2,21 +2,25 @@ package node import ( "context" - "fmt" + "github.com/cristalhq/jwt" "github.com/filecoin-project/go-jsonrpc/auth" logging "github.com/ipfs/go-log/v2" + + "github.com/celestiaorg/celestia-node/libs/authtoken" ) -const APIVersion = "v0.2.0" +const APIVersion = "v0.2.1" type module struct { - tp Type + tp Type + signer jwt.Signer } -func newModule(tp Type) Module { +func newModule(tp Type, signer jwt.Signer) Module { return &module{ - tp: tp, + tp: tp, + signer: signer, } } @@ -38,10 +42,10 @@ func (m *module) LogLevelSet(_ context.Context, name, level string) error { return logging.SetLogLevel(name, level) } -func (m *module) AuthVerify(context.Context, string) ([]auth.Permission, error) { - return []auth.Permission{}, fmt.Errorf("not implemented") +func (m *module) AuthVerify(_ context.Context, token string) ([]auth.Permission, error) { + return authtoken.ExtractSignedPermissions(m.signer, token) } -func (m *module) AuthNew(context.Context, []auth.Permission) ([]byte, error) { - return nil, fmt.Errorf("not implemented") +func (m *module) AuthNew(_ context.Context, permissions []auth.Permission) (string, error) { + return authtoken.NewSignedJWT(m.signer, permissions) } diff --git a/nodebuilder/node/mocks/api.go b/nodebuilder/node/mocks/api.go index 98df713429..d8789a771c 100644 --- a/nodebuilder/node/mocks/api.go +++ b/nodebuilder/node/mocks/api.go @@ -37,10 +37,10 @@ func (m *MockModule) EXPECT() *MockModuleMockRecorder { } // AuthNew mocks base method. -func (m *MockModule) AuthNew(arg0 context.Context, arg1 []auth.Permission) ([]byte, error) { +func (m *MockModule) AuthNew(arg0 context.Context, arg1 []auth.Permission) (string, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AuthNew", arg0, arg1) - ret0, _ := ret[0].([]byte) + ret0, _ := ret[0].(string) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/nodebuilder/node/module.go b/nodebuilder/node/module.go index e7ae88182e..5abfad8e5f 100644 --- a/nodebuilder/node/module.go +++ b/nodebuilder/node/module.go @@ -9,7 +9,7 @@ func ConstructModule(tp Type) fx.Option { return fx.Module( "node", fx.Provide(func(secret jwt.Signer) Module { - return newModule(tp) + return newModule(tp, secret) }), fx.Provide(secret), ) diff --git a/nodebuilder/node/node.go b/nodebuilder/node/node.go index c33f73ef58..18ce93615b 100644 --- a/nodebuilder/node/node.go +++ b/nodebuilder/node/node.go @@ -20,7 +20,7 @@ type Module interface { // AuthVerify returns the permissions assigned to the given token. AuthVerify(ctx context.Context, token string) ([]auth.Permission, error) // AuthNew signs and returns a new token with the given permissions. - AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error) + AuthNew(ctx context.Context, perms []auth.Permission) (string, error) } var _ Module = (*API)(nil) @@ -30,7 +30,7 @@ type API struct { Info func(context.Context) (Info, error) `perm:"admin"` LogLevelSet func(ctx context.Context, name, level string) error `perm:"admin"` AuthVerify func(ctx context.Context, token string) ([]auth.Permission, error) `perm:"admin"` - AuthNew func(ctx context.Context, perms []auth.Permission) ([]byte, error) `perm:"admin"` + AuthNew func(ctx context.Context, perms []auth.Permission) (string, error) `perm:"admin"` } } @@ -46,6 +46,6 @@ func (api *API) AuthVerify(ctx context.Context, token string) ([]auth.Permission return api.Internal.AuthVerify(ctx, token) } -func (api *API) AuthNew(ctx context.Context, perms []auth.Permission) ([]byte, error) { +func (api *API) AuthNew(ctx context.Context, perms []auth.Permission) (string, error) { return api.Internal.AuthNew(ctx, perms) } diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index bd5d1da811..e6775419c7 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -34,7 +34,7 @@ func TestLifecycle(t *testing.T) { require.NotNil(t, node.Host) require.NotNil(t, node.HeaderServ) require.NotNil(t, node.StateServ) - require.NotNil(t, node.AdminSigner) + require.NotNil(t, node.AdminServ) require.Equal(t, tt.tp, node.Type) ctx, cancel := context.WithCancel(context.Background()) diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 13cf083eee..37504ec3b3 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -13,29 +13,64 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" - "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) -func TestGetByHeight(t *testing.T) { +func TestNodeModule(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second)) - // start a bridge node bridge := sw.NewBridgeNode() err := bridge.Start(ctx) require.NoError(t, err) - signer := bridge.AdminSigner bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() - jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + writePerms := []auth.Permission{"public", "read", "write"} + adminPerms := []auth.Permission{"public", "read", "write", "admin"} + jwt, err := bridge.AdminServ.AuthNew(ctx, adminPerms) + require.NoError(t, err) + + client, err := client.NewClient(ctx, bridgeAddr, jwt) + require.NoError(t, err) + + info, err := client.Node.Info(ctx) + require.NoError(t, err) + require.Equal(t, info.APIVersion, node.APIVersion) + + perms, err := client.Node.AuthVerify(ctx, jwt) + require.NoError(t, err) + require.Equal(t, perms, adminPerms) + + writeJWT, err := client.Node.AuthNew(ctx, writePerms) + require.NoError(t, err) + + perms, err = client.Node.AuthVerify(ctx, writeJWT) + require.NoError(t, err) + require.Equal(t, perms, writePerms) + +} + +func TestGetByHeight(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) + + sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second)) + + // start a bridge node + bridge := sw.NewBridgeNode() + err := bridge.Start(ctx) + require.NoError(t, err) + + adminPerms := []auth.Permission{"public", "read", "write", "admin"} + jwt, err := bridge.AdminServ.AuthNew(ctx, adminPerms) require.NoError(t, err) + bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() client, err := client.NewClient(ctx, bridgeAddr, jwt) require.NoError(t, err) @@ -66,12 +101,11 @@ func TestBlobRPC(t *testing.T) { err := bridge.Start(ctx) require.NoError(t, err) - signer := bridge.AdminSigner - bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() - - jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + adminPerms := []auth.Permission{"public", "read", "write", "admin"} + jwt, err := bridge.AdminServ.AuthNew(ctx, adminPerms) require.NoError(t, err) + bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() client, err := client.NewClient(ctx, bridgeAddr, jwt) require.NoError(t, err) From 258dc78a2d29fece0782a7fe0f0cc1a95562c285 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 16 Jun 2023 05:55:39 -0400 Subject: [PATCH 052/388] fix: parseNamespace for <= 10 byte namespace IDs (#2325) Co-authored-by: Ryan --- cmd/celestia/rpc.go | 68 +++++++++++++++++++-------------- cmd/celestia/rpc_test.go | 82 ++++++++++++++++++++++++++++++++++++++++ 2 files changed, 121 insertions(+), 29 deletions(-) create mode 100644 cmd/celestia/rpc_test.go diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 767fca872c..18851dc797 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -16,6 +16,7 @@ import ( "github.com/spf13/cobra" + appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/api/rpc/client" @@ -112,17 +113,17 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[0] = root // 2. NamespaceID - nID, err := parseNamespace(params[1]) + nID, err := parseV0NamespaceID(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) + panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) } parsedParams[1] = nID case "Submit": // 1. NamespaceID var err error - nID, err := parseNamespace(params[0]) + nID, err := parseV0NamespaceID(params[0]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) + panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) } // 2. Blob data var blobData []byte @@ -162,9 +163,9 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[1] = num // 3. NamespaceID - nID, err := parseNamespace(params[2]) + nID, err := parseV0NamespaceID(params[2]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) + panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) } // 4. Blob data var blobData []byte @@ -201,9 +202,9 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[0] = num // 2. NamespaceID - nID, err := parseNamespace(params[1]) + nID, err := parseV0NamespaceID(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) + panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) } parsedParams[1] = nID // 3: Commitment @@ -221,9 +222,9 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[0] = num // 2. NamespaceID - nID, err := parseNamespace(params[1]) + nID, err := parseV0NamespaceID(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) + panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) } parsedParams[1] = []namespace.ID{nID} return parsedParams @@ -368,7 +369,7 @@ func sendJSONRPCRequest(namespace, method string, params []interface{}) { rawResponseJSON, err := parseJSON(string(responseBody)) if err != nil { - panic(err) + log.Fatalf("Error parsing JSON-RPC response: %v", err) } if printRequest { output, err := json.MarshalIndent(outputWithRequest{ @@ -418,32 +419,41 @@ func parseSignatureForHelpstring(methodSig reflect.StructField) string { return simplifiedSignature } -func parseNamespace(param string) (namespace.ID, error) { - var nID []byte - var err error +// parseV0NamespaceID parses a namespace ID from a base64 or hex string. The param +// is expected to be the user-specified portion of a v0 namespace ID (i.e. the +// last 10 bytes). +func parseV0NamespaceID(param string) (namespace.ID, error) { + userBytes, err := decodeToBytes(param) + if err != nil { + return nil, err + } + + if len(userBytes) > appns.NamespaceVersionZeroIDSize { + return nil, fmt.Errorf( + "namespace ID %v is too large to be a v0 namespace, want <= %v bytes", + userBytes, appns.NamespaceVersionZeroIDSize, + ) + } + // if the namespace ID is <= 10 bytes, left pad it with 0s + return share.NewNamespaceV0(userBytes) +} + +// decodeToBytes decodes a Base64 or hex input string into a byte slice. +func decodeToBytes(param string) ([]byte, error) { if strings.HasPrefix(param, "0x") { decoded, err := hex.DecodeString(param[2:]) if err != nil { return nil, fmt.Errorf("error decoding namespace ID: %w", err) } - nID = decoded - } else { - // otherwise, it's just a base64 string - nID, err = base64.StdEncoding.DecodeString(param) - if err != nil { - return nil, fmt.Errorf("error decoding namespace ID: %w", err) - } + return decoded, nil } - // if the namespace ID is 8 bytes, add v0 share + namespace prefix and zero pad - if len(nID) == 8 { - nID, err = share.NewNamespaceV0(nID) - if err != nil { - return nil, err - } + // otherwise, it's just a base64 string + decoded, err := base64.StdEncoding.DecodeString(param) + if err != nil { + return nil, fmt.Errorf("error decoding namespace ID: %w", err) } - return nID, nil + return decoded, nil } - func parseJSON(param string) (json.RawMessage, error) { var raw json.RawMessage err := json.Unmarshal([]byte(param), &raw) diff --git a/cmd/celestia/rpc_test.go b/cmd/celestia/rpc_test.go new file mode 100644 index 0000000000..9380a8b01e --- /dev/null +++ b/cmd/celestia/rpc_test.go @@ -0,0 +1,82 @@ +package main + +import ( + "testing" + + "github.com/stretchr/testify/assert" + + "github.com/celestiaorg/nmt/namespace" +) + +func Test_parseNamespaceID(t *testing.T) { + type testCase struct { + name string + param string + want namespace.ID + wantErr bool + } + testCases := []testCase{ + { + param: "0x0c204d39600fddd3", + name: "8 byte hex encoded namespace ID gets left padded", + want: namespace.ID{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, + }, + wantErr: false, + }, + { + name: "10 byte hex encoded namespace ID", + param: "0x42690c204d39600fddd3", + want: namespace.ID{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x42, 0x69, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, + }, + wantErr: false, + }, + { + name: "29 byte hex encoded namespace ID", + param: "0x0000000000000000000000000000000000000001010101010101010101", + want: namespace.ID{ + 0x0, // namespace version + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // v0 ID prefix + 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // namespace ID + }, + wantErr: true, + }, + { + name: "11 byte hex encoded namespace ID returns error", + param: "0x42690c204d39600fddd3a3", + want: namespace.ID{}, + wantErr: true, + }, + { + name: "10 byte base64 encoded namespace ID", + param: "QmkMIE05YA/d0w==", + want: namespace.ID{ + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, + 0x0, 0x0, 0x0, 0x0, 0x42, 0x69, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, + }, + wantErr: false, + }, + { + name: "not base64 or hex encoded namespace ID returns error", + param: "5748493939429", + want: namespace.ID{}, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := parseV0NamespaceID(tc.param) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + + } +} From fd1818446ff6f0e9f3c14d6cdf1996c33b7ecb3b Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Fri, 16 Jun 2023 19:34:24 +0800 Subject: [PATCH 053/388] add kind:testing label (#2375) ## Overview Add testing tag for pure testing PRs --- .github/workflows/labels.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/labels.yml b/.github/workflows/labels.yml index bed2b3352c..b9d4351bbd 100644 --- a/.github/workflows/labels.yml +++ b/.github/workflows/labels.yml @@ -16,4 +16,4 @@ jobs: with: mode: minimum count: 1 - labels: "kind:fix, kind:misc, kind:break!, kind:refactor, kind:feat, kind:deps, kind:docs, kind:ci, kind:chore" # yamllint disable-line rule:line-length + labels: "kind:fix, kind:misc, kind:break!, kind:refactor, kind:feat, kind:deps, kind:docs, kind:ci, kind:chore, kind:testing" # yamllint disable-line rule:line-length From 6654cdf4994dbd381efd0d6a29688c731177c855 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 16 Jun 2023 15:20:40 +0200 Subject: [PATCH 054/388] feat(libs/pidstore): Implement `PeerIDStore` (#2274) This PR implements a `PeerIDStore` that will be useful for both dumping useful peers inside of celestia-node's use of p2p.Exchange as well as inside of discovery. Related to https://github.com/celestiaorg/celestia-node/issues/1851 --------- Co-authored-by: Hlib Kanunnikov --- libs/pidstore/pidstore.go | 67 ++++++++++++++++++++++++++++++++++ libs/pidstore/pidstore_test.go | 59 ++++++++++++++++++++++++++++++ 2 files changed, 126 insertions(+) create mode 100644 libs/pidstore/pidstore.go create mode 100644 libs/pidstore/pidstore_test.go diff --git a/libs/pidstore/pidstore.go b/libs/pidstore/pidstore.go new file mode 100644 index 0000000000..2d4eb870a8 --- /dev/null +++ b/libs/pidstore/pidstore.go @@ -0,0 +1,67 @@ +package pidstore + +import ( + "context" + "encoding/json" + "fmt" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + logging "github.com/ipfs/go-log/v2" + "github.com/libp2p/go-libp2p/core/peer" +) + +var ( + storePrefix = datastore.NewKey("pidstore") + peersKey = datastore.NewKey("peers") + + log = logging.Logger("pidstore") +) + +// PeerIDStore is used to store/load peers to/from disk. +type PeerIDStore struct { + ds datastore.Datastore +} + +// NewPeerIDStore creates a new peer ID store backed by the given datastore. +func NewPeerIDStore(ds datastore.Datastore) *PeerIDStore { + return &PeerIDStore{ + ds: namespace.Wrap(ds, storePrefix), + } +} + +// Load loads the peers from datastore and returns them. +func (p *PeerIDStore) Load(ctx context.Context) ([]peer.ID, error) { + log.Debug("Loading peers") + + bin, err := p.ds.Get(ctx, peersKey) + if err != nil { + return nil, fmt.Errorf("pidstore: loading peers from datastore: %w", err) + } + + var peers []peer.ID + err = json.Unmarshal(bin, &peers) + if err != nil { + return nil, fmt.Errorf("pidstore: unmarshalling peer IDs: %w", err) + } + + log.Infow("Loaded peers from disk", "amount", len(peers)) + return peers, nil +} + +// Put persists the given peer IDs to the datastore. +func (p *PeerIDStore) Put(ctx context.Context, peers []peer.ID) error { + log.Debugw("Persisting peers to disk", "amount", len(peers)) + + bin, err := json.Marshal(peers) + if err != nil { + return fmt.Errorf("pidstore: marshal peerlist: %w", err) + } + + if err = p.ds.Put(ctx, peersKey, bin); err != nil { + return fmt.Errorf("pidstore: error writing to datastore: %w", err) + } + + log.Infow("Persisted peers successfully", "amount", len(peers)) + return nil +} diff --git a/libs/pidstore/pidstore_test.go b/libs/pidstore/pidstore_test.go new file mode 100644 index 0000000000..eafceff3fe --- /dev/null +++ b/libs/pidstore/pidstore_test.go @@ -0,0 +1,59 @@ +package pidstore + +import ( + "context" + "crypto/rand" + "crypto/rsa" + "testing" + "time" + + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/sync" + "github.com/libp2p/go-libp2p/core/crypto" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPutLoad(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer t.Cleanup(cancel) + + peerstore := NewPeerIDStore(sync.MutexWrap(datastore.NewMapDatastore())) + + ids, err := generateRandomPeerList(10) + require.NoError(t, err) + + err = peerstore.Put(ctx, ids) + require.NoError(t, err) + + retrievedPeerlist, err := peerstore.Load(ctx) + require.NoError(t, err) + + assert.Equal(t, len(ids), len(retrievedPeerlist)) + assert.Equal(t, ids, retrievedPeerlist) +} + +func generateRandomPeerList(length int) ([]peer.ID, error) { + peerlist := make([]peer.ID, length) + for i := range peerlist { + key, err := rsa.GenerateKey(rand.Reader, 2096) + if err != nil { + return nil, err + } + + _, pubkey, err := crypto.KeyPairFromStdKey(key) + if err != nil { + return nil, err + } + + peerID, err := peer.IDFromPublicKey(pubkey) + if err != nil { + return nil, err + } + + peerlist[i] = peerID + } + + return peerlist, nil +} From 30f061699af303fd1a0a214fc60d520d83175b37 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Wed, 21 Jun 2023 11:08:56 -0400 Subject: [PATCH 055/388] Update ci_release.yml (#2384) ## Overview I missed adding the `merge_group` trigger when enabling the merge queue. oops. Adding it now :-) ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- .github/workflows/ci_release.yml | 1 + .github/workflows/docker-build-publish.yml | 1 + 2 files changed, 2 insertions(+) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 654c3c0a34..653add9256 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -1,5 +1,6 @@ name: CI and Release on: + merge_group: push: branches: - main diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 0ddebbd750..c922e8b901 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -2,6 +2,7 @@ name: Docker Build & Publish # Trigger on all push events, new semantic version tags, and all PRs on: + merge_group: push: branches: - "**" From de88cffff715980177e9a2cfee4686ca440106b3 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 21 Jun 2023 17:24:24 +0200 Subject: [PATCH 056/388] feat(gateway): enabling CORS * on gateway (#2373) Closes #2263 Co-authored-by: Hlib Kanunnikov --- api/gateway/middleware.go | 8 ++++++++ api/gateway/server_test.go | 31 +++++++++++++++++++++++++++++-- 2 files changed, 37 insertions(+), 2 deletions(-) diff --git a/api/gateway/middleware.go b/api/gateway/middleware.go index 498b9c5d64..2c88b34185 100644 --- a/api/gateway/middleware.go +++ b/api/gateway/middleware.go @@ -18,9 +18,17 @@ func (h *Handler) RegisterMiddleware(srv *Server) { setContentType, checkPostDisabled(h.state), wrapRequestContext, + enableCors, ) } +func enableCors(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.Header().Set("Access-Control-Allow-Origin", "*") + next.ServeHTTP(w, r) + }) +} + func setContentType(next http.Handler) http.Handler { return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { w.Header().Add("Content-Type", "application/json") diff --git a/api/gateway/server_test.go b/api/gateway/server_test.go index e98d7a6091..cb8e3d17ae 100644 --- a/api/gateway/server_test.go +++ b/api/gateway/server_test.go @@ -12,8 +12,12 @@ import ( "github.com/stretchr/testify/require" ) +const ( + address = "localhost" + port = "0" +) + func TestServer(t *testing.T) { - address, port := "localhost", "0" server := NewServer(address, port) ctx, cancel := context.WithCancel(context.Background()) @@ -42,10 +46,33 @@ func TestServer(t *testing.T) { require.NoError(t, err) } +func TestCorsEnabled(t *testing.T) { + server := NewServer(address, port) + server.RegisterMiddleware(enableCors) + + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + err := server.Start(ctx) + require.NoError(t, err) + + // register ping handler + ping := new(ping) + server.RegisterHandlerFunc("/ping", ping.ServeHTTP, http.MethodGet) + + url := fmt.Sprintf("http://%s/ping", server.ListenAddr()) + + resp, err := http.Get(url) + require.NoError(t, err) + defer resp.Body.Close() + + require.NoError(t, err) + require.Equal(t, resp.Header.Get("Access-Control-Allow-Origin"), "*") +} + // TestServer_contextLeakProtection tests to ensure a context // deadline was added by the context wrapper middleware server-side. func TestServer_contextLeakProtection(t *testing.T) { - address, port := "localhost", "0" server := NewServer(address, port) server.RegisterMiddleware(wrapRequestContext) From 9adb61eb1d3dbe54e2c7e13cbd51e9de9205fc47 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 21 Jun 2023 17:26:41 +0200 Subject: [PATCH 057/388] share: introduce Namespace type (#2367) Mostly copied from the app's Namespace type with multiple modifications. In preparation for https://github.com/celestiaorg/celestia-node/issues/2301. We want a more straightforward Namespace type from what the app provides: * Repsents 1:1 mapping of the serialized form of Namespace [version byte + namespace id] * So that it does not allocate on the hot path when getting data by namespace * and had simpler json serialization --- share/namespace.go | 151 ++++++++++++++++++++++++++++++++++++++++ share/namespace_test.go | 84 ++++++++++++++++++++++ share/nid.go | 1 + 3 files changed, 236 insertions(+) create mode 100644 share/namespace.go create mode 100644 share/namespace_test.go diff --git a/share/namespace.go b/share/namespace.go new file mode 100644 index 0000000000..0af1d1a60e --- /dev/null +++ b/share/namespace.go @@ -0,0 +1,151 @@ +package share + +import ( + "bytes" + "encoding/hex" + "fmt" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" + "github.com/celestiaorg/nmt/namespace" +) + +// Various reserved namespaces. +var ( + MaxReservedNamespace = Namespace(appns.MaxReservedNamespace.Bytes()) + ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) + TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) + ReservedPaddingNamespace = Namespace(appns.ReservedPaddingNamespace.Bytes()) + TxNamespace = Namespace(appns.TxNamespace.Bytes()) + PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) +) + +// Namespace represents namespace of a Share. +// Consists of version byte and namespace ID. +type Namespace []byte + +// NamespaceFromBytes converts bytes into Namespace and validates it. +func NamespaceFromBytes(b []byte) (Namespace, error) { + n := Namespace(b) + return n, n.Validate() +} + +// Version reports version of the Namespace. +func (n Namespace) Version() byte { + return n[appns.NamespaceVersionSize-1] +} + +// ID reports ID of the Namespace. +func (n Namespace) ID() namespace.ID { + return namespace.ID(n[appns.NamespaceVersionSize:]) +} + +// ToNMT converts the whole Namespace(both Version and ID parts) into NMT's namespace.ID +// NOTE: Once https://github.com/celestiaorg/nmt/issues/206 is closed Namespace should become NNT's +// type. +func (n Namespace) ToNMT() namespace.ID { + return namespace.ID(n) +} + +// ToAppNamespace converts the Namespace to App's definition of Namespace. +// TODO: Unify types between node and app +func (n Namespace) ToAppNamespace() appns.Namespace { + return appns.Namespace{Version: n.Version(), ID: n.ID()} +} + +// Len reports the total length of the namespace. +func (n Namespace) Len() int { + return len(n) +} + +// String stringifies the Namespace. +func (n Namespace) String() string { + return hex.EncodeToString(n) +} + +// Equals compares two Namespaces. +func (n Namespace) Equals(target Namespace) bool { + return bytes.Equal(n, target) +} + +// Validate checks if the namespace is correct. +func (n Namespace) Validate() error { + if n.Len() != NamespaceSize { + return fmt.Errorf("invalid namespace length: expected %d, got %d", NamespaceSize, n.Len()) + } + if n.Version() != appns.NamespaceVersionZero && n.Version() != appns.NamespaceVersionMax { + return fmt.Errorf("invalid namespace version %v", n.Version()) + } + if len(n.ID()) != appns.NamespaceIDSize { + return fmt.Errorf("invalid namespace id length: expected %d, got %d", appns.NamespaceIDSize, n.ID().Size()) + } + if n.Version() == appns.NamespaceVersionZero && !bytes.HasPrefix(n.ID(), appns.NamespaceVersionZeroPrefix) { + return fmt.Errorf("invalid namespace id: expect %d leading zeroes", len(appns.NamespaceVersionZeroPrefix)) + } + return nil +} + +// ValidateDataNamespace checks if the Namespace contains real/useful data. +func (n Namespace) ValidateDataNamespace() error { + if err := n.Validate(); err != nil { + return err + } + if n.Equals(ParitySharesNamespace) || n.Equals(TailPaddingNamespace) { + return fmt.Errorf("invalid data namespace(%s): parity and tail padding namespace are forbidden", n) + } + return nil +} + +// ValidateBlobNamespace checks if the Namespace is valid blob namespace. +func (n Namespace) ValidateBlobNamespace() error { + if err := n.ValidateDataNamespace(); err != nil { + return err + } + if bytes.Compare(n, MaxReservedNamespace) < 1 { + return fmt.Errorf("invalid blob namespace(%s): reserved namespaces are forbidden", n) + } + return nil +} + +// IsAboveMax checks if the namespace is above the maximum namespace of the given hash. +func (n Namespace) IsAboveMax(nodeHash []byte) bool { + return !n.IsLessOrEqual(nodeHash[n.Len() : n.Len()*2]) +} + +// IsBelowMin checks if the target namespace is below the minimum namespace of the given hash. +func (n Namespace) IsBelowMin(nodeHash []byte) bool { + return n.IsLess(nodeHash[:n.Len()]) +} + +// IsOutsideRange checks if the namespace is outside the min-max range of the given hashes. +func (n Namespace) IsOutsideRange(leftNodeHash, rightNodeHash []byte) bool { + return n.IsBelowMin(leftNodeHash) || n.IsAboveMax(rightNodeHash) +} + +// Repeat copies the Namespace t times. +func (n Namespace) Repeat(t int) []Namespace { + ns := make([]Namespace, t) + for i := 0; i < t; i++ { + ns[i] = n + } + return ns +} + +// IsLess reports if the Namespace is less than the target. +func (n Namespace) IsLess(target Namespace) bool { + return bytes.Compare(n, target) == -1 +} + +// IsLessOrEqual reports if the Namespace is less than the target. +func (n Namespace) IsLessOrEqual(target Namespace) bool { + return bytes.Compare(n, target) < 1 +} + +// IsGreater reports if the Namespace is greater than the target. +func (n Namespace) IsGreater(target Namespace) bool { + return bytes.Compare(n, target) == 1 +} + +// IsGreaterOrEqualThan reports if the Namespace is greater or equal than the target. +func (n Namespace) IsGreaterOrEqualThan(target Namespace) bool { + return bytes.Compare(n, target) > -1 +} diff --git a/share/namespace_test.go b/share/namespace_test.go new file mode 100644 index 0000000000..8cc61b379b --- /dev/null +++ b/share/namespace_test.go @@ -0,0 +1,84 @@ +package share + +import ( + "bytes" + "testing" + + "github.com/stretchr/testify/assert" + + appns "github.com/celestiaorg/celestia-app/pkg/namespace" +) + +var ( + validID = append( + appns.NamespaceVersionZeroPrefix, + bytes.Repeat([]byte{1}, appns.NamespaceVersionZeroIDSize)..., + ) + tooShortID = append(appns.NamespaceVersionZeroPrefix, []byte{1}...) + tooLongID = append(appns.NamespaceVersionZeroPrefix, bytes.Repeat([]byte{1}, NamespaceSize)...) + invalidPrefixID = bytes.Repeat([]byte{1}, NamespaceSize) +) + +func TestFrom(t *testing.T) { + type testCase struct { + name string + bytes []byte + wantErr bool + want Namespace + } + validNamespace := []byte{} + validNamespace = append(validNamespace, appns.NamespaceVersionZero) + validNamespace = append(validNamespace, appns.NamespaceVersionZeroPrefix...) + validNamespace = append(validNamespace, bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)...) + parityNamespace := bytes.Repeat([]byte{0xFF}, NamespaceSize) + + testCases := []testCase{ + { + name: "valid namespace", + bytes: validNamespace, + wantErr: false, + want: append([]byte{appns.NamespaceVersionZero}, validID...), + }, + { + name: "parity namespace", + bytes: parityNamespace, + wantErr: false, + want: append([]byte{appns.NamespaceVersionMax}, bytes.Repeat([]byte{0xFF}, appns.NamespaceIDSize)...), + }, + { + name: "unsupported version", + bytes: append([]byte{1}, append( + appns.NamespaceVersionZeroPrefix, + bytes.Repeat([]byte{1}, NamespaceSize-len(appns.NamespaceVersionZeroPrefix))..., + )...), + wantErr: true, + }, + { + name: "unsupported id: too short", + bytes: append([]byte{appns.NamespaceVersionZero}, tooShortID...), + wantErr: true, + }, + { + name: "unsupported id: too long", + bytes: append([]byte{appns.NamespaceVersionZero}, tooLongID...), + wantErr: true, + }, + { + name: "unsupported id: invalid prefix", + bytes: append([]byte{appns.NamespaceVersionZero}, invalidPrefixID...), + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := NamespaceFromBytes(tc.bytes) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + }) + } +} diff --git a/share/nid.go b/share/nid.go index b7fd4e5836..7d960cc9e1 100644 --- a/share/nid.go +++ b/share/nid.go @@ -10,6 +10,7 @@ import ( // NewNamespaceV0 takes a variable size byte slice and creates a version 0 Namespace ID. // The byte slice must be <= 10 bytes. // If it is less than 10 bytes, it will be left padded to size 10 with 0s. +// TODO: Adapt for Namespace in the integration PR func NewNamespaceV0(subNId []byte) (namespace.ID, error) { if lnid := len(subNId); lnid > appns.NamespaceVersionZeroIDSize { return nil, fmt.Errorf("namespace id must be <= %v, but it was %v bytes", appns.NamespaceVersionZeroIDSize, lnid) From ce8ccb3677dd0498b5230ca5c3080a10003aa8d2 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 21 Jun 2023 17:28:02 +0200 Subject: [PATCH 058/388] feat(header): Check app version in eh validation and return err on mismatch (#2138) As recommended by @cmwaters, we should ensure (as a rudimentary check) that the headers being passed to us from core match in app versions as upgraded app versions can indicate breaking changes. We should error out if the app version does not match. This PR is still draft until celestia-app has a version out where they export the AppVersion as a const that we can depend on. Eventually we should support several versions via #2137 but for now, this is a good enough check. TODO: - [x] depend on const from app - [x] test --------- Co-authored-by: Hlib Kanunnikov --- go.mod | 2 +- go.sum | 4 ++-- header/header.go | 6 ++++++ header/headertest/verify_test.go | 8 ++++++++ 4 files changed, 17 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5f5fe104a0..aef01d86a8 100644 --- a/go.mod +++ b/go.mod @@ -330,7 +330,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 diff --git a/go.sum b/go.sum index e548acff6a..065c88774f 100644 --- a/go.sum +++ b/go.sum @@ -346,8 +346,8 @@ github.com/celestiaorg/celestia-app v1.0.0-rc2 h1:/u7eespYtBpQtBSz3P8/rKfz9rW7QO github.com/celestiaorg/celestia-app v1.0.0-rc2/go.mod h1:uiTWKTtRpVwvSiFDl2zausrU1ZBHBWgk7z52pfzJqJU= github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 h1:nmr9O5BflgNR1aWehs1ZFw4obA//M/+g+SrSMK9sOBA= github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27/go.mod h1:GVo91Wifg9KL/nFx9nPkpl0UIFdvvs4fhnly9GhGxZU= -github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11 h1:Rd5EvJx1nG3KurBspVN51RVmvif0Lp2UVURbG2ad3Cs= -github.com/celestiaorg/cosmos-sdk v1.13.0-sdk-v0.46.11/go.mod h1:xCG6OUkJy5KUMEg20Zk010lra9XjkmKS3+bk0wp7bd8= +github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11 h1:xDLC0ZvIwj9S2gTs9b8EespilL/u/vOGcqJuHz1r/eA= +github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11/go.mod h1:IwvD2nN3vEMkjxhTw/SF5tyJ0+x3GB0EdyQJyteK06U= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= diff --git a/header/header.go b/header/header.go index 25c3b0e5cf..e59d3802c1 100644 --- a/header/header.go +++ b/header/header.go @@ -10,6 +10,7 @@ import ( tmjson "github.com/tendermint/tendermint/libs/json" core "github.com/tendermint/tendermint/types" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/rsmt2d" @@ -115,6 +116,11 @@ func (eh *ExtendedHeader) Validate() error { return fmt.Errorf("ValidateBasic error on RawHeader at height %d: %w", eh.Height(), err) } + if eh.RawHeader.Version.App != appconsts.LatestVersion { + return fmt.Errorf("app version mismatch, expected: %d, got %d", appconsts.LatestVersion, + eh.RawHeader.Version.App) + } + err = eh.Commit.ValidateBasic() if err != nil { return fmt.Errorf("ValidateBasic error on Commit at height %d: %w", eh.Height(), err) diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 78da779a54..177352a4cf 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -8,6 +8,7 @@ import ( "github.com/stretchr/testify/assert" tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/celestiaorg/celestia-app/pkg/appconsts" libhead "github.com/celestiaorg/go-header" ) @@ -80,6 +81,13 @@ func TestVerify(t *testing.T) { }, err: true, }, + { + prepare: func() libhead.Header { + untrustedAdj.RawHeader.Version.App = appconsts.LatestVersion + 1 + return untrustedAdj + }, + err: true, + }, } for i, test := range tests { From 367fc66c6f9c68bcbea4906e335b7207d6854a31 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 21 Jun 2023 23:43:05 +0800 Subject: [PATCH 059/388] chore(share/getters): add unit test for ctxwithsplittimeout (#2374) ## Overview Adds extra safety. Co-authored-by: Hlib Kanunnikov --- share/getters/utils_test.go | 115 ++++++++++++++++++++++++++++++++++++ 1 file changed, 115 insertions(+) diff --git a/share/getters/utils_test.go b/share/getters/utils_test.go index 73e9400010..db8cd0f8cc 100644 --- a/share/getters/utils_test.go +++ b/share/getters/utils_test.go @@ -1,11 +1,14 @@ package getters import ( + "context" "errors" "fmt" "testing" + "time" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" ) func Test_ErrorContains(t *testing.T) { @@ -101,3 +104,115 @@ func Test_ErrorContains(t *testing.T) { }) } } + +func Test_ctxWithSplitTimeout(t *testing.T) { + type args struct { + ctxTimeout time.Duration + splitFactor []int + minTimeout time.Duration + } + tests := []struct { + name string + args args + want time.Duration + }{ + { + name: "ctxTimeout > minTimeout, splitFactor <= 0", + args: args{ + ctxTimeout: 3 * time.Minute, + splitFactor: []int{-1, 0}, + minTimeout: time.Minute, + }, + want: time.Minute, + }, + { + name: "ctxTimeout > minTimeout, splitFactor = 1", + args: args{ + ctxTimeout: 3 * time.Minute, + splitFactor: []int{1}, + minTimeout: time.Minute, + }, + want: 3 * time.Minute, + }, + { + name: "ctxTimeout > minTimeout, splitFactor = 2", + args: args{ + ctxTimeout: 3 * time.Minute, + splitFactor: []int{2}, + minTimeout: time.Minute, + }, + want: 3 * time.Minute / 2, + }, + { + name: "ctxTimeout > minTimeout, resulted timeout limited by minTimeout", + args: args{ + ctxTimeout: 3 * time.Minute, + splitFactor: []int{3, 4, 5}, + minTimeout: time.Minute, + }, + want: time.Minute, + }, + { + name: "ctxTimeout < minTimeout", + args: args{ + ctxTimeout: time.Minute, + splitFactor: []int{-1, 0, 1, 2, 3}, + minTimeout: 2 * time.Minute, + }, + want: time.Minute, + }, + { + name: "minTimeout = 0, splitFactor <= 1", + args: args{ + ctxTimeout: time.Minute, + splitFactor: []int{-1, 0, 1}, + minTimeout: 0, + }, + want: time.Minute, + }, + { + name: "minTimeout = 0, splitFactor > 1", + args: args{ + ctxTimeout: time.Minute, + splitFactor: []int{2}, + minTimeout: 0, + }, + want: time.Minute / 2, + }, + { + name: "no context timeout", + args: args{ + ctxTimeout: 0, + splitFactor: []int{-1, 0, 1, 2}, + minTimeout: time.Minute, + }, + want: 0, + }, + { + name: "no context timeout, minTimeout = 0", + args: args{ + ctxTimeout: 0, + splitFactor: []int{-1, 0, 1, 2}, + minTimeout: 0, + }, + want: 0, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + for _, sf := range tt.args.splitFactor { + ctx, cancel := context.WithTimeout(context.Background(), tt.args.ctxTimeout) + t.Cleanup(cancel) + got, _ := ctxWithSplitTimeout(ctx, sf, tt.args.minTimeout) + dl, ok := got.Deadline() + if !ok { + require.Equal(t, tt.want, 0) + continue + } + d := time.Until(dl) + require.True(t, d <= tt.want+time.Second) + require.True(t, d >= tt.want-time.Second) + } + }) + } +} From b222475b1d707e8c4e4cef9b67a5e47348c90bea Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 21 Jun 2023 11:12:33 -0500 Subject: [PATCH 060/388] chore: bump celestia-app rc4 (#2371) ## Overview I'm getting a failed unit test, `TestBlobModule`, locally but I'm also getting that on `main` ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords Co-authored-by: Hlib Kanunnikov --- go.mod | 54 +++++++++++++++++--------------- go.sum | 99 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 79 insertions(+), 74 deletions(-) diff --git a/go.mod b/go.mod index aef01d86a8..31420f8745 100644 --- a/go.mod +++ b/go.mod @@ -6,17 +6,17 @@ replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1 require ( cosmossdk.io/errors v1.0.0-beta.7 - cosmossdk.io/math v1.0.0-beta.3 + cosmossdk.io/math v1.0.0-rc.0 github.com/BurntSushi/toml v1.3.0 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc2 + github.com/celestiaorg/celestia-app v1.0.0-rc4 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.16.0 github.com/celestiaorg/rsmt2d v0.9.0 - github.com/cosmos/cosmos-sdk v0.46.11 + github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 github.com/dgraph-io/badger/v2 v2.2007.4 @@ -62,7 +62,7 @@ require ( github.com/spf13/cobra v1.6.1 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 - github.com/tendermint/tendermint v0.34.24 + github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/otel v1.13.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.34.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 @@ -77,20 +77,20 @@ require ( golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/sync v0.2.0 golang.org/x/text v0.9.0 - google.golang.org/grpc v1.53.0 + google.golang.org/grpc v1.55.0 google.golang.org/protobuf v1.30.0 ) require ( - cloud.google.com/go v0.107.0 // indirect - cloud.google.com/go/compute v1.15.1 // indirect + cloud.google.com/go v0.110.0 // indirect + cloud.google.com/go/compute v1.18.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.8.0 // indirect - cloud.google.com/go/storage v1.27.0 // indirect + cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/storage v1.28.1 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect - github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d // indirect + github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/StackExchange/wmi v1.2.1 // indirect @@ -116,10 +116,10 @@ require ( github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/cosmos-proto v1.0.0-alpha8 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect - github.com/cosmos/gogoproto v1.4.2 // indirect + github.com/cosmos/gogoproto v1.4.10 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect - github.com/cosmos/iavl v0.19.5 // indirect - github.com/cosmos/ibc-go/v6 v6.1.1 // indirect + github.com/cosmos/iavl v0.19.6 // indirect + github.com/cosmos/ibc-go/v6 v6.2.0 // indirect github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect @@ -158,7 +158,7 @@ require ( github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/gateway v1.1.0 // indirect - github.com/golang/glog v1.0.0 // indirect + github.com/golang/glog v1.1.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect @@ -168,7 +168,7 @@ require ( github.com/google/orderedcode v0.0.1 // indirect github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/google/uuid v1.3.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -221,7 +221,7 @@ require ( github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/reedsolomon v1.11.1 // indirect github.com/koron/go-ssdp v0.0.4 // indirect - github.com/lib/pq v1.10.6 // indirect + github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect @@ -259,14 +259,14 @@ require ( github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.9.5 // indirect - github.com/pelletier/go-toml/v2 v2.0.5 // indirect + github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect - github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 // indirect + github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.39.0 // indirect + github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect github.com/pyroscope-io/godeltaprof v0.1.0 // indirect github.com/quic-go/qpack v0.4.0 // indirect @@ -280,7 +280,7 @@ require ( github.com/regen-network/cosmos-proto v0.3.1 // indirect github.com/rivo/uniseg v0.4.2 // indirect github.com/rs/cors v1.8.2 // indirect - github.com/rs/zerolog v1.27.0 // indirect + github.com/rs/zerolog v1.29.1 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil v3.21.6+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect @@ -289,7 +289,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.14.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect - github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 // indirect + github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tendermint/tm-db v0.6.7 // indirect @@ -312,14 +312,14 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.4.0 // indirect + golang.org/x/oauth2 v0.6.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.103.0 // indirect + google.golang.org/api v0.110.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f // indirect + google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -330,8 +330,10 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 + // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app + github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28 ) diff --git a/go.sum b/go.sum index 065c88774f..3bf1455309 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w9 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.107.0 h1:qkj22L7bgkl6vIeZDlOY2po43Mx/TIa2Wsa7VR+PEww= -cloud.google.com/go v0.107.0/go.mod h1:wpc2eNrD7hXUTy8EKS10jkxpZBjASrORK7goS+3YX2I= +cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= +cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -75,8 +75,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.15.1 h1:7UGq3QknM33pw5xATlpzeoomNxsacIVvTqTTvbfajmE= -cloud.google.com/go/compute v1.15.1/go.mod h1:bjjoF/NtFUrkD/urWfdHaKuOPDR5nWIs63rR+SXhcpA= +cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= +cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -116,13 +116,13 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.8.0 h1:E2osAkZzxI/+8pZcxVLcDtAQx/u+hZXVryUaYQ5O0Kk= -cloud.google.com/go/iam v0.8.0/go.mod h1:lga0/y3iH6CX7sYqypWJ33hf7kkfXJag67naqGESjkE= +cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= +cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.3.0 h1:NjljC+FYPV3uh5/OwWT6pVU+doBqMg2x/rZlE+CamDs= +cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= @@ -179,8 +179,9 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= -cloud.google.com/go/storage v1.27.0 h1:YOO045NZI9RKfCj1c5A/ZtuuENUc8OAW+gHdGnDgyMQ= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= +cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= +cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -195,8 +196,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= cosmossdk.io/errors v1.0.0-beta.7 h1:gypHW76pTQGVnHKo6QBkb4yFOJjC+sUGRc5Al3Odj1w= cosmossdk.io/errors v1.0.0-beta.7/go.mod h1:mz6FQMJRku4bY7aqS/Gwfcmr/ue91roMEKAmDUDpBfE= -cosmossdk.io/math v1.0.0-beta.3 h1:TbZxSopz2LqjJ7aXYfn7nJSb8vNaBklW6BLpcei1qwM= -cosmossdk.io/math v1.0.0-beta.3/go.mod h1:3LYasri3Zna4XpbrTNdKsWmD5fHHkaNAod/mNT9XdE4= +cosmossdk.io/math v1.0.0-rc.0 h1:ml46ukocrAAoBpYKMidF0R2tQJ1Uxfns0yH8wqgMAFc= +cosmossdk.io/math v1.0.0-rc.0/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -223,8 +224,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/toml v1.3.0 h1:Ws8e5YmnrGEHzZEzg0YvK/7COGYtTC5PbaH9oSSbgfA= github.com/BurntSushi/toml v1.3.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d h1:nalkkPQcITbvhmL4+C4cKA87NW0tfm3Kl9VXRoPywFg= -github.com/ChainSafe/go-schnorrkel v0.0.0-20200405005733-88cbf1b4c40d/go.mod h1:URdX5+vg25ts3aCh8H5IFZybJYKWhJHYMTnf+ULtoC4= +github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM= +github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= @@ -338,16 +339,17 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= +github.com/bufbuild/protocompile v0.1.0 h1:HjgJBI85hY/qmW5tw/66sNDZ7z0UDdVSi/5r40WHw4s= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc2 h1:/u7eespYtBpQtBSz3P8/rKfz9rW7QOxkH8ebh8T4VxI= -github.com/celestiaorg/celestia-app v1.0.0-rc2/go.mod h1:uiTWKTtRpVwvSiFDl2zausrU1ZBHBWgk7z52pfzJqJU= -github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27 h1:nmr9O5BflgNR1aWehs1ZFw4obA//M/+g+SrSMK9sOBA= -github.com/celestiaorg/celestia-core v1.21.2-tm-v0.34.27/go.mod h1:GVo91Wifg9KL/nFx9nPkpl0UIFdvvs4fhnly9GhGxZU= -github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11 h1:xDLC0ZvIwj9S2gTs9b8EespilL/u/vOGcqJuHz1r/eA= -github.com/celestiaorg/cosmos-sdk v1.14.0-sdk-v0.46.11/go.mod h1:IwvD2nN3vEMkjxhTw/SF5tyJ0+x3GB0EdyQJyteK06U= +github.com/celestiaorg/celestia-app v1.0.0-rc4 h1:S48+XMsLnk0GkNHrUc5c+rlyyHLIl0ZJ/zh92jmSSTk= +github.com/celestiaorg/celestia-app v1.0.0-rc4/go.mod h1:i+jE3Hh8IQxlZnE5XSBlYF0PCNPvD3v5g8KmY095PEA= +github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28 h1:idHJK9i4WCkYOf5PXVWZbOs8pWkCiHHQGI4MZr0iMtQ= +github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28/go.mod h1:LOxHW9nA++/9U8TgvTyKo9TO3F09sWv8asKQs00m73U= +github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 h1:vaQKgaOm0w58JAvOgn2iDohqjH7kvvRqVKiMcBDWifA= +github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13/go.mod h1:G9XkhOJZde36FH0kt/1ayg4ZaioZEQmmRfMa/zQig0I= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= @@ -441,7 +443,6 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= -github.com/coreos/go-systemd/v22 v22.3.3-0.20220203105225-a9a7ef127534/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -455,14 +456,14 @@ github.com/cosmos/cosmos-sdk/api v0.1.0/go.mod h1:CupqQBskAOiTXO1XDZ/wrtWzN/wTxU github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= -github.com/cosmos/gogoproto v1.4.2 h1:UeGRcmFW41l0G0MiefWhkPEVEwvu78SZsHBvI78dAYw= -github.com/cosmos/gogoproto v1.4.2/go.mod h1:cLxOsn1ljAHSV527CHOtaIP91kK6cCrZETRBrkzItWU= +github.com/cosmos/gogoproto v1.4.10 h1:QH/yT8X+c0F4ZDacDv3z+xE3WU1P1Z3wQoLMBRJoKuI= +github.com/cosmos/gogoproto v1.4.10/go.mod h1:3aAZzeRWpAwr+SS/LLkICX2/kDFyaYVzckBDzygIxek= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= -github.com/cosmos/iavl v0.19.5 h1:rGA3hOrgNxgRM5wYcSCxgQBap7fW82WZgY78V9po/iY= -github.com/cosmos/iavl v0.19.5/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= -github.com/cosmos/ibc-go/v6 v6.1.1 h1:oqqMNyjj6SLQF8rvgCaDGwfdITEIsbhs8F77/8xvRIo= -github.com/cosmos/ibc-go/v6 v6.1.1/go.mod h1:NL17FpFAaWjRFVb1T7LUKuOoMSsATPpu+Icc4zL5/Ik= +github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU= +github.com/cosmos/iavl v0.19.6/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= +github.com/cosmos/ibc-go/v6 v6.2.0 h1:HKS5WNxQrlmjowHb73J9LqlNJfvTnvkbhXZ9QzNTU7Q= +github.com/cosmos/ibc-go/v6 v6.2.0/go.mod h1:+S3sxcNwOhgraYDJAhIFDg5ipXHaUnJrg7tOQqGyWlc= github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA= github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= @@ -697,8 +698,9 @@ github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ= github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= +github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= +github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -778,8 +780,8 @@ github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPg github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= -github.com/google/martian/v3 v3.2.1 h1:d8MncMlErDFTwQGBK1xhv026j9kqhvw1Qv9IbWT1VLQ= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= +github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= github.com/google/orderedcode v0.0.1 h1:UzfcAexk9Vhv8+9pNOgRu41f16lHq725vPwnSeiG/Us= github.com/google/orderedcode v0.0.1/go.mod h1:iVyU4/qPKHY5h/wSd6rZZCDcLJNxiWO6dvsYES2Sb20= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -808,8 +810,9 @@ github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= -github.com/googleapis/enterprise-certificate-proxy v0.2.0 h1:y8Yozv7SZtlU//QXbezB6QkpuE6jMD2/gfzk4AftXjs= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= +github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= +github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -1101,7 +1104,7 @@ github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1C github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= -github.com/jhump/protoreflect v1.12.1-0.20220721211354-060cc04fc18b h1:izTof8BKh/nE1wrKOrloNA5q4odOarjf+Xpe+4qow98= +github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -1183,8 +1186,8 @@ github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2 github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= -github.com/lib/pq v1.10.6 h1:jbk+ZieJ0D7EVGJYpL9QTz7/YW6UHbmdnZWYyK5cdBs= -github.com/lib/pq v1.10.6/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= +github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= +github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/libp2p/go-addr-util v0.0.1/go.mod h1:4ac6O7n9rIAKB1dnd+s8IbbMXkt+oBpzX4/+RACcnlQ= github.com/libp2p/go-addr-util v0.0.2/go.mod h1:Ecd6Fb3yIuLzq4bD7VcywcVSBtefcAwnUISBM3WG15E= github.com/libp2p/go-addr-util v0.1.0/go.mod h1:6I3ZYuFr2O/9D+SoyM0zEw0EF3YkldtTX406BpdQMqw= @@ -1701,15 +1704,16 @@ github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtP github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= -github.com/pelletier/go-toml/v2 v2.0.5 h1:ipoSadvV8oGUjnUbMub59IDPPwfxF694nG/jwbMiyQg= -github.com/pelletier/go-toml/v2 v2.0.5/go.mod h1:OMHamSCAODeSsVrwwvcJOaoN0LIUIaFVNZzmWyNfXas= +github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= +github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 h1:1/WtZae0yGtPq+TI6+Tv1WTxkukpXeMlviSxvL7SRgk= github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9/go.mod h1:x3N5drFsm2uilKKuuYo6LdyD8vZAW55sH/9w+pbo1sw= github.com/peterh/liner v1.0.1-0.20180619022028-8c1271fcf47f/go.mod h1:xIteQHvHuaLYG9IFj6mSxM0fCKrs34IrEQUhOYuGPHc= github.com/peterh/liner v1.1.1-0.20190123174540-a2c9a5303de7/go.mod h1:CRroGNssyjTd/qIG2FyxByd2S8JEAZXBl4qUrZf8GS0= -github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5 h1:q2e307iGHPdTGp0hoxKjt1H5pDo6utceo3dQVK3I5XQ= github.com/petermattis/goid v0.0.0-20180202154549-b0b1615b78e5/go.mod h1:jvVRKCrJTQWu0XVbaOlby/2lO20uSCHEMzzplHXte1o= +github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 h1:hDSdbBuw3Lefr6R18ax0tZ2BJeNB3NehB3trOwYBsdU= +github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08/go.mod h1:pxMtw7cyUw6B2bRH0ZBANSPg+AoSud1I1iyJHI69jH4= github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= @@ -1763,8 +1767,8 @@ github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= -github.com/prometheus/common v0.39.0 h1:oOyhkDq05hPZKItWVBkJ6g6AtGxi+fy7F4JvUV8uhsI= -github.com/prometheus/common v0.39.0/go.mod h1:6XBZ7lYdLCbkAVhwRsWTZn+IN5AB9F/NXd5w0BbEX0Y= +github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= +github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1820,10 +1824,10 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.3.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= -github.com/rs/zerolog v1.27.0 h1:1T7qCieN22GVc8S4Q2yuexzBb1EqjbgjSH9RohbMjKs= -github.com/rs/zerolog v1.27.0/go.mod h1:7frBqO0oezxmnO7GF86FY++uy8I0Tk/If5ni1G9Qc0U= +github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= +github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -1933,7 +1937,6 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= -github.com/syndtr/goleveldb v1.0.0/go.mod h1:ZVVdQEZoIme9iO1Ch2Jdy24qqXrMMOU6lpPAyBWyWuQ= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -2318,8 +2321,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.4.0 h1:NF0gk8LVPg1Ml7SSbGyySuoxdsXitj7TvgvuRxIMc/M= -golang.org/x/oauth2 v0.4.0/go.mod h1:RznEsdpjGAINPTOF0UH/t+xJ75L18YO3Ho6Pyn+uRec= +golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= +golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2634,8 +2637,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ= -google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0= +google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU= +google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2759,8 +2762,8 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f h1:BWUVssLB0HVOSY78gIdvk1dTVYtT1y8SBWtPYuTJ/6w= -google.golang.org/genproto v0.0.0-20230110181048-76db0878b65f/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= +google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2807,8 +2810,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.53.0 h1:LAv2ds7cmFV/XTS3XG1NneeENYrXGmorPxsBbptIjNc= -google.golang.org/grpc v1.53.0/go.mod h1:OnIrk0ipVdj4N5d9IUoFUx72/VlD7+jUsHwZgwSMQpw= +google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= +google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From f32c9039194037b15df85aef4f158b3e39c90e3d Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 22 Jun 2023 16:16:24 +0800 Subject: [PATCH 061/388] feat(share/getter) Add support for Non-inclusion proofs (#2256) ## Overview This PR implements support for non-inclusion proof for getters and shrex protocol. Requires https://github.com/celestiaorg/celestia-node/pull/2242 to be merged first --- blob/service.go | 3 +- share/availability/light/availability_test.go | 5 +- share/get_test.go | 2 +- share/getter.go | 6 +- share/getters/cascade.go | 4 +- share/getters/getter_test.go | 12 +- share/getters/shrex.go | 13 +- share/getters/shrex_test.go | 50 ++++++-- share/getters/store.go | 4 - share/getters/utils.go | 8 +- share/ipld/namespace_data.go | 2 +- share/p2p/shrexnd/client.go | 50 ++++---- share/p2p/shrexnd/exchange_test.go | 7 +- share/p2p/shrexnd/pb/share.pb.go | 112 +++++++++++++----- share/p2p/shrexnd/pb/share.proto | 3 +- share/p2p/shrexnd/server.go | 32 ++--- 16 files changed, 197 insertions(+), 116 deletions(-) diff --git a/blob/service.go b/blob/service.go index 0b43493e27..0876a45a0c 100644 --- a/blob/service.go +++ b/blob/service.go @@ -193,8 +193,7 @@ func (s *Service) getByCommitment( namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, nID) if err != nil { - if errors.Is(err, share.ErrNamespaceNotFound) || - errors.Is(err, share.ErrNotFound) { + if errors.Is(err, share.ErrNotFound) { err = ErrBlobNotFound } return nil, nil, err diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 4980af031e..ca72596816 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -141,8 +141,9 @@ func TestService_GetSharesByNamespaceNotFound(t *testing.T) { getter, root := GetterWithRandSquare(t, 1) root.RowRoots = nil - _, err := getter.GetSharesByNamespace(context.Background(), root, namespace.RandomNamespace().Bytes()) - assert.ErrorIs(t, err, share.ErrNamespaceNotFound) + emptyShares, err := getter.GetSharesByNamespace(context.Background(), root, namespace.RandomNamespace().Bytes()) + require.NoError(t, err) + require.Empty(t, emptyShares.Flatten()) } func BenchmarkService_GetSharesByNamespace(b *testing.B) { diff --git a/share/get_test.go b/share/get_test.go index 25711e5a65..1b9df69be3 100644 --- a/share/get_test.go +++ b/share/get_test.go @@ -237,7 +237,7 @@ func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 16) + shares := RandShares(t, 1024) // set all shares to the same namespace id nids, err := randomNids(5) diff --git a/share/getter.go b/share/getter.go index 6ba729c71a..15f384a2c4 100644 --- a/share/getter.go +++ b/share/getter.go @@ -17,9 +17,6 @@ import ( var ( // ErrNotFound is used to indicate that requested data could not be found. ErrNotFound = errors.New("share: data not found") - // ErrNamespaceNotFound is returned by GetSharesByNamespace when data for requested root does - // not include any shares from the given namespace - ErrNamespaceNotFound = errors.New("share: namespace not found in data") ) // Getter interface provides a set of accessors for shares by the Root. @@ -35,6 +32,9 @@ type Getter interface { // GetSharesByNamespace gets all shares from an EDS within the given namespace. // Shares are returned in a row-by-row order if the namespace spans multiple rows. + // Inclusion of returned data could be verified using Verify method on NamespacedShares. + // If no shares are found for target namespace non-inclusion could be also verified by calling + // Verify method. GetSharesByNamespace(context.Context, *Root, namespace.ID) (NamespacedShares, error) } diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 1a0d8fb274..01540de926 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -122,8 +122,8 @@ func cascadeGetters[V any]( getCtx, cancel := ctxWithSplitTimeout(ctx, len(getters)-i, 0) val, getErr := get(getCtx, getter) cancel() - if getErr == nil || errors.Is(getErr, share.ErrNamespaceNotFound) { - return val, getErr + if getErr == nil { + return val, nil } if errors.Is(getErr, errOperationNotSupported) { diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index c9bf82031a..20b7e25191 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -138,8 +138,9 @@ func TestStoreGetter(t *testing.T) { // nid not found nID = make([]byte, namespace.NamespaceSize) - _, err = sg.GetSharesByNamespace(ctx, &dah, nID) - require.ErrorIs(t, err, share.ErrNamespaceNotFound) + emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + require.NoError(t, err) + require.Empty(t, emptyShares.Flatten()) // root not found root := share.Root{} @@ -216,13 +217,14 @@ func TestIPLDGetter(t *testing.T) { // nid not found nID = make([]byte, namespace.NamespaceSize) emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) - require.ErrorIs(t, err, share.ErrNamespaceNotFound) + require.NoError(t, err) require.Nil(t, emptyShares) // nid doesnt exist in root root := share.Root{} - _, err = sg.GetSharesByNamespace(ctx, &root, nID) - require.ErrorIs(t, err, share.ErrNamespaceNotFound) + emptyShares, err = sg.GetSharesByNamespace(ctx, &root, nID) + require.NoError(t, err) + require.Empty(t, emptyShares.Flatten()) }) } diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 58a85abc0c..99da2f8758 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -191,7 +191,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( // verify that the namespace could exist inside the roots before starting network requests roots := filterRootsByNamespace(root, id) if len(roots) == 0 { - return nil, share.ErrNamespaceNotFound + return nil, nil } for { @@ -218,18 +218,15 @@ func (sg *ShrexGetter) GetSharesByNamespace( cancel() switch { case getErr == nil: - if getErr = nd.Verify(root, id); getErr != nil { + // both inclusion and non-inclusion cases needs verification + if verErr := nd.Verify(root, id); verErr != nil { + getErr = verErr setStatus(peers.ResultBlacklistPeer) break } setStatus(peers.ResultNoop) sg.metrics.recordNDAttempt(ctx, attempt, true) - return nd, getErr - case errors.Is(getErr, share.ErrNamespaceNotFound): - // TODO: will be merged with first case once non-inclusion proofs are ready - setStatus(peers.ResultNoop) - sg.metrics.recordNDAttempt(ctx, attempt, true) - return nd, getErr + return nd, nil case errors.Is(getErr, context.DeadlineExceeded), errors.Is(getErr, context.Canceled): setStatus(peers.ResultCooldownPeer) diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index db60e0138a..72ac78e69b 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -15,8 +15,8 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" libhead "github.com/celestiaorg/go-header" + "github.com/celestiaorg/nmt" nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" @@ -24,6 +24,7 @@ import ( "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" @@ -88,12 +89,12 @@ func TestShrexGetter(t *testing.T) { require.ErrorIs(t, err, share.ErrNotFound) }) - t.Run("ND_namespace_not_found", func(t *testing.T) { + t.Run("ND_namespace_not_included", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) // generate test data - eds, dah, nID := generateTestEDS(t) + eds, dah, maxNID := generateTestEDS(t) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), @@ -101,10 +102,43 @@ func TestShrexGetter(t *testing.T) { }) // corrupt NID - nID[4]++ + nID := make([]byte, ipld.NamespaceSize) + copy(nID, maxNID) + nID[ipld.NamespaceSize-1]-- // pray for last byte to not be 0x00 + // check for namespace to be between max and min namespace in root + require.Len(t, filterRootsByNamespace(&dah, maxNID), 1) - _, err := getter.GetSharesByNamespace(ctx, &dah, nID) - require.ErrorIs(t, err, share.ErrNamespaceNotFound) + emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) + require.NoError(t, err) + // no shares should be returned + require.Empty(t, emptyShares.Flatten()) + require.Nil(t, emptyShares.Verify(&dah, nID)) + }) + + t.Run("ND_namespace_not_in_dah", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, time.Second) + t.Cleanup(cancel) + + // generate test data + eds, dah, maxNID := generateTestEDS(t) + require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ + DataHash: dah.Hash(), + Height: 1, + }) + + // corrupt NID + nID := make([]byte, ipld.NamespaceSize) + copy(nID, maxNID) + nID[ipld.NamespaceSize-1]++ // pray for last byte to not be 0xFF + // check for namespace to be not in root + require.Len(t, filterRootsByNamespace(&dah, nID), 0) + + emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) + require.NoError(t, err) + // no shares should be returned + require.Empty(t, emptyShares.Flatten()) + require.Nil(t, emptyShares.Verify(&dah, nID)) }) t.Run("EDS_Available", func(t *testing.T) { @@ -166,8 +200,8 @@ func newStore(t *testing.T) (*eds.Store, error) { func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, nmtnamespace.ID) { eds := share.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) - randNID := dah.RowRoots[(len(dah.RowRoots)-1)/2][:namespace.NamespaceSize] - return eds, dah, randNID + max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], ipld.NamespaceSize) + return eds, dah, max } func testManager( diff --git a/share/getters/store.go b/share/getters/store.go index 91200b78f3..f39f41fc80 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -122,10 +122,6 @@ func (sg *StoreGetter) GetSharesByNamespace( // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) shares, err = collectSharesByNamespace(ctx, blockGetter, root, nID) - if errors.Is(err, ipld.ErrNodeNotFound) { - // convert error to satisfy getter interface contract - err = share.ErrNotFound - } if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve shares by namespace: %w", err) } diff --git a/share/getters/utils.go b/share/getters/utils.go index c0b882f330..79438204bd 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -59,7 +59,7 @@ func collectSharesByNamespace( rootCIDs := filterRootsByNamespace(root, nID) if len(rootCIDs) == 0 { - return nil, share.ErrNamespaceNotFound + return nil, nil } errGroup, ctx := errgroup.WithContext(ctx) @@ -83,12 +83,6 @@ func collectSharesByNamespace( if err := errGroup.Wait(); err != nil { return nil, err } - - // return ErrNamespaceNotFound if no shares are found for the namespace.ID - if len(rootCIDs) == 1 && len(shares[0].Shares) == 0 { - return nil, share.ErrNamespaceNotFound - } - return shares, nil } diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index b38e5884b5..f86a9803ac 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -137,7 +137,7 @@ func (n *NamespaceData) addProof(d direction, cid cid.Cid, depth int) { // Leaves returns retrieved leaves within the bounds in case `WithLeaves` option was passed, // otherwise nil will be returned. func (n *NamespaceData) Leaves() []ipld.Node { - if n.leaves == nil || n.noLeaves() { + if n.leaves == nil || n.noLeaves() || n.isAbsentNamespace.Load() { return nil } return n.leaves[n.bounds.lowest : n.bounds.highest+1] diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index ab407126de..b59bb347a7 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -56,7 +56,7 @@ func (c *Client) RequestND( ) (share.NamespacedShares, error) { shares, err := c.doRequest(ctx, root, nID, peer) if err == nil { - return shares, err + return shares, nil } if errors.Is(err, context.DeadlineExceeded) || errors.Is(err, context.Canceled) { c.metrics.ObserveRequests(ctx, 1, p2p.StatusTimeout) @@ -71,7 +71,7 @@ func (c *Client) RequestND( return nil, context.DeadlineExceeded } } - if err != p2p.ErrNotFound && err != share.ErrNamespaceNotFound { + if err != p2p.ErrNotFound { log.Warnw("client-nd: peer returned err", "err", err) } return nil, err @@ -119,19 +119,11 @@ func (c *Client) doRequest( return nil, fmt.Errorf("client-nd: reading response: %w", err) } - if err = c.statusToErr(ctx, resp.Status); err != nil { - return nil, fmt.Errorf("client-nd: response code is not OK: %w", err) - } - - shares, err := convertToNamespacedShares(resp.Rows) - if err != nil { - return nil, fmt.Errorf("client-nd: converting response to shares: %w", err) - } - return shares, nil + return c.convertResponse(ctx, resp) } // convertToNamespacedShares converts proto Rows to share.NamespacedShares -func convertToNamespacedShares(rows []*pb.Row) (share.NamespacedShares, error) { +func convertToNamespacedShares(rows []*pb.Row) share.NamespacedShares { shares := make([]share.NamespacedRow, 0, len(rows)) for _, row := range rows { var proof *nmt.Proof @@ -150,7 +142,24 @@ func convertToNamespacedShares(rows []*pb.Row) (share.NamespacedShares, error) { Proof: proof, }) } - return shares, nil + return shares +} + +func convertToNonInclusionProofs(rows []*pb.Row) share.NamespacedShares { + shares := make([]share.NamespacedRow, 0, len(rows)) + for _, row := range rows { + proof := nmt.NewAbsenceProof( + int(row.Proof.Start), + int(row.Proof.End), + row.Proof.Nodes, + row.Proof.Hashleaf, + ipld.NMTIgnoreMaxNamespace, + ) + shares = append(shares, share.NamespacedRow{ + Proof: &proof, + }) + } + return shares } func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) { @@ -181,22 +190,23 @@ func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) } } -func (c *Client) statusToErr(ctx context.Context, code pb.StatusCode) error { - switch code { +func (c *Client) convertResponse( + ctx context.Context, resp pb.GetSharesByNamespaceResponse) (share.NamespacedShares, error) { + switch resp.Status { case pb.StatusCode_OK: c.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) - return nil + return convertToNamespacedShares(resp.Rows), nil + case pb.StatusCode_NAMESPACE_NOT_FOUND: + return convertToNonInclusionProofs(resp.Rows), nil case pb.StatusCode_NOT_FOUND: c.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - return p2p.ErrNotFound - case pb.StatusCode_NAMESPACE_NOT_FOUND: - return share.ErrNamespaceNotFound + return nil, p2p.ErrNotFound case pb.StatusCode_INVALID: log.Debug("client-nd: invalid request") fallthrough case pb.StatusCode_INTERNAL: fallthrough default: - return p2p.ErrInvalidResponse + return nil, p2p.ErrInvalidResponse } } diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 8c5a132fdc..25528595d7 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -49,8 +49,9 @@ func TestExchange_RequestND_NotFound(t *testing.T) { require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) randNID := dah.RowRoots[(len(dah.RowRoots)-1)/2][:namespace.NamespaceSize] - _, err := client.RequestND(ctx, &dah, randNID, server.host.ID()) - require.ErrorIs(t, err, share.ErrNamespaceNotFound) + emptyShares, err := client.RequestND(ctx, &dah, randNID, server.host.ID()) + require.NoError(t, err) + require.Empty(t, emptyShares.Flatten()) }) } @@ -119,7 +120,7 @@ func (m notFoundGetter) GetEDS( func (m notFoundGetter) GetSharesByNamespace( _ context.Context, _ *share.Root, _ nmtnamespace.ID, ) (share.NamespacedShares, error) { - return nil, share.ErrNamespaceNotFound + return nil, nil } func newStore(t *testing.T) *eds.Store { diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go index d902570410..7e19bebc09 100644 --- a/share/p2p/shrexnd/pb/share.pb.go +++ b/share/p2p/shrexnd/pb/share.pb.go @@ -213,9 +213,10 @@ func (m *Row) GetProof() *Proof { } type Proof struct { - Start int64 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` - End int64 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` - Nodes [][]byte `protobuf:"bytes,3,rep,name=Nodes,proto3" json:"Nodes,omitempty"` + Start int64 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` + End int64 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` + Nodes [][]byte `protobuf:"bytes,3,rep,name=nodes,proto3" json:"nodes,omitempty"` + Hashleaf []byte `protobuf:"bytes,4,opt,name=hashleaf,proto3" json:"hashleaf,omitempty"` } func (m *Proof) Reset() { *m = Proof{} } @@ -272,6 +273,13 @@ func (m *Proof) GetNodes() [][]byte { return nil } +func (m *Proof) GetHashleaf() []byte { + if m != nil { + return m.Hashleaf + } + return nil +} + func init() { proto.RegisterEnum("share.p2p.shrex.nd.StatusCode", StatusCode_name, StatusCode_value) proto.RegisterType((*GetSharesByNamespaceRequest)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceRequest") @@ -283,32 +291,33 @@ func init() { func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } var fileDescriptor_ed9f13149b0de397 = []byte{ - // 386 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xc1, 0xae, 0x93, 0x40, - 0x14, 0x86, 0x81, 0xb9, 0xc5, 0x7b, 0x0f, 0x68, 0xc8, 0x68, 0xbc, 0x98, 0x6b, 0x48, 0x65, 0xd5, - 0x68, 0x02, 0x09, 0x26, 0xee, 0x69, 0x8b, 0x4a, 0xac, 0xd3, 0x66, 0x5a, 0x75, 0x65, 0x08, 0x95, - 0x31, 0xb8, 0x90, 0x19, 0x99, 0x69, 0xaa, 0x6b, 0x5f, 0xc0, 0xc7, 0x72, 0xd9, 0xa5, 0x4b, 0xd3, - 0xbe, 0x88, 0x61, 0xa8, 0x76, 0x61, 0x77, 0xfc, 0xff, 0xf9, 0xce, 0x7f, 0xce, 0x21, 0x03, 0x43, - 0x59, 0x97, 0x2d, 0x8b, 0x45, 0x22, 0x62, 0x59, 0xb7, 0xec, 0x6b, 0x53, 0xc5, 0x62, 0x1d, 0x6b, - 0x33, 0x12, 0x2d, 0x57, 0x1c, 0xe3, 0xa3, 0x48, 0x44, 0xa4, 0x89, 0xa8, 0xa9, 0xc2, 0xf7, 0x70, - 0xf3, 0x82, 0xa9, 0x65, 0x57, 0x90, 0xe3, 0x6f, 0xa4, 0xfc, 0xcc, 0xa4, 0x28, 0x3f, 0x30, 0xca, - 0xbe, 0x6c, 0x98, 0x54, 0xf8, 0x06, 0xae, 0x5a, 0xce, 0x55, 0x51, 0x97, 0xb2, 0xf6, 0xcd, 0xa1, - 0x39, 0x72, 0xe9, 0x65, 0x67, 0xbc, 0x2c, 0x65, 0x8d, 0x1f, 0x81, 0xdb, 0xfc, 0x6d, 0x28, 0x3e, - 0x55, 0xbe, 0xa5, 0xeb, 0xce, 0x3f, 0x2f, 0xaf, 0xc2, 0xef, 0x26, 0x3c, 0x3c, 0x9f, 0x2f, 0x05, - 0x6f, 0x24, 0xc3, 0xcf, 0xc0, 0x96, 0xaa, 0x54, 0x1b, 0xa9, 0xd3, 0xef, 0x24, 0x41, 0xf4, 0xff, - 0x92, 0xd1, 0x52, 0x13, 0x13, 0x5e, 0x31, 0x7a, 0xa4, 0xf1, 0x13, 0xb8, 0x68, 0xf9, 0x56, 0xfa, - 0xd6, 0x10, 0x8d, 0x9c, 0xe4, 0xfa, 0x5c, 0x17, 0xe5, 0x5b, 0xaa, 0xa1, 0x90, 0x00, 0xa2, 0x7c, - 0x8b, 0xef, 0x83, 0xad, 0xb1, 0x6e, 0x16, 0x1a, 0xb9, 0xf4, 0xa8, 0x70, 0x0c, 0x03, 0xd1, 0x72, - 0xfe, 0x51, 0x1f, 0xe0, 0x24, 0x0f, 0xce, 0x85, 0x2d, 0x3a, 0x80, 0xf6, 0x5c, 0x98, 0xc1, 0x40, - 0x6b, 0x7c, 0x0f, 0x06, 0x52, 0x95, 0xad, 0xd2, 0xcb, 0x23, 0xda, 0x0b, 0xec, 0x01, 0x62, 0x4d, - 0xff, 0x3b, 0x10, 0xed, 0x3e, 0x3b, 0x8e, 0xf0, 0x8a, 0x49, 0x1f, 0xe9, 0xc1, 0xbd, 0x78, 0xfc, - 0x0e, 0xe0, 0x74, 0x19, 0x76, 0xe0, 0x56, 0x4e, 0xde, 0xa6, 0xb3, 0x7c, 0xea, 0x19, 0xd8, 0x06, - 0x6b, 0xfe, 0xca, 0x33, 0xf1, 0x6d, 0xb8, 0x22, 0xf3, 0x55, 0xf1, 0x7c, 0xfe, 0x86, 0x4c, 0x3d, - 0x0b, 0xbb, 0x70, 0x99, 0x93, 0x55, 0x46, 0x49, 0x3a, 0xf3, 0x10, 0xbe, 0x86, 0xbb, 0x24, 0x7d, - 0x9d, 0x2d, 0x17, 0xe9, 0x24, 0x2b, 0x4e, 0xd8, 0xc5, 0xd8, 0xff, 0xb9, 0x0f, 0xcc, 0xdd, 0x3e, - 0x30, 0x7f, 0xef, 0x03, 0xf3, 0xc7, 0x21, 0x30, 0x76, 0x87, 0xc0, 0xf8, 0x75, 0x08, 0x8c, 0xb5, - 0xad, 0x5f, 0xc2, 0xd3, 0x3f, 0x01, 0x00, 0x00, 0xff, 0xff, 0xba, 0xc1, 0x4e, 0xec, 0x2d, 0x02, - 0x00, 0x00, + // 401 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcf, 0x6e, 0xd3, 0x40, + 0x10, 0xc6, 0xe3, 0x6c, 0x62, 0xd2, 0xb1, 0x41, 0xd6, 0x80, 0xa8, 0xa1, 0xc8, 0x0a, 0x3e, 0x45, + 0x20, 0xd9, 0x92, 0x91, 0xb8, 0xbb, 0x6d, 0x00, 0x8b, 0xb2, 0xa9, 0x36, 0x05, 0x4e, 0xc8, 0xda, + 0xe2, 0xad, 0x8c, 0x04, 0xde, 0xc5, 0xbb, 0x55, 0xe0, 0xcc, 0x0b, 0xf0, 0x58, 0x1c, 0x7b, 0xe4, + 0x88, 0x92, 0x17, 0x41, 0x5e, 0xa7, 0xf4, 0x40, 0x6e, 0xfb, 0xcd, 0xfc, 0xe6, 0xdf, 0xa7, 0x85, + 0xa9, 0xae, 0x79, 0x2b, 0x52, 0x95, 0xa9, 0x54, 0xd7, 0xad, 0xf8, 0xd6, 0x54, 0xa9, 0x3a, 0x4f, + 0x6d, 0x30, 0x51, 0xad, 0x34, 0x12, 0x71, 0x2b, 0x32, 0x95, 0x58, 0x22, 0x69, 0xaa, 0xf8, 0x03, + 0x1c, 0xbc, 0x14, 0x66, 0xd9, 0x25, 0xf4, 0xe1, 0x77, 0xca, 0xbf, 0x08, 0xad, 0xf8, 0x47, 0xc1, + 0xc4, 0xd7, 0x4b, 0xa1, 0x0d, 0x1e, 0xc0, 0x5e, 0x2b, 0xa5, 0x29, 0x6b, 0xae, 0xeb, 0xd0, 0x99, + 0x3a, 0x33, 0x9f, 0x4d, 0xba, 0xc0, 0x2b, 0xae, 0x6b, 0x7c, 0x0c, 0x7e, 0x73, 0x5d, 0x50, 0x7e, + 0xaa, 0xc2, 0xa1, 0xcd, 0x7b, 0xff, 0x62, 0x45, 0x15, 0xff, 0x70, 0xe0, 0xd1, 0xee, 0xfe, 0x5a, + 0xc9, 0x46, 0x0b, 0x7c, 0x0e, 0xae, 0x36, 0xdc, 0x5c, 0x6a, 0xdb, 0xfd, 0x4e, 0x16, 0x25, 0xff, + 0x2f, 0x99, 0x2c, 0x2d, 0x71, 0x24, 0x2b, 0xc1, 0xb6, 0x34, 0x3e, 0x85, 0x51, 0x2b, 0x57, 0x3a, + 0x1c, 0x4e, 0xc9, 0xcc, 0xcb, 0xf6, 0x77, 0x55, 0x31, 0xb9, 0x62, 0x16, 0x8a, 0x29, 0x10, 0x26, + 0x57, 0x78, 0x1f, 0x5c, 0x8b, 0x75, 0xb3, 0xc8, 0xcc, 0x67, 0x5b, 0x85, 0x29, 0x8c, 0x55, 0x2b, + 0xe5, 0x85, 0x3d, 0xc0, 0xcb, 0x1e, 0xec, 0x6a, 0x76, 0xda, 0x01, 0xac, 0xe7, 0x62, 0x0e, 0x63, + 0xab, 0xf1, 0x1e, 0x8c, 0xb5, 0xe1, 0xad, 0xb1, 0xcb, 0x13, 0xd6, 0x0b, 0x0c, 0x80, 0x88, 0xa6, + 0xb7, 0x83, 0xb0, 0xee, 0xd9, 0x71, 0x8d, 0xac, 0x84, 0x0e, 0x89, 0x1d, 0xdc, 0x0b, 0x7c, 0x08, + 0x93, 0xce, 0xd7, 0xcf, 0x82, 0x5f, 0x84, 0xa3, 0xde, 0xdb, 0x6b, 0xfd, 0xe4, 0x3d, 0xc0, 0xcd, + 0xd5, 0xe8, 0xc1, 0xad, 0x82, 0xbe, 0xcb, 0x4f, 0x8a, 0xe3, 0x60, 0x80, 0x2e, 0x0c, 0x17, 0xaf, + 0x03, 0x07, 0x6f, 0xc3, 0x1e, 0x5d, 0x9c, 0x95, 0x2f, 0x16, 0x6f, 0xe9, 0x71, 0x30, 0x44, 0x1f, + 0x26, 0x05, 0x3d, 0x9b, 0x33, 0x9a, 0x9f, 0x04, 0x04, 0xf7, 0xe1, 0x2e, 0xcd, 0xdf, 0xcc, 0x97, + 0xa7, 0xf9, 0xd1, 0xbc, 0xbc, 0xc1, 0x46, 0x87, 0xe1, 0xaf, 0x75, 0xe4, 0x5c, 0xad, 0x23, 0xe7, + 0xcf, 0x3a, 0x72, 0x7e, 0x6e, 0xa2, 0xc1, 0xd5, 0x26, 0x1a, 0xfc, 0xde, 0x44, 0x83, 0x73, 0xd7, + 0xfe, 0x92, 0x67, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x3e, 0x78, 0x01, 0x49, 0x02, 0x00, + 0x00, } func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { @@ -454,6 +463,13 @@ func (m *Proof) MarshalToSizedBuffer(dAtA []byte) (int, error) { _ = i var l int _ = l + if len(m.Hashleaf) > 0 { + i -= len(m.Hashleaf) + copy(dAtA[i:], m.Hashleaf) + i = encodeVarintShare(dAtA, i, uint64(len(m.Hashleaf))) + i-- + dAtA[i] = 0x22 + } if len(m.Nodes) > 0 { for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { i -= len(m.Nodes[iNdEx]) @@ -559,6 +575,10 @@ func (m *Proof) Size() (n int) { n += 1 + l + sovShare(uint64(l)) } } + l = len(m.Hashleaf) + if l > 0 { + n += 1 + l + sovShare(uint64(l)) + } return n } @@ -1006,6 +1026,40 @@ func (m *Proof) Unmarshal(dAtA []byte) error { m.Nodes = append(m.Nodes, make([]byte, postIndex-iNdEx)) copy(m.Nodes[len(m.Nodes)-1], dAtA[iNdEx:postIndex]) iNdEx = postIndex + case 4: + if wireType != 2 { + return fmt.Errorf("proto: wrong wireType = %d for field Hashleaf", wireType) + } + var byteLen int + for shift := uint(0); ; shift += 7 { + if shift >= 64 { + return ErrIntOverflowShare + } + if iNdEx >= l { + return io.ErrUnexpectedEOF + } + b := dAtA[iNdEx] + iNdEx++ + byteLen |= int(b&0x7F) << shift + if b < 0x80 { + break + } + } + if byteLen < 0 { + return ErrInvalidLengthShare + } + postIndex := iNdEx + byteLen + if postIndex < 0 { + return ErrInvalidLengthShare + } + if postIndex > l { + return io.ErrUnexpectedEOF + } + m.Hashleaf = append(m.Hashleaf[:0], dAtA[iNdEx:postIndex]...) + if m.Hashleaf == nil { + m.Hashleaf = []byte{} + } + iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipShare(dAtA[iNdEx:]) diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto index 3d6a896641..09d77d29f3 100644 --- a/share/p2p/shrexnd/pb/share.proto +++ b/share/p2p/shrexnd/pb/share.proto @@ -28,5 +28,6 @@ message Row { message Proof { int64 start = 1; int64 end = 2; - repeated bytes Nodes = 3; + repeated bytes nodes = 3; + bytes hashleaf = 4; } diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 67f64b8393..72c5231eff 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -136,9 +136,6 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre logger.Warn("server: nd not found") srv.respondNotFoundError(ctx, logger, stream) return - case errors.Is(err, share.ErrNamespaceNotFound): - srv.respondNamespaceNotFoundError(ctx, logger, stream) - return case err != nil: logger.Errorw("server: retrieving shares", "err", err) srv.respondInternalError(ctx, logger, stream) @@ -170,15 +167,6 @@ func (srv *Server) respondNotFoundError(ctx context.Context, srv.respond(ctx, logger, stream, resp) } -// respondNamespaceNotFoundError sends a namespace not found response to client -func (srv *Server) respondNamespaceNotFoundError(ctx context.Context, - logger *zap.SugaredLogger, stream network.Stream) { - resp := &pb.GetSharesByNamespaceResponse{ - Status: pb.StatusCode_NAMESPACE_NOT_FOUND, - } - srv.respond(ctx, logger, stream, resp) -} - // respondInternalError sends internal error response to client func (srv *Server) respondInternalError(ctx context.Context, logger *zap.SugaredLogger, stream network.Stream) { @@ -192,22 +180,26 @@ func (srv *Server) respondInternalError(ctx context.Context, func namespacedSharesToResponse(shares share.NamespacedShares) *pb.GetSharesByNamespaceResponse { rows := make([]*pb.Row, 0, len(shares)) for _, row := range shares { - proof := &pb.Proof{ - Start: int64(row.Proof.Start()), - End: int64(row.Proof.End()), - Nodes: row.Proof.Nodes(), - } - row := &pb.Row{ Shares: row.Shares, - Proof: proof, + Proof: &pb.Proof{ + Start: int64(row.Proof.Start()), + End: int64(row.Proof.End()), + Nodes: row.Proof.Nodes(), + Hashleaf: row.Proof.LeafHash(), + }, } rows = append(rows, row) } + status := pb.StatusCode_OK + if len(shares) == 0 || (len(shares) == 1 && len(shares[0].Shares) == 0) { + status = pb.StatusCode_NAMESPACE_NOT_FOUND + } + return &pb.GetSharesByNamespaceResponse{ - Status: pb.StatusCode_OK, + Status: status, Rows: rows, } } From 8b28ad799688b5cbfe22dc2b4449f15c248badba Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 23 Jun 2023 17:16:06 +0200 Subject: [PATCH 062/388] fix(api/gateway): Handle err from tx properly to avoid panic (#2393) Fixes a panic that can occur if txResp is nil and also returns actual error in the case of nil txResp. Reported by @tuxcanfly --- api/gateway/state.go | 17 +++++++++-------- 1 file changed, 9 insertions(+), 8 deletions(-) diff --git a/api/gateway/state.go b/api/gateway/state.go index b584b00d36..f97a7da38e 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -150,10 +150,15 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } // perform request - txResp, txerr := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*blob.Blob{constructedBlob}) - if txerr != nil && txResp == nil { - // no tx data to return - writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) + txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*blob.Blob{constructedBlob}) + if err != nil { + if txResp == nil { + // no tx data to return + writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) + return + } + // if error returned, change status from 200 to 206 + w.WriteHeader(http.StatusPartialContent) } bs, err := json.Marshal(&txResp) @@ -162,10 +167,6 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { return } - // if error returned, change status from 200 to 206 - if txerr != nil { - w.WriteHeader(http.StatusPartialContent) - } _, err = w.Write(bs) if err != nil { log.Errorw("writing response", "endpoint", submitPFBEndpoint, "err", err) From 9264c1aec14f81e55f280aad670ab280260698d1 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Fri, 23 Jun 2023 15:43:58 -0500 Subject: [PATCH 063/388] chore: bump celestia-app to v1.0.0-rc5 (#2395) ## Overview bumps app to rc5 [full changelog](https://github.com/celestiaorg/celestia-app/compare/v1.0.0-rc4...v1.0.0-rc5) ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 31420f8745..0568717120 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/BurntSushi/toml v1.3.0 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc4 + github.com/celestiaorg/celestia-app v1.0.0-rc5 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 github.com/celestiaorg/go-libp2p-messenger v0.2.0 diff --git a/go.sum b/go.sum index 3bf1455309..425611c7df 100644 --- a/go.sum +++ b/go.sum @@ -344,8 +344,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc4 h1:S48+XMsLnk0GkNHrUc5c+rlyyHLIl0ZJ/zh92jmSSTk= -github.com/celestiaorg/celestia-app v1.0.0-rc4/go.mod h1:i+jE3Hh8IQxlZnE5XSBlYF0PCNPvD3v5g8KmY095PEA= +github.com/celestiaorg/celestia-app v1.0.0-rc5 h1:PjKIp91CQgqyP4MLI3KeF8E4MPHtBPlNoRPvGIMWjHw= +github.com/celestiaorg/celestia-app v1.0.0-rc5/go.mod h1:i+jE3Hh8IQxlZnE5XSBlYF0PCNPvD3v5g8KmY095PEA= github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28 h1:idHJK9i4WCkYOf5PXVWZbOs8pWkCiHHQGI4MZr0iMtQ= github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28/go.mod h1:LOxHW9nA++/9U8TgvTyKo9TO3F09sWv8asKQs00m73U= github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 h1:vaQKgaOm0w58JAvOgn2iDohqjH7kvvRqVKiMcBDWifA= From f79651c29cd4b9b4473139d92f0473c8667d76f6 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Mon, 26 Jun 2023 12:03:14 +0200 Subject: [PATCH 064/388] deps: bump libp2p (#2399) Change logs https://github.com/libp2p/go-libp2p/releases/tag/v0.28.1 --- go.mod | 2 +- go.sum | 4 ++-- nodebuilder/blob/mocks/api.go | 3 ++- nodebuilder/das/mocks/api.go | 3 ++- nodebuilder/fraud/mocks/api.go | 3 ++- nodebuilder/header/mocks/api.go | 3 ++- nodebuilder/node/mocks/api.go | 3 ++- nodebuilder/share/mocks/api.go | 3 ++- nodebuilder/state/mocks/api.go | 5 +++-- share/mocks/getter.go | 3 ++- 10 files changed, 20 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 0568717120..73509c56c3 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.10.0 github.com/ipld/go-car v0.6.0 - github.com/libp2p/go-libp2p v0.28.0 + github.com/libp2p/go-libp2p v0.28.1 github.com/libp2p/go-libp2p-kad-dht v0.21.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 diff --git a/go.sum b/go.sum index 425611c7df..28113d7847 100644 --- a/go.sum +++ b/go.sum @@ -1213,8 +1213,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= -github.com/libp2p/go-libp2p v0.28.0 h1:zO8cY98nJiPzZpFv5w5gqqb8aVzt4ukQ0nVOSaaKhJ8= -github.com/libp2p/go-libp2p v0.28.0/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= +github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= +github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index f99d1d8168..b7c61aa450 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + blob "github.com/celestiaorg/celestia-node/blob" namespace "github.com/celestiaorg/nmt/namespace" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/das/mocks/api.go b/nodebuilder/das/mocks/api.go index c4046e90e8..68ffaf3c8c 100644 --- a/nodebuilder/das/mocks/api.go +++ b/nodebuilder/das/mocks/api.go @@ -8,8 +8,9 @@ import ( context "context" reflect "reflect" - das "github.com/celestiaorg/celestia-node/das" gomock "github.com/golang/mock/gomock" + + das "github.com/celestiaorg/celestia-node/das" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index ba88131695..399f8746e1 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + fraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" fraud0 "github.com/celestiaorg/go-fraud" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/header/mocks/api.go b/nodebuilder/header/mocks/api.go index 02529a8ef9..7d6661ff5d 100644 --- a/nodebuilder/header/mocks/api.go +++ b/nodebuilder/header/mocks/api.go @@ -8,10 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + header "github.com/celestiaorg/celestia-node/header" header0 "github.com/celestiaorg/go-header" sync "github.com/celestiaorg/go-header/sync" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/node/mocks/api.go b/nodebuilder/node/mocks/api.go index d8789a771c..14357316dc 100644 --- a/nodebuilder/node/mocks/api.go +++ b/nodebuilder/node/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" - node "github.com/celestiaorg/celestia-node/nodebuilder/node" auth "github.com/filecoin-project/go-jsonrpc/auth" gomock "github.com/golang/mock/gomock" + + node "github.com/celestiaorg/celestia-node/nodebuilder/node" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 586c6dab4b..1b26273c0f 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -8,11 +8,12 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index dbd1d5dabe..754920dee2 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -9,12 +9,13 @@ import ( reflect "reflect" math "cosmossdk.io/math" - blob "github.com/celestiaorg/celestia-node/blob" - state "github.com/celestiaorg/celestia-node/state" types "github.com/cosmos/cosmos-sdk/types" types0 "github.com/cosmos/cosmos-sdk/x/staking/types" gomock "github.com/golang/mock/gomock" types1 "github.com/tendermint/tendermint/types" + + blob "github.com/celestiaorg/celestia-node/blob" + state "github.com/celestiaorg/celestia-node/state" ) // MockModule is a mock of Module interface. diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 1c73c9170d..12c36cb015 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -8,11 +8,12 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" - gomock "github.com/golang/mock/gomock" ) // MockGetter is a mock of Getter interface. From 637072e8f6349a4919da9b5e5d8b4637c3058ecc Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 27 Jun 2023 15:17:02 +0200 Subject: [PATCH 065/388] fix(libs/utils): Use valid ip4 address (#2394) Fixes an issue where `localhost` was not accepted as a valid `--gateway.addr`. Uses net.ResolveIPAddr instead. Found by @tuxcanfly --- libs/utils/address.go | 19 +++++++++---------- 1 file changed, 9 insertions(+), 10 deletions(-) diff --git a/libs/utils/address.go b/libs/utils/address.go index a8170e44b9..ae52a03b16 100644 --- a/libs/utils/address.go +++ b/libs/utils/address.go @@ -29,15 +29,14 @@ func ValidateAddr(addr string) (string, error) { return addr, err } - if ip := net.ParseIP(addr); ip == nil { - addrs, err := net.LookupHost(addr) - if err != nil { - return addr, fmt.Errorf("could not resolve %v: %w", addr, err) - } - if len(addrs) == 0 { - return addr, fmt.Errorf("no IP addresses found for DNS record: %v", addr) - } - addr = addrs[0] + ip := net.ParseIP(addr) + if ip != nil { + return addr, nil } - return addr, nil + + resolved, err := net.ResolveIPAddr("ip4", addr) + if err != nil { + return addr, err + } + return resolved.String(), nil } From 0267e21fd1a500bb94960b6a1a1a38dae9a3084d Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 27 Jun 2023 15:17:40 +0200 Subject: [PATCH 066/388] feat: adding spans to shrex getter (#2404) Self explanatory --- share/getters/shrex.go | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 99da2f8758..08006073d6 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -12,10 +12,12 @@ import ( "go.opentelemetry.io/otel/metric/instrument" "go.opentelemetry.io/otel/metric/instrument/syncint64" "go.opentelemetry.io/otel/metric/unit" + "go.opentelemetry.io/otel/trace" "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/p2p" "github.com/celestiaorg/celestia-node/share/p2p/peers" @@ -128,6 +130,13 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex attempt int err error ) + ctx, span := tracer.Start(ctx, "shrex/get-eds", trace.WithAttributes( + attribute.String("root", root.String()), + )) + defer func() { + utils.SetStatusAndEnd(span, err) + }() + for { if ctx.Err() != nil { sg.metrics.recordEDSAttempt(ctx, attempt, false) @@ -187,6 +196,13 @@ func (sg *ShrexGetter) GetSharesByNamespace( attempt int err error ) + ctx, span := tracer.Start(ctx, "shrex/get-shares-by-namespace", trace.WithAttributes( + attribute.String("root", root.String()), + attribute.String("nid", hex.EncodeToString(id)), + )) + defer func() { + utils.SetStatusAndEnd(span, err) + }() // verify that the namespace could exist inside the roots before starting network requests roots := filterRootsByNamespace(root, id) From ccae150a8999d124ab0d799f779a32568e518953 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Tue, 27 Jun 2023 15:37:55 +0200 Subject: [PATCH 067/388] refactor!(share): integrate new Namespace type (#2388) ## Notable Changes - `namespace.ID` to `share.Namespace` - Changes every comment namespace ID mentioning to just namespace. I renamed every such mention besides in ADRs. I don't want to touch ADRs here, as they need a more holistic re-review and up-to-date catchup. - Uses all the utility methods on the type, where suitable - Namespace constructor now only creates Blob Namespaces. For other reserved namespaces, the predefined globals should be used. - Uses namespace.ValidateDataNamespace everywhere data is requested. This is guarantees we verify the namespace are 100% valid and forbids requesting parity and padding namespaces. - Restricts PFBs for reserved namespaces - Reversed the dependency from `share -> share/ipld` to `share/ipld -> share` - NewBlobV0 constructor. Similar to NewNamespaceV0 - `sharetest` pkg for share related testing utilities - `edstest` pkg for eds related testing utilities ## Follow-ups - `blobtest` pkg to generate node's blob type ## Refs * Substitutes zombie PR https://github.com/celestiaorg/celestia-node/pull/2376. I push to the branch, but GH does not see commits. * Based on https://github.com/celestiaorg/celestia-node/pull/2367 * Closes https://github.com/celestiaorg/celestia-node/issues/2301 * Closes #2309 * Blocked on https://github.com/celestiaorg/celestia-node/pull/2256 --- api/docgen/examples.go | 4 +- api/gateway/endpoints.go | 8 +- api/gateway/share.go | 24 +- api/gateway/share_test.go | 9 +- api/gateway/state.go | 6 +- api/gateway/state_test.go | 2 +- blob/blob.go | 57 +++-- blob/blob_test.go | 10 +- blob/blobtest/testing.go | 8 +- blob/helper.go | 4 +- blob/service.go | 45 ++-- blob/service_test.go | 60 ++--- cmd/celestia/rpc.go | 53 ++--- cmd/celestia/rpc_test.go | 19 +- header/headertest/testing.go | 15 +- nodebuilder/blob/blob.go | 37 ++- nodebuilder/blob/mocks/api.go | 10 +- nodebuilder/share/constructors.go | 11 + nodebuilder/share/mocks/api.go | 3 +- nodebuilder/share/module.go | 2 +- nodebuilder/share/share.go | 7 +- nodebuilder/tests/api_test.go | 2 +- nodebuilder/tests/blob_test.go | 11 +- share/availability.go | 2 +- share/availability/light/availability_test.go | 30 +-- share/availability/test/testing.go | 6 +- share/doc.go | 2 +- share/eds/byzantine/bad_encoding.go | 8 +- share/eds/byzantine/bad_encoding_test.go | 32 ++- share/eds/byzantine/share_proof.go | 4 +- share/eds/byzantine/share_proof_test.go | 10 +- share/eds/eds.go | 14 +- share/eds/eds_test.go | 14 +- share/eds/edstest/testing.go | 31 +++ share/eds/ods_test.go | 2 +- share/eds/retriever.go | 2 +- share/eds/retriever_test.go | 46 +--- share/eds/store_test.go | 5 +- share/empty.go | 74 +++--- share/getter.go | 24 +- share/getters/cascade.go | 8 +- share/getters/getter_test.go | 91 ++++---- share/getters/ipld.go | 15 +- share/getters/shrex.go | 19 +- share/getters/shrex_test.go | 53 +++-- share/getters/store.go | 15 +- share/getters/tee.go | 8 +- share/getters/testing.go | 6 +- share/getters/utils.go | 31 +-- share/helpers.go | 58 +++++ share/{ => ipld}/add.go | 45 +--- share/ipld/get.go | 4 +- share/{get.go => ipld/get_shares.go} | 27 ++- .../{get_test.go => ipld/get_shares_test.go} | 214 +++++++++--------- share/ipld/namespace_data.go | 36 +-- share/ipld/nmt.go | 40 +--- share/ipld/nmt_test.go | 43 +--- share/mocks/getter.go | 3 +- share/namespace.go | 31 ++- share/namespace_test.go | 53 +++++ share/nid.go | 29 --- share/nid_test.go | 56 ----- share/p2p/shrexeds/exchange_test.go | 8 +- share/p2p/shrexnd/client.go | 15 +- share/p2p/shrexnd/exchange_test.go | 20 +- share/p2p/shrexnd/pb/share.pb.go | 77 ++++--- share/p2p/shrexnd/pb/share.proto | 2 +- share/p2p/shrexnd/server.go | 10 +- share/share.go | 21 +- share/sharetest/testing.go | 58 +++++ share/test_helpers.go | 65 ------ state/core_access.go | 7 +- 72 files changed, 915 insertions(+), 966 deletions(-) create mode 100644 share/eds/edstest/testing.go create mode 100644 share/helpers.go rename share/{ => ipld}/add.go (61%) rename share/{get.go => ipld/get_shares.go} (69%) rename share/{get_test.go => ipld/get_shares_test.go} (64%) delete mode 100644 share/nid.go delete mode 100644 share/nid_test.go create mode 100644 share/sharetest/testing.go delete mode 100644 share/test_helpers.go diff --git a/api/docgen/examples.go b/api/docgen/examples.go index 80a8c64d93..3456880c4f 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -133,13 +133,13 @@ func init() { } addToExampleValues(addrInfo) - namespace, err := share.NewNamespaceV0([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}) + namespace, err := share.NewBlobNamespaceV0([]byte{0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}) if err != nil { panic(err) } addToExampleValues(namespace) - generatedBlob, err := blob.NewBlob(0, namespace, []byte("This is an example of some blob data")) + generatedBlob, err := blob.NewBlobV0(namespace, []byte("This is an example of some blob data")) if err != nil { panic(err) } diff --git a/api/gateway/endpoints.go b/api/gateway/endpoints.go index 0ae93b112c..9600138909 100644 --- a/api/gateway/endpoints.go +++ b/api/gateway/endpoints.go @@ -33,13 +33,13 @@ func (h *Handler) RegisterEndpoints(rpc *Server, deprecatedEndpointsEnabled bool rpc.RegisterHandlerFunc(submitTxEndpoint, h.handleSubmitTx, http.MethodPost) // share endpoints - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, nIDKey, heightKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, namespaceKey, heightKey), h.handleSharesByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, nIDKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, namespaceKey), h.handleSharesByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, nIDKey, heightKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, namespaceKey, heightKey), h.handleDataByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, nIDKey), + rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, namespaceKey), h.handleDataByNamespaceRequest, http.MethodGet) // DAS endpoints diff --git a/api/gateway/share.go b/api/gateway/share.go index db5ed37286..c9dec071f3 100644 --- a/api/gateway/share.go +++ b/api/gateway/share.go @@ -10,7 +10,6 @@ import ( "github.com/gorilla/mux" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/share" ) @@ -20,7 +19,7 @@ const ( namespacedDataEndpoint = "/namespaced_data" ) -var nIDKey = "nid" +var namespaceKey = "nid" // NamespacedSharesResponse represents the response to a // SharesByNamespace request. @@ -37,12 +36,12 @@ type NamespacedDataResponse struct { } func (h *Handler) handleSharesByNamespaceRequest(w http.ResponseWriter, r *http.Request) { - height, nID, err := parseGetByNamespaceArgs(r) + height, namespace, err := parseGetByNamespaceArgs(r) if err != nil { writeError(w, http.StatusBadRequest, namespacedSharesEndpoint, err) return } - shares, err := h.getShares(r.Context(), height, nID) + shares, err := h.getShares(r.Context(), height, namespace) if err != nil { writeError(w, http.StatusInternalServerError, namespacedSharesEndpoint, err) return @@ -62,12 +61,12 @@ func (h *Handler) handleSharesByNamespaceRequest(w http.ResponseWriter, r *http. } func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Request) { - height, nID, err := parseGetByNamespaceArgs(r) + height, namespace, err := parseGetByNamespaceArgs(r) if err != nil { writeError(w, http.StatusBadRequest, namespacedDataEndpoint, err) return } - shares, err := h.getShares(r.Context(), height, nID) + shares, err := h.getShares(r.Context(), height, namespace) if err != nil { writeError(w, http.StatusInternalServerError, namespacedDataEndpoint, err) return @@ -91,13 +90,13 @@ func (h *Handler) handleDataByNamespaceRequest(w http.ResponseWriter, r *http.Re } } -func (h *Handler) getShares(ctx context.Context, height uint64, nID namespace.ID) ([]share.Share, error) { +func (h *Handler) getShares(ctx context.Context, height uint64, namespace share.Namespace) ([]share.Share, error) { header, err := h.header.GetByHeight(ctx, height) if err != nil { return nil, err } - shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, nID) + shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, namespace) if err != nil { return nil, err } @@ -124,7 +123,7 @@ func dataFromShares(input []share.Share) (data [][]byte, err error) { return data, nil } -func parseGetByNamespaceArgs(r *http.Request) (height uint64, nID namespace.ID, err error) { +func parseGetByNamespaceArgs(r *http.Request) (height uint64, namespace share.Namespace, err error) { vars := mux.Vars(r) // if a height was given, parse it, otherwise get namespaced shares/data from the latest header if strHeight, ok := vars[heightKey]; ok { @@ -133,11 +132,10 @@ func parseGetByNamespaceArgs(r *http.Request) (height uint64, nID namespace.ID, return 0, nil, err } } - hexNID := vars[nIDKey] - nID, err = hex.DecodeString(hexNID) + hexNamespace := vars[namespaceKey] + namespace, err = hex.DecodeString(hexNamespace) if err != nil { return 0, nil, err } - - return height, nID, nil + return height, namespace, namespace.ValidateForData() } diff --git a/api/gateway/share_test.go b/api/gateway/share_test.go index 16cf606680..9b12240f62 100644 --- a/api/gateway/share_test.go +++ b/api/gateway/share_test.go @@ -8,8 +8,9 @@ import ( coretypes "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" + + "github.com/celestiaorg/celestia-node/share/sharetest" ) func Test_dataFromShares(t *testing.T) { @@ -19,13 +20,13 @@ func Test_dataFromShares(t *testing.T) { []byte("BEEEEAHP"), } - ns := namespace.RandomBlobNamespace() + ns := sharetest.RandV0Namespace() sss := shares.NewSparseShareSplitter() for _, data := range testData { b := coretypes.Blob{ Data: data, - NamespaceID: ns.ID, - NamespaceVersion: ns.Version, + NamespaceID: ns.ID(), + NamespaceVersion: ns.Version(), ShareVersion: appconsts.ShareVersionZero, } err := sss.Write(b) diff --git a/api/gateway/state.go b/api/gateway/state.go index f97a7da38e..69900b0bfc 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -9,8 +9,6 @@ import ( "github.com/cosmos/cosmos-sdk/types" "github.com/gorilla/mux" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/state" ) @@ -131,7 +129,7 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) return } - nID, err := hex.DecodeString(req.NamespaceID) + namespace, err := hex.DecodeString(req.NamespaceID) if err != nil { writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) return @@ -143,7 +141,7 @@ func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { } fee := types.NewInt(req.Fee) - constructedBlob, err := blob.NewBlob(appconsts.DefaultShareVersion, nID, data) + constructedBlob, err := blob.NewBlobV0(namespace, data) if err != nil { writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) return diff --git a/api/gateway/state_test.go b/api/gateway/state_test.go index a613471a04..aa9196cc8d 100644 --- a/api/gateway/state_test.go +++ b/api/gateway/state_test.go @@ -34,7 +34,7 @@ func TestHandleSubmitPFB(t *testing.T) { mock.EXPECT().SubmitPayForBlob(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). Return(&txResponse, timedErr) - ns, err := share.NewNamespaceV0([]byte("abc")) + ns, err := share.NewBlobNamespaceV0([]byte("abc")) require.NoError(t, err) hexNs := hex.EncodeToString(ns[:]) diff --git a/blob/blob.go b/blob/blob.go index 9771714cb9..e9ad2b6255 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -5,11 +5,13 @@ import ( "encoding/json" "fmt" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -101,41 +103,51 @@ type Blob struct { types.Blob `json:"blob"` Commitment Commitment `json:"commitment"` + + // the celestia-node's namespace type + // this is to avoid converting to and from app's type + namespace share.Namespace } -// NewBlob constructs a new blob from the provided namespace.ID and data. -func NewBlob(shareVersion uint8, namespace namespace.ID, data []byte) (*Blob, error) { - if len(namespace) != appns.NamespaceSize { - return nil, fmt.Errorf("invalid size of the namespace id. got:%d, want:%d", len(namespace), appns.NamespaceSize) - } +// NewBlobV0 constructs a new blob from the provided Namespace and data. +// The blob will be formatted as v0 shares. +func NewBlobV0(namespace share.Namespace, data []byte) (*Blob, error) { + return NewBlob(appconsts.ShareVersionZero, namespace, data) +} - ns, err := appns.New(namespace[appns.NamespaceVersionSize-1], namespace[appns.NamespaceVersionSize:]) - if err != nil { +// NewBlob constructs a new blob from the provided Namespace, data and share version. +func NewBlob(shareVersion uint8, namespace share.Namespace, data []byte) (*Blob, error) { + if len(data) == 0 || len(data) > appconsts.DefaultMaxBytes { + return nil, fmt.Errorf("blob data must be > 0 && <= %d, but it was %d bytes", appconsts.DefaultMaxBytes, len(data)) + } + if err := namespace.ValidateForBlob(); err != nil { return nil, err } - blob, err := types.NewBlob(ns, data, shareVersion) - if err != nil { - return nil, err + blob := tmproto.Blob{ + NamespaceId: namespace.ID(), + Data: data, + ShareVersion: uint32(shareVersion), + NamespaceVersion: uint32(namespace.Version()), } - com, err := types.CreateCommitment(blob) + com, err := types.CreateCommitment(&blob) if err != nil { return nil, err } - return &Blob{Blob: *blob, Commitment: com}, nil + return &Blob{Blob: blob, Commitment: com, namespace: namespace}, nil } // Namespace returns blob's namespace. -func (b *Blob) Namespace() namespace.ID { - return append([]byte{uint8(b.NamespaceVersion)}, b.NamespaceId...) +func (b *Blob) Namespace() share.Namespace { + return b.namespace } type jsonBlob struct { - Namespace namespace.ID `json:"namespace"` - Data []byte `json:"data"` - ShareVersion uint32 `json:"share_version"` - Commitment Commitment `json:"commitment"` + Namespace share.Namespace `json:"namespace"` + Data []byte `json:"data"` + ShareVersion uint32 `json:"share_version"` + Commitment Commitment `json:"commitment"` } func (b *Blob) MarshalJSON() ([]byte, error) { @@ -155,10 +167,11 @@ func (b *Blob) UnmarshalJSON(data []byte) error { return err } - b.Blob.NamespaceVersion = uint32(blob.Namespace[0]) - b.Blob.NamespaceId = blob.Namespace[1:] + b.Blob.NamespaceVersion = uint32(blob.Namespace.Version()) + b.Blob.NamespaceId = blob.Namespace.ID() b.Blob.Data = blob.Data b.Blob.ShareVersion = blob.ShareVersion b.Commitment = blob.Commitment + b.namespace = blob.Namespace return nil } diff --git a/blob/blob_test.go b/blob/blob_test.go index 3aabd6559b..85486ad125 100644 --- a/blob/blob_test.go +++ b/blob/blob_test.go @@ -8,14 +8,13 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/types" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/blob/blobtest" ) func TestBlob(t *testing.T) { - appBlobs, err := blobtest.GenerateBlobs([]int{1}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{1}, false) require.NoError(t, err) blob, err := convertBlobs(appBlobs...) require.NoError(t, err) @@ -42,12 +41,9 @@ func TestBlob(t *testing.T) { }, }, { - name: "verify nID", + name: "verify namespace", expectedRes: func(t *testing.T) { - ns, err := appns.New( - blob[0].Namespace()[appns.NamespaceVersionSize-1], - blob[0].Namespace()[appns.NamespaceVersionSize:], - ) + ns := blob[0].Namespace().ToAppNamespace() require.NoError(t, err) require.NoError(t, apptypes.ValidateBlobNamespace(ns)) }, diff --git a/blob/blobtest/testing.go b/blob/blobtest/testing.go index 395ef4167a..a22f22f790 100644 --- a/blob/blobtest/testing.go +++ b/blob/blobtest/testing.go @@ -11,14 +11,16 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -func GenerateBlobs(sizes []int, sameNID bool) ([]types.Blob, error) { +// GenerateV0Blobs is a test utility producing v0 share formatted blobs with the +// requested size and random namespaces. +func GenerateV0Blobs(sizes []int, sameNamespace bool) ([]types.Blob, error) { blobs := make([]types.Blob, 0, len(sizes)) for _, size := range sizes { size := rawBlobSize(appconsts.FirstSparseShareContentSize * size) appBlob := testfactory.GenerateRandomBlob(size) - if !sameNID { - nid, err := share.NewNamespaceV0(tmrand.Bytes(7)) + if !sameNamespace { + nid, err := share.NewBlobNamespaceV0(tmrand.Bytes(7)) if err != nil { return nil, err } diff --git a/blob/helper.go b/blob/helper.go index 1fef41dc22..341080f42f 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -19,8 +19,8 @@ func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { } appShares := make([]shares.Share, 0, len(rawShares)) - for _, sh := range rawShares { - bShare, err := shares.NewShare(sh) + for _, shr := range rawShares { + bShare, err := shares.NewShare(shr) if err != nil { return nil, err } diff --git a/blob/service.go b/blob/service.go index 0876a45a0c..960f8c6c01 100644 --- a/blob/service.go +++ b/blob/service.go @@ -12,7 +12,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" @@ -72,8 +71,8 @@ func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { } // Get retrieves all the blobs for given namespaces at the given height by commitment. -func (s *Service) Get(ctx context.Context, height uint64, nID namespace.ID, commitment Commitment) (*Blob, error) { - blob, _, err := s.getByCommitment(ctx, height, nID, commitment) +func (s *Service) Get(ctx context.Context, height uint64, ns share.Namespace, commitment Commitment) (*Blob, error) { + blob, _, err := s.getByCommitment(ctx, height, ns, commitment) if err != nil { return nil, err } @@ -85,10 +84,10 @@ func (s *Service) Get(ctx context.Context, height uint64, nID namespace.ID, comm func (s *Service) GetProof( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment Commitment, ) (*Proof, error) { - _, proof, err := s.getByCommitment(ctx, height, nID, commitment) + _, proof, err := s.getByCommitment(ctx, height, namespace, commitment) if err != nil { return nil, err } @@ -97,29 +96,29 @@ func (s *Service) GetProof( // GetAll returns all blobs under the given namespaces at the given height. // GetAll can return blobs and an error in case if some requests failed. -func (s *Service) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID) ([]*Blob, error) { +func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share.Namespace) ([]*Blob, error) { header, err := s.headerGetter(ctx, height) if err != nil { return nil, err } var ( - resultBlobs = make([][]*Blob, len(nIDs)) - resultErr = make([]error, len(nIDs)) + resultBlobs = make([][]*Blob, len(namespaces)) + resultErr = make([]error, len(namespaces)) ) wg := sync.WaitGroup{} - for i, nID := range nIDs { + for i, namespace := range namespaces { wg.Add(1) - go func(i int, nID namespace.ID) { + go func(i int, namespace share.Namespace) { defer wg.Done() - blobs, err := s.getBlobs(ctx, nID, header.DAH) + blobs, err := s.getBlobs(ctx, namespace, header.DAH) if err != nil { - resultErr[i] = fmt.Errorf("getting blobs for nID(%s): %s", nID.String(), err) + resultErr[i] = fmt.Errorf("getting blobs for namespace(%s): %s", namespace.String(), err) return } resultBlobs[i] = blobs - }(i, nID) + }(i, namespace) } wg.Wait() @@ -143,7 +142,7 @@ func (s *Service) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID func (s *Service) Included( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, proof *Proof, com Commitment, ) (bool, error) { @@ -156,7 +155,7 @@ func (s *Service) Included( // but we have to guarantee that all our stored subtree roots will be on the same height(e.g. one // level above shares). // TODO(@vgonkivs): rework the implementation to perform all verification without network requests. - _, resProof, err := s.getByCommitment(ctx, height, nID, com) + _, resProof, err := s.getByCommitment(ctx, height, namespace, com) switch err { case nil: case ErrBlobNotFound: @@ -172,12 +171,12 @@ func (s *Service) Included( func (s *Service) getByCommitment( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment Commitment, ) (*Blob, *Proof, error) { log.Infow("requesting blob", "height", height, - "nID", nID.String()) + "namespace", namespace.String()) header, err := s.headerGetter(ctx, height) if err != nil { @@ -191,7 +190,7 @@ func (s *Service) getByCommitment( blobShare *shares.Share ) - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, nID) + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, namespace) if err != nil { if errors.Is(err, share.ErrNotFound) { err = ErrBlobNotFound @@ -209,8 +208,8 @@ func (s *Service) getByCommitment( // reconstruct the `blobShare` from the first rawShare in range // in order to get blob's length(first share will contain this info) if blobShare == nil { - for i, sh := range rawShares { - bShare, err := shares.NewShare(sh) + for i, shr := range rawShares { + bShare, err := shares.NewShare(shr) if err != nil { return nil, nil, err } @@ -278,10 +277,10 @@ func (s *Service) getByCommitment( return nil, nil, ErrBlobNotFound } -// getBlobs retrieves the DAH and fetches all shares from the requested namespace.ID and converts +// getBlobs retrieves the DAH and fetches all shares from the requested Namespace and converts // them to Blobs. -func (s *Service) getBlobs(ctx context.Context, nID namespace.ID, root *share.Root) ([]*Blob, error) { - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, root, nID) +func (s *Service) getBlobs(ctx context.Context, namespace share.Namespace, root *share.Root) ([]*Blob, error) { + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, root, namespace) if err != nil { return nil, err } diff --git a/blob/service_test.go b/blob/service_test.go index ee6e982fe8..51475178d2 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -14,17 +14,15 @@ import ( "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/go-header/store" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/blob/blobtest" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) func TestBlobService_Get(t *testing.T) { @@ -37,12 +35,12 @@ func TestBlobService_Get(t *testing.T) { blobSize3 = 12 ) - appBlobs, err := blobtest.GenerateBlobs([]int{blobSize0, blobSize1}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{blobSize0, blobSize1}, false) require.NoError(t, err) blobs0, err := convertBlobs(appBlobs...) require.NoError(t, err) - appBlobs, err = blobtest.GenerateBlobs([]int{blobSize2, blobSize3}, true) + appBlobs, err = blobtest.GenerateV0Blobs([]int{blobSize2, blobSize3}, true) require.NoError(t, err) blobs1, err := convertBlobs(appBlobs...) require.NoError(t, err) @@ -71,9 +69,9 @@ func TestBlobService_Get(t *testing.T) { }, }, { - name: "get all with the same nID", + name: "get all with the same namespace", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, []namespace.ID{blobs1[0].Namespace()}) + b, err := service.GetAll(ctx, 1, []share.Namespace{blobs1[0].Namespace()}) return b, err }, expectedResult: func(res interface{}, err error) { @@ -91,9 +89,9 @@ func TestBlobService_Get(t *testing.T) { }, }, { - name: "get all with different nIDs", + name: "get all with different namespaces", doFn: func() (interface{}, error) { - b, err := service.GetAll(ctx, 1, []namespace.ID{blobs0[0].Namespace(), blobs0[1].Namespace()}) + b, err := service.GetAll(ctx, 1, []share.Namespace{blobs0[0].Namespace(), blobs0[1].Namespace()}) return b, err }, expectedResult: func(res interface{}, err error) { @@ -126,7 +124,7 @@ func TestBlobService_Get(t *testing.T) { { name: "get invalid blob", doFn: func() (interface{}, error) { - appBlob, err := blobtest.GenerateBlobs([]int{10}, false) + appBlob, err := blobtest.GenerateV0Blobs([]int{10}, false) require.NoError(t, err) blob, err := convertBlobs(appBlob...) require.NoError(t, err) @@ -157,13 +155,13 @@ func TestBlobService_Get(t *testing.T) { proof, ok := res.(*Proof) assert.True(t, ok) - verifyFn := func(t *testing.T, rawShares [][]byte, proof *Proof, nID namespace.ID) { + verifyFn := func(t *testing.T, rawShares [][]byte, proof *Proof, namespace share.Namespace) { for _, row := range header.DAH.RowRoots { to := 0 for _, p := range *proof { from := to to = p.End() - p.Start() + from - eq := p.VerifyInclusion(sha256.New(), nID, rawShares[from:to], row) + eq := p.VerifyInclusion(sha256.New(), namespace.ToNMT(), rawShares[from:to], row) if eq == true { return } @@ -209,7 +207,7 @@ func TestBlobService_Get(t *testing.T) { { name: "not included", doFn: func() (interface{}, error) { - appBlob, err := blobtest.GenerateBlobs([]int{10}, false) + appBlob, err := blobtest.GenerateV0Blobs([]int{10}, false) require.NoError(t, err) blob, err := convertBlobs(appBlob...) require.NoError(t, err) @@ -256,8 +254,8 @@ func TestBlobService_Get(t *testing.T) { { name: "get all not found", doFn: func() (interface{}, error) { - nID := tmrand.Bytes(appconsts.NamespaceSize) - return service.GetAll(ctx, 1, []namespace.ID{nID}) + namespace := share.Namespace(tmrand.Bytes(share.NamespaceSize)) + return service.GetAll(ctx, 1, []share.Namespace{namespace}) }, expectedResult: func(i interface{}, err error) { blobs, ok := i.([]*Blob) @@ -297,23 +295,19 @@ func TestBlobService_Get(t *testing.T) { } } -// TestService_GetSingleBlobWithoutPadding creates two blobs with the same nID +// TestService_GetSingleBlobWithoutPadding creates two blobs with the same namespace // But to satisfy the rule of eds creating, padding namespace share is placed between // blobs. Test ensures that blob service will skip padding share and return the correct blob. func TestService_GetSingleBlobWithoutPadding(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) - appBlob, err := blobtest.GenerateBlobs([]int{9, 5}, true) + appBlob, err := blobtest.GenerateV0Blobs([]int{9, 5}, true) require.NoError(t, err) blobs, err := convertBlobs(appBlob...) require.NoError(t, err) - ns1, err := appns.New(blobs[0].Namespace()[0], blobs[0].Namespace()[appns.NamespaceVersionSize:]) - require.NoError(t, err) - - ns2, err := appns.New(blobs[1].Namespace()[0], blobs[1].Namespace()[appns.NamespaceVersionSize:]) - require.NoError(t, err) + ns1, ns2 := blobs[0].Namespace().ToAppNamespace(), blobs[1].Namespace().ToAppNamespace() padding0, err := shares.NamespacePaddingShare(ns1) require.NoError(t, err) @@ -332,7 +326,7 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) - eds, err := share.AddShares(ctx, rawShares, bs) + eds, err := ipld.AddShares(ctx, rawShares, bs) require.NoError(t, err) h := headertest.ExtendedHeaderFromEDS(t, 1, eds) @@ -353,22 +347,12 @@ func TestService_GetAllWithoutPadding(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) - appBlob, err := blobtest.GenerateBlobs([]int{9, 5}, true) + appBlob, err := blobtest.GenerateV0Blobs([]int{9, 5}, true) require.NoError(t, err) blobs, err := convertBlobs(appBlob...) require.NoError(t, err) - ns1, err := appns.New( - blobs[0].Namespace()[appns.NamespaceVersionSize-1], - blobs[0].Namespace()[appns.NamespaceVersionSize:], - ) - require.NoError(t, err) - - ns2, err := appns.New( - blobs[1].Namespace()[appns.NamespaceVersionSize-1], - blobs[1].Namespace()[appns.NamespaceVersionSize:], - ) - require.NoError(t, err) + ns1, ns2 := blobs[0].Namespace().ToAppNamespace(), blobs[1].Namespace().ToAppNamespace() padding0, err := shares.NamespacePaddingShare(ns1) require.NoError(t, err) @@ -393,7 +377,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) - eds, err := share.AddShares(ctx, rawShares, bs) + eds, err := ipld.AddShares(ctx, rawShares, bs) require.NoError(t, err) h := headertest.ExtendedHeaderFromEDS(t, 1, eds) @@ -406,7 +390,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { service := NewService(nil, getters.NewIPLDGetter(bs), fn) - _, err = service.GetAll(ctx, 1, []namespace.ID{blobs[0].Namespace(), blobs[1].Namespace()}) + _, err = service.GetAll(ctx, 1, []share.Namespace{blobs[0].Namespace(), blobs[1].Namespace()}) require.NoError(t, err) } @@ -417,7 +401,7 @@ func createService(ctx context.Context, t *testing.T, blobs []*Blob) *Service { require.NoError(t, err) rawShares, err := BlobsToShares(blobs...) require.NoError(t, err) - eds, err := share.AddShares(ctx, rawShares, bs) + eds, err := ipld.AddShares(ctx, rawShares, bs) require.NoError(t, err) h := headertest.ExtendedHeaderFromEDS(t, 1, eds) diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 18851dc797..169ca4b8d1 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -16,9 +16,6 @@ import ( "github.com/spf13/cobra" - appns "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/share" @@ -112,18 +109,18 @@ func parseParams(method string, params []string) []interface{} { panic(fmt.Errorf("couldn't parse share root as json: %v", err)) } parsedParams[0] = root - // 2. NamespaceID - nID, err := parseV0NamespaceID(params[1]) + // 2. Namespace + namespace, err := parseV0Namespace(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - parsedParams[1] = nID + parsedParams[1] = namespace case "Submit": - // 1. NamespaceID + // 1. Namespace var err error - nID, err := parseV0NamespaceID(params[0]) + namespace, err := parseV0Namespace(params[0]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } // 2. Blob data var blobData []byte @@ -146,7 +143,7 @@ func parseParams(method string, params []string) []interface{} { panic("Error decoding blob data: base64 string could not be decoded.") } } - parsedBlob, err := blob.NewBlob(0, nID, blobData) + parsedBlob, err := blob.NewBlobV0(namespace, blobData) if err != nil { panic(fmt.Sprintf("Error creating blob: %v", err)) } @@ -162,10 +159,10 @@ func parseParams(method string, params []string) []interface{} { panic("Error parsing gas limit: uint64 could not be parsed.") } parsedParams[1] = num - // 3. NamespaceID - nID, err := parseV0NamespaceID(params[2]) + // 3. Namespace + namespace, err := parseV0Namespace(params[2]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } // 4. Blob data var blobData []byte @@ -188,7 +185,7 @@ func parseParams(method string, params []string) []interface{} { panic("Error decoding blob: base64 string could not be decoded.") } } - parsedBlob, err := blob.NewBlob(0, nID, blobData) + parsedBlob, err := blob.NewBlobV0(namespace, blobData) if err != nil { panic(fmt.Sprintf("Error creating blob: %v", err)) } @@ -202,11 +199,11 @@ func parseParams(method string, params []string) []interface{} { } parsedParams[0] = num // 2. NamespaceID - nID, err := parseV0NamespaceID(params[1]) + namespace, err := parseV0Namespace(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - parsedParams[1] = nID + parsedParams[1] = namespace // 3: Commitment commitment, err := base64.StdEncoding.DecodeString(params[2]) if err != nil { @@ -221,12 +218,12 @@ func parseParams(method string, params []string) []interface{} { panic("Error parsing gas limit: uint64 could not be parsed.") } parsedParams[0] = num - // 2. NamespaceID - nID, err := parseV0NamespaceID(params[1]) + // 2. Namespace + namespace, err := parseV0Namespace(params[1]) if err != nil { - panic(fmt.Sprintf("Error parsing namespace ID: %v", err)) + panic(fmt.Sprintf("Error parsing namespace: %v", err)) } - parsedParams[1] = []namespace.ID{nID} + parsedParams[1] = []share.Namespace{namespace} return parsedParams case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": var err error @@ -419,23 +416,17 @@ func parseSignatureForHelpstring(methodSig reflect.StructField) string { return simplifiedSignature } -// parseV0NamespaceID parses a namespace ID from a base64 or hex string. The param +// parseV0Namespace parses a namespace from a base64 or hex string. The param // is expected to be the user-specified portion of a v0 namespace ID (i.e. the // last 10 bytes). -func parseV0NamespaceID(param string) (namespace.ID, error) { +func parseV0Namespace(param string) (share.Namespace, error) { userBytes, err := decodeToBytes(param) if err != nil { return nil, err } - if len(userBytes) > appns.NamespaceVersionZeroIDSize { - return nil, fmt.Errorf( - "namespace ID %v is too large to be a v0 namespace, want <= %v bytes", - userBytes, appns.NamespaceVersionZeroIDSize, - ) - } // if the namespace ID is <= 10 bytes, left pad it with 0s - return share.NewNamespaceV0(userBytes) + return share.NewBlobNamespaceV0(userBytes) } // decodeToBytes decodes a Base64 or hex input string into a byte slice. diff --git a/cmd/celestia/rpc_test.go b/cmd/celestia/rpc_test.go index 9380a8b01e..53087646a7 100644 --- a/cmd/celestia/rpc_test.go +++ b/cmd/celestia/rpc_test.go @@ -5,21 +5,21 @@ import ( "github.com/stretchr/testify/assert" - "github.com/celestiaorg/nmt/namespace" + "github.com/celestiaorg/celestia-node/share" ) func Test_parseNamespaceID(t *testing.T) { type testCase struct { name string param string - want namespace.ID + want share.Namespace wantErr bool } testCases := []testCase{ { param: "0x0c204d39600fddd3", name: "8 byte hex encoded namespace ID gets left padded", - want: namespace.ID{ + want: share.Namespace{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, }, @@ -28,7 +28,7 @@ func Test_parseNamespaceID(t *testing.T) { { name: "10 byte hex encoded namespace ID", param: "0x42690c204d39600fddd3", - want: namespace.ID{ + want: share.Namespace{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x69, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, }, @@ -37,7 +37,7 @@ func Test_parseNamespaceID(t *testing.T) { { name: "29 byte hex encoded namespace ID", param: "0x0000000000000000000000000000000000000001010101010101010101", - want: namespace.ID{ + want: share.Namespace{ 0x0, // namespace version 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // v0 ID prefix 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, // namespace ID @@ -47,13 +47,13 @@ func Test_parseNamespaceID(t *testing.T) { { name: "11 byte hex encoded namespace ID returns error", param: "0x42690c204d39600fddd3a3", - want: namespace.ID{}, + want: share.Namespace{}, wantErr: true, }, { name: "10 byte base64 encoded namespace ID", param: "QmkMIE05YA/d0w==", - want: namespace.ID{ + want: share.Namespace{ 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x42, 0x69, 0xc, 0x20, 0x4d, 0x39, 0x60, 0xf, 0xdd, 0xd3, }, @@ -62,14 +62,14 @@ func Test_parseNamespaceID(t *testing.T) { { name: "not base64 or hex encoded namespace ID returns error", param: "5748493939429", - want: namespace.ID{}, + want: share.Namespace{}, wantErr: true, }, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got, err := parseV0NamespaceID(tc.param) + got, err := parseV0Namespace(tc.param) if tc.wantErr { assert.Error(t, err) return @@ -77,6 +77,5 @@ func Test_parseNamespaceID(t *testing.T) { assert.NoError(t, err) assert.Equal(t, tc.want, got) }) - } } diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 3e0da71d69..fbc71f92f9 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -26,7 +26,8 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" ) var log = logging.Logger("headertest") @@ -349,17 +350,15 @@ func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedData func CreateFraudExtHeader( t *testing.T, eh *header.ExtendedHeader, - dag blockservice.BlockService, + serv blockservice.BlockService, ) (*header.ExtendedHeader, *rsmt2d.ExtendedDataSquare) { - extended := share.RandEDS(t, 2) - shares := share.ExtractEDS(extended) - copy(shares[0][share.NamespaceSize:], shares[1][share.NamespaceSize:]) - extended, err := share.ImportShares(context.Background(), shares, dag) + square := edstest.RandByzantineEDS(t, 16) + err := ipld.ImportEDS(context.Background(), square, serv) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(extended) + dah := da.NewDataAvailabilityHeader(square) eh.DAH = &dah eh.RawHeader.DataHash = dah.Hash() - return eh, extended + return eh, square } type Subscriber struct { diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index 7cbe312856..aae502824c 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -3,9 +3,8 @@ package blob import ( "context" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/share" ) var _ Module = (*API)(nil) @@ -19,23 +18,23 @@ type Module interface { // Uses default wallet registered on the Node. Submit(_ context.Context, _ []*blob.Blob) (height uint64, _ error) // Get retrieves the blob by commitment under the given namespace and height. - Get(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Blob, error) + Get(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Blob, error) // GetAll returns all blobs under the given namespaces and height. - GetAll(_ context.Context, height uint64, _ []namespace.ID) ([]*blob.Blob, error) + GetAll(_ context.Context, height uint64, _ []share.Namespace) ([]*blob.Blob, error) // GetProof retrieves proofs in the given namespaces at the given height by commitment. - GetProof(_ context.Context, height uint64, _ namespace.ID, _ blob.Commitment) (*blob.Proof, error) + GetProof(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Proof, error) // Included checks whether a blob's given commitment(Merkle subtree root) is included at // given height and under the namespace. - Included(_ context.Context, height uint64, _ namespace.ID, _ *blob.Proof, _ blob.Commitment) (bool, error) + Included(_ context.Context, height uint64, _ share.Namespace, _ *blob.Proof, _ blob.Commitment) (bool, error) } type API struct { Internal struct { - Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` - Get func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Blob, error) `perm:"read"` - GetAll func(context.Context, uint64, []namespace.ID) ([]*blob.Blob, error) `perm:"read"` - GetProof func(context.Context, uint64, namespace.ID, blob.Commitment) (*blob.Proof, error) `perm:"read"` - Included func(context.Context, uint64, namespace.ID, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` + Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` + Get func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Blob, error) `perm:"read"` + GetAll func(context.Context, uint64, []share.Namespace) ([]*blob.Blob, error) `perm:"read"` + GetProof func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Proof, error) `perm:"read"` + Included func(context.Context, uint64, share.Namespace, *blob.Proof, blob.Commitment) (bool, error) `perm:"read"` } } @@ -46,31 +45,31 @@ func (api *API) Submit(ctx context.Context, blobs []*blob.Blob) (uint64, error) func (api *API) Get( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment blob.Commitment, ) (*blob.Blob, error) { - return api.Internal.Get(ctx, height, nID, commitment) + return api.Internal.Get(ctx, height, namespace, commitment) } -func (api *API) GetAll(ctx context.Context, height uint64, nIDs []namespace.ID) ([]*blob.Blob, error) { - return api.Internal.GetAll(ctx, height, nIDs) +func (api *API) GetAll(ctx context.Context, height uint64, namespaces []share.Namespace) ([]*blob.Blob, error) { + return api.Internal.GetAll(ctx, height, namespaces) } func (api *API) GetProof( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, commitment blob.Commitment, ) (*blob.Proof, error) { - return api.Internal.GetProof(ctx, height, nID, commitment) + return api.Internal.GetProof(ctx, height, namespace, commitment) } func (api *API) Included( ctx context.Context, height uint64, - nID namespace.ID, + namespace share.Namespace, proof *blob.Proof, commitment blob.Commitment, ) (bool, error) { - return api.Internal.Included(ctx, height, nID, proof, commitment) + return api.Internal.Included(ctx, height, namespace, proof, commitment) } diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index b7c61aa450..5cd34b74b6 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -11,7 +11,7 @@ import ( gomock "github.com/golang/mock/gomock" blob "github.com/celestiaorg/celestia-node/blob" - namespace "github.com/celestiaorg/nmt/namespace" + share "github.com/celestiaorg/celestia-node/share" ) // MockModule is a mock of Module interface. @@ -38,7 +38,7 @@ func (m *MockModule) EXPECT() *MockModuleMockRecorder { } // Get mocks base method. -func (m *MockModule) Get(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 blob.Commitment) (*blob.Blob, error) { +func (m *MockModule) Get(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 blob.Commitment) (*blob.Blob, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*blob.Blob) @@ -53,7 +53,7 @@ func (mr *MockModuleMockRecorder) Get(arg0, arg1, arg2, arg3 interface{}) *gomoc } // GetAll mocks base method. -func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 []namespace.ID) ([]*blob.Blob, error) { +func (m *MockModule) GetAll(arg0 context.Context, arg1 uint64, arg2 []share.Namespace) ([]*blob.Blob, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetAll", arg0, arg1, arg2) ret0, _ := ret[0].([]*blob.Blob) @@ -68,7 +68,7 @@ func (mr *MockModuleMockRecorder) GetAll(arg0, arg1, arg2 interface{}) *gomock.C } // GetProof mocks base method. -func (m *MockModule) GetProof(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 blob.Commitment) (*blob.Proof, error) { +func (m *MockModule) GetProof(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 blob.Commitment) (*blob.Proof, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetProof", arg0, arg1, arg2, arg3) ret0, _ := ret[0].(*blob.Proof) @@ -83,7 +83,7 @@ func (mr *MockModuleMockRecorder) GetProof(arg0, arg1, arg2, arg3 interface{}) * } // Included mocks base method. -func (m *MockModule) Included(arg0 context.Context, arg1 uint64, arg2 namespace.ID, arg3 *blob.Proof, arg4 blob.Commitment) (bool, error) { +func (m *MockModule) Included(arg0 context.Context, arg1 uint64, arg2 share.Namespace, arg3 *blob.Proof, arg4 blob.Commitment) (bool, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Included", arg0, arg1, arg2, arg3, arg4) ret0, _ := ret[0].(bool) diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 1913b3d576..5cb0e41e53 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/filecoin-project/dagstore" + "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" @@ -18,6 +19,7 @@ import ( "github.com/celestiaorg/celestia-node/share/availability/light" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" ) @@ -60,6 +62,15 @@ func ensureEmptyCARExists(ctx context.Context, store *eds.Store) error { return err } +// ensureEmptyEDSInBS checks if the given DAG contains an empty block data square. +// If it does not, it stores an empty block. This optimization exists to prevent +// redundant storing of empty block data so that it is only stored once and returned +// upon request for a block with an empty data square. +func ensureEmptyEDSInBS(ctx context.Context, bServ blockservice.BlockService) error { + _, err := ipld.AddShares(ctx, share.EmptyBlockShares(), bServ) + return err +} + func lightGetter( shrexGetter *getters.ShrexGetter, ipldGetter *getters.IPLDGetter, diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 1b26273c0f..66baa23301 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -12,7 +12,6 @@ import ( da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" - namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" ) @@ -70,7 +69,7 @@ func (mr *MockModuleMockRecorder) GetShare(arg0, arg1, arg2, arg3 interface{}) * } // GetSharesByNamespace mocks base method. -func (m *MockModule) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 namespace.ID) (share.NamespacedShares, error) { +func (m *MockModule) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 share.Namespace) (share.NamespacedShares, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSharesByNamespace", arg0, arg1, arg2) ret0, _ := ret[0].(share.NamespacedShares) diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 6dfb155bb0..b924cf8167 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -176,7 +176,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option } }), shrexGetterComponents, - fx.Invoke(share.EnsureEmptySquareExists), + fx.Invoke(ensureEmptyEDSInBS), fx.Provide(getters.NewIPLDGetter), fx.Provide(lightGetter), // shrexsub broadcaster stub for daser diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index 0c703f9a15..def3bb4d0f 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -3,7 +3,6 @@ package share import ( "context" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -40,7 +39,7 @@ type Module interface { GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) // GetSharesByNamespace gets all shares from an EDS within the given namespace. // Shares are returned in a row-by-row order if the namespace spans multiple rows. - GetSharesByNamespace(ctx context.Context, root *share.Root, namespace namespace.ID) (share.NamespacedShares, error) + GetSharesByNamespace(ctx context.Context, root *share.Root, namespace share.Namespace) (share.NamespacedShares, error) } // API is a wrapper around Module for the RPC. @@ -61,7 +60,7 @@ type API struct { GetSharesByNamespace func( ctx context.Context, root *share.Root, - namespace namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) `perm:"public"` } } @@ -85,7 +84,7 @@ func (api *API) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedD func (api *API) GetSharesByNamespace( ctx context.Context, root *share.Root, - namespace namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) { return api.Internal.GetSharesByNamespace(ctx, root, namespace) } diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 37504ec3b3..1bc1c261de 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -109,7 +109,7 @@ func TestBlobRPC(t *testing.T) { client, err := client.NewClient(ctx, bridgeAddr, jwt) require.NoError(t, err) - appBlobs, err := blobtest.GenerateBlobs([]int{8}, false) + appBlobs, err := blobtest.GenerateV0Blobs([]int{8}, false) require.NoError(t, err) newBlob, err := blob.NewBlob( diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 537e320c5a..6a7db50c7c 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -11,12 +11,11 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" + "github.com/celestiaorg/celestia-node/share" ) func TestBlobModule(t *testing.T) { @@ -24,9 +23,9 @@ func TestBlobModule(t *testing.T) { t.Cleanup(cancel) sw := swamp.NewSwamp(t) - appBlobs0, err := blobtest.GenerateBlobs([]int{8, 4}, true) + appBlobs0, err := blobtest.GenerateV0Blobs([]int{8, 4}, true) require.NoError(t, err) - appBlobs1, err := blobtest.GenerateBlobs([]int{4}, false) + appBlobs1, err := blobtest.GenerateV0Blobs([]int{4}, false) require.NoError(t, err) blobs := make([]*blob.Blob, 0, len(appBlobs0)+len(appBlobs1)) @@ -79,7 +78,7 @@ func TestBlobModule(t *testing.T) { { name: "GetAll", doFn: func(t *testing.T) { - newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, []namespace.ID{blobs[0].Namespace()}) + newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, []share.Namespace{blobs[0].Namespace()}) require.NoError(t, err) require.Len(t, newBlobs, len(appBlobs0)) require.True(t, bytes.Equal(blobs[0].Commitment, newBlobs[0].Commitment)) @@ -106,7 +105,7 @@ func TestBlobModule(t *testing.T) { { name: "Not Found", doFn: func(t *testing.T) { - appBlob, err := blobtest.GenerateBlobs([]int{4}, false) + appBlob, err := blobtest.GenerateV0Blobs([]int{4}, false) require.NoError(t, err) newBlob, err := blob.NewBlob( appBlob[0].ShareVersion, diff --git a/share/availability.go b/share/availability.go index 9538573114..b6b44271c8 100644 --- a/share/availability.go +++ b/share/availability.go @@ -4,7 +4,7 @@ import ( "context" "errors" - da "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/da" ) // ErrNotAvailable is returned whenever DA sampling fails. diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index ca72596816..1709c3f4b7 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -10,11 +10,11 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestSharesAvailable(t *testing.T) { @@ -81,23 +81,23 @@ func TestService_GetSharesByNamespace(t *testing.T) { t.Run("size: "+strconv.Itoa(tt.squareSize), func(t *testing.T) { getter, bServ := EmptyGetter() totalShares := tt.squareSize * tt.squareSize - randShares := share.RandShares(t, totalShares) + randShares := sharetest.RandShares(t, totalShares) idx1 := (totalShares - 1) / 2 idx2 := totalShares / 2 if tt.expectedShareCount > 1 { - // make it so that two rows have the same namespace ID - copy(randShares[idx2][:namespace.NamespaceSize], randShares[idx1][:namespace.NamespaceSize]) + // make it so that two rows have the same namespace + copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) } root := availability_test.FillBS(t, bServ, randShares) - randNID := randShares[idx1][:namespace.NamespaceSize] + randNamespace := share.GetNamespace(randShares[idx1]) - shares, err := getter.GetSharesByNamespace(context.Background(), root, randNID) + shares, err := getter.GetSharesByNamespace(context.Background(), root, randNamespace) require.NoError(t, err) - require.NoError(t, shares.Verify(root, randNID)) + require.NoError(t, shares.Verify(root, randNamespace)) flattened := shares.Flatten() assert.Len(t, flattened, tt.expectedShareCount) for _, value := range flattened { - assert.Equal(t, randNID, []byte(share.ID(value))) + assert.Equal(t, randNamespace, share.GetNamespace(value)) } if tt.expectedShareCount > 1 { // idx1 is always smaller than idx2 @@ -105,14 +105,14 @@ func TestService_GetSharesByNamespace(t *testing.T) { assert.Equal(t, randShares[idx2], flattened[1]) } }) - t.Run("last two rows of a 4x4 square that have the same namespace ID have valid NMT proofs", func(t *testing.T) { + t.Run("last two rows of a 4x4 square that have the same namespace have valid NMT proofs", func(t *testing.T) { squareSize := 4 totalShares := squareSize * squareSize getter, bServ := EmptyGetter() - randShares := share.RandShares(t, totalShares) - lastNID := randShares[totalShares-1][:namespace.NamespaceSize] + randShares := sharetest.RandShares(t, totalShares) + lastNID := share.GetNamespace(randShares[totalShares-1]) for i := totalShares / 2; i < totalShares; i++ { - copy(randShares[i][:namespace.NamespaceSize], lastNID) + copy(share.GetNamespace(randShares[i]), lastNID) } root := availability_test.FillBS(t, bServ, randShares) @@ -141,7 +141,7 @@ func TestService_GetSharesByNamespaceNotFound(t *testing.T) { getter, root := GetterWithRandSquare(t, 1) root.RowRoots = nil - emptyShares, err := getter.GetSharesByNamespace(context.Background(), root, namespace.RandomNamespace().Bytes()) + emptyShares, err := getter.GetSharesByNamespace(context.Background(), root, sharetest.RandV0Namespace()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) } @@ -159,11 +159,11 @@ func BenchmarkService_GetSharesByNamespace(b *testing.B) { b.Run(strconv.Itoa(tt.amountShares), func(b *testing.B) { t := &testing.T{} getter, root := GetterWithRandSquare(t, tt.amountShares) - randNID := root.RowRoots[(len(root.RowRoots)-1)/2][:8] + randNamespace := root.RowRoots[(len(root.RowRoots)-1)/2][:share.NamespaceSize] root.RowRoots[(len(root.RowRoots) / 2)] = root.RowRoots[(len(root.RowRoots)-1)/2] b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := getter.GetSharesByNamespace(context.Background(), root, randNID) + _, err := getter.GetSharesByNamespace(context.Background(), root, randNamespace) require.NoError(t, err) } }) diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index b2fb780bd4..19f63f114a 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -20,17 +20,19 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) // RandFillBS fills the given BlockService with a random block of a given size. func RandFillBS(t *testing.T, n int, bServ blockservice.BlockService) *share.Root { - shares := share.RandShares(t, n*n) + shares := sharetest.RandShares(t, n*n) return FillBS(t, bServ, shares) } // FillBS fills the given BlockService with the given shares. func FillBS(t *testing.T, bServ blockservice.BlockService, shares []share.Share) *share.Root { - eds, err := share.AddShares(context.TODO(), shares, bServ) + eds, err := ipld.AddShares(context.TODO(), shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(eds) return &dah diff --git a/share/doc.go b/share/doc.go index 7ef8c9f7bf..97229932a7 100644 --- a/share/doc.go +++ b/share/doc.go @@ -4,7 +4,7 @@ block data. Though this package contains several useful methods for getting specific shares and/or sampling them at random, a particularly useful method is GetSharesByNamespace which retrieves -all shares of block data of the given namespace.ID from the block associated with the given +all shares of block data of the given Namespace from the block associated with the given DataAvailabilityHeader (DAH, but referred to as Root within this package). This package also contains declaration of the Availability interface. Implementations of diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 4e673ed8b8..0672096b25 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -143,17 +143,17 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { // verify that Merkle proofs correspond to particular shares. shares := make([][]byte, len(merkleRowRoots)) - for index, share := range p.Shares { - if share == nil { + for index, shr := range p.Shares { + if shr == nil { continue } // validate inclusion of the share into one of the DAHeader roots - if ok := share.Validate(ipld.MustCidFromNamespacedSha256(root)); !ok { + if ok := shr.Validate(ipld.MustCidFromNamespacedSha256(root)); !ok { return fmt.Errorf("fraud: invalid proof: incorrect share received at index %d", index) } // NMTree commits the additional namespace while rsmt2d does not know about, so we trim it // this is ugliness from NMTWrapper that we have to embrace ¯\_(ツ)_/¯ - shares[index] = share.Share[ipld.NamespaceSize:] + shares[index] = share.GetData(shr.Share) } odsWidth := uint64(len(merkleRowRoots) / 2) diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 6e413b17a2..5f6adac595 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -3,8 +3,10 @@ package byzantine import ( "context" "testing" + "time" mdutils "github.com/ipfs/go-merkledag/test" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" core "github.com/tendermint/tendermint/types" @@ -12,10 +14,34 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) +func TestBadEncodingFraudProof(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer t.Cleanup(cancel) + bServ := mdutils.Bserv() + + square := edstest.RandByzantineEDS(t, 16) + dah := da.NewDataAvailabilityHeader(square) + err := ipld.ImportEDS(ctx, square, bServ) + require.NoError(t, err) + + var errRsmt2d *rsmt2d.ErrByzantineData + err = square.Repair(dah.RowRoots, dah.ColumnRoots) + require.ErrorAs(t, err, &errRsmt2d) + + errByz := NewErrByzantine(ctx, bServ, &dah, errRsmt2d) + + befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) + err = befp.Validate(&header.ExtendedHeader{ + DAH: &dah, + }) + assert.NoError(t, err) +} + // TestIncorrectBadEncodingFraudProof asserts that BEFP is not generated for the correct data func TestIncorrectBadEncodingFraudProof(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) @@ -24,9 +50,9 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { bServ := mdutils.Bserv() squareSize := 8 - shares := share.RandShares(t, squareSize*squareSize) + shares := sharetest.RandShares(t, squareSize*squareSize) - eds, err := share.AddShares(ctx, shares, bServ) + eds, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(eds) diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index 86eaad54a8..b8e39ee1d3 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -45,8 +45,8 @@ func NewShareWithProof(index int, share share.Share, pathToLeaf []cid.Cid) *Shar func (s *ShareWithProof) Validate(root cid.Cid) bool { return s.Proof.VerifyInclusion( sha256.New(), // TODO(@Wondertan): This should be defined somewhere globally - share.ID(s.Share), - [][]byte{share.Data(s.Share)}, + share.GetNamespace(s.Share).ToNMT(), + [][]byte{share.GetData(s.Share)}, ipld.NamespacedSha256FromCID(root), ) } diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go index 9cffe6eb18..0f63d4f0c9 100644 --- a/share/eds/byzantine/share_proof_test.go +++ b/share/eds/byzantine/share_proof_test.go @@ -12,8 +12,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestGetProof(t *testing.T) { @@ -23,8 +23,8 @@ func TestGetProof(t *testing.T) { defer cancel() bServ := mdutils.Bserv() - shares := share.RandShares(t, width*width) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, width*width) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(in) @@ -59,8 +59,8 @@ func TestGetProofs(t *testing.T) { defer cancel() bServ := mdutils.Bserv() - shares := share.RandShares(t, width*width) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, width*width) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(in) diff --git a/share/eds/eds.go b/share/eds/eds.go index 4e96fd684e..239ba4a0ad 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -18,9 +18,7 @@ import ( "github.com/ipld/go-car/util" "github.com/minio/sha256-simd" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -114,7 +112,7 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io. return &writingSession{ eds: eds, store: store, - hasher: nmt.NewNmtHasher(sha256.New(), ipld.NamespaceSize, ipld.NMTIgnoreMaxNamespace), + hasher: nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace), w: w, }, nil } @@ -219,13 +217,13 @@ func getQuadrantCells(eds *rsmt2d.ExtendedDataSquare, i, j uint) [][]byte { // prependNamespace adds the namespace to the passed share if in the first quadrant, // otherwise it adds the ParitySharesNamespace to the beginning. -func prependNamespace(quadrant int, share []byte) []byte { - namespacedShare := make([]byte, 0, appconsts.NamespaceSize+appconsts.ShareSize) +func prependNamespace(quadrant int, shr share.Share) []byte { + namespacedShare := make([]byte, 0, share.NamespaceSize+share.Size) switch quadrant { case 0: - return append(append(namespacedShare, share[:ipld.NamespaceSize]...), share...) + return append(append(namespacedShare, share.GetNamespace(shr)...), shr...) case 1, 2, 3: - return append(append(namespacedShare, namespace.ParitySharesNamespace.Bytes()...), share...) + return append(append(namespacedShare, share.ParitySharesNamespace...), shr...) default: panic("invalid quadrant") } @@ -273,7 +271,7 @@ func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d } // the stored first quadrant shares are wrapped with the namespace twice. // we cut it off here, because it is added again while importing to the tree below - shares[i] = block.RawData()[ipld.NamespaceSize:] + shares[i] = share.GetData(block.RawData()) } eds, err = rsmt2d.ComputeExtendedDataSquare( diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index ea0f06c138..0ef211ec6f 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -18,10 +18,10 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -57,7 +57,7 @@ func TestQuadrantOrder(t *testing.T) { res := quadrantOrder(eds) for _, s := range res { - require.Len(t, s, testShareSize+namespace.NamespaceSize) + require.Len(t, s, testShareSize+share.NamespaceSize) } for q := 0; q < 4; q++ { @@ -101,7 +101,7 @@ func TestWriteEDSStartsWithLeaves(t *testing.T) { block, err := reader.Next() require.NoError(t, err, "error getting first block") - require.Equal(t, block.RawData()[ipld.NamespaceSize:], eds.GetCell(0, 0)) + require.Equal(t, share.GetData(block.RawData()), eds.GetCell(0, 0)) } func TestWriteEDSIncludesRoots(t *testing.T) { @@ -193,7 +193,7 @@ func TestReadEDS(t *testing.T) { func TestReadEDSContentIntegrityMismatch(t *testing.T) { writeRandomEDS(t) - dah := da.NewDataAvailabilityHeader(share.RandEDS(t, 4)) + dah := da.NewDataAvailabilityHeader(edstest.RandEDS(t, 4)) f := openWrittenEDS(t) defer f.Close() @@ -208,7 +208,7 @@ func BenchmarkReadWriteEDS(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) for originalDataWidth := 4; originalDataWidth <= 64; originalDataWidth *= 2 { - eds := share.RandEDS(b, originalDataWidth) + eds := edstest.RandEDS(b, originalDataWidth) dah := da.NewDataAvailabilityHeader(eds) b.Run(fmt.Sprintf("Writing %dx%d", originalDataWidth, originalDataWidth), func(b *testing.B) { b.ReportAllocs() @@ -242,7 +242,7 @@ func writeRandomEDS(t *testing.T) *rsmt2d.ExtendedDataSquare { f, err := os.OpenFile("test.car", os.O_WRONLY|os.O_CREATE, 0600) require.NoError(t, err, "error opening file") - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) err = WriteEDS(ctx, eds, f) require.NoError(t, err, "error writing EDS to file") f.Close() @@ -276,7 +276,7 @@ func createTestData(t *testing.T, testDir string) { //nolint:unused f, err := os.OpenFile("example.car", os.O_WRONLY|os.O_CREATE, 0600) require.NoError(t, err, "opening file") - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) err = WriteEDS(ctx, eds, f) require.NoError(t, err, "writing EDS to file") f.Close() diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go new file mode 100644 index 0000000000..2b6d9ef78d --- /dev/null +++ b/share/eds/edstest/testing.go @@ -0,0 +1,31 @@ +package edstest + +import ( + "testing" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/sharetest" +) + +func RandByzantineEDS(t *testing.T, size int) *rsmt2d.ExtendedDataSquare { + eds := RandEDS(t, size) + shares := share.ExtractEDS(eds) + copy(share.GetData(shares[0]), share.GetData(shares[1])) // corrupting eds + eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + require.NoError(t, err, "failure to recompute the extended data square") + return eds +} + +// RandEDS generates EDS filled with the random data with the given size for original square. It +// uses require.TestingT to be able to take both a *testing.T and a *testing.B. +func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { + shares := sharetest.RandShares(t, size*size) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + require.NoError(t, err, "failure to recompute the extended data square") + return eds +} diff --git a/share/eds/ods_test.go b/share/eds/ods_test.go index 75fad3022e..eb243a4022 100644 --- a/share/eds/ods_test.go +++ b/share/eds/ods_test.go @@ -52,7 +52,7 @@ func TestODSReader(t *testing.T) { assert.NoError(t, err) // check that original data from eds is same as data from reader - assert.Equal(t, original, block.RawData()[share.NamespaceSize:]) + assert.Equal(t, original, share.GetData(block.RawData())) } } diff --git a/share/eds/retriever.go b/share/eds/retriever.go index b2dcc4ff7a..b3cf363056 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -256,7 +256,7 @@ func (rs *retrievalSession) doRequest(ctx context.Context, q *quadrant) { // and go get shares of left or the right side of the whole col/row axis // the left or the right side of the tree represent some portion of the quadrant // which we put into the rs.square share-by-share by calculating shares' indexes using q.index - share.GetShares(ctx, rs.bget, nd.Links()[q.x].Cid, size, func(j int, share share.Share) { + ipld.GetShares(ctx, rs.bget, nd.Links()[q.x].Cid, size, func(j int, share share.Share) { // NOTE: Each share can appear twice here, for a Row and Col, respectively. // These shares are always equal, and we allow only the first one to be written // in the square. diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index e90216de13..ca689aa5ec 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -2,11 +2,9 @@ package eds import ( "context" - "errors" "testing" "time" - "github.com/ipfs/go-blockservice" mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -16,11 +14,11 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestRetriever_Retrieve(t *testing.T) { @@ -48,8 +46,8 @@ func TestRetriever_Retrieve(t *testing.T) { tc := tc t.Run(tc.name, func(t *testing.T) { // generate EDS - shares := share.RandShares(t, tc.squareSize*tc.squareSize) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, tc.squareSize*tc.squareSize) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) // limit with timeout, specifically retrieval @@ -70,8 +68,8 @@ func TestRetriever_ByzantineError(t *testing.T) { defer cancel() bserv := mdutils.Bserv() - shares := share.ExtractEDS(share.RandEDS(t, width)) - _, err := share.ImportShares(ctx, shares, bserv) + shares := share.ExtractEDS(edstest.RandEDS(t, width)) + _, err := ipld.ImportShares(ctx, shares, bserv) require.NoError(t, err) // corrupt shares so that eds erasure coding does not match @@ -109,8 +107,8 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { r := NewRetriever(bServ) // generate EDS - shares := share.RandShares(t, squareSize*squareSize) - in, err := share.AddShares(ctx, shares, bServ) + shares := sharetest.RandShares(t, squareSize*squareSize) + in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(in) @@ -126,31 +124,3 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { _, err = ses.Reconstruct(ctx) assert.NoError(t, err) } - -func TestFraudProofValidation(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) - defer t.Cleanup(cancel) - bServ := mdutils.Bserv() - - var errByz *byzantine.ErrByzantine - faultHeader, err := generateByzantineError(ctx, t, bServ) - require.True(t, errors.As(err, &errByz)) - - p := byzantine.CreateBadEncodingProof([]byte("hash"), uint64(faultHeader.Height()), errByz) - err = p.Validate(faultHeader) - require.NoError(t, err) -} - -func generateByzantineError( - ctx context.Context, - t *testing.T, - bServ blockservice.BlockService, -) (*header.ExtendedHeader, error) { - store := headertest.NewStore(t) - h, err := store.GetByHeight(ctx, 1) - require.NoError(t, err) - - faultHeader, _ := headertest.CreateFraudExtHeader(t, h, bServ) - _, err = NewRetriever(bServ).Retrieve(ctx, faultHeader.DAH) - return faultHeader, err -} diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 6c6d7d10b2..b1dc3986b7 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -17,6 +17,7 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" ) func TestEDSStore(t *testing.T) { @@ -77,7 +78,7 @@ func TestEDSStore(t *testing.T) { original := eds.GetCell(uint(i), uint(j)) block, err := carReader.Next() assert.NoError(t, err) - assert.Equal(t, original, block.RawData()[share.NamespaceSize:]) + assert.Equal(t, original, share.GetData(block.RawData())) } } }) @@ -261,7 +262,7 @@ func newStore(t *testing.T) (*Store, error) { } func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) return eds, dah diff --git a/share/empty.go b/share/empty.go index 0b7ea2e775..b0e1e6e6ae 100644 --- a/share/empty.go +++ b/share/empty.go @@ -2,10 +2,8 @@ package share import ( "bytes" - "context" "fmt" - - "github.com/ipfs/go-blockservice" + "sync" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" @@ -13,19 +11,48 @@ import ( "github.com/celestiaorg/rsmt2d" ) +// EmptyRoot returns Root of the empty block EDS. +func EmptyRoot() *Root { + initEmpty() + return emptyBlockRoot +} + +// EmptyExtendedDataSquare returns the EDS of the empty block data square. +func EmptyExtendedDataSquare() *rsmt2d.ExtendedDataSquare { + initEmpty() + return emptyBlockEDS +} + +// EmptyBlockShares returns the shares of the empty block. +func EmptyBlockShares() []Share { + initEmpty() + return emptyBlockShares +} + var ( - emptyRoot *Root - emptyEDS *rsmt2d.ExtendedDataSquare + emptyMu sync.Mutex + emptyBlockRoot *Root + emptyBlockEDS *rsmt2d.ExtendedDataSquare + emptyBlockShares []Share ) -func init() { +// initEmpty enables lazy initialization for constant empty block data. +func initEmpty() { + emptyMu.Lock() + defer emptyMu.Unlock() + if emptyBlockRoot != nil { + return + } + // compute empty block EDS and DAH for it - shares := emptyDataSquare() - eds, err := da.ExtendShares(shares) + result := shares.TailPaddingShares(appconsts.MinShareCount) + emptyBlockShares = shares.ToBytes(result) + + eds, err := da.ExtendShares(emptyBlockShares) if err != nil { panic(fmt.Errorf("failed to create empty EDS: %w", err)) } - emptyEDS = eds + emptyBlockEDS = eds dah := da.NewDataAvailabilityHeader(eds) minDAH := da.MinDataAvailabilityHeader() @@ -33,33 +60,8 @@ func init() { panic(fmt.Sprintf("mismatch in calculated minimum DAH and minimum DAH from celestia-app, "+ "expected %s, got %s", minDAH.String(), dah.String())) } - emptyRoot = &dah + emptyBlockRoot = &dah // precompute Hash, so it's cached internally to avoid potential races - emptyRoot.Hash() -} - -// EmptyRoot returns Root of an empty EDS. -func EmptyRoot() *Root { - return emptyRoot -} - -// EnsureEmptySquareExists checks if the given DAG contains an empty block data square. -// If it does not, it stores an empty block. This optimization exists to prevent -// redundant storing of empty block data so that it is only stored once and returned -// upon request for a block with an empty data square. Ref: header/constructors.go#L56 -func EnsureEmptySquareExists(ctx context.Context, bServ blockservice.BlockService) (*rsmt2d.ExtendedDataSquare, error) { - shares := emptyDataSquare() - return AddShares(ctx, shares, bServ) -} - -// EmptyExtendedDataSquare returns the EDS of the empty block data square. -func EmptyExtendedDataSquare() *rsmt2d.ExtendedDataSquare { - return emptyEDS -} - -// emptyDataSquare returns the minimum size data square filled with tail padding. -func emptyDataSquare() [][]byte { - result := shares.TailPaddingShares(appconsts.MinShareCount) - return shares.ToBytes(result) + emptyBlockRoot.Hash() } diff --git a/share/getter.go b/share/getter.go index 15f384a2c4..7caa954a24 100644 --- a/share/getter.go +++ b/share/getter.go @@ -8,10 +8,7 @@ import ( "github.com/minio/sha256-simd" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share/ipld" ) var ( @@ -35,7 +32,7 @@ type Getter interface { // Inclusion of returned data could be verified using Verify method on NamespacedShares. // If no shares are found for target namespace non-inclusion could be also verified by calling // Verify method. - GetSharesByNamespace(context.Context, *Root, namespace.ID) (NamespacedShares, error) + GetSharesByNamespace(context.Context, *Root, Namespace) (NamespacedShares, error) } // NamespacedShares represents all shares with proofs within a specific namespace of an EDS. @@ -57,10 +54,10 @@ type NamespacedRow struct { } // Verify validates NamespacedShares by checking every row with nmt inclusion proof. -func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { - originalRoots := make([][]byte, 0) +func (ns NamespacedShares) Verify(root *Root, namespace Namespace) error { + var originalRoots [][]byte for _, row := range root.RowRoots { - if !ipld.NamespaceIsOutsideRange(row, row, nID) { + if !namespace.IsOutsideRange(row, row) { originalRoots = append(originalRoots, row) } } @@ -72,7 +69,7 @@ func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { for i, row := range ns { // verify row data against row hash from original root - if !row.verify(originalRoots[i], nID) { + if !row.verify(originalRoots[i], namespace) { return fmt.Errorf("row verification failed: row %d doesn't match original root: %s", i, root.String()) } } @@ -80,17 +77,18 @@ func (ns NamespacedShares) Verify(root *Root, nID namespace.ID) error { } // verify validates the row using nmt inclusion proof. -func (row *NamespacedRow) verify(rowRoot []byte, nID namespace.ID) bool { +func (row *NamespacedRow) verify(rowRoot []byte, namespace Namespace) bool { // construct nmt leaves from shares by prepending namespace leaves := make([][]byte, 0, len(row.Shares)) - for _, sh := range row.Shares { - leaves = append(leaves, append(sh[:NamespaceSize], sh...)) + for _, shr := range row.Shares { + leaves = append(leaves, append(GetNamespace(shr), shr...)) } // verify namespace return row.Proof.VerifyNamespace( sha256.New(), - nID, + namespace.ToNMT(), leaves, - rowRoot) + rowRoot, + ) } diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 01540de926..fe99bed4c3 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -2,13 +2,11 @@ package getters import ( "context" - "encoding/hex" "errors" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -67,16 +65,16 @@ func (cg *CascadeGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d. func (cg *CascadeGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - id namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) { ctx, span := tracer.Start(ctx, "cascade/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(id)), + attribute.String("namespace", namespace.String()), )) defer span.End() get := func(ctx context.Context, get share.Getter) (share.NamespacedShares, error) { - return get.GetSharesByNamespace(ctx, root, id) + return get.GetSharesByNamespace(ctx, root, namespace) } return cascadeGetters(ctx, cg.getters, get) diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 20b7e25191..7af8af2f26 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -14,12 +14,14 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestTeeGetter(t *testing.T) { @@ -39,8 +41,8 @@ func TestTeeGetter(t *testing.T) { tg := NewTeeGetter(ig, edsStore) t.Run("TeesToEDSStore", func(t *testing.T) { - eds, dah := randomEDS(t) - _, err := share.ImportShares(ctx, eds.Flattened(), bServ) + randEds, dah := randomEDS(t) + _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) require.NoError(t, err) // eds store doesn't have the EDS yet @@ -50,7 +52,7 @@ func TestTeeGetter(t *testing.T) { retrievedEDS, err := tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(eds, retrievedEDS)) + require.True(t, share.EqualEDS(randEds, retrievedEDS)) // eds store now has the EDS and it can be retrieved ok, err = edsStore.Has(ctx, dah.Hash()) @@ -58,22 +60,22 @@ func TestTeeGetter(t *testing.T) { assert.NoError(t, err) finalEDS, err := edsStore.Get(ctx, dah.Hash()) assert.NoError(t, err) - require.True(t, share.EqualEDS(eds, finalEDS)) + require.True(t, share.EqualEDS(randEds, finalEDS)) }) t.Run("ShardAlreadyExistsDoesntError", func(t *testing.T) { - eds, dah := randomEDS(t) - _, err := share.ImportShares(ctx, eds.Flattened(), bServ) + randEds, dah := randomEDS(t) + _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) require.NoError(t, err) retrievedEDS, err := tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(eds, retrievedEDS)) + require.True(t, share.EqualEDS(randEds, retrievedEDS)) // no error should be returned, even though the EDS identified by the DAH already exists retrievedEDS, err = tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(eds, retrievedEDS)) + require.True(t, share.EqualEDS(randEds, retrievedEDS)) }) } @@ -92,16 +94,16 @@ func TestStoreGetter(t *testing.T) { sg := NewStoreGetter(edsStore) t.Run("GetShare", func(t *testing.T) { - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - squareSize := int(eds.Width()) + squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { share, err := sg.GetShare(ctx, &dah, i, j) require.NoError(t, err) - assert.Equal(t, eds.GetCell(uint(i), uint(j)), share) + assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } @@ -112,13 +114,13 @@ func TestStoreGetter(t *testing.T) { }) t.Run("GetEDS", func(t *testing.T) { - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(eds, retrievedEDS)) + assert.True(t, share.EqualEDS(randEds, retrievedEDS)) // root not found root := share.Root{} @@ -127,24 +129,24 @@ func TestStoreGetter(t *testing.T) { }) t.Run("GetSharesByNamespace", func(t *testing.T) { - eds, nID, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - shares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, nID)) + require.NoError(t, shares.Verify(&dah, namespace)) assert.Len(t, shares.Flatten(), 2) - // nid not found - nID = make([]byte, namespace.NamespaceSize) - emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + // namespace not found + randNamespace := sharetest.RandV0Namespace() + emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, randNamespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) // root not found root := share.Root{} - _, err = sg.GetSharesByNamespace(ctx, &root, nID) + _, err = sg.GetSharesByNamespace(ctx, &root, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) } @@ -168,16 +170,16 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - squareSize := int(eds.Width()) + squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { share, err := sg.GetShare(ctx, &dah, i, j) require.NoError(t, err) - assert.Equal(t, eds.GetCell(uint(i), uint(j)), share) + assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } @@ -191,47 +193,46 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(eds, retrievedEDS)) + assert.True(t, share.EqualEDS(randEds, retrievedEDS)) }) t.Run("GetSharesByNamespace", func(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds, nID, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), eds) + randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) // first check that shares are returned correctly if they exist - shares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, nID)) + require.NoError(t, shares.Verify(&dah, namespace)) assert.Len(t, shares.Flatten(), 2) - // nid not found - nID = make([]byte, namespace.NamespaceSize) - emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, nID) + // namespace not found + randNamespace := sharetest.RandV0Namespace() + emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, randNamespace) require.NoError(t, err) - require.Nil(t, emptyShares) + require.Empty(t, emptyShares.Flatten()) // nid doesnt exist in root root := share.Root{} - emptyShares, err = sg.GetSharesByNamespace(ctx, &root, nID) + emptyShares, err = sg.GetSharesByNamespace(ctx, &root, namespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) } func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) - return eds, dah } @@ -239,7 +240,7 @@ func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { // middle that share a namespace. func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedDataSquare, []byte, share.Root) { n := size * size - randShares := share.RandShares(t, n) + randShares := sharetest.RandShares(t, n) idx1 := (n - 1) / 2 idx2 := n / 2 @@ -251,7 +252,7 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData // D _ _ _ // _ _ _ _ // where the D shares have a common namespace. - copy(randShares[idx2][:share.NamespaceSize], randShares[idx1][:share.NamespaceSize]) + copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) eds, err := rsmt2d.ComputeExtendedDataSquare( randShares, @@ -261,5 +262,5 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData require.NoError(t, err, "failure to recompute the extended data square") dah := da.NewDataAvailabilityHeader(eds) - return eds, randShares[idx1][:share.NamespaceSize], dah + return eds, share.GetNamespace(randShares[idx1]), dah } diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 04a1f4f728..92d1c55678 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "sync" @@ -12,7 +11,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -55,7 +53,7 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in // wrap the blockservice in a session if it has been signaled in the context. blockGetter := getGetter(ctx, ig.bServ) - s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) + s, err := ipld.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound @@ -90,24 +88,23 @@ func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d func (ig *IPLDGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(nID)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - err = verifyNIDSize(nID) - if err != nil { - return nil, fmt.Errorf("getter/ipld: invalid namespace ID: %w", err) + if err = namespace.ValidateForData(); err != nil { + return nil, err } // wrap the blockservice in a session if it has been signaled in the context. blockGetter := getGetter(ctx, ig.bServ) - shares, err = collectSharesByNamespace(ctx, blockGetter, root, nID) + shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 08006073d6..bb5e7ca7a7 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "time" @@ -14,7 +13,6 @@ import ( "go.opentelemetry.io/otel/metric/unit" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -190,22 +188,25 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex func (sg *ShrexGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - id namespace.ID, + namespace share.Namespace, ) (share.NamespacedShares, error) { + if err := namespace.ValidateForData(); err != nil { + return nil, err + } var ( attempt int err error ) ctx, span := tracer.Start(ctx, "shrex/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(id)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() // verify that the namespace could exist inside the roots before starting network requests - roots := filterRootsByNamespace(root, id) + roots := filterRootsByNamespace(root, namespace) if len(roots) == 0 { return nil, nil } @@ -221,7 +222,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( if getErr != nil { log.Debugw("nd: couldn't find peer", "hash", root.String(), - "nid", hex.EncodeToString(id), + "namespace", namespace.String(), "err", getErr, "finished (s)", time.Since(start)) sg.metrics.recordNDAttempt(ctx, attempt, false) @@ -230,12 +231,12 @@ func (sg *ShrexGetter) GetSharesByNamespace( reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - nd, getErr := sg.ndClient.RequestND(reqCtx, root, id, peer) + nd, getErr := sg.ndClient.RequestND(reqCtx, root, namespace, peer) cancel() switch { case getErr == nil: // both inclusion and non-inclusion cases needs verification - if verErr := nd.Verify(root, id); verErr != nil { + if verErr := nd.Verify(root, namespace); verErr != nil { getErr = verErr setStatus(peers.ResultBlacklistPeer) break @@ -260,7 +261,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( } log.Debugw("nd: request failed", "hash", root.String(), - "nid", hex.EncodeToString(id), + "namespace", namespace.String(), "peer", peer.String(), "attempt", attempt, "err", getErr, diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 72ac78e69b..e1628d8725 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -17,14 +17,13 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/nmt" - nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" @@ -62,16 +61,16 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, nID := generateTestEDS(t) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + randEDS, dah, namespace := generateTestEDS(t) + require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - got, err := getter.GetSharesByNamespace(ctx, &dah, nID) + got, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) - require.NoError(t, got.Verify(&dah, nID)) + require.NoError(t, got.Verify(&dah, namespace)) }) t.Run("ND_err_not_found", func(t *testing.T) { @@ -79,13 +78,13 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - _, dah, nID := generateTestEDS(t) + _, dah, namespace := generateTestEDS(t) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - _, err := getter.GetSharesByNamespace(ctx, &dah, nID) + _, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -94,7 +93,7 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, maxNID := generateTestEDS(t) + eds, dah, maxNamespace := generateTestEDS(t) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), @@ -102,11 +101,11 @@ func TestShrexGetter(t *testing.T) { }) // corrupt NID - nID := make([]byte, ipld.NamespaceSize) - copy(nID, maxNID) - nID[ipld.NamespaceSize-1]-- // pray for last byte to not be 0x00 + nID := make([]byte, share.NamespaceSize) + copy(nID, maxNamespace) + nID[share.NamespaceSize-1]-- // check for namespace to be between max and min namespace in root - require.Len(t, filterRootsByNamespace(&dah, maxNID), 1) + require.Len(t, filterRootsByNamespace(&dah, maxNamespace), 1) emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) require.NoError(t, err) @@ -120,25 +119,25 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, maxNID := generateTestEDS(t) + eds, dah, maxNamesapce := generateTestEDS(t) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - // corrupt NID - nID := make([]byte, ipld.NamespaceSize) - copy(nID, maxNID) - nID[ipld.NamespaceSize-1]++ // pray for last byte to not be 0xFF + // corrupt namespace + namespace := make([]byte, share.NamespaceSize) + copy(namespace, maxNamesapce) + namespace[share.NamespaceSize-1]++ // check for namespace to be not in root - require.Len(t, filterRootsByNamespace(&dah, nID), 0) + require.Len(t, filterRootsByNamespace(&dah, namespace), 0) - emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) + emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(&dah, nID)) + require.Nil(t, emptyShares.Verify(&dah, namespace)) }) t.Run("EDS_Available", func(t *testing.T) { @@ -146,8 +145,8 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, _ := generateTestEDS(t) - require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) + randEDS, dah, _ := generateTestEDS(t) + require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, @@ -155,7 +154,7 @@ func TestShrexGetter(t *testing.T) { got, err := getter.GetEDS(ctx, &dah) require.NoError(t, err) - require.Equal(t, eds.Flattened(), got.Flattened()) + require.Equal(t, randEDS.Flattened(), got.Flattened()) }) t.Run("EDS_ctx_deadline", func(t *testing.T) { @@ -197,10 +196,10 @@ func newStore(t *testing.T) (*eds.Store, error) { return eds.NewStore(tmpDir, ds) } -func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, nmtnamespace.ID) { - eds := share.RandEDS(t, 4) +func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, share.Namespace) { + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) - max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], ipld.NamespaceSize) + max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], share.NamespaceSize) return eds, dah, max } diff --git a/share/getters/store.go b/share/getters/store.go index f39f41fc80..0b88d89155 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -2,14 +2,12 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -58,7 +56,7 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) - s, err := share.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) + s, err := ipld.GetShare(ctx, blockGetter, root, leaf, len(dah.RowRoots)) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound @@ -95,19 +93,18 @@ func (sg *StoreGetter) GetEDS(ctx context.Context, root *share.Root) (data *rsmt func (sg *StoreGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "store/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(nID)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - err = verifyNIDSize(nID) - if err != nil { - return nil, fmt.Errorf("getter/store: invalid namespace ID: %w", err) + if err = namespace.ValidateForData(); err != nil { + return nil, err } bs, err := sg.store.CARBlockstore(ctx, root.Hash()) @@ -121,7 +118,7 @@ func (sg *StoreGetter) GetSharesByNamespace( // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) - shares, err = collectSharesByNamespace(ctx, blockGetter, root, nID) + shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve shares by namespace: %w", err) } diff --git a/share/getters/tee.go b/share/getters/tee.go index 50e2e1b55d..213f05f90c 100644 --- a/share/getters/tee.go +++ b/share/getters/tee.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" @@ -10,7 +9,6 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" @@ -72,15 +70,15 @@ func (tg *TeeGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d. func (tg *TeeGetter) GetSharesByNamespace( ctx context.Context, root *share.Root, - id namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "tee/get-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(id)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - return tg.getter.GetSharesByNamespace(ctx, root, id) + return tg.getter.GetSharesByNamespace(ctx, root, namespace) } diff --git a/share/getters/testing.go b/share/getters/testing.go index a90b937a51..4734557d7f 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -6,15 +6,15 @@ import ( "testing" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" ) // TestGetter provides a testing SingleEDSGetter and the root of the EDS it holds. func TestGetter(t *testing.T) (share.Getter, *share.Root) { - eds := share.RandEDS(t, 8) + eds := edstest.RandEDS(t, 8) dah := da.NewDataAvailabilityHeader(eds) return &SingleEDSGetter{ EDS: eds, @@ -46,7 +46,7 @@ func (seg *SingleEDSGetter) GetEDS(_ context.Context, root *share.Root) (*rsmt2d } // GetSharesByNamespace returns NamespacedShares from a kept EDS if the correct root is given. -func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *share.Root, namespace.ID, +func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *share.Root, share.Namespace, ) (share.NamespacedShares, error) { panic("SingleEDSGetter: GetSharesByNamespace is not implemented") } diff --git a/share/getters/utils.go b/share/getters/utils.go index 79438204bd..94b1cfe5ee 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -2,7 +2,6 @@ package getters import ( "context" - "encoding/hex" "errors" "fmt" "time" @@ -15,8 +14,6 @@ import ( "go.opentelemetry.io/otel/trace" "golang.org/x/sync/errgroup" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" @@ -30,34 +27,33 @@ var ( ) // filterRootsByNamespace returns the row roots from the given share.Root that contain the passed -// namespace ID. -func filterRootsByNamespace(root *share.Root, nID namespace.ID) []cid.Cid { +// namespace. +func filterRootsByNamespace(root *share.Root, namespace share.Namespace) []cid.Cid { rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) for _, row := range root.RowRoots { - if !ipld.NamespaceIsOutsideRange(row, row, nID) { + if !namespace.IsOutsideRange(row, row) { rowRootCIDs = append(rowRootCIDs, ipld.MustCidFromNamespacedSha256(row)) } } return rowRootCIDs } -// collectSharesByNamespace collects NamespaceShares within the given namespace ID from the given -// share.Root. +// collectSharesByNamespace collects NamespaceShares within the given namespace from share.Root. func collectSharesByNamespace( ctx context.Context, bg blockservice.BlockGetter, root *share.Root, - nID namespace.ID, + namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( attribute.String("root", root.String()), - attribute.String("nid", hex.EncodeToString(nID)), + attribute.String("namespace", namespace.String()), )) defer func() { utils.SetStatusAndEnd(span, err) }() - rootCIDs := filterRootsByNamespace(root, nID) + rootCIDs := filterRootsByNamespace(root, namespace) if len(rootCIDs) == 0 { return nil, nil } @@ -68,13 +64,13 @@ func collectSharesByNamespace( // shadow loop variables, to ensure correct values are captured i, rootCID := i, rootCID errGroup.Go(func() error { - row, proof, err := share.GetSharesByNamespace(ctx, bg, rootCID, nID, len(root.RowRoots)) + row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rootCID, namespace, len(root.RowRoots)) shares[i] = share.NamespacedRow{ Shares: row, Proof: proof, } if err != nil { - return fmt.Errorf("retrieving nID %x for row %x: %w", nID, rootCID, err) + return fmt.Errorf("retrieving shares by namespace %s for row %x: %w", namespace.String(), rootCID, err) } return nil }) @@ -83,15 +79,8 @@ func collectSharesByNamespace( if err := errGroup.Wait(); err != nil { return nil, err } - return shares, nil -} -func verifyNIDSize(nID namespace.ID) error { - if len(nID) != share.NamespaceSize { - return fmt.Errorf("expected namespace ID of size %d, got %d", - share.NamespaceSize, len(nID)) - } - return nil + return shares, nil } // ctxWithSplitTimeout will split timeout stored in context by splitFactor and return the result if diff --git a/share/helpers.go b/share/helpers.go new file mode 100644 index 0000000000..0d729fd9b2 --- /dev/null +++ b/share/helpers.go @@ -0,0 +1,58 @@ +package share + +import ( + "bytes" + + "github.com/celestiaorg/rsmt2d" +) + +// TODO(Wondertan): All these helpers should be methods on rsmt2d.EDS + +// ExtractODS returns the original shares of the given ExtendedDataSquare. This +// is a helper function for circumstances where AddShares must be used after the EDS has already +// been generated. +func ExtractODS(eds *rsmt2d.ExtendedDataSquare) []Share { + origWidth := eds.Width() / 2 + origShares := make([][]byte, origWidth*origWidth) + for i := uint(0); i < origWidth; i++ { + row := eds.Row(i) + for j := uint(0); j < origWidth; j++ { + origShares[(i*origWidth)+j] = row[j] + } + } + return origShares +} + +// ExtractEDS takes an EDS and extracts all shares from it in a flattened slice(row by row). +func ExtractEDS(eds *rsmt2d.ExtendedDataSquare) []Share { + flattenedEDSSize := eds.Width() * eds.Width() + out := make([][]byte, flattenedEDSSize) + count := 0 + for i := uint(0); i < eds.Width(); i++ { + for _, share := range eds.Row(i) { + out[count] = share + count++ + } + } + return out +} + +// EqualEDS check whether two given EDSes are equal. +// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and +// Golang practices +func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool { + if a.Width() != b.Width() { + return false + } + + for i := uint(0); i < a.Width(); i++ { + ar, br := a.Row(i), b.Row(i) + for j := 0; j < len(ar); j++ { + if !bytes.Equal(ar[j], br[j]) { + return false + } + } + } + + return true +} diff --git a/share/add.go b/share/ipld/add.go similarity index 61% rename from share/add.go rename to share/ipld/add.go index 02016cadf6..fddfe8e99e 100644 --- a/share/add.go +++ b/share/ipld/add.go @@ -1,4 +1,4 @@ -package share +package ipld import ( "context" @@ -11,14 +11,14 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share" ) // AddShares erasures and extends shares to blockservice.BlockService using the provided // ipld.NodeAdder. func AddShares( ctx context.Context, - shares []Share, + shares []share.Share, adder blockservice.BlockService, ) (*rsmt2d.ExtendedDataSquare, error) { if len(shares) == 0 { @@ -26,12 +26,12 @@ func AddShares( } squareSize := int(utils.SquareSize(len(shares))) // create nmt adder wrapping batch adder with calculated size - batchAdder := ipld.NewNmtNodeAdder(ctx, adder, ipld.MaxSizeBatchOption(squareSize*2)) + batchAdder := NewNmtNodeAdder(ctx, adder, MaxSizeBatchOption(squareSize*2)) // create the nmt wrapper to generate row and col commitments // recompute the eds eds, err := rsmt2d.ComputeExtendedDataSquare( shares, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareSize), nmt.NodeVisitor(batchAdder.Visit)), ) @@ -55,11 +55,11 @@ func ImportShares( } squareSize := int(utils.SquareSize(len(shares))) // create nmt adder wrapping batch adder with calculated size - batchAdder := ipld.NewNmtNodeAdder(ctx, adder, ipld.MaxSizeBatchOption(squareSize*2)) + batchAdder := NewNmtNodeAdder(ctx, adder, MaxSizeBatchOption(squareSize*2)) // recompute the eds eds, err := rsmt2d.ImportExtendedDataSquare( shares, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareSize/2), nmt.NodeVisitor(batchAdder.Visit)), ) @@ -72,31 +72,8 @@ func ImportShares( return eds, batchAdder.Commit() } -// ExtractODS returns the original shares of the given ExtendedDataSquare. This -// is a helper function for circumstances where AddShares must be used after the EDS has already -// been generated. -func ExtractODS(eds *rsmt2d.ExtendedDataSquare) []Share { - origWidth := eds.Width() / 2 - origShares := make([][]byte, origWidth*origWidth) - for i := uint(0); i < origWidth; i++ { - row := eds.Row(i) - for j := uint(0); j < origWidth; j++ { - origShares[(i*origWidth)+j] = row[j] - } - } - return origShares -} - -// ExtractEDS takes an EDS and extracts all shares from it in a flattened slice(row by row). -func ExtractEDS(eds *rsmt2d.ExtendedDataSquare) []Share { - flattenedEDSSize := eds.Width() * eds.Width() - out := make([][]byte, flattenedEDSSize) - count := 0 - for i := uint(0); i < eds.Width(); i++ { - for _, share := range eds.Row(i) { - out[count] = share - count++ - } - } - return out +func ImportEDS(ctx context.Context, square *rsmt2d.ExtendedDataSquare, adder blockservice.BlockService) error { + shares := share.ExtractEDS(square) + _, err := ImportShares(ctx, shares, adder) + return err } diff --git a/share/ipld/get.go b/share/ipld/get.go index 6d7de35c27..35f601853d 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -12,6 +12,8 @@ import ( ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/codes" + + "github.com/celestiaorg/celestia-node/share" ) // NumWorkersLimit sets global limit for workers spawned by GetShares. @@ -26,7 +28,7 @@ import ( // // TODO(@Wondertan): This assumes we have parallelized DASer implemented. Sync the values once it is shipped. // TODO(@Wondertan): Allow configuration of values without global state. -var NumWorkersLimit = MaxSquareSize * MaxSquareSize / 2 * NumConcurrentSquares +var NumWorkersLimit = share.MaxSquareSize * share.MaxSquareSize / 2 * NumConcurrentSquares // NumConcurrentSquares limits the amount of squares that are fetched // concurrently/simultaneously. diff --git a/share/get.go b/share/ipld/get_shares.go similarity index 69% rename from share/get.go rename to share/ipld/get_shares.go index 9ec4c3c2c5..0bed240fdc 100644 --- a/share/get.go +++ b/share/ipld/get_shares.go @@ -1,4 +1,4 @@ -package share +package ipld import ( "context" @@ -8,9 +8,8 @@ import ( format "github.com/ipfs/go-ipld-format" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share" ) // GetShare fetches and returns the data for leaf `leafIndex` of root `rootCid`. @@ -20,8 +19,8 @@ func GetShare( rootCid cid.Cid, leafIndex int, totalLeafs int, // this corresponds to the extended square width -) (Share, error) { - nd, err := ipld.GetLeaf(ctx, bGetter, rootCid, leafIndex, totalLeafs) +) (share.Share, error) { + nd, err := GetLeaf(ctx, bGetter, rootCid, leafIndex, totalLeafs) if err != nil { return nil, err } @@ -32,30 +31,30 @@ func GetShare( // GetShares walks the tree of a given root and puts shares into the given 'put' func. // Does not return any error, and returns/unblocks only on success // (got all shares) or on context cancellation. -func GetShares(ctx context.Context, bGetter blockservice.BlockGetter, root cid.Cid, shares int, put func(int, Share)) { +func GetShares(ctx context.Context, bg blockservice.BlockGetter, root cid.Cid, shares int, put func(int, share.Share)) { ctx, span := tracer.Start(ctx, "get-shares") defer span.End() putNode := func(i int, leaf format.Node) { put(i, leafToShare(leaf)) } - ipld.GetLeaves(ctx, bGetter, root, shares, putNode) + GetLeaves(ctx, bg, root, shares, putNode) } // GetSharesByNamespace walks the tree of a given root and returns its shares within the given -// namespace.ID. If a share could not be retrieved, err is not nil, and the returned array +// Namespace. If a share could not be retrieved, err is not nil, and the returned array // contains nil shares in place of the shares it was unable to retrieve. func GetSharesByNamespace( ctx context.Context, bGetter blockservice.BlockGetter, root cid.Cid, - nID namespace.ID, + namespace share.Namespace, maxShares int, -) ([]Share, *nmt.Proof, error) { +) ([]share.Share, *nmt.Proof, error) { ctx, span := tracer.Start(ctx, "get-shares-by-namespace") defer span.End() - data := ipld.NewNamespaceData(maxShares, nID, ipld.WithLeaves(), ipld.WithProofs()) + data := NewNamespaceData(maxShares, namespace, WithLeaves(), WithProofs()) err := data.CollectLeavesByNamespace(ctx, bGetter, root) if err != nil { return nil, nil, err @@ -63,7 +62,7 @@ func GetSharesByNamespace( leaves := data.Leaves() - shares := make([]Share, len(leaves)) + shares := make([]share.Share, len(leaves)) for i, leaf := range leaves { if leaf != nil { shares[i] = leafToShare(leaf) @@ -73,8 +72,8 @@ func GetSharesByNamespace( } // leafToShare converts an NMT leaf into a Share. -func leafToShare(nd format.Node) Share { +func leafToShare(nd format.Node) share.Share { // * Additional namespace is prepended so that parity data can be identified with a parity // namespace, which we cut off - return nd.RawData()[NamespaceSize:] + return share.GetData(nd.RawData()) } diff --git a/share/get_test.go b/share/ipld/get_shares_test.go similarity index 64% rename from share/get_test.go rename to share/ipld/get_shares_test.go index 1b9df69be3..cd26f759b3 100644 --- a/share/get_test.go +++ b/share/ipld/get_shares_test.go @@ -1,9 +1,8 @@ -package share +package ipld import ( "bytes" "context" - "crypto/rand" "errors" mrand "math/rand" "sort" @@ -23,11 +22,12 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestGetShare(t *testing.T) { @@ -38,14 +38,14 @@ func TestGetShare(t *testing.T) { bServ := mdutils.Bserv() // generate random shares for the nmt - shares := RandShares(t, size*size) + shares := sharetest.RandShares(t, size*size) eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) for i, leaf := range shares { row := i / size pos := i - (size * row) - share, err := GetShare(ctx, bServ, ipld.MustCidFromNamespacedSha256(eds.RowRoots()[row]), pos, size*2) + share, err := GetShare(ctx, bServ, MustCidFromNamespacedSha256(eds.RowRoots()[row]), pos, size*2) require.NoError(t, err) assert.Equal(t, leaf, share) } @@ -58,12 +58,12 @@ func TestBlockRecovery(t *testing.T) { extendedShareCount := extendedSquareWidth * extendedSquareWidth // generate test data - quarterShares := RandShares(t, shareCount) - allShares := RandShares(t, shareCount) + quarterShares := sharetest.RandShares(t, shareCount) + allShares := sharetest.RandShares(t, shareCount) testCases := []struct { name string - shares []Share + shares []share.Share expectErr bool errString string d int // number of shares to delete @@ -79,25 +79,29 @@ func TestBlockRecovery(t *testing.T) { t.Run(tc.name, func(t *testing.T) { squareSize := utils.SquareSize(len(tc.shares)) - eds, err := rsmt2d.ComputeExtendedDataSquare(tc.shares, DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) + testEds, err := rsmt2d.ComputeExtendedDataSquare( + tc.shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(squareSize), + ) require.NoError(t, err) // calculate roots using the first complete square - rowRoots := eds.RowRoots() - colRoots := eds.ColRoots() + rowRoots := testEds.RowRoots() + colRoots := testEds.ColRoots() - flat := ExtractEDS(eds) + flat := share.ExtractEDS(testEds) // recover a partially complete square rdata := removeRandShares(flat, tc.d) - eds, err = rsmt2d.ImportExtendedDataSquare( + testEds, err = rsmt2d.ImportExtendedDataSquare( rdata, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize), ) require.NoError(t, err) - err = eds.Repair(rowRoots, colRoots) + err = testEds.Repair(rowRoots, colRoots) if tc.expectErr { require.Error(t, err) require.Contains(t, err.Error(), tc.errString) @@ -105,27 +109,27 @@ func TestBlockRecovery(t *testing.T) { } assert.NoError(t, err) - reds, err := rsmt2d.ImportExtendedDataSquare(rdata, DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) + reds, err := rsmt2d.ImportExtendedDataSquare(rdata, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) require.NoError(t, err) // check that the squares are equal - assert.Equal(t, ExtractEDS(eds), ExtractEDS(reds)) + assert.Equal(t, share.ExtractEDS(testEds), share.ExtractEDS(reds)) }) } } func Test_ConvertEDStoShares(t *testing.T) { squareWidth := 16 - shares := RandShares(t, squareWidth*squareWidth) + shares := sharetest.RandShares(t, squareWidth*squareWidth) // compute extended square - eds, err := rsmt2d.ComputeExtendedDataSquare( + testEds, err := rsmt2d.ComputeExtendedDataSquare( shares, - DefaultRSMT2DCodec(), + share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(squareWidth)), ) require.NoError(t, err) - resshares := ExtractODS(eds) + resshares := share.ExtractODS(testEds) require.Equal(t, shares, resshares) } @@ -150,29 +154,29 @@ func TestGetSharesByNamespace(t *testing.T) { bServ := mdutils.Bserv() var tests = []struct { - rawData []Share + rawData []share.Share }{ - {rawData: RandShares(t, 4)}, - {rawData: RandShares(t, 16)}, + {rawData: sharetest.RandShares(t, 4)}, + {rawData: sharetest.RandShares(t, 16)}, } for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - // choose random nID from rand shares + // choose random namespace from rand shares expected := tt.rawData[len(tt.rawData)/2] - nID := expected[:NamespaceSize] + namespace := share.GetNamespace(expected) - // change rawData to contain several shares with same nID + // change rawData to contain several shares with same namespace tt.rawData[(len(tt.rawData)/2)+1] = expected // put raw data in BlockService eds, err := AddShares(ctx, tt.rawData, bServ) require.NoError(t, err) - var shares []Share + var shares []share.Share for _, row := range eds.RowRoots() { - rcid := ipld.MustCidFromNamespacedSha256(row) - rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots())) - if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + rcid := MustCidFromNamespacedSha256(row) + rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(eds.RowRoots())) + if errors.Is(err, ErrNamespaceOutsideRange) { continue } require.NoError(t, err) @@ -193,13 +197,12 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 16) + shares := sharetest.RandShares(t, 16) // set all shares to the same namespace id - nid := shares[0][:NamespaceSize] - - for _, nspace := range shares { - copy(nspace[:NamespaceSize], nid) + namespace := share.GetNamespace(shares[0]) + for _, shr := range shares { + copy(share.GetNamespace(shr), namespace) } eds, err := AddShares(ctx, shares, bServ) @@ -208,28 +211,28 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { roots := eds.RowRoots() // remove the second share from the first row - rcid := ipld.MustCidFromNamespacedSha256(roots[0]) - node, err := ipld.GetNode(ctx, bServ, rcid) + rcid := MustCidFromNamespacedSha256(roots[0]) + node, err := GetNode(ctx, bServ, rcid) require.NoError(t, err) // Left side of the tree contains the original shares - data, err := ipld.GetNode(ctx, bServ, node.Links()[0].Cid) + data, err := GetNode(ctx, bServ, node.Links()[0].Cid) require.NoError(t, err) // Second share is the left side's right child - l, err := ipld.GetNode(ctx, bServ, data.Links()[0].Cid) + l, err := GetNode(ctx, bServ, data.Links()[0].Cid) require.NoError(t, err) - r, err := ipld.GetNode(ctx, bServ, l.Links()[1].Cid) + r, err := GetNode(ctx, bServ, l.Links()[1].Cid) require.NoError(t, err) err = bServ.DeleteBlock(ctx, r.Cid()) require.NoError(t, err) - namespaceData := ipld.NewNamespaceData(len(shares), nid, ipld.WithLeaves()) + namespaceData := NewNamespaceData(len(shares), namespace, WithLeaves()) err = namespaceData.CollectLeavesByNamespace(ctx, bServ, rcid) + require.Error(t, err) leaves := namespaceData.Leaves() assert.Nil(t, leaves[1]) assert.Equal(t, 4, len(leaves)) - require.Error(t, err) } func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { @@ -237,42 +240,42 @@ func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 1024) + shares := sharetest.RandShares(t, 1024) - // set all shares to the same namespace id - nids, err := randomNids(5) + // set all shares to the same namespace + namespaces, err := randomNamespaces(5) require.NoError(t, err) - minNid := nids[0] - minIncluded := nids[1] - midNid := nids[2] - maxIncluded := nids[3] - maxNid := nids[4] + minNamespace := namespaces[0] + minIncluded := namespaces[1] + midNamespace := namespaces[2] + maxIncluded := namespaces[3] + maxNamespace := namespaces[4] secondNamespaceFrom := mrand.Intn(len(shares)-2) + 1 - for i, nspace := range shares { + for i, shr := range shares { if i < secondNamespaceFrom { - copy(nspace[:NamespaceSize], minIncluded) + copy(share.GetNamespace(shr), minIncluded) continue } - copy(nspace[:NamespaceSize], maxIncluded) + copy(share.GetNamespace(shr), maxIncluded) } var tests = []struct { - name string - data []Share - missingNid []byte - isAbsence bool + name string + data []share.Share + missingNamespace share.Namespace + isAbsence bool }{ - {name: "Namespace id less than the minimum namespace in data", data: shares, missingNid: minNid}, - {name: "Namespace id greater than the maximum namespace in data", data: shares, missingNid: maxNid}, - {name: "Namespace id in range but still missing", data: shares, missingNid: midNid, isAbsence: true}, + {name: "Namespace less than the minimum namespace in data", data: shares, missingNamespace: minNamespace}, + {name: "Namespace greater than the maximum namespace in data", data: shares, missingNamespace: maxNamespace}, + {name: "Namespace in range but still missing", data: shares, missingNamespace: midNamespace, isAbsence: true}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) - assertNoRowContainsNID(ctx, t, bServ, eds, tt.missingNid, tt.isAbsence) + assertNoRowContainsNID(ctx, t, bServ, eds, tt.missingNamespace, tt.isAbsence) }) } } @@ -282,10 +285,10 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi t.Cleanup(cancel) bServ := mdutils.Bserv() - shares := RandShares(t, 16) + shares := sharetest.RandShares(t, 16) // set all shares to the same namespace and data but the last one - nid := shares[0][:NamespaceSize] + namespace := share.GetNamespace(shares[0]) commonNamespaceData := shares[0] for i, nspace := range shares { @@ -300,10 +303,10 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi require.NoError(t, err) for _, row := range eds.RowRoots() { - rcid := ipld.MustCidFromNamespacedSha256(row) - data := ipld.NewNamespaceData(len(shares), nid, ipld.WithLeaves()) + rcid := MustCidFromNamespacedSha256(row) + data := NewNamespaceData(len(shares), namespace, WithLeaves()) err := data.CollectLeavesByNamespace(ctx, bServ, rcid) - if errors.Is(err, ipld.ErrNamespaceOutsideRange) { + if errors.Is(err, ErrNamespaceOutsideRange) { continue } assert.Nil(t, err) @@ -311,7 +314,7 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi for _, node := range leaves { // test that the data returned by collectLeavesByNamespace for nid // matches the commonNamespaceData that was copied across almost all data - assert.Equal(t, commonNamespaceData, node.RawData()[NamespaceSize:]) + assert.Equal(t, commonNamespaceData, share.GetData(node.RawData())) } } } @@ -322,11 +325,11 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { bServ := mdutils.Bserv() var tests = []struct { - rawData []Share + rawData []share.Share }{ - {rawData: RandShares(t, 4)}, - {rawData: RandShares(t, 16)}, - {rawData: RandShares(t, 64)}, + {rawData: sharetest.RandShares(t, 4)}, + {rawData: sharetest.RandShares(t, 16)}, + {rawData: sharetest.RandShares(t, 64)}, } for i, tt := range tests { @@ -341,9 +344,9 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { } expected := tt.rawData[from] - nID := namespace.ID(expected[:NamespaceSize]) + namespace := share.GetNamespace(expected) - // change rawData to contain several shares with same nID + // change rawData to contain several shares with same namespace for i := from; i <= to; i++ { tt.rawData[i] = expected } @@ -352,12 +355,12 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { eds, err := AddShares(ctx, tt.rawData, bServ) require.NoError(t, err) - var shares []Share + var shares []share.Share for _, row := range eds.RowRoots() { - rcid := ipld.MustCidFromNamespacedSha256(row) - rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, nID, len(eds.RowRoots())) - if ipld.NamespaceIsOutsideRange(row, row, nID) { - require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange) + rcid := MustCidFromNamespacedSha256(row) + rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(eds.RowRoots())) + if namespace.IsOutsideRange(row, row) { + require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue } require.NoError(t, err) @@ -368,24 +371,24 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { // construct nodes from shares by prepending namespace var leaves [][]byte - for _, sh := range rowShares { - leaves = append(leaves, append(sh[:NamespaceSize], sh...)) + for _, shr := range rowShares { + leaves = append(leaves, append(share.GetNamespace(shr), shr...)) } // verify namespace verified := proof.VerifyNamespace( sha256.New(), - nID, + namespace.ToNMT(), leaves, - ipld.NamespacedSha256FromCID(rcid)) + NamespacedSha256FromCID(rcid)) require.True(t, verified) // verify inclusion verified = proof.VerifyInclusion( sha256.New(), - nID, + namespace.ToNMT(), rowShares, - ipld.NamespacedSha256FromCID(rcid)) + NamespacedSha256FromCID(rcid)) require.True(t, verified) } } @@ -409,7 +412,7 @@ func TestBatchSize(t *testing.T) { {"8", 8}, {"16", 16}, {"32", 32}, - // {"64", 64}, // test case too large for CI with race detector + {"64", 64}, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -418,8 +421,8 @@ func TestBatchSize(t *testing.T) { bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - eds := RandEDS(t, tt.origWidth) - _, err := AddShares(ctx, ExtractODS(eds), blockservice.New(bs, offline.Exchange(bs))) + randEds := edstest.RandEDS(t, tt.origWidth) + _, err := AddShares(ctx, share.ExtractODS(randEds), blockservice.New(bs, offline.Exchange(bs))) require.NoError(t, err) out, err := bs.AllKeysChan(ctx) @@ -430,7 +433,7 @@ func TestBatchSize(t *testing.T) { count++ } extendedWidth := tt.origWidth * 2 - assert.Equalf(t, count, ipld.BatchSize(extendedWidth), "batchSize(%v)", extendedWidth) + assert.Equalf(t, count, BatchSize(extendedWidth), "batchSize(%v)", extendedWidth) }) } } @@ -440,57 +443,52 @@ func assertNoRowContainsNID( t *testing.T, bServ blockservice.BlockService, eds *rsmt2d.ExtendedDataSquare, - nID namespace.ID, + namespace share.Namespace, isAbsent bool, ) { rowRootCount := len(eds.RowRoots()) // get all row root cids rowRootCIDs := make([]cid.Cid, rowRootCount) for i, rowRoot := range eds.RowRoots() { - rowRootCIDs[i] = ipld.MustCidFromNamespacedSha256(rowRoot) + rowRootCIDs[i] = MustCidFromNamespacedSha256(rowRoot) } - // for each row root cid check if the minNID exists + // for each row root cid check if the min namespace exists var absentCount, foundAbsenceRows int for _, rowRoot := range eds.RowRoots() { var outsideRange bool - if !ipld.NamespaceIsOutsideRange(rowRoot, rowRoot, nID) { - // nID does belong to namespace range of the row + if !namespace.IsOutsideRange(rowRoot, rowRoot) { + // namespace does belong to namespace range of the row absentCount++ } else { outsideRange = true } - data := ipld.NewNamespaceData(rowRootCount, nID, ipld.WithProofs()) - rootCID := ipld.MustCidFromNamespacedSha256(rowRoot) + data := NewNamespaceData(rowRootCount, namespace, WithProofs()) + rootCID := MustCidFromNamespacedSha256(rowRoot) err := data.CollectLeavesByNamespace(ctx, bServ, rootCID) if outsideRange { - require.ErrorIs(t, err, ipld.ErrNamespaceOutsideRange) + require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue } require.NoError(t, err) // if no error returned, check absence proof foundAbsenceRows++ - verified := data.Proof().VerifyNamespace(sha256.New(), nID, nil, rowRoot) + verified := data.Proof().VerifyNamespace(sha256.New(), namespace.ToNMT(), nil, rowRoot) require.True(t, verified) } if isAbsent { require.Equal(t, foundAbsenceRows, absentCount) - // there should be max 1 row that has namespace range containing nID + // there should be max 1 row that has namespace range containing namespace require.LessOrEqual(t, absentCount, 1) } } -func randomNids(total int) ([]namespace.ID, error) { - namespaces := make([]namespace.ID, total) +func randomNamespaces(total int) ([]share.Namespace, error) { + namespaces := make([]share.Namespace, total) for i := range namespaces { - nid := make([]byte, NamespaceSize) - _, err := rand.Read(nid) - if err != nil { - return nil, err - } - namespaces[i] = nid + namespaces[i] = sharetest.RandV0Namespace() } sort.Slice(namespaces, func(i, j int) bool { return bytes.Compare(namespaces[i], namespaces[j]) < 0 }) return namespaces, nil diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index f86a9803ac..d776da219d 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -2,7 +2,6 @@ package ipld import ( "context" - "encoding/hex" "errors" "fmt" "sync" @@ -15,11 +14,12 @@ import ( "go.opentelemetry.io/otel/codes" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/share" ) var ErrNamespaceOutsideRange = errors.New("share/ipld: " + - "target namespace id is outside of namespace range for the given root") + "target namespace is outside of namespace range for the given root") // Option is the functional option that is applied to the NamespaceData instance // to configure data that needs to be stored. @@ -48,20 +48,20 @@ type NamespaceData struct { bounds fetchedBounds maxShares int - nID namespace.ID + namespace share.Namespace isAbsentNamespace atomic.Bool absenceProofLeaf ipld.Node } -func NewNamespaceData(maxShares int, nID namespace.ID, options ...Option) *NamespaceData { +func NewNamespaceData(maxShares int, namespace share.Namespace, options ...Option) *NamespaceData { data := &NamespaceData{ // we don't know where in the tree the leaves in the namespace are, // so we keep track of the bounds to return the correct slice // maxShares acts as a sentinel to know if we find any leaves bounds: fetchedBounds{int64(maxShares), 0}, maxShares: maxShares, - nID: nID, + namespace: namespace, } for _, opt := range options { @@ -71,8 +71,8 @@ func NewNamespaceData(maxShares int, nID namespace.ID, options ...Option) *Names } func (n *NamespaceData) validate(rootCid cid.Cid) error { - if len(n.nID) != NamespaceSize { - return fmt.Errorf("expected namespace ID of size %d, got %d", NamespaceSize, len(n.nID)) + if err := n.namespace.Validate(); err != nil { + return err } if n.leaves == nil && n.proofs == nil { @@ -80,7 +80,7 @@ func (n *NamespaceData) validate(rootCid cid.Cid) error { } root := NamespacedSha256FromCID(rootCid) - if NamespaceIsOutsideRange(root, root, n.nID) { + if n.namespace.IsOutsideRange(root, root) { return ErrNamespaceOutsideRange } return nil @@ -180,7 +180,7 @@ func (n *NamespaceData) Proof() *nmt.Proof { } // CollectLeavesByNamespace collects leaves and corresponding proof that could be used to verify -// leaves inclusion. It returns as many leaves from the given root with the given namespace.ID as +// leaves inclusion. It returns as many leaves from the given root with the given Namespace as // it can retrieve. If no shares are found, it returns error as nil. A // non-nil error means that only partial data is returned, because at least one share retrieval // failed. The following implementation is based on `GetShares`. @@ -197,7 +197,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( defer span.End() span.SetAttributes( - attribute.String("namespace", hex.EncodeToString(n.nID)), + attribute.String("namespace", n.namespace.String()), attribute.String("root", root.String()), ) @@ -245,7 +245,7 @@ func (n *NamespaceData) CollectLeavesByNamespace( retrievalErr = err }) log.Errorw("could not retrieve IPLD node", - "nID", hex.EncodeToString(n.nID), + "namespace", n.namespace.String(), "pos", j.sharePos, "err", err, ) @@ -301,17 +301,17 @@ func (n *NamespaceData) collectNDWithProofs(j job, links []*ipld.Link) []job { var nextJobs []job // check if target namespace is outside of boundaries of both links - if NamespaceIsOutsideRange(leftLink, rightLink, n.nID) { + if n.namespace.IsOutsideRange(leftLink, rightLink) { log.Fatalf("target namespace outside of boundaries of links at depth: %v", j.depth) } - if !NamespaceIsAboveMax(leftLink, n.nID) { + if !n.namespace.IsAboveMax(leftLink) { // namespace is within the range of left link nextJobs = append(nextJobs, j.next(left, leftCid, false)) } else { - // proof is on the left side, if the nID is on the right side of the range of left link + // proof is on the left side, if the namespace is on the right side of the range of left link n.addProof(left, leftCid, j.depth) - if NamespaceIsBelowMin(rightLink, n.nID) { + if n.namespace.IsBelowMin(rightLink) { // namespace is not included in either links, convert to absence collector n.isAbsentNamespace.Store(true) nextJobs = append(nextJobs, j.next(right, rightCid, true)) @@ -319,11 +319,11 @@ func (n *NamespaceData) collectNDWithProofs(j job, links []*ipld.Link) []job { } } - if !NamespaceIsBelowMin(rightLink, n.nID) { + if !n.namespace.IsBelowMin(rightLink) { // namespace is within the range of right link nextJobs = append(nextJobs, j.next(right, rightCid, false)) } else { - // proof is on the right side, if the nID is on the left side of the range of right link + // proof is on the right side, if the namespace is on the left side of the range of right link n.addProof(right, rightCid, j.depth) } return nextJobs diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index df140ef8c7..485642dde4 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -11,7 +11,7 @@ import ( "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - blocks "github.com/ipfs/go-libipfs/blocks" + "github.com/ipfs/go-libipfs/blocks" logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" mhcore "github.com/multiformats/go-multihash/core" @@ -20,7 +20,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" + + "github.com/celestiaorg/celestia-node/share" ) var ( @@ -39,17 +40,14 @@ const ( // that contain an NMT node (inner and leaf nodes). sha256NamespaceFlagged = 0x7701 - // NamespaceSize is a system-wide size for NMT namespaces. - NamespaceSize = appconsts.NamespaceSize - // NmtHashSize is the size of a digest created by an NMT in bytes. - NmtHashSize = 2*NamespaceSize + sha256.Size + NmtHashSize = 2*share.NamespaceSize + sha256.Size // innerNodeSize is the size of data in inner nodes. innerNodeSize = NmtHashSize * 2 // leafNodeSize is the size of data in leaf nodes. - leafNodeSize = NamespaceSize + appconsts.ShareSize + leafNodeSize = share.NamespaceSize + appconsts.ShareSize // cidPrefixSize is the size of the prepended buffer of the CID encoding // for NamespacedSha256. For more information, see: @@ -57,21 +55,15 @@ const ( cidPrefixSize = 4 // NMTIgnoreMaxNamespace is currently used value for IgnoreMaxNamespace option in NMT. - // IgnoreMaxNamespace defines whether the largest possible namespace.ID MAX_NID should be 'ignored'. + // IgnoreMaxNamespace defines whether the largest possible Namespace MAX_NID should be 'ignored'. // If set to true, this allows for shorter proofs in particular use-cases. NMTIgnoreMaxNamespace = true ) -var ( - // MaxSquareSize is currently the maximum size supported for unerasured data in - // rsmt2d.ExtendedDataSquare. - MaxSquareSize = appconsts.SquareSizeUpperBound(appconsts.LatestVersion) -) - func init() { // required for Bitswap to hash and verify inbound data correctly mhcore.Register(sha256NamespaceFlagged, func() hash.Hash { - nh := nmt.NewNmtHasher(sha256.New(), NamespaceSize, true) + nh := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, true) nh.Reset() return nh }) @@ -182,21 +174,3 @@ func Translate(dah *da.DataAvailabilityHeader, row, col int) (cid.Cid, int) { func NamespacedSha256FromCID(cid cid.Cid) []byte { return cid.Hash()[cidPrefixSize:] } - -// NamespaceIsAboveMax checks if the target namespace is above the maximum namespace for a given -// node hash. -func NamespaceIsAboveMax(nodeHash []byte, target namespace.ID) bool { - return !target.LessOrEqual(nmt.MaxNamespace(nodeHash, target.Size())) -} - -// NamespaceIsBelowMin checks if the target namespace is below the minimum namespace for a given -// node hash. -func NamespaceIsBelowMin(nodeHash []byte, target namespace.ID) bool { - return target.Less(nmt.MinNamespace(nodeHash, target.Size())) -} - -// NamespaceIsOutsideRange checks if the target namespace is outside the range defined by the left -// and right nodes -func NamespaceIsOutsideRange(leftNodeHash, rightNodeHash []byte, target namespace.ID) bool { - return NamespaceIsBelowMin(leftNodeHash, target) || NamespaceIsAboveMax(rightNodeHash, target) -} diff --git a/share/ipld/nmt_test.go b/share/ipld/nmt_test.go index b52a75c150..aa125ab3c7 100644 --- a/share/ipld/nmt_test.go +++ b/share/ipld/nmt_test.go @@ -1,36 +1,32 @@ package ipld import ( - "bytes" - "crypto/rand" - "sort" "strconv" "testing" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share/eds/edstest" ) // TestNamespaceFromCID checks that deriving the Namespaced hash from // the given CID works correctly. func TestNamespaceFromCID(t *testing.T) { var tests = []struct { - randData [][]byte + eds *rsmt2d.ExtendedDataSquare }{ // note that the number of shares must be a power of two - {randData: generateRandNamespacedRawData(4, appconsts.NamespaceSize, appconsts.ShareSize-appconsts.NamespaceSize)}, - {randData: generateRandNamespacedRawData(16, appconsts.NamespaceSize, appconsts.ShareSize-appconsts.NamespaceSize)}, + {eds: edstest.RandEDS(t, 4)}, + {eds: edstest.RandEDS(t, 16)}, } for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - // create DAH from rand data - eds, err := da.ExtendShares(tt.randData) - require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(eds) + dah := da.NewDataAvailabilityHeader(tt.eds) // check to make sure NamespacedHash is correctly derived from CID for _, row := range dah.RowRoots { c, err := CidFromNamespacedSha256(row) @@ -42,28 +38,3 @@ func TestNamespaceFromCID(t *testing.T) { }) } } - -// generateRandNamespacedRawData returns random namespaced raw data for testing -// purposes. Note that this does not check that total is a power of two. -func generateRandNamespacedRawData(total, nidSize, leafSize uint32) [][]byte { - data := make([][]byte, total) - for i := uint32(0); i < total; i++ { - nid := make([]byte, nidSize) - - _, _ = rand.Read(nid) - data[i] = nid - } - sortByteArrays(data) - for i := uint32(0); i < total; i++ { - d := make([]byte, leafSize) - - _, _ = rand.Read(d) - data[i] = append(data[i], d...) - } - - return data -} - -func sortByteArrays(src [][]byte) { - sort.Slice(src, func(i, j int) bool { return bytes.Compare(src[i], src[j]) < 0 }) -} diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 12c36cb015..2a1b84efd5 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -12,7 +12,6 @@ import ( da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" - namespace "github.com/celestiaorg/nmt/namespace" rsmt2d "github.com/celestiaorg/rsmt2d" ) @@ -70,7 +69,7 @@ func (mr *MockGetterMockRecorder) GetShare(arg0, arg1, arg2, arg3 interface{}) * } // GetSharesByNamespace mocks base method. -func (m *MockGetter) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 namespace.ID) (share.NamespacedShares, error) { +func (m *MockGetter) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 share.Namespace) (share.NamespacedShares, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSharesByNamespace", arg0, arg1, arg2) ret0, _ := ret[0].(share.NamespacedShares) diff --git a/share/namespace.go b/share/namespace.go index 0af1d1a60e..433529a57f 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -9,6 +9,9 @@ import ( "github.com/celestiaorg/nmt/namespace" ) +// NamespaceSize is a system-wide size for NMT namespaces. +const NamespaceSize = appns.NamespaceSize + // Various reserved namespaces. var ( MaxReservedNamespace = Namespace(appns.MaxReservedNamespace.Bytes()) @@ -17,12 +20,30 @@ var ( ReservedPaddingNamespace = Namespace(appns.ReservedPaddingNamespace.Bytes()) TxNamespace = Namespace(appns.TxNamespace.Bytes()) PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) + ISRNamespace = Namespace(appns.IntermediateStateRootsNamespace.Bytes()) ) // Namespace represents namespace of a Share. // Consists of version byte and namespace ID. type Namespace []byte +// NewBlobNamespaceV0 takes a variable size byte slice and creates a valid version 0 Blob Namespace. +// The byte slice must be <= 10 bytes. +// If it is less than 10 bytes, it will be left padded to size 10 with 0s. +// Use predefined namespaces above, if non-blob namespace is needed. +func NewBlobNamespaceV0(id []byte) (Namespace, error) { + if len(id) == 0 || len(id) > appns.NamespaceVersionZeroIDSize { + return nil, fmt.Errorf( + "namespace id must be > 0 && <= %d, but it was %d bytes", appns.NamespaceVersionZeroIDSize, len(id)) + } + + n := make(Namespace, NamespaceSize) + // version and zero padding are already set as zero, + // so simply copying subNID to the end is enough to comply the V0 spec + copy(n[len(n)-len(id):], id) + return n, n.ValidateForBlob() +} + // NamespaceFromBytes converts bytes into Namespace and validates it. func NamespaceFromBytes(b []byte) (Namespace, error) { n := Namespace(b) @@ -84,8 +105,8 @@ func (n Namespace) Validate() error { return nil } -// ValidateDataNamespace checks if the Namespace contains real/useful data. -func (n Namespace) ValidateDataNamespace() error { +// ValidateForData checks if the Namespace is of real/useful data. +func (n Namespace) ValidateForData() error { if err := n.Validate(); err != nil { return err } @@ -95,9 +116,9 @@ func (n Namespace) ValidateDataNamespace() error { return nil } -// ValidateBlobNamespace checks if the Namespace is valid blob namespace. -func (n Namespace) ValidateBlobNamespace() error { - if err := n.ValidateDataNamespace(); err != nil { +// ValidateForBlob checks if the Namespace is valid blob namespace. +func (n Namespace) ValidateForBlob() error { + if err := n.ValidateForData(); err != nil { return err } if bytes.Compare(n, MaxReservedNamespace) < 1 { diff --git a/share/namespace_test.go b/share/namespace_test.go index 8cc61b379b..62746e4b4e 100644 --- a/share/namespace_test.go +++ b/share/namespace_test.go @@ -19,6 +19,59 @@ var ( invalidPrefixID = bytes.Repeat([]byte{1}, NamespaceSize) ) +func TestNewNamespaceV0(t *testing.T) { + type testCase struct { + name string + subNid []byte + expected Namespace + wantErr bool + } + testCases := []testCase{ + { + name: "8 byte id, gets left padded", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, + expected: Namespace{ + 0x0, + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros + 0x0, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}, // id with left padding + wantErr: false, + }, + { + name: "10 byte id, no padding", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10}, + expected: Namespace{ + 0x0, // version + 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros + 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}, // id + wantErr: false, + }, + { + name: "11 byte id", + subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10, 0x11}, + expected: []byte{}, + wantErr: true, + }, + { + name: "nil id", + subNid: nil, + expected: []byte{}, + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + got, err := NewBlobNamespaceV0(tc.subNid) + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + assert.Equal(t, tc.expected, got) + }) + } +} + func TestFrom(t *testing.T) { type testCase struct { name string diff --git a/share/nid.go b/share/nid.go deleted file mode 100644 index 7d960cc9e1..0000000000 --- a/share/nid.go +++ /dev/null @@ -1,29 +0,0 @@ -package share - -import ( - "fmt" - - appns "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/nmt/namespace" -) - -// NewNamespaceV0 takes a variable size byte slice and creates a version 0 Namespace ID. -// The byte slice must be <= 10 bytes. -// If it is less than 10 bytes, it will be left padded to size 10 with 0s. -// TODO: Adapt for Namespace in the integration PR -func NewNamespaceV0(subNId []byte) (namespace.ID, error) { - if lnid := len(subNId); lnid > appns.NamespaceVersionZeroIDSize { - return nil, fmt.Errorf("namespace id must be <= %v, but it was %v bytes", appns.NamespaceVersionZeroIDSize, lnid) - } - - id := make([]byte, appns.NamespaceIDSize) - leftPaddingOffset := appns.NamespaceVersionZeroIDSize - len(subNId) - copy(id[appns.NamespaceVersionZeroPrefixSize+leftPaddingOffset:], subNId) - - appID, err := appns.New(appns.NamespaceVersionZero, id) - if err != nil { - return nil, err - } - - return appID.Bytes(), nil -} diff --git a/share/nid_test.go b/share/nid_test.go deleted file mode 100644 index 8f83d430e3..0000000000 --- a/share/nid_test.go +++ /dev/null @@ -1,56 +0,0 @@ -package share - -import ( - "testing" - - "github.com/stretchr/testify/assert" - - "github.com/celestiaorg/nmt/namespace" -) - -func TestNewNamespaceV0(t *testing.T) { - type testCase struct { - name string - subNid []byte - expected namespace.ID - wantErr bool - } - testCases := []testCase{ - { - name: "8 byte subNid, gets left padded", - subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08}, - expected: namespace.ID{ - 0x0, - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros - 0x0, 0x0, 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8}, // id with left padding - wantErr: false, - }, - { - name: "10 byte subNid, no padding", - subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10}, - expected: namespace.ID{ - 0x0, // version - 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, // filled zeros - 0x1, 0x2, 0x3, 0x4, 0x5, 0x6, 0x7, 0x8, 0x9, 0x10}, // id - wantErr: false, - }, - { - name: "11 byte subNid", - subNid: []byte{0x01, 0x02, 0x03, 0x04, 0x05, 0x06, 0x07, 0x08, 0x9, 0x10, 0x11}, - expected: []byte{}, - wantErr: true, - }, - } - - for _, tc := range testCases { - t.Run(tc.name, func(t *testing.T) { - got, err := NewNamespaceV0(tc.subNid) - if tc.wantErr { - assert.Error(t, err) - return - } - assert.NoError(t, err) - assert.Equal(t, tc.expected, got) - }) - } -} diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index b0e11e3587..21fe9f77a1 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -16,8 +16,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" ) @@ -34,7 +34,7 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is immediately available t.Run("EDS_Available", func(t *testing.T) { - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) err = store.Put(ctx, dah.Hash(), eds) require.NoError(t, err) @@ -47,7 +47,7 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is unavailable initially, but is found after multiple requests t.Run("EDS_AvailableAfterDelay", func(t *testing.T) { storageDelay := time.Second - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) go func() { time.Sleep(storageDelay) @@ -76,7 +76,7 @@ func TestExchange_RequestEDS(t *testing.T) { t.Run("EDS_err_not_found", func(t *testing.T) { timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) _, err := client.RequestEDS(timeoutCtx, dah.Hash(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index b59bb347a7..7f12c21cf2 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -15,7 +15,6 @@ import ( "github.com/celestiaorg/go-libp2p-messenger/serde" "github.com/celestiaorg/nmt" - "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" @@ -51,10 +50,14 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { func (c *Client) RequestND( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, peer peer.ID, ) (share.NamespacedShares, error) { - shares, err := c.doRequest(ctx, root, nID, peer) + if err := namespace.ValidateForData(); err != nil { + return nil, err + } + + shares, err := c.doRequest(ctx, root, namespace, peer) if err == nil { return shares, nil } @@ -80,7 +83,7 @@ func (c *Client) RequestND( func (c *Client) doRequest( ctx context.Context, root *share.Root, - nID namespace.ID, + namespace share.Namespace, peerID peer.ID, ) (share.NamespacedShares, error) { stream, err := c.host.NewStream(ctx, peerID, c.protocolID) @@ -92,8 +95,8 @@ func (c *Client) doRequest( c.setStreamDeadlines(ctx, stream) req := &pb.GetSharesByNamespaceRequest{ - RootHash: root.Hash(), - NamespaceId: nID, + RootHash: root.Hash(), + Namespace: namespace, } _, err = serde.Write(stream, req) diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 25528595d7..e8d3e439c0 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -14,13 +14,13 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/namespace" - nmtnamespace "github.com/celestiaorg/nmt/namespace" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestExchange_RequestND_NotFound(t *testing.T) { @@ -35,8 +35,8 @@ func TestExchange_RequestND_NotFound(t *testing.T) { t.Cleanup(cancel) root := share.Root{} - nID := make([]byte, namespace.NamespaceSize) - _, err := client.RequestND(ctx, &root, nID, server.host.ID()) + namespace := sharetest.RandV0Namespace() + _, err := client.RequestND(ctx, &root, namespace, server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) @@ -44,12 +44,12 @@ func TestExchange_RequestND_NotFound(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - eds := share.RandEDS(t, 4) + eds := edstest.RandEDS(t, 4) dah := da.NewDataAvailabilityHeader(eds) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) - randNID := dah.RowRoots[(len(dah.RowRoots)-1)/2][:namespace.NamespaceSize] - emptyShares, err := client.RequestND(ctx, &dah, randNID, server.host.ID()) + randNamespace := dah.RowRoots[(len(dah.RowRoots)-1)/2][:share.NamespaceSize] + emptyShares, err := client.RequestND(ctx, &dah, randNamespace, server.host.ID()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) @@ -92,13 +92,13 @@ func TestExchange_RequestND(t *testing.T) { // take server concurrency slots with blocked requests for i := 0; i < rateLimit; i++ { go func(i int) { - client.RequestND(ctx, nil, nil, server.host.ID()) //nolint:errcheck + client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) //nolint:errcheck }(i) } // wait until all server slots are taken wg.Wait() - _, err = client.RequestND(ctx, nil, nil, server.host.ID()) + _, err = client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) } @@ -118,7 +118,7 @@ func (m notFoundGetter) GetEDS( } func (m notFoundGetter) GetSharesByNamespace( - _ context.Context, _ *share.Root, _ nmtnamespace.ID, + _ context.Context, _ *share.Root, _ share.Namespace, ) (share.NamespacedShares, error) { return nil, nil } diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go index 7e19bebc09..e4c7656b4e 100644 --- a/share/p2p/shrexnd/pb/share.pb.go +++ b/share/p2p/shrexnd/pb/share.pb.go @@ -57,8 +57,8 @@ func (StatusCode) EnumDescriptor() ([]byte, []int) { } type GetSharesByNamespaceRequest struct { - RootHash []byte `protobuf:"bytes,1,opt,name=root_hash,json=rootHash,proto3" json:"root_hash,omitempty"` - NamespaceId []byte `protobuf:"bytes,2,opt,name=namespace_id,json=namespaceId,proto3" json:"namespace_id,omitempty"` + RootHash []byte `protobuf:"bytes,1,opt,name=root_hash,json=rootHash,proto3" json:"root_hash,omitempty"` + Namespace []byte `protobuf:"bytes,2,opt,name=namespace,proto3" json:"namespace,omitempty"` } func (m *GetSharesByNamespaceRequest) Reset() { *m = GetSharesByNamespaceRequest{} } @@ -101,9 +101,9 @@ func (m *GetSharesByNamespaceRequest) GetRootHash() []byte { return nil } -func (m *GetSharesByNamespaceRequest) GetNamespaceId() []byte { +func (m *GetSharesByNamespaceRequest) GetNamespace() []byte { if m != nil { - return m.NamespaceId + return m.Namespace } return nil } @@ -291,33 +291,32 @@ func init() { func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } var fileDescriptor_ed9f13149b0de397 = []byte{ - // 401 bytes of a gzipped FileDescriptorProto + // 396 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcf, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0xe3, 0x6c, 0x62, 0xd2, 0xb1, 0x41, 0xd6, 0x80, 0xa8, 0xa1, 0xc8, 0x0a, 0x3e, 0x45, - 0x20, 0xd9, 0x92, 0x91, 0xb8, 0xbb, 0x6d, 0x00, 0x8b, 0xb2, 0xa9, 0x36, 0x05, 0x4e, 0xc8, 0xda, - 0xe2, 0xad, 0x8c, 0x04, 0xde, 0xc5, 0xbb, 0x55, 0xe0, 0xcc, 0x0b, 0xf0, 0x58, 0x1c, 0x7b, 0xe4, - 0x88, 0x92, 0x17, 0x41, 0x5e, 0xa7, 0xf4, 0x40, 0x6e, 0xfb, 0xcd, 0xfc, 0xe6, 0xdf, 0xa7, 0x85, - 0xa9, 0xae, 0x79, 0x2b, 0x52, 0x95, 0xa9, 0x54, 0xd7, 0xad, 0xf8, 0xd6, 0x54, 0xa9, 0x3a, 0x4f, - 0x6d, 0x30, 0x51, 0xad, 0x34, 0x12, 0x71, 0x2b, 0x32, 0x95, 0x58, 0x22, 0x69, 0xaa, 0xf8, 0x03, - 0x1c, 0xbc, 0x14, 0x66, 0xd9, 0x25, 0xf4, 0xe1, 0x77, 0xca, 0xbf, 0x08, 0xad, 0xf8, 0x47, 0xc1, - 0xc4, 0xd7, 0x4b, 0xa1, 0x0d, 0x1e, 0xc0, 0x5e, 0x2b, 0xa5, 0x29, 0x6b, 0xae, 0xeb, 0xd0, 0x99, - 0x3a, 0x33, 0x9f, 0x4d, 0xba, 0xc0, 0x2b, 0xae, 0x6b, 0x7c, 0x0c, 0x7e, 0x73, 0x5d, 0x50, 0x7e, - 0xaa, 0xc2, 0xa1, 0xcd, 0x7b, 0xff, 0x62, 0x45, 0x15, 0xff, 0x70, 0xe0, 0xd1, 0xee, 0xfe, 0x5a, - 0xc9, 0x46, 0x0b, 0x7c, 0x0e, 0xae, 0x36, 0xdc, 0x5c, 0x6a, 0xdb, 0xfd, 0x4e, 0x16, 0x25, 0xff, - 0x2f, 0x99, 0x2c, 0x2d, 0x71, 0x24, 0x2b, 0xc1, 0xb6, 0x34, 0x3e, 0x85, 0x51, 0x2b, 0x57, 0x3a, - 0x1c, 0x4e, 0xc9, 0xcc, 0xcb, 0xf6, 0x77, 0x55, 0x31, 0xb9, 0x62, 0x16, 0x8a, 0x29, 0x10, 0x26, - 0x57, 0x78, 0x1f, 0x5c, 0x8b, 0x75, 0xb3, 0xc8, 0xcc, 0x67, 0x5b, 0x85, 0x29, 0x8c, 0x55, 0x2b, - 0xe5, 0x85, 0x3d, 0xc0, 0xcb, 0x1e, 0xec, 0x6a, 0x76, 0xda, 0x01, 0xac, 0xe7, 0x62, 0x0e, 0x63, - 0xab, 0xf1, 0x1e, 0x8c, 0xb5, 0xe1, 0xad, 0xb1, 0xcb, 0x13, 0xd6, 0x0b, 0x0c, 0x80, 0x88, 0xa6, - 0xb7, 0x83, 0xb0, 0xee, 0xd9, 0x71, 0x8d, 0xac, 0x84, 0x0e, 0x89, 0x1d, 0xdc, 0x0b, 0x7c, 0x08, - 0x93, 0xce, 0xd7, 0xcf, 0x82, 0x5f, 0x84, 0xa3, 0xde, 0xdb, 0x6b, 0xfd, 0xe4, 0x3d, 0xc0, 0xcd, - 0xd5, 0xe8, 0xc1, 0xad, 0x82, 0xbe, 0xcb, 0x4f, 0x8a, 0xe3, 0x60, 0x80, 0x2e, 0x0c, 0x17, 0xaf, - 0x03, 0x07, 0x6f, 0xc3, 0x1e, 0x5d, 0x9c, 0x95, 0x2f, 0x16, 0x6f, 0xe9, 0x71, 0x30, 0x44, 0x1f, - 0x26, 0x05, 0x3d, 0x9b, 0x33, 0x9a, 0x9f, 0x04, 0x04, 0xf7, 0xe1, 0x2e, 0xcd, 0xdf, 0xcc, 0x97, - 0xa7, 0xf9, 0xd1, 0xbc, 0xbc, 0xc1, 0x46, 0x87, 0xe1, 0xaf, 0x75, 0xe4, 0x5c, 0xad, 0x23, 0xe7, - 0xcf, 0x3a, 0x72, 0x7e, 0x6e, 0xa2, 0xc1, 0xd5, 0x26, 0x1a, 0xfc, 0xde, 0x44, 0x83, 0x73, 0xd7, - 0xfe, 0x92, 0x67, 0x7f, 0x03, 0x00, 0x00, 0xff, 0xff, 0xd2, 0x3e, 0x78, 0x01, 0x49, 0x02, 0x00, - 0x00, + 0x10, 0xc6, 0x6d, 0x6f, 0x62, 0x92, 0x49, 0x40, 0xd6, 0x80, 0xa8, 0xa1, 0x95, 0x15, 0xf9, 0x14, + 0x81, 0x64, 0x4b, 0x46, 0xe2, 0xee, 0xb6, 0x01, 0x22, 0xca, 0xa6, 0xda, 0x94, 0x3f, 0xb7, 0x68, + 0x8b, 0xb7, 0xf2, 0x01, 0xbc, 0x8b, 0x77, 0xa3, 0xc0, 0x99, 0x17, 0xe0, 0xb1, 0x38, 0xe6, 0xc8, + 0x11, 0x25, 0x2f, 0x82, 0xbc, 0x4e, 0x94, 0x03, 0xb9, 0xf9, 0x9b, 0xf9, 0xcd, 0x37, 0x9f, 0x47, + 0x0b, 0x23, 0x5d, 0xf2, 0x5a, 0xa4, 0x2a, 0x53, 0xa9, 0x2e, 0x6b, 0xf1, 0xbd, 0x2a, 0x52, 0x75, + 0x9b, 0xda, 0x62, 0xa2, 0x6a, 0x69, 0x24, 0xe2, 0x4e, 0x64, 0x2a, 0xb1, 0x44, 0x52, 0x15, 0xf1, + 0x27, 0x38, 0x7d, 0x2d, 0xcc, 0xbc, 0x69, 0xe8, 0xf3, 0x1f, 0x94, 0x7f, 0x15, 0x5a, 0xf1, 0xcf, + 0x82, 0x89, 0x6f, 0x4b, 0xa1, 0x0d, 0x9e, 0x42, 0xbf, 0x96, 0xd2, 0x2c, 0x4a, 0xae, 0xcb, 0xd0, + 0x1d, 0xb9, 0xe3, 0x21, 0xeb, 0x35, 0x85, 0x37, 0x5c, 0x97, 0x78, 0x06, 0xfd, 0x6a, 0x3f, 0x10, + 0x7a, 0xb6, 0x79, 0x28, 0xc4, 0x3f, 0x5d, 0x38, 0x3b, 0x6e, 0xad, 0x95, 0xac, 0xb4, 0xc0, 0x97, + 0xe0, 0x6b, 0xc3, 0xcd, 0x52, 0x5b, 0xe3, 0x07, 0x59, 0x94, 0xfc, 0x9f, 0x2f, 0x99, 0x5b, 0xe2, + 0x42, 0x16, 0x82, 0xed, 0x68, 0x7c, 0x0e, 0x9d, 0x5a, 0xae, 0x74, 0xe8, 0x8d, 0xc8, 0x78, 0x90, + 0x9d, 0x1c, 0x9b, 0x62, 0x72, 0xc5, 0x2c, 0x14, 0x53, 0x20, 0x4c, 0xae, 0xf0, 0x31, 0xf8, 0x16, + 0x6b, 0x76, 0x91, 0xf1, 0x90, 0xed, 0x14, 0xa6, 0xd0, 0x55, 0xb5, 0x94, 0x77, 0x36, 0xfe, 0x20, + 0x7b, 0x72, 0xcc, 0xec, 0xba, 0x01, 0x58, 0xcb, 0xc5, 0x1c, 0xba, 0x56, 0xe3, 0x23, 0xe8, 0x6a, + 0xc3, 0x6b, 0x63, 0xc3, 0x13, 0xd6, 0x0a, 0x0c, 0x80, 0x88, 0xaa, 0xb0, 0x6e, 0x84, 0x35, 0x9f, + 0x0d, 0x57, 0xc9, 0x42, 0xe8, 0x90, 0xd8, 0xc5, 0xad, 0xc0, 0xa7, 0xd0, 0x6b, 0x4e, 0xfa, 0x45, + 0xf0, 0xbb, 0xb0, 0xd3, 0x9e, 0x75, 0xaf, 0x9f, 0x7d, 0x04, 0x38, 0xfc, 0x35, 0x0e, 0xe0, 0xde, + 0x94, 0x7e, 0xc8, 0xaf, 0xa6, 0x97, 0x81, 0x83, 0x3e, 0x78, 0xb3, 0xb7, 0x81, 0x8b, 0xf7, 0xa1, + 0x4f, 0x67, 0x37, 0x8b, 0x57, 0xb3, 0xf7, 0xf4, 0x32, 0xf0, 0x70, 0x08, 0xbd, 0x29, 0xbd, 0x99, + 0x30, 0x9a, 0x5f, 0x05, 0x04, 0x4f, 0xe0, 0x21, 0xcd, 0xdf, 0x4d, 0xe6, 0xd7, 0xf9, 0xc5, 0x64, + 0x71, 0xc0, 0x3a, 0xe7, 0xe1, 0xef, 0x4d, 0xe4, 0xae, 0x37, 0x91, 0xfb, 0x77, 0x13, 0xb9, 0xbf, + 0xb6, 0x91, 0xb3, 0xde, 0x46, 0xce, 0x9f, 0x6d, 0xe4, 0xdc, 0xfa, 0xf6, 0x81, 0xbc, 0xf8, 0x17, + 0x00, 0x00, 0xff, 0xff, 0x0b, 0x93, 0xfd, 0x1b, 0x44, 0x02, 0x00, 0x00, } func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { @@ -340,10 +339,10 @@ func (m *GetSharesByNamespaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, er _ = i var l int _ = l - if len(m.NamespaceId) > 0 { - i -= len(m.NamespaceId) - copy(dAtA[i:], m.NamespaceId) - i = encodeVarintShare(dAtA, i, uint64(len(m.NamespaceId))) + if len(m.Namespace) > 0 { + i -= len(m.Namespace) + copy(dAtA[i:], m.Namespace) + i = encodeVarintShare(dAtA, i, uint64(len(m.Namespace))) i-- dAtA[i] = 0x12 } @@ -513,7 +512,7 @@ func (m *GetSharesByNamespaceRequest) Size() (n int) { if l > 0 { n += 1 + l + sovShare(uint64(l)) } - l = len(m.NamespaceId) + l = len(m.Namespace) if l > 0 { n += 1 + l + sovShare(uint64(l)) } @@ -653,7 +652,7 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { iNdEx = postIndex case 2: if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field NamespaceId", wireType) + return fmt.Errorf("proto: wrong wireType = %d for field Namespace", wireType) } var byteLen int for shift := uint(0); ; shift += 7 { @@ -680,9 +679,9 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { if postIndex > l { return io.ErrUnexpectedEOF } - m.NamespaceId = append(m.NamespaceId[:0], dAtA[iNdEx:postIndex]...) - if m.NamespaceId == nil { - m.NamespaceId = []byte{} + m.Namespace = append(m.Namespace[:0], dAtA[iNdEx:postIndex]...) + if m.Namespace == nil { + m.Namespace = []byte{} } iNdEx = postIndex default: diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto index 09d77d29f3..306865d17f 100644 --- a/share/p2p/shrexnd/pb/share.proto +++ b/share/p2p/shrexnd/pb/share.proto @@ -4,7 +4,7 @@ package share.p2p.shrex.nd; message GetSharesByNamespaceRequest{ bytes root_hash = 1; - bytes namespace_id = 2; + bytes namespace = 2; } message GetSharesByNamespaceResponse{ diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 72c5231eff..d61e6b2e68 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -17,7 +17,6 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p" pb "github.com/celestiaorg/celestia-node/share/p2p/shrexnd/pb" ) @@ -100,7 +99,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre stream.Reset() //nolint:errcheck return } - logger = logger.With("namespaceId", hex.EncodeToString(req.NamespaceId), "hash", share.DataHash(req.RootHash).String()) + logger = logger.With("namespace", hex.EncodeToString(req.Namespace), "hash", share.DataHash(req.RootHash).String()) logger.Debugw("server: new request") err = stream.CloseRead() @@ -130,7 +129,7 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre return } - shares, err := srv.getter.GetSharesByNamespace(ctx, dah, req.NamespaceId) + shares, err := srv.getter.GetSharesByNamespace(ctx, dah, req.Namespace) switch { case errors.Is(err, share.ErrNotFound): logger.Warn("server: nd not found") @@ -148,13 +147,12 @@ func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stre // validateRequest checks correctness of the request func validateRequest(req pb.GetSharesByNamespaceRequest) error { - if len(req.NamespaceId) != ipld.NamespaceSize { - return fmt.Errorf("incorrect namespace id length: %v", len(req.NamespaceId)) + if err := share.Namespace(req.Namespace).ValidateForData(); err != nil { + return err } if len(req.RootHash) != sha256.Size { return fmt.Errorf("incorrect root hash length: %v", len(req.RootHash)) } - return nil } diff --git a/share/share.go b/share/share.go index 0178054a9f..02ccd73909 100644 --- a/share/share.go +++ b/share/share.go @@ -4,32 +4,23 @@ import ( "bytes" "fmt" - "go.opentelemetry.io/otel" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/nmt/namespace" - - "github.com/celestiaorg/celestia-node/share/ipld" ) var ( - tracer = otel.Tracer("share") - // DefaultRSMT2DCodec sets the default rsmt2d.Codec for shares. DefaultRSMT2DCodec = appconsts.DefaultCodec ) const ( - // NamespaceSize is a system-wide size for NMT namespaces. - NamespaceSize = appconsts.NamespaceSize - // Size is a system-wide size of a share, including both data and namespace ID + // Size is a system-wide size of a share, including both data and namespace GetNamespace Size = appconsts.ShareSize ) var ( // MaxSquareSize is currently the maximum size supported for unerasured data in // rsmt2d.ExtendedDataSquare. - MaxSquareSize = ipld.MaxSquareSize + MaxSquareSize = appconsts.SquareSizeUpperBound(appconsts.LatestVersion) ) // Share contains the raw share data without the corresponding namespace. @@ -38,13 +29,13 @@ var ( // on it. type Share = []byte -// ID gets the namespace ID from the share. -func ID(s Share) namespace.ID { +// GetNamespace slices Namespace out of the Share. +func GetNamespace(s Share) Namespace { return s[:NamespaceSize] } -// Data gets data from the share. -func Data(s Share) []byte { +// GetData slices out data of the Share. +func GetData(s Share) []byte { return s[NamespaceSize:] } diff --git a/share/sharetest/testing.go b/share/sharetest/testing.go new file mode 100644 index 0000000000..50eb0eb003 --- /dev/null +++ b/share/sharetest/testing.go @@ -0,0 +1,58 @@ +package sharetest + +import ( + "bytes" + "math/rand" + "sort" + "sync" + "time" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/namespace" + + "github.com/celestiaorg/celestia-node/share" +) + +// RandShares generate 'total' amount of shares filled with random data. It uses require.TestingT +// to be able to take both a *testing.T and a *testing.B. +func RandShares(t require.TestingT, total int) []share.Share { + if total&(total-1) != 0 { + t.Errorf("total must be power of 2: %d", total) + t.FailNow() + } + + shares := make([]share.Share, total) + for i := range shares { + shr := make([]byte, share.Size) + copy(share.GetNamespace(shr), RandV0Namespace()) + rndMu.Lock() + _, err := rnd.Read(share.GetData(shr)) + rndMu.Unlock() + require.NoError(t, err) + shares[i] = shr + } + sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) + + return shares +} + +// RandV0Namespace generates random valid data namespace for testing purposes. +func RandV0Namespace() share.Namespace { + rb := make([]byte, namespace.NamespaceVersionZeroIDSize) + rndMu.Lock() + rnd.Read(rb) + rndMu.Unlock() + for { + namespace, _ := share.NewBlobNamespaceV0(rb) + if err := namespace.ValidateForData(); err != nil { + continue + } + return namespace + } +} + +var ( + rnd = rand.New(rand.NewSource(time.Now().Unix())) //nolint:gosec + rndMu sync.Mutex +) diff --git a/share/test_helpers.go b/share/test_helpers.go deleted file mode 100644 index c02bfc55ac..0000000000 --- a/share/test_helpers.go +++ /dev/null @@ -1,65 +0,0 @@ -package share - -import ( - "bytes" - "crypto/rand" - "sort" - - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/namespace" - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/rsmt2d" -) - -// EqualEDS check whether two given EDSes are equal. -// TODO(Wondertan): Move to rsmt2d -// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and -// Golang practices -func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool { - if a.Width() != b.Width() { - return false - } - - for i := uint(0); i < a.Width(); i++ { - ar, br := a.Row(i), b.Row(i) - for j := 0; j < len(ar); j++ { - if !bytes.Equal(ar[j], br[j]) { - return false - } - } - } - - return true -} - -// RandEDS generates EDS filled with the random data with the given size for original square. It -// uses require.TestingT to be able to take both a *testing.T and a *testing.B. -func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { - shares := RandShares(t, size*size) - // recompute the eds - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) - require.NoError(t, err, "failure to recompute the extended data square") - return eds -} - -// RandShares generate 'total' amount of shares filled with random data. It uses require.TestingT -// to be able to take both a *testing.T and a *testing.B. -func RandShares(t require.TestingT, total int) []Share { - if total&(total-1) != 0 { - t.Errorf("total must be power of 2: %d", total) - t.FailNow() - } - - shares := make([]Share, total) - for i := range shares { - share := make([]byte, Size) - copy(share[:NamespaceSize], namespace.RandomNamespace().Bytes()) - _, err := rand.Read(share[NamespaceSize:]) - require.NoError(t, err) - shares[i] = share - } - sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) - - return shares -} diff --git a/state/core_access.go b/state/core_access.go index 7b59f3e714..3fefaa3ed9 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -167,8 +167,11 @@ func (ca *CoreAccessor) SubmitPayForBlob( } appblobs := make([]*apptypes.Blob, len(blobs)) - for i, blob := range blobs { - appblobs[i] = &blob.Blob + for i, b := range blobs { + if err := b.Namespace().ValidateForBlob(); err != nil { + return nil, err + } + appblobs[i] = &b.Blob } response, err := appblob.SubmitPayForBlob( From 4d986940d662649eba4b0d3daa85268318204ff2 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 27 Jun 2023 15:43:55 +0200 Subject: [PATCH 068/388] chore(go.mod): Bump nmt (#2405) Self-explanatory --- go.mod | 2 +- go.sum | 8 ++++---- share/eds/eds.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 73509c56c3..3d38242260 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.7 github.com/celestiaorg/go-libp2p-messenger v0.2.0 - github.com/celestiaorg/nmt v0.16.0 + github.com/celestiaorg/nmt v0.17.0 github.com/celestiaorg/rsmt2d v0.9.0 github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 diff --git a/go.sum b/go.sum index 28113d7847..d8cfbe3806 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.16.0 h1:4CX6d1Uwf1C+tGcAWskPve0HCDTnI4Ey8ffjiDwcGH0= -github.com/celestiaorg/nmt v0.16.0/go.mod h1:GfwIvQPhUakn1modWxJ+rv8dUjJzuXg5H+MLFM1o7nY= +github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= +github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.9.0 h1:kon78I748ZqjNzI8OAqPN+2EImuZuanj/6gTh8brX3o= @@ -1950,14 +1950,14 @@ github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= -github.com/tidwall/gjson v1.14.0 h1:6aeJ0bzojgWLa82gDQHcx3S0Lr/O51I9bJ5nv6JFx5w= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= +github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= -github.com/tidwall/pretty v1.2.0 h1:RWIZEg2iJ8/g6fDDYzMpobmaoGh5OLl4AXtGUGPcqCs= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= +github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= diff --git a/share/eds/eds.go b/share/eds/eds.go index 239ba4a0ad..544f3c2438 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -35,7 +35,7 @@ var ErrEmptySquare = errors.New("share: importing empty data") type writingSession struct { eds *rsmt2d.ExtendedDataSquare store bstore.Blockstore // caches inner nodes (proofs) while we walk the nmt tree. - hasher *nmt.Hasher + hasher *nmt.NmtHasher w io.Writer } From 9ee50806244e5f2eeb296c3353cc8344f9f78fea Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 29 Jun 2023 23:32:15 +0800 Subject: [PATCH 069/388] tests(share/eds): add store basic benchmarks (#2407) ## Overview Implements basic benchmarks for eds store implementation. --- share/eds/store_test.go | 43 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/share/eds/store_test.go b/share/eds/store_test.go index b1dc3986b7..7f2f548f63 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -253,6 +253,49 @@ func Test_CachedAccessor(t *testing.T) { assert.Equal(t, firstBlock, secondBlock) } +func BenchmarkStore(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + b.Cleanup(cancel) + + tmpDir := b.TempDir() + ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) + edsStore, err := NewStore(tmpDir, ds) + require.NoError(b, err) + err = edsStore.Start(ctx) + require.NoError(b, err) + + // BenchmarkStore/bench_put_128-10 10 3231859283 ns/op (~3sec) + b.Run("bench put 128", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + // pause the timer for initializing test data + b.StopTimer() + eds := edstest.RandEDS(b, 128) + dah := da.NewDataAvailabilityHeader(eds) + b.StartTimer() + + err = edsStore.Put(ctx, dah.Hash(), eds) + require.NoError(b, err) + } + }) + + // BenchmarkStore/bench_read_128-10 14 78970661 ns/op (~70ms) + b.Run("bench read 128", func(b *testing.B) { + b.ResetTimer() + for i := 0; i < b.N; i++ { + // pause the timer for initializing test data + b.StopTimer() + eds := edstest.RandEDS(b, 128) + dah := da.NewDataAvailabilityHeader(eds) + _ = edsStore.Put(ctx, dah.Hash(), eds) + b.StartTimer() + + _, err := edsStore.Get(ctx, dah.Hash()) + require.NoError(b, err) + } + }) +} + func newStore(t *testing.T) (*Store, error) { t.Helper() From 061b78881d7d3b968fafaff5ec2744f75b775954 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ramon=20Ma=C3=B1es?= <32740567+jrmanes@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:22:10 +0200 Subject: [PATCH 070/388] chore!: bump arabica-9 (#2417) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hello team! This PR contains the updates for arabica-9: - the genesis hash - chain-id - bootstrapper multiaddresses thanks in advance! 🚀 Jose Ramon Mañes Signed-off-by: Jose Ramon Mañes --- nodebuilder/p2p/bootstrap.go | 8 ++++---- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index c534aa4fba..376d95fb14 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -38,10 +38,10 @@ func bootstrappersFor(net Network) ([]string, error) { // NOTE: Every time we add a new long-running network, its bootstrap peers have to be added here. var bootstrapList = map[Network][]string{ Arabica: { - "/dns4/da-bridge-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWDXkXARv79Dtn5xrGBgJePtCzCsEwWR7eGWnx9ZCyUyD6", - "/dns4/da-bridge-arabica-8-2.celestia-arabica.com/tcp/2121/p2p/12D3KooWPu8qKmmNgYFMBsTkLBa1m3D9Cy9ReCAoQLqxEn9MHD1i", - "/dns4/da-full-1-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWEmeFodzypdTBTcw8Yub6WZRT4h1UgFtwCwwq6wS5Dtqm", - "/dns4/da-full-2-arabica-8.celestia-arabica.com/tcp/2121/p2p/12D3KooWCs3wFmqwPn1u8pNU4BGsvLsob1ShTzvps8qEtTRuuuK5", + "/dns4/da-bridge-arabica-9.celestia-arabica.com/tcp/2121/p2p/12D3KooWBLvsfkbovAH74DbGGxHPpVW7DkvKdbQxhorrkv9tfGZU", + "/dns4/da-bridge-arabica-9-2.celestia-arabica.com/tcp/2121/p2p/12D3KooWNjJSk8JcY7VoLEjGGUz8CXp9Bxt495zXmdmccjaMPgHf", + "/dns4/da-full-1-arabica-9.celestia-arabica.com/tcp/2121/p2p/12D3KooWFUK2Z4WPsQN3p5n8tgBigxP32gbmABUet2UMby2Ha9ZK", + "/dns4/da-full-2-arabica-9.celestia-arabica.com/tcp/2121/p2p/12D3KooWKnmwsimoghxUT1DXr7f8yXbWCfmXDk4UGbQDsAks9XsN", }, Mocha: { "/dns4/bootstr-mocha-1.celestia-mocha.com/tcp/2121/p2p/12D3KooWDRSJMbH3PS4dRDa11H7Tk615aqTUgkeEKz4pwd4sS6fN", diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index 0a36dc54cc..e027cfdb1f 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -23,7 +23,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ - Arabica: "E5D620B5BE7873222DCD83464C285FD0F215C209393E7481F9A5979280AD6CA2", + Arabica: "7A5FABB19713D732D967B1DA84FA0DF5E87A7B62302D783F78743E216C1A3550", Mocha: "1181AF8EAE5DDF3CBBFF3BF3CC44C5B795DF5094F5A0CC0AE52921ECCA0AF3C8", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index dd04cd377c..276fc83501 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -12,7 +12,7 @@ const ( // DefaultNetwork is the default network of the current build. DefaultNetwork = Mocha // Arabica testnet. See: celestiaorg/networks. - Arabica Network = "arabica-8" + Arabica Network = "arabica-9" // Mocha testnet. See: celestiaorg/networks. Mocha Network = "mocha-2" // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. From 44786bb2e6d6c98f853d185413cbb66567fa5aba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ramon=20Ma=C3=B1es?= <32740567+jrmanes@users.noreply.github.com> Date: Fri, 30 Jun 2023 13:53:55 +0200 Subject: [PATCH 071/388] feat: update Go base alpine - add branch in makefile (#2414) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hello team! This PR contains a couple of things: - Update the base version of Go Alpine to `golang:1.20-alpine3.18` - In the Makefile, I've added the following line: ```Makefile main.semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" ``` Basically what it does is, in case this commit doesn't contain any tag, it uses the branch name. At the moment we are having some issues using the flag `version`, the output command is: ![Screenshot 2023-06-29 at 17 24 22](https://github.com/celestiaorg/celestia-node/assets/32740567/b6b6f948-fcec-4157-920e-0a9863b8d11f) With this change, we'll have something like: ![Screenshot 2023-06-29 at 17 08 17](https://github.com/celestiaorg/celestia-node/assets/32740567/7d2de807-f014-444d-8917-6ec45a9d9750) *Already tested* Let me know what you think about it and if it's worth it 😊 Thanks in advance! 🚀 Jose Ramon Mañes --------- Signed-off-by: Jose Ramon Mañes Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- Dockerfile | 2 +- Makefile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 9346c559d1..6e9934dcb1 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.20-alpine3.17 as builder +FROM docker.io/golang:1.20-alpine3.18 as builder # hadolint ignore=DL3018 RUN apk update && apk add --no-cache \ diff --git a/Makefile b/Makefile index 8f4d1b42a9..310e65cbe0 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SHELL=/usr/bin/env bash PROJECTNAME=$(shell basename "$(PWD)") -LDFLAGS=-ldflags="-X 'main.buildTime=$(shell date)' -X 'main.lastCommit=$(shell git rev-parse HEAD)' -X 'main.semanticVersion=$(shell git describe --tags --dirty=-dev)'" +LDFLAGS=-ldflags="-X 'main.buildTime=$(shell date)' -X 'main.lastCommit=$(shell git rev-parse HEAD)' -X 'main.semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" ifeq (${PREFIX},) PREFIX := /usr/local endif From 1a37094a633ba62dbf95eba03d0bcd23d18494d1 Mon Sep 17 00:00:00 2001 From: Mustafa Al-Bassam Date: Mon, 3 Jul 2023 11:10:46 +0100 Subject: [PATCH 072/388] doc(share/p2p/shrexnd): Improve accuracy of RequestND description (#2418) `RequestND` doesn't actually verify the inclusion proofs; this is done later in `GetSharesByNamespace`: https://github.com/celestiaorg/celestia-node/blob/main/share/getters/shrex.go#L237 --- share/p2p/shrexnd/client.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index 7f12c21cf2..74162f5981 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -46,7 +46,7 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { } // RequestND requests namespaced data from the given peer. -// Returns valid data with its verified inclusion against the share.Root. +// Returns shares with unverified inclusion proofs against the share.Root. func (c *Client) RequestND( ctx context.Context, root *share.Root, From beaf6dbdc73fd43b73a98578330a7a5ad422c3c8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 3 Jul 2023 14:28:28 +0200 Subject: [PATCH 073/388] chore(deps): bump alpine from 3.18.0 to 3.18.2 (#2380) Bumps alpine from 3.18.0 to 3.18.2. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=alpine&package-manager=docker&previous-version=3.18.0&new-version=3.18.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index 6e9934dcb1..2d07babc94 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . RUN make build && make cel-key -FROM docker.io/alpine:3.18.0 +FROM docker.io/alpine:3.18.2 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From 0d10cedaa0d0d117b18b618d1d3ccbe456097495 Mon Sep 17 00:00:00 2001 From: Alex Kiss Date: Wed, 5 Jul 2023 22:50:29 +0100 Subject: [PATCH 074/388] chore(nodebuilder/p2p)! Upgrade to mocha-3 (#2432) ## Overview Update to mocha-3 chain name when init is called. ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index e027cfdb1f..1fb6b10a55 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -24,7 +24,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ Arabica: "7A5FABB19713D732D967B1DA84FA0DF5E87A7B62302D783F78743E216C1A3550", - Mocha: "1181AF8EAE5DDF3CBBFF3BF3CC44C5B795DF5094F5A0CC0AE52921ECCA0AF3C8", + Mocha: "79A97034D569C4199A867439B1B7B77D4E1E1D9697212755E1CE6D920CDBB541", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", } diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index 276fc83501..b1c3a5fbb7 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -14,7 +14,7 @@ const ( // Arabica testnet. See: celestiaorg/networks. Arabica Network = "arabica-9" // Mocha testnet. See: celestiaorg/networks. - Mocha Network = "mocha-2" + Mocha Network = "mocha-3" // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. BlockspaceRace Network = "blockspacerace-0" // Private can be used to set up any private network, including local testing setups. From 387c160d76bcf74b2d3e4ecc7be017fe419ed3b3 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 6 Jul 2023 13:05:11 +0300 Subject: [PATCH 075/388] share/befp: add benchmarks for fraud proof creation and validation (#2400) ## Overview Add benchmarks for BEFP creation and validation BEFP was created for the first row. ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- header/headertest/testing.go | 2 +- share/eds/retriever_test.go | 122 +++++++++++++++++++++++++++++++++++ 2 files changed, 123 insertions(+), 1 deletion(-) diff --git a/header/headertest/testing.go b/header/headertest/testing.go index fbc71f92f9..ceeb9bb164 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -352,7 +352,7 @@ func CreateFraudExtHeader( eh *header.ExtendedHeader, serv blockservice.BlockService, ) (*header.ExtendedHeader, *rsmt2d.ExtendedDataSquare) { - square := edstest.RandByzantineEDS(t, 16) + square := edstest.RandByzantineEDS(t, len(eh.DAH.RowRoots)) err := ipld.ImportEDS(context.Background(), square, serv) require.NoError(t, err) dah := da.NewDataAvailabilityHeader(square) diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index ca689aa5ec..c90697862f 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -2,9 +2,12 @@ package eds import ( "context" + "errors" + "fmt" "testing" "time" + "github.com/ipfs/go-blockservice" mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -14,6 +17,8 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/byzantine" "github.com/celestiaorg/celestia-node/share/eds/edstest" @@ -124,3 +129,120 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { _, err = ses.Reconstruct(ctx) assert.NoError(t, err) } + +func TestFraudProofValidation(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer t.Cleanup(cancel) + bServ := mdutils.Bserv() + + odsSize := []int{2, 4, 16, 32, 64, 128} + for _, size := range odsSize { + t.Run(fmt.Sprintf("ods size:%d", size), func(t *testing.T) { + var errByz *byzantine.ErrByzantine + faultHeader, err := generateByzantineError(ctx, t, size, bServ) + require.True(t, errors.As(err, &errByz)) + + p := byzantine.CreateBadEncodingProof([]byte("hash"), uint64(faultHeader.Height()), errByz) + err = p.Validate(faultHeader) + require.NoError(t, err) + }) + } +} + +func generateByzantineError( + ctx context.Context, + t *testing.T, + odsSize int, + bServ blockservice.BlockService, +) (*header.ExtendedHeader, error) { + eds := edstest.RandByzantineEDS(t, odsSize) + err := ipld.ImportEDS(ctx, eds, bServ) + require.NoError(t, err) + h := headertest.ExtendedHeaderFromEDS(t, 1, eds) + _, err = NewRetriever(bServ).Retrieve(ctx, h.DAH) + + return h, err +} + +/* +BenchmarkBEFPValidation/ods_size:2 31273 38819 ns/op 68052 B/op 366 allocs/op +BenchmarkBEFPValidation/ods_size:4 14664 80439 ns/op 135892 B/op 894 allocs/op +BenchmarkBEFPValidation/ods_size:16 2850 386178 ns/op 587890 B/op 4945 allocs/op +BenchmarkBEFPValidation/ods_size:32 1399 874490 ns/op 1233399 B/op 11284 allocs/op +BenchmarkBEFPValidation/ods_size:64 619 2047540 ns/op 2578008 B/op 25364 allocs/op +BenchmarkBEFPValidation/ods_size:128 259 4934375 ns/op 5418406 B/op 56345 allocs/op +*/ +func BenchmarkBEFPValidation(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) + defer b.Cleanup(cancel) + bServ := mdutils.Bserv() + r := NewRetriever(bServ) + t := &testing.T{} + odsSize := []int{2, 4, 16, 32, 64, 128} + for _, size := range odsSize { + b.Run(fmt.Sprintf("ods size:%d", size), func(b *testing.B) { + b.ResetTimer() + b.StopTimer() + eds := edstest.RandByzantineEDS(t, size) + err := ipld.ImportEDS(ctx, eds, bServ) + require.NoError(t, err) + h := headertest.ExtendedHeaderFromEDS(t, 1, eds) + _, err = r.Retrieve(ctx, h.DAH) + var errByz *byzantine.ErrByzantine + require.ErrorAs(t, err, &errByz) + b.StartTimer() + + for i := 0; i < b.N; i++ { + b.ReportAllocs() + p := byzantine.CreateBadEncodingProof([]byte("hash"), uint64(h.Height()), errByz) + err = p.Validate(h) + require.NoError(b, err) + } + }) + } +} + +/* +BenchmarkNewErrByzantineData/ods_size:2 29605 38846 ns/op 49518 B/op 579 allocs/op +BenchmarkNewErrByzantineData/ods_size:4 11380 105302 ns/op 134967 B/op 1571 allocs/op +BenchmarkNewErrByzantineData/ods_size:16 1902 631086 ns/op 830199 B/op 9601 allocs/op +BenchmarkNewErrByzantineData/ods_size:32 756 1530985 ns/op 1985272 B/op 22901 allocs/op +BenchmarkNewErrByzantineData/ods_size:64 340 3445544 ns/op 4767053 B/op 54704 allocs/op +BenchmarkNewErrByzantineData/ods_size:128 132 8740678 ns/op 11991093 B/op 136584 allocs/op +*/ +func BenchmarkNewErrByzantineData(b *testing.B) { + odsSize := []int{2, 4, 16, 32, 64, 128} + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + bServ := mdutils.Bserv() + r := NewRetriever(bServ) + t := &testing.T{} + for _, size := range odsSize { + b.Run(fmt.Sprintf("ods size:%d", size), func(b *testing.B) { + b.StopTimer() + eds := edstest.RandByzantineEDS(t, size) + err := ipld.ImportEDS(ctx, eds, bServ) + require.NoError(t, err) + h := headertest.ExtendedHeaderFromEDS(t, 1, eds) + ses, err := r.newSession(ctx, h.DAH) + require.NoError(t, err) + + select { + case <-ctx.Done(): + b.Fatal(ctx.Err()) + case <-ses.Done(): + } + + _, err = ses.Reconstruct(ctx) + assert.NoError(t, err) + var errByz *rsmt2d.ErrByzantineData + require.ErrorAs(t, err, &errByz) + b.StartTimer() + + for i := 0; i < b.N; i++ { + err = byzantine.NewErrByzantine(ctx, bServ, h.DAH, errByz) + require.NotNil(t, err) + } + }) + } +} From 37123381add941b02c7de2e087bb8fdb581bd4ad Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 6 Jul 2023 16:25:48 +0200 Subject: [PATCH 076/388] deps: bump go-header (#2436) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 3d38242260..36ed440e4a 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc5 github.com/celestiaorg/go-fraud v0.1.0 - github.com/celestiaorg/go-header v0.2.7 + github.com/celestiaorg/go-header v0.2.10 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.17.0 github.com/celestiaorg/rsmt2d v0.9.0 diff --git a/go.sum b/go.sum index d8cfbe3806..471f6c47d2 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,8 @@ github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPc github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= github.com/celestiaorg/go-fraud v0.1.0/go.mod h1:yoNM35cKMAkt5Mi/Qx3Wi9bnPilLi8n6RpHZVglTUDs= -github.com/celestiaorg/go-header v0.2.7 h1:r0X9Dl7lqBkQpwG3ekQHC61n/QdwO6epuIxDkQ4YX4o= -github.com/celestiaorg/go-header v0.2.7/go.mod h1:i9OpY70+PJ1xPw1IgMfF0Pk6vBD6VWPmjY3bgubJBcU= +github.com/celestiaorg/go-header v0.2.10 h1:KaULxmQ/WJGBeN+rEzTqEEwLD3adkqeD3RM+H8GRHkg= +github.com/celestiaorg/go-header v0.2.10/go.mod h1:i9OpY70+PJ1xPw1IgMfF0Pk6vBD6VWPmjY3bgubJBcU= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= From e1b4c9c545cd3fc20651d171885399893ca0043b Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Thu, 6 Jul 2023 10:45:52 -0500 Subject: [PATCH 077/388] chore: bump celestia-app to v1.0.0-rc8 (#2434) ## Overview this PR bumps node to rc8 ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- blob/helper.go | 2 +- core/eds.go | 20 ++++------------- core/eds_test.go | 17 ++++++++++----- core/exchange.go | 4 ++-- core/header_test.go | 2 +- core/listener.go | 2 +- go.mod | 22 +++++++++---------- go.sum | 44 +++++++++++++++++++------------------- nodebuilder/p2p/genesis.go | 2 +- 9 files changed, 55 insertions(+), 60 deletions(-) diff --git a/blob/helper.go b/blob/helper.go index 341080f42f..35f0abc540 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -74,7 +74,7 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { return val <= 0 }) - rawShares, err := shares.SplitBlobs(0, nil, b, false) + rawShares, err := shares.SplitBlobs(b...) if err != nil { return nil, err } diff --git a/core/eds.go b/core/eds.go index dc9b2b4a9e..c87d947e35 100644 --- a/core/eds.go +++ b/core/eds.go @@ -7,9 +7,7 @@ import ( "github.com/filecoin-project/dagstore" "github.com/tendermint/tendermint/types" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/square" + "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -19,22 +17,12 @@ import ( // extendBlock extends the given block data, returning the resulting // ExtendedDataSquare (EDS). If there are no transactions in the block, // nil is returned in place of the eds. -func extendBlock(data types.Data) (*rsmt2d.ExtendedDataSquare, error) { - if len(data.Txs) == 0 && data.SquareSize == uint64(1) { +func extendBlock(data types.Data, appVersion uint64) (*rsmt2d.ExtendedDataSquare, error) { + if app.IsEmptyBlock(data, appVersion) { return nil, nil } - sqr, err := square.Construct(data.Txs.ToSliceOfBytes(), appconsts.LatestVersion, share.MaxSquareSize) - if err != nil { - return nil, err - } - - shares := make([][]byte, len(sqr)) - for i, s := range sqr { - shares[i] = s.ToBytes() - } - - return da.ExtendShares(shares) + return app.ExtendBlock(data, appVersion) } // storeEDS will only store extended block if it is not empty and doesn't already exist. diff --git a/core/eds_test.go b/core/eds_test.go index 6bc04c96c4..2cc6f1c7cc 100644 --- a/core/eds_test.go +++ b/core/eds_test.go @@ -7,6 +7,8 @@ import ( "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/types" + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-node/share" @@ -21,7 +23,7 @@ func TestTrulyEmptySquare(t *testing.T) { SquareSize: 1, } - eds, err := extendBlock(data) + eds, err := extendBlock(data, appconsts.LatestVersion) require.NoError(t, err) assert.Nil(t, eds) } @@ -32,14 +34,19 @@ func TestTrulyEmptySquare(t *testing.T) { // construction of the square is deterministic, and the rules which dictate the // square size do not allow for empty block data. However, should that ever // occur, we need to ensure that the correct data root is generated. -func TestNonEmptySquareWithZeroTxs(t *testing.T) { +func TestEmptySquareWithZeroTxs(t *testing.T) { data := types.Data{ - Txs: []types.Tx{}, - SquareSize: 16, + Txs: []types.Tx{}, } - eds, err := extendBlock(data) + eds, err := extendBlock(data, appconsts.LatestVersion) + require.Nil(t, eds) + require.NoError(t, err) + + // force extend the square using an empty block and compare with the min DAH + eds, err = app.ExtendBlock(data, appconsts.LatestVersion) require.NoError(t, err) + dah := da.NewDataAvailabilityHeader(eds) assert.Equal(t, share.EmptyRoot().Hash(), dah.Hash()) } diff --git a/core/exchange.go b/core/exchange.go index 70ca47c56f..a3e98c0307 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -105,7 +105,7 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende } // extend block data - eds, err := extendBlock(block.Data) + eds, err := extendBlock(block.Data, block.Header.Version.App) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", &block.Height, err) } @@ -142,7 +142,7 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 log.Debugw("fetched signed block from core", "height", b.Header.Height) // extend block data - eds, err := extendBlock(b.Data) + eds, err := extendBlock(b.Data, b.Header.Version.App) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", b.Header.Height, err) } diff --git a/core/header_test.go b/core/header_test.go index 57f53d3661..6315fbc143 100644 --- a/core/header_test.go +++ b/core/header_test.go @@ -30,7 +30,7 @@ func TestMakeExtendedHeaderForEmptyBlock(t *testing.T) { comm, val, err := fetcher.GetBlockInfo(ctx, &height) require.NoError(t, err) - eds, err := extendBlock(b.Data) + eds, err := extendBlock(b.Data, b.Header.Version.App) require.NoError(t, err) headerExt, err := header.MakeExtendedHeader(ctx, &b.Header, comm, val, eds) diff --git a/core/listener.go b/core/listener.go index 24d83cda12..7ca408e4fb 100644 --- a/core/listener.go +++ b/core/listener.go @@ -150,7 +150,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS attribute.Int64("height", b.Header.Height), ) // extend block data - eds, err := extendBlock(b.Data) + eds, err := extendBlock(b.Data, b.Header.Version.App) if err != nil { return fmt.Errorf("extending block data: %w", err) } diff --git a/go.mod b/go.mod index 36ed440e4a..65d1436afc 100644 --- a/go.mod +++ b/go.mod @@ -7,10 +7,10 @@ replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1 require ( cosmossdk.io/errors v1.0.0-beta.7 cosmossdk.io/math v1.0.0-rc.0 - github.com/BurntSushi/toml v1.3.0 + github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc5 + github.com/celestiaorg/celestia-app v1.0.0-rc8 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.10 github.com/celestiaorg/go-libp2p-messenger v0.2.0 @@ -77,15 +77,15 @@ require ( golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/sync v0.2.0 golang.org/x/text v0.9.0 - google.golang.org/grpc v1.55.0 - google.golang.org/protobuf v1.30.0 + google.golang.org/grpc v1.56.1 + google.golang.org/protobuf v1.31.0 ) require ( cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.18.0 // indirect + cloud.google.com/go/compute v1.19.1 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.12.0 // indirect + cloud.google.com/go/iam v0.13.0 // indirect cloud.google.com/go/storage v1.28.1 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -169,7 +169,7 @@ require ( github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.0 // indirect + github.com/googleapis/gax-go/v2 v2.7.1 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect @@ -312,14 +312,14 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.6.0 // indirect + golang.org/x/oauth2 v0.7.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect - google.golang.org/api v0.110.0 // indirect + google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 // indirect + google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect @@ -335,5 +335,5 @@ replace ( github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28 ) diff --git a/go.sum b/go.sum index 471f6c47d2..94ac7c3259 100644 --- a/go.sum +++ b/go.sum @@ -75,8 +75,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.18.0 h1:FEigFqoDbys2cvFkZ9Fjq4gnHBP55anJ0yQyau2f9oY= -cloud.google.com/go/compute v1.18.0/go.mod h1:1X7yHxec2Ga+Ss6jPyjxRxpu2uu7PLgsOVXvgU0yacs= +cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= +cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -116,8 +116,8 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.12.0 h1:DRtTY29b75ciH6Ov1PHb4/iat2CLCvrOm40Q0a6DFpE= -cloud.google.com/go/iam v0.12.0/go.mod h1:knyHGviacl11zrtZUoDuYpDgLjvr28sLQaG0YB2GYAY= +cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= +cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -221,8 +221,8 @@ github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSu github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v1.3.0 h1:Ws8e5YmnrGEHzZEzg0YvK/7COGYtTC5PbaH9oSSbgfA= -github.com/BurntSushi/toml v1.3.0/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= +github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= +github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRrLeDnvGIM= github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4= @@ -344,10 +344,10 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc5 h1:PjKIp91CQgqyP4MLI3KeF8E4MPHtBPlNoRPvGIMWjHw= -github.com/celestiaorg/celestia-app v1.0.0-rc5/go.mod h1:i+jE3Hh8IQxlZnE5XSBlYF0PCNPvD3v5g8KmY095PEA= -github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28 h1:idHJK9i4WCkYOf5PXVWZbOs8pWkCiHHQGI4MZr0iMtQ= -github.com/celestiaorg/celestia-core v1.22.0-tm-v0.34.28/go.mod h1:LOxHW9nA++/9U8TgvTyKo9TO3F09sWv8asKQs00m73U= +github.com/celestiaorg/celestia-app v1.0.0-rc8 h1:bBQY8tR3DCc8uA7c2ney4AeWwEn9Ob1lc1QryGKVF2M= +github.com/celestiaorg/celestia-app v1.0.0-rc8/go.mod h1:X0R6s+LvfusZu+jBj/2SbTm4Nb/H1R2MD1CnR4fwQno= +github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28 h1:G7/rq6xTnuFf3XsVZEcl/Sa6vtagm9NQNhaUaSgjvy0= +github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 h1:vaQKgaOm0w58JAvOgn2iDohqjH7kvvRqVKiMcBDWifA= github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13/go.mod h1:G9XkhOJZde36FH0kt/1ayg4ZaioZEQmmRfMa/zQig0I= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= @@ -824,8 +824,8 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.0 h1:IcsPKeInNvYi7eqSaDjiZqDDKu5rsmunY0Y1YupQSSQ= -github.com/googleapis/gax-go/v2 v2.7.0/go.mod h1:TEop28CZZQ2y+c0VxMUmu1lV+fQx57QpBWsYpwqHJx8= +github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= +github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= @@ -2321,8 +2321,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.6.0 h1:Lh8GPgSKBfWSwFvtuWOfeI3aAAnbXTSutYxJiOJFgIw= -golang.org/x/oauth2 v0.6.0/go.mod h1:ycmewcwgD4Rpr3eZJLSB4Kyyljb3qDh40vJ8STE5HKw= +golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= +golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2637,8 +2637,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.110.0 h1:l+rh0KYUooe9JGbGVx71tbFo4SMbMTXK3I3ia2QSEeU= -google.golang.org/api v0.110.0/go.mod h1:7FC4Vvx1Mooxh8C5HWjzZHcavuS2f6pmJpZx60ca7iI= +google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= +google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2762,8 +2762,8 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4 h1:DdoeryqhaXp1LtT/emMP1BRJPHHKFi5akj/nbx/zNTA= -google.golang.org/genproto v0.0.0-20230306155012-7f2fa6fef1f4/go.mod h1:NWraEVixdDnqcqQ30jipen1STv2r/n24Wb7twVTGR4s= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= +google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2810,8 +2810,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.55.0 h1:3Oj82/tFSCeUrRTg/5E/7d/W5A1tj6Ky1ABAuZuv5ag= -google.golang.org/grpc v1.55.0/go.mod h1:iYEXKGkEBhg1PjZQvoYEVPTDkHo1/bjTnfwTeGONTY8= +google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= +google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2828,8 +2828,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.30.0 h1:kPPoIgf3TsEvrm0PFe15JQ+570QVxYzEvvHqChK+cng= -google.golang.org/protobuf v1.30.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= +google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index 1fb6b10a55..cbe5628d9f 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -24,7 +24,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ Arabica: "7A5FABB19713D732D967B1DA84FA0DF5E87A7B62302D783F78743E216C1A3550", - Mocha: "79A97034D569C4199A867439B1B7B77D4E1E1D9697212755E1CE6D920CDBB541", + Mocha: "831B81ADDC5CE999EBB0C150B778F76DAAD9E09DF75FACF164B1F11DCE93E2E1", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", } From eaa58358b21f9ffbb580de48071e2315e55a2b9c Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 6 Jul 2023 19:07:02 +0200 Subject: [PATCH 078/388] deps: bump go-header (#2438) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 65d1436afc..70e8de6cc5 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc8 github.com/celestiaorg/go-fraud v0.1.0 - github.com/celestiaorg/go-header v0.2.10 + github.com/celestiaorg/go-header v0.2.11 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.17.0 github.com/celestiaorg/rsmt2d v0.9.0 diff --git a/go.sum b/go.sum index 94ac7c3259..ac183c1c6d 100644 --- a/go.sum +++ b/go.sum @@ -354,8 +354,8 @@ github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPc github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= github.com/celestiaorg/go-fraud v0.1.0/go.mod h1:yoNM35cKMAkt5Mi/Qx3Wi9bnPilLi8n6RpHZVglTUDs= -github.com/celestiaorg/go-header v0.2.10 h1:KaULxmQ/WJGBeN+rEzTqEEwLD3adkqeD3RM+H8GRHkg= -github.com/celestiaorg/go-header v0.2.10/go.mod h1:i9OpY70+PJ1xPw1IgMfF0Pk6vBD6VWPmjY3bgubJBcU= +github.com/celestiaorg/go-header v0.2.11 h1:dLpuUfpGNxFfJNw3Ar3SqWc+AeyT1DlTP5mLjx9Ths8= +github.com/celestiaorg/go-header v0.2.11/go.mod h1:i9OpY70+PJ1xPw1IgMfF0Pk6vBD6VWPmjY3bgubJBcU= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= From 579a06078fc858887a4425a615434e11534c170e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ramon=20Ma=C3=B1es?= <32740567+jrmanes@users.noreply.github.com> Date: Wed, 12 Jul 2023 17:29:54 +0200 Subject: [PATCH 079/388] feat: use context in the common pipeline (#2447) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hello team! I've created a new version in the common pipeline, this one should fix the issues with the forks and we'll have the version output that we're currently missing when using the docker containers. If this works, we could close the PR: #2421 I've tested on my end and seems to be working crossed_fingers PR where we've done the fix: celestiaorg/.github#65 Thanks in advance! rocket Jose Ramon Mañes celestiaorg/devops#370 --- .github/workflows/docker-build-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index c922e8b901..2ce01c81df 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -18,6 +18,6 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.0 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.2 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile From 60a949e8d06ee142244fb2f20e622d34db02b462 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 12 Jul 2023 17:35:35 +0200 Subject: [PATCH 080/388] fix(header): temporary disable skipping verification (#2456) Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- header/headertest/verify_test.go | 8 -------- header/verify.go | 8 -------- nodebuilder/p2p/genesis.go | 2 +- 3 files changed, 1 insertion(+), 17 deletions(-) diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 177352a4cf..33bcf72642 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -37,14 +37,6 @@ func TestVerify(t *testing.T) { }, err: true, }, - { - prepare: func() libhead.Header { - untrusted := *untrustedNonAdj - untrusted.Commit = NewTestSuite(t, 2).Commit(RandRawHeader(t)) - return &untrusted - }, - err: true, - }, { prepare: func() libhead.Header { untrusted := *untrustedAdj diff --git a/header/verify.go b/header/verify.go index 18e7f91ea6..827f6c1d1b 100644 --- a/header/verify.go +++ b/header/verify.go @@ -5,8 +5,6 @@ import ( "fmt" "time" - "github.com/tendermint/tendermint/light" - libhead "github.com/celestiaorg/go-header" ) @@ -47,12 +45,6 @@ func (eh *ExtendedHeader) Verify(untrusted libhead.Header) error { return nil } - // Ensure that untrusted commit has enough of trusted commit's power. - err := eh.ValidatorSet.VerifyCommitLightTrusting(eh.ChainID(), untrst.Commit, light.DefaultTrustLevel) - if err != nil { - return &libhead.VerifyError{Reason: err} - } - return nil } diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index cbe5628d9f..1fb6b10a55 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -24,7 +24,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ Arabica: "7A5FABB19713D732D967B1DA84FA0DF5E87A7B62302D783F78743E216C1A3550", - Mocha: "831B81ADDC5CE999EBB0C150B778F76DAAD9E09DF75FACF164B1F11DCE93E2E1", + Mocha: "79A97034D569C4199A867439B1B7B77D4E1E1D9697212755E1CE6D920CDBB541", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", } From c8fa853ee92494de98781b679f494f17c9ad87a2 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 12 Jul 2023 18:54:07 +0300 Subject: [PATCH 081/388] share/befp: rework collecting and verifying the BEFP (#2408) * Use an orthogonal axis to collect and verify BEFP instead of collecting and verifying it against a single Merkle Root(where ErrByzantine occurred) * collect only 1/2 of the shares instead of all available shares from the row/col --- share/eds/byzantine/bad_encoding.go | 58 +++++++++++++++++------------ share/eds/byzantine/byzantine.go | 46 +++++++++++++++++------ share/eds/byzantine/share_proof.go | 34 ++++++++++++----- 3 files changed, 94 insertions(+), 44 deletions(-) diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 0672096b25..ae77026acf 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -2,7 +2,6 @@ package byzantine import ( "bytes" - "errors" "fmt" "github.com/celestiaorg/celestia-app/pkg/wrapper" @@ -113,50 +112,58 @@ func (p *BadEncodingProof) UnmarshalBinary(data []byte) error { func (p *BadEncodingProof) Validate(hdr libhead.Header) error { header, ok := hdr.(*header.ExtendedHeader) if !ok { - panic(fmt.Sprintf("invalid header type: expected %T, got %T", header, hdr)) + panic(fmt.Sprintf("invalid header type received during BEFP validation: expected %T, got %T", header, hdr)) } if header.Height() != int64(p.BlockHeight) { - return errors.New("fraud: incorrect block height") + return fmt.Errorf("incorrect block height during BEFP validation: expected %d, got %d", + p.BlockHeight, header.Height(), + ) } - merkleRowRoots := header.DAH.RowRoots - merkleColRoots := header.DAH.ColumnRoots - if len(merkleRowRoots) != len(merkleColRoots) { + + if len(header.DAH.RowRoots) != len(header.DAH.ColumnRoots) { // NOTE: This should never happen as callers of this method should not feed it with a // malformed extended header. panic(fmt.Sprintf( - "fraud: invalid extended header: length of row and column roots do not match. (rowRoots=%d) (colRoots=%d)", - len(merkleRowRoots), - len(merkleColRoots)), + "invalid extended header: length of row and column roots do not match. (rowRoots=%d) (colRoots=%d)", + len(header.DAH.RowRoots), + len(header.DAH.ColumnRoots)), ) } - if int(p.Index) >= len(merkleRowRoots) { - return fmt.Errorf("fraud: invalid proof: index out of bounds (%d >= %d)", int(p.Index), len(merkleRowRoots)) + + // merkleRoots are the roots against which we are going to check the inclusion of the received + // shares. Changing the order of the roots to prove the shares relative to the orthogonal axis, + // because inside the rsmt2d library rsmt2d.Row = 0 and rsmt2d.Col = 1 + merkleRoots := header.DAH.RowRoots + if p.Axis == rsmt2d.Row { + merkleRoots = header.DAH.ColumnRoots } - if len(merkleRowRoots) != len(p.Shares) { - return fmt.Errorf("fraud: invalid proof: incorrect number of shares %d != %d", len(p.Shares), len(merkleRowRoots)) + if int(p.Index) >= len(merkleRoots) { + return fmt.Errorf("invalid %s proof: index out of bounds (%d >= %d)", + BadEncoding, int(p.Index), len(merkleRoots), + ) } - - root := merkleRowRoots[p.Index] - if p.Axis == rsmt2d.Col { - root = merkleColRoots[p.Index] + if len(p.Shares) != len(merkleRoots) { + return fmt.Errorf("invalid %s proof: incorrect number of shares %d != %d", + BadEncoding, len(p.Shares), len(merkleRoots), + ) } // verify that Merkle proofs correspond to particular shares. - shares := make([][]byte, len(merkleRowRoots)) + shares := make([][]byte, len(merkleRoots)) for index, shr := range p.Shares { if shr == nil { continue } // validate inclusion of the share into one of the DAHeader roots - if ok := shr.Validate(ipld.MustCidFromNamespacedSha256(root)); !ok { - return fmt.Errorf("fraud: invalid proof: incorrect share received at index %d", index) + if ok := shr.Validate(ipld.MustCidFromNamespacedSha256(merkleRoots[index])); !ok { + return fmt.Errorf("invalid %s proof: incorrect share received at index %d", BadEncoding, index) } // NMTree commits the additional namespace while rsmt2d does not know about, so we trim it // this is ugliness from NMTWrapper that we have to embrace ¯\_(ツ)_/¯ shares[index] = share.GetData(shr.Share) } - odsWidth := uint64(len(merkleRowRoots) / 2) + odsWidth := uint64(len(merkleRoots) / 2) codec := share.DefaultRSMT2DCodec() // rebuild a row or col. @@ -183,10 +190,15 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { return err } + // root is a merkle root of the row/col where ErrByzantine occurred + root := header.DAH.RowRoots[p.Index] + if p.Axis == rsmt2d.Col { + root = header.DAH.ColumnRoots[p.Index] + } + // comparing rebuilt Merkle Root of bad row/col with respective Merkle Root of row/col from block. if bytes.Equal(expectedRoot, root) { - return errors.New("fraud: invalid proof: recomputed Merkle root matches the DAH's row/column root") + return fmt.Errorf("invalid %s proof: recomputed Merkle root matches the DAH's row/column root", BadEncoding) } - return nil } diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index b9c8ef414f..0fcd78273e 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -5,6 +5,7 @@ import ( "fmt" "github.com/ipfs/go-blockservice" + "golang.org/x/sync/errgroup" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" @@ -35,17 +36,41 @@ func NewErrByzantine( dah *da.DataAvailabilityHeader, errByz *rsmt2d.ErrByzantineData, ) *ErrByzantine { - root := [][][]byte{ - dah.RowRoots, + // changing the order to collect proofs against an orthogonal axis + roots := [][][]byte{ dah.ColumnRoots, - }[errByz.Axis][errByz.Index] - sharesWithProof, err := GetProofsForShares( - ctx, - bGetter, - ipld.MustCidFromNamespacedSha256(root), - errByz.Shares, - ) - if err != nil { + dah.RowRoots, + }[errByz.Axis] + + sharesWithProof := make([]*ShareWithProof, len(errByz.Shares)) + sharesAmount := 0 + + errGr, ctx := errgroup.WithContext(ctx) + for index, share := range errByz.Shares { + // skip further shares if we already requested half of them, which is enough to recompute the row + // or col + if sharesAmount == len(dah.RowRoots)/2 { + break + } + + if share == nil { + continue + } + sharesAmount++ + + index := index + errGr.Go(func() error { + share, err := getProofsAt( + ctx, bGetter, + ipld.MustCidFromNamespacedSha256(roots[index]), + int(errByz.Index), len(errByz.Shares), + ) + sharesWithProof[index] = share + return err + }) + } + + if err := errGr.Wait(); err != nil { // Fatal as rsmt2d proved that error is byzantine, // but we cannot properly collect the proof, // so verification will fail and thus services won't be stopped @@ -53,7 +78,6 @@ func NewErrByzantine( // TODO(@Wondertan): Find a better way to handle log.Fatalw("getting proof for ErrByzantine", "err", err) } - return &ErrByzantine{ Index: uint32(errByz.Index), Shares: sharesWithProof, diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index b8e39ee1d3..d6aa9dad51 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -78,24 +78,38 @@ func GetProofsForShares( proofs := make([]*ShareWithProof, len(shares)) for index, share := range shares { if share != nil { - proof := make([]cid.Cid, 0) - // TODO(@vgonkivs): Combine GetLeafData and GetProof in one function as the are traversing the same - // tree. Add options that will control what data will be fetched. - s, err := ipld.GetLeaf(ctx, bGetter, root, index, len(shares)) + proof, err := getProofsAt(ctx, bGetter, root, index, len(shares)) if err != nil { return nil, err } - proof, err = ipld.GetProof(ctx, bGetter, root, proof, index, len(shares)) - if err != nil { - return nil, err - } - proofs[index] = NewShareWithProof(index, s.RawData(), proof) + proofs[index] = proof } } - return proofs, nil } +func getProofsAt( + ctx context.Context, + bGetter blockservice.BlockGetter, + root cid.Cid, + index, + total int, +) (*ShareWithProof, error) { + proof := make([]cid.Cid, 0) + // TODO(@vgonkivs): Combine GetLeafData and GetProof in one function as the are traversing the same + // tree. Add options that will control what data will be fetched. + node, err := ipld.GetLeaf(ctx, bGetter, root, index, total) + if err != nil { + return nil, err + } + + proof, err = ipld.GetProof(ctx, bGetter, root, proof, index, total) + if err != nil { + return nil, err + } + return NewShareWithProof(index, node.RawData(), proof), nil +} + func ProtoToShare(protoShares []*pb.Share) []*ShareWithProof { shares := make([]*ShareWithProof, len(protoShares)) for i, share := range protoShares { From 7fe4fee1b1602c20dffdae69f4e15c820d0f6bc8 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 12 Jul 2023 11:32:54 -0500 Subject: [PATCH 082/388] chore: bump celestia app v1.0.0-rc9 (#2458) Co-authored-by: Rootul Patel Co-authored-by: Wondertan --- core/eds_test.go | 3 +- go.mod | 8 ++-- go.sum | 16 ++++---- header/header.go | 10 ++++- header/headertest/testing.go | 6 ++- nodebuilder/share/constructors.go | 7 +++- nodebuilder/share/share_test.go | 3 +- nodebuilder/tests/swamp/config.go | 6 +-- share/availability/light/availability_test.go | 3 +- share/availability/test/testing.go | 3 +- share/eds/byzantine/bad_encoding_test.go | 8 ++-- share/eds/byzantine/share_proof_test.go | 6 ++- share/eds/eds.go | 24 ++++++++++-- share/eds/eds_test.go | 36 +++++++++++++----- share/eds/ods_test.go | 14 ++++++- share/eds/retriever.go | 8 ++-- share/eds/retriever_test.go | 9 +++-- share/eds/store_test.go | 11 ++++-- share/empty.go | 5 ++- share/getters/getter_test.go | 6 ++- share/getters/shrex_test.go | 3 +- share/getters/testing.go | 9 ++++- share/ipld/add.go | 10 ++++- share/ipld/get_shares_test.go | 38 +++++++++++++------ share/ipld/nmt_test.go | 3 +- share/p2p/shrexeds/exchange_test.go | 11 ++++-- share/p2p/shrexnd/exchange_test.go | 3 +- 27 files changed, 188 insertions(+), 81 deletions(-) diff --git a/core/eds_test.go b/core/eds_test.go index 2cc6f1c7cc..6a2026ee58 100644 --- a/core/eds_test.go +++ b/core/eds_test.go @@ -47,6 +47,7 @@ func TestEmptySquareWithZeroTxs(t *testing.T) { eds, err = app.ExtendBlock(data, appconsts.LatestVersion) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) assert.Equal(t, share.EmptyRoot().Hash(), dah.Hash()) } diff --git a/go.mod b/go.mod index 70e8de6cc5..f7af093ece 100644 --- a/go.mod +++ b/go.mod @@ -10,12 +10,12 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc8 + github.com/celestiaorg/celestia-app v1.0.0-rc9 github.com/celestiaorg/go-fraud v0.1.0 github.com/celestiaorg/go-header v0.2.11 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.17.0 - github.com/celestiaorg/rsmt2d v0.9.0 + github.com/celestiaorg/rsmt2d v0.10.0 github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 @@ -330,10 +330,10 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 ) diff --git a/go.sum b/go.sum index ac183c1c6d..7d3cc73274 100644 --- a/go.sum +++ b/go.sum @@ -344,12 +344,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc8 h1:bBQY8tR3DCc8uA7c2ney4AeWwEn9Ob1lc1QryGKVF2M= -github.com/celestiaorg/celestia-app v1.0.0-rc8/go.mod h1:X0R6s+LvfusZu+jBj/2SbTm4Nb/H1R2MD1CnR4fwQno= -github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28 h1:G7/rq6xTnuFf3XsVZEcl/Sa6vtagm9NQNhaUaSgjvy0= -github.com/celestiaorg/celestia-core v1.23.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= -github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13 h1:vaQKgaOm0w58JAvOgn2iDohqjH7kvvRqVKiMcBDWifA= -github.com/celestiaorg/cosmos-sdk v1.15.0-sdk-v0.46.13/go.mod h1:G9XkhOJZde36FH0kt/1ayg4ZaioZEQmmRfMa/zQig0I= +github.com/celestiaorg/celestia-app v1.0.0-rc9 h1:6xDYE+OziXO/rLeYy/MutnJpE8M2sIPryZ/ifSWUmdc= +github.com/celestiaorg/celestia-app v1.0.0-rc9/go.mod h1:aGFnIIdA30DtFzznYbcfMdNnXiUebfEUkkrQu8imC3I= +github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHVxcTzhzQW44KgTcooR3OxnK4= +github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= +github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13 h1:N1PrCWcYkaODeIQyyVBmDKDTwiQWZ31bgtTEYIGeby8= +github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= @@ -366,8 +366,8 @@ github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSL github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= -github.com/celestiaorg/rsmt2d v0.9.0 h1:kon78I748ZqjNzI8OAqPN+2EImuZuanj/6gTh8brX3o= -github.com/celestiaorg/rsmt2d v0.9.0/go.mod h1:E06nDxfoeBDltWRvTR9dLviiUZI5/6mLXAuhSJzz3Iw= +github.com/celestiaorg/rsmt2d v0.10.0 h1:8dprr6CW5mCk5YPnbiLdirojw9YsJOE+XB+GORb8sT0= +github.com/celestiaorg/rsmt2d v0.10.0/go.mod h1:BiCZkCJfhDHUEOJKXUeu+CudjluecKvRTqHcuxKvodc= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= diff --git a/header/header.go b/header/header.go index e59d3802c1..0e386be8b5 100644 --- a/header/header.go +++ b/header/header.go @@ -75,12 +75,18 @@ func MakeExtendedHeader( vals *core.ValidatorSet, eds *rsmt2d.ExtendedDataSquare, ) (*ExtendedHeader, error) { - var dah DataAvailabilityHeader + var ( + dah DataAvailabilityHeader + err error + ) switch eds { case nil: dah = EmptyDAH() default: - dah = da.NewDataAvailabilityHeader(eds) + dah, err = da.NewDataAvailabilityHeader(eds) + } + if err != nil { + return nil, err } eh := &ExtendedHeader{ diff --git a/header/headertest/testing.go b/header/headertest/testing.go index ceeb9bb164..b20d389452 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -325,7 +325,8 @@ func FraudMaker(t *testing.T, faultHeight int64, bServ blockservice.BlockService func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedDataSquare) *header.ExtendedHeader { valSet, vals := RandValidatorSet(10, 10) gen := RandRawHeader(t) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) gen.DataHash = dah.Hash() gen.ValidatorsHash = valSet.Hash() @@ -355,7 +356,8 @@ func CreateFraudExtHeader( square := edstest.RandByzantineEDS(t, len(eh.DAH.RowRoots)) err := ipld.ImportEDS(context.Background(), square, serv) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(square) + dah, err := da.NewDataAvailabilityHeader(square) + require.NoError(t, err) eh.DAH = &dah eh.RawHeader.DataHash = dah.Hash() return eh, square diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 5cb0e41e53..a1b7e39713 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -53,9 +53,12 @@ func newModule(getter share.Getter, avail share.Availability) Module { // ensureEmptyCARExists adds an empty EDS to the provided EDS store. func ensureEmptyCARExists(ctx context.Context, store *eds.Store) error { emptyEDS := share.EmptyExtendedDataSquare() - emptyDAH := da.NewDataAvailabilityHeader(emptyEDS) + emptyDAH, err := da.NewDataAvailabilityHeader(emptyEDS) + if err != nil { + return err + } - err := store.Put(ctx, emptyDAH.Hash(), emptyEDS) + err = store.Put(ctx, emptyDAH.Hash(), emptyEDS) if errors.Is(err, dagstore.ErrShardExists) { return nil } diff --git a/nodebuilder/share/share_test.go b/nodebuilder/share/share_test.go index 388fd9af07..7c440a6dbf 100644 --- a/nodebuilder/share/share_test.go +++ b/nodebuilder/share/share_test.go @@ -27,7 +27,8 @@ func Test_EmptyCARExists(t *testing.T) { require.NoError(t, err) eds := share.EmptyExtendedDataSquare() - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) // add empty EDS to store err = ensureEmptyCARExists(ctx, edsStore) diff --git a/nodebuilder/tests/swamp/config.go b/nodebuilder/tests/swamp/config.go index 630920609f..e6c06b40cc 100644 --- a/nodebuilder/tests/swamp/config.go +++ b/nodebuilder/tests/swamp/config.go @@ -15,8 +15,8 @@ type Config struct { // 100ms func DefaultConfig() *Config { cfg := core.DefaultTestConfig() - // target height duration lower than this tend to be flakier - cfg.Tendermint.Consensus.TargetHeightDuration = 200 * time.Millisecond + // timeout commit lower than this tend to be flakier + cfg.Tendermint.Consensus.TimeoutCommit = 200 * time.Millisecond return &Config{ cfg, } @@ -31,7 +31,7 @@ func WithBlockTime(t time.Duration) Option { // for empty block c.Tendermint.Consensus.CreateEmptyBlocksInterval = t // for filled block - c.Tendermint.Consensus.TargetHeightDuration = t + c.Tendermint.Consensus.TimeoutCommit = t c.Tendermint.Consensus.SkipTimeoutCommit = false } } diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 1709c3f4b7..48813a33f9 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -132,7 +132,8 @@ func TestGetShares(t *testing.T) { eds, err := getter.GetEDS(ctx, dah) require.NoError(t, err) - gotDAH := da.NewDataAvailabilityHeader(eds) + gotDAH, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) require.True(t, dah.Equals(&gotDAH)) } diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 19f63f114a..27d6669061 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -34,7 +34,8 @@ func RandFillBS(t *testing.T, n int, bServ blockservice.BlockService) *share.Roo func FillBS(t *testing.T, bServ blockservice.BlockService, shares []share.Share) *share.Root { eds, err := ipld.AddShares(context.TODO(), shares, bServ) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) return &dah } diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 5f6adac595..49cf64c2c2 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -25,8 +25,9 @@ func TestBadEncodingFraudProof(t *testing.T) { bServ := mdutils.Bserv() square := edstest.RandByzantineEDS(t, 16) - dah := da.NewDataAvailabilityHeader(square) - err := ipld.ImportEDS(ctx, square, bServ) + dah, err := da.NewDataAvailabilityHeader(square) + require.NoError(t, err) + err = ipld.ImportEDS(ctx, square, bServ) require.NoError(t, err) var errRsmt2d *rsmt2d.ErrByzantineData @@ -55,7 +56,8 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { eds, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) // get an arbitrary row row := uint(squareSize / 2) diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go index 0f63d4f0c9..db1db64f80 100644 --- a/share/eds/byzantine/share_proof_test.go +++ b/share/eds/byzantine/share_proof_test.go @@ -27,7 +27,8 @@ func TestGetProof(t *testing.T) { in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(in) + dah, err := da.NewDataAvailabilityHeader(in) + require.NoError(t, err) var tests = []struct { roots [][]byte }{ @@ -63,7 +64,8 @@ func TestGetProofs(t *testing.T) { in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(in) + dah, err := da.NewDataAvailabilityHeader(in) + require.NoError(t, err) for _, root := range dah.ColumnRoots { rootCid := ipld.MustCidFromNamespacedSha256(root) data := make([][]byte, 0, in.Width()) diff --git a/share/eds/eds.go b/share/eds/eds.go index 544f3c2438..cc775491c9 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -102,7 +102,10 @@ func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io. return nil, fmt.Errorf("recomputing data square: %w", err) } // compute roots - eds.RowRoots() + _, err = eds.RowRoots() + if err != nil { + return nil, fmt.Errorf("computing row roots: %w", err) + } // commit the batch to DAG err = batchAdder.Commit() if err != nil { @@ -231,8 +234,18 @@ func prependNamespace(quadrant int, shr share.Share) []byte { // rootsToCids converts the EDS's Row and Column roots to CIDs. func rootsToCids(eds *rsmt2d.ExtendedDataSquare) ([]cid.Cid, error) { - var err error - roots := append(eds.RowRoots(), eds.ColRoots()...) + rowRoots, err := eds.RowRoots() + if err != nil { + return nil, err + } + colRoots, err := eds.ColRoots() + if err != nil { + return nil, err + } + + roots := make([][]byte, 0, len(rowRoots)+len(colRoots)) + roots = append(roots, rowRoots...) + roots = append(roots, colRoots...) rootCids := make([]cid.Cid, len(roots)) for i, r := range roots { rootCids[i], err = ipld.CidFromNamespacedSha256(r) @@ -283,7 +296,10 @@ func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d return nil, fmt.Errorf("share: computing eds: %w", err) } - newDah := da.NewDataAvailabilityHeader(eds) + newDah, err := da.NewDataAvailabilityHeader(eds) + if err != nil { + return nil, err + } if !bytes.Equal(newDah.Hash(), root) { return nil, fmt.Errorf( "share: content integrity mismatch: imported root %s doesn't match expected root %s", diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index 0ef211ec6f..af870f60b3 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -167,14 +167,25 @@ func TestInnerNodeBatchSize(t *testing.T) { func TestReadWriteRoundtrip(t *testing.T) { eds := writeRandomEDS(t) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) f := openWrittenEDS(t) defer f.Close() loaded, err := ReadEDS(context.Background(), f, dah.Hash()) require.NoError(t, err, "error reading EDS from file") - require.Equal(t, eds.RowRoots(), loaded.RowRoots()) - require.Equal(t, eds.ColRoots(), loaded.ColRoots()) + + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + loadedRowRoots, err := loaded.RowRoots() + require.NoError(t, err) + require.Equal(t, rowRoots, loadedRowRoots) + + colRoots, err := eds.ColRoots() + require.NoError(t, err) + loadedColRoots, err := loaded.ColRoots() + require.NoError(t, err) + require.Equal(t, colRoots, loadedColRoots) } func TestReadEDS(t *testing.T) { @@ -187,17 +198,22 @@ func TestReadEDS(t *testing.T) { loaded, err := ReadEDS(context.Background(), f, dah.Hash()) require.NoError(t, err, "error reading EDS from file") - require.Equal(t, dah.RowRoots, loaded.RowRoots()) - require.Equal(t, dah.ColumnRoots, loaded.ColRoots()) + rowRoots, err := loaded.RowRoots() + require.NoError(t, err) + require.Equal(t, dah.RowRoots, rowRoots) + colRoots, err := loaded.ColRoots() + require.NoError(t, err) + require.Equal(t, dah.ColumnRoots, colRoots) } func TestReadEDSContentIntegrityMismatch(t *testing.T) { writeRandomEDS(t) - dah := da.NewDataAvailabilityHeader(edstest.RandEDS(t, 4)) + dah, err := da.NewDataAvailabilityHeader(edstest.RandEDS(t, 4)) + require.NoError(t, err) f := openWrittenEDS(t) defer f.Close() - _, err := ReadEDS(context.Background(), f, dah.Hash()) + _, err = ReadEDS(context.Background(), f, dah.Hash()) require.ErrorContains(t, err, "share: content integrity mismatch: imported root") } @@ -209,7 +225,8 @@ func BenchmarkReadWriteEDS(b *testing.B) { b.Cleanup(cancel) for originalDataWidth := 4; originalDataWidth <= 64; originalDataWidth *= 2 { eds := edstest.RandEDS(b, originalDataWidth) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) b.Run(fmt.Sprintf("Writing %dx%d", originalDataWidth, originalDataWidth), func(b *testing.B) { b.ReportAllocs() for i := 0; i < b.N; i++ { @@ -280,7 +297,8 @@ func createTestData(t *testing.T, testDir string) { //nolint:unused err = WriteEDS(ctx, eds, f) require.NoError(t, err, "writing EDS to file") f.Close() - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) header, err := json.MarshalIndent(dah, "", "") require.NoError(t, err, "marshaling example root") diff --git a/share/eds/ods_test.go b/share/eds/ods_test.go index eb243a4022..5b6ed5568b 100644 --- a/share/eds/ods_test.go +++ b/share/eds/ods_test.go @@ -89,6 +89,16 @@ func TestODSReaderReconstruction(t *testing.T) { // reconstruct EDS from ODSReader loaded, err := ReadEDS(ctx, odsR, dah.Hash()) assert.NoError(t, err) - require.Equal(t, eds.RowRoots(), loaded.RowRoots()) - require.Equal(t, eds.ColRoots(), loaded.ColRoots()) + + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + loadedRowRoots, err := loaded.RowRoots() + require.NoError(t, err) + require.Equal(t, rowRoots, loadedRowRoots) + + colRoots, err := eds.ColRoots() + require.NoError(t, err) + loadedColRoots, err := loaded.ColRoots() + require.NoError(t, err) + require.Equal(t, colRoots, loadedColRoots) } diff --git a/share/eds/retriever.go b/share/eds/retriever.go index b3cf363056..2483a37a92 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -127,7 +127,7 @@ func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHead return &tree } - square, err := rsmt2d.ImportExtendedDataSquare(make([][]byte, size*size), share.DefaultRSMT2DCodec(), treeFn) + square, err := rsmt2d.NewExtendedDataSquare(share.DefaultRSMT2DCodec(), treeFn, uint(size), share.Size) if err != nil { return nil, err } @@ -283,10 +283,12 @@ func (rs *retrievalSession) doRequest(ctx context.Context, q *quadrant) { if rs.isReconstructed() { return } - if rs.square.GetCell(uint(x), uint(y)) != nil { + if err := rs.square.SetCell(uint(x), uint(y), share); err != nil { + // safe to ignore as: + // * share size already verified + // * the same share might come from either Row or Col return } - rs.square.SetCell(uint(x), uint(y), share) // if we have >= 1/4 of the square we can start trying to Reconstruct // TODO(@Wondertan): This is not an ideal way to know when to start // reconstruction and can cause idle reconstruction tries in some cases, diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index c90697862f..2277f894a1 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -59,7 +59,8 @@ func TestRetriever_Retrieve(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Minute*5) // the timeout is big for the max size which is long defer cancel() - dah := da.NewDataAvailabilityHeader(in) + dah, err := da.NewDataAvailabilityHeader(in) + require.NoError(t, err) out, err := r.Retrieve(ctx, &dah) require.NoError(t, err) assert.True(t, share.EqualEDS(in, out)) @@ -93,7 +94,8 @@ func TestRetriever_ByzantineError(t *testing.T) { require.NoError(t, err) // ensure we rcv an error - dah := da.NewDataAvailabilityHeader(attackerEDS) + dah, err := da.NewDataAvailabilityHeader(attackerEDS) + require.NoError(t, err) r := NewRetriever(bserv) _, err = r.Retrieve(ctx, &dah) var errByz *byzantine.ErrByzantine @@ -116,7 +118,8 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { in, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) - dah := da.NewDataAvailabilityHeader(in) + dah, err := da.NewDataAvailabilityHeader(in) + require.NoError(t, err) ses, err := r.newSession(ctx, &dah) require.NoError(t, err) diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 7f2f548f63..491b63c48a 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -271,7 +271,8 @@ func BenchmarkStore(b *testing.B) { // pause the timer for initializing test data b.StopTimer() eds := edstest.RandEDS(b, 128) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) b.StartTimer() err = edsStore.Put(ctx, dah.Hash(), eds) @@ -286,11 +287,12 @@ func BenchmarkStore(b *testing.B) { // pause the timer for initializing test data b.StopTimer() eds := edstest.RandEDS(b, 128) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) _ = edsStore.Put(ctx, dah.Hash(), eds) b.StartTimer() - _, err := edsStore.Get(ctx, dah.Hash()) + _, err = edsStore.Get(ctx, dah.Hash()) require.NoError(b, err) } }) @@ -306,7 +308,8 @@ func newStore(t *testing.T) (*Store, error) { func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) return eds, dah } diff --git a/share/empty.go b/share/empty.go index b0e1e6e6ae..07d48f2f07 100644 --- a/share/empty.go +++ b/share/empty.go @@ -54,7 +54,10 @@ func initEmpty() { } emptyBlockEDS = eds - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + if err != nil { + panic(fmt.Errorf("failed to create empty DAH: %w", err)) + } minDAH := da.MinDataAvailabilityHeader() if !bytes.Equal(minDAH.Hash(), dah.Hash()) { panic(fmt.Sprintf("mismatch in calculated minimum DAH and minimum DAH from celestia-app, "+ diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 7af8af2f26..571129f029 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -232,7 +232,8 @@ func TestIPLDGetter(t *testing.T) { func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) return eds, dah } @@ -260,7 +261,8 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData wrapper.NewConstructor(uint64(size)), ) require.NoError(t, err, "failure to recompute the extended data square") - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) return eds, share.GetNamespace(randShares[idx1]), dah } diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index e1628d8725..0dbe7e44db 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -198,7 +198,8 @@ func newStore(t *testing.T) (*eds.Store, error) { func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, share.Namespace) { eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], share.NamespaceSize) return eds, dah, max } diff --git a/share/getters/testing.go b/share/getters/testing.go index 4734557d7f..3a15bb5bf2 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -7,6 +7,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" + "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/edstest" @@ -15,7 +16,8 @@ import ( // TestGetter provides a testing SingleEDSGetter and the root of the EDS it holds. func TestGetter(t *testing.T) (share.Getter, *share.Root) { eds := edstest.RandEDS(t, 8) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) return &SingleEDSGetter{ EDS: eds, }, &dah @@ -52,7 +54,10 @@ func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *share.Root, s } func (seg *SingleEDSGetter) checkRoot(root *share.Root) error { - dah := da.NewDataAvailabilityHeader(seg.EDS) + dah, err := da.NewDataAvailabilityHeader(seg.EDS) + if err != nil { + return err + } if !root.Equals(&dah) { return fmt.Errorf("unknown EDS: have %s, asked %s", dah.String(), root.String()) } diff --git a/share/ipld/add.go b/share/ipld/add.go index fddfe8e99e..5807a320fd 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -39,7 +39,10 @@ func AddShares( return nil, fmt.Errorf("failure to recompute the extended data square: %w", err) } // compute roots - eds.RowRoots() + _, err = eds.RowRoots() + if err != nil { + return nil, err + } // commit the batch to ipfs return eds, batchAdder.Commit() } @@ -67,7 +70,10 @@ func ImportShares( return nil, fmt.Errorf("failure to recompute the extended data square: %w", err) } // compute roots - eds.RowRoots() + _, err = eds.RowRoots() + if err != nil { + return nil, err + } // commit the batch to DAG return eds, batchAdder.Commit() } diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index cd26f759b3..7ee7704fc0 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -45,7 +45,9 @@ func TestGetShare(t *testing.T) { for i, leaf := range shares { row := i / size pos := i - (size * row) - share, err := GetShare(ctx, bServ, MustCidFromNamespacedSha256(eds.RowRoots()[row]), pos, size*2) + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + share, err := GetShare(ctx, bServ, MustCidFromNamespacedSha256(rowRoots[row]), pos, size*2) require.NoError(t, err) assert.Equal(t, leaf, share) } @@ -87,8 +89,10 @@ func TestBlockRecovery(t *testing.T) { require.NoError(t, err) // calculate roots using the first complete square - rowRoots := testEds.RowRoots() - colRoots := testEds.ColRoots() + rowRoots, err := testEds.RowRoots() + require.NoError(t, err) + colRoots, err := testEds.ColRoots() + require.NoError(t, err) flat := share.ExtractEDS(testEds) @@ -173,9 +177,11 @@ func TestGetSharesByNamespace(t *testing.T) { require.NoError(t, err) var shares []share.Share - for _, row := range eds.RowRoots() { + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + for _, row := range rowRoots { rcid := MustCidFromNamespacedSha256(row) - rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(eds.RowRoots())) + rowShares, _, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(rowRoots)) if errors.Is(err, ErrNamespaceOutsideRange) { continue } @@ -208,7 +214,8 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) - roots := eds.RowRoots() + roots, err := eds.RowRoots() + require.NoError(t, err) // remove the second share from the first row rcid := MustCidFromNamespacedSha256(roots[0]) @@ -302,7 +309,10 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi eds, err := AddShares(ctx, shares, bServ) require.NoError(t, err) - for _, row := range eds.RowRoots() { + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + + for _, row := range rowRoots { rcid := MustCidFromNamespacedSha256(row) data := NewNamespaceData(len(shares), namespace, WithLeaves()) err := data.CollectLeavesByNamespace(ctx, bServ, rcid) @@ -356,9 +366,11 @@ func TestGetSharesWithProofsByNamespace(t *testing.T) { require.NoError(t, err) var shares []share.Share - for _, row := range eds.RowRoots() { + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + for _, row := range rowRoots { rcid := MustCidFromNamespacedSha256(row) - rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(eds.RowRoots())) + rowShares, proof, err := GetSharesByNamespace(ctx, bServ, rcid, namespace, len(rowRoots)) if namespace.IsOutsideRange(row, row) { require.ErrorIs(t, err, ErrNamespaceOutsideRange) continue @@ -446,16 +458,18 @@ func assertNoRowContainsNID( namespace share.Namespace, isAbsent bool, ) { - rowRootCount := len(eds.RowRoots()) + rowRoots, err := eds.RowRoots() + require.NoError(t, err) + rowRootCount := len(rowRoots) // get all row root cids rowRootCIDs := make([]cid.Cid, rowRootCount) - for i, rowRoot := range eds.RowRoots() { + for i, rowRoot := range rowRoots { rowRootCIDs[i] = MustCidFromNamespacedSha256(rowRoot) } // for each row root cid check if the min namespace exists var absentCount, foundAbsenceRows int - for _, rowRoot := range eds.RowRoots() { + for _, rowRoot := range rowRoots { var outsideRange bool if !namespace.IsOutsideRange(rowRoot, rowRoot) { // namespace does belong to namespace range of the row diff --git a/share/ipld/nmt_test.go b/share/ipld/nmt_test.go index aa125ab3c7..77268d7112 100644 --- a/share/ipld/nmt_test.go +++ b/share/ipld/nmt_test.go @@ -26,7 +26,8 @@ func TestNamespaceFromCID(t *testing.T) { for i, tt := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { - dah := da.NewDataAvailabilityHeader(tt.eds) + dah, err := da.NewDataAvailabilityHeader(tt.eds) + require.NoError(t, err) // check to make sure NamespacedHash is correctly derived from CID for _, row := range dah.RowRoots { c, err := CidFromNamespacedSha256(row) diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index 21fe9f77a1..b1d2e0ad18 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -35,7 +35,8 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is immediately available t.Run("EDS_Available", func(t *testing.T) { eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) err = store.Put(ctx, dah.Hash(), eds) require.NoError(t, err) @@ -48,7 +49,8 @@ func TestExchange_RequestEDS(t *testing.T) { t.Run("EDS_AvailableAfterDelay", func(t *testing.T) { storageDelay := time.Second eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) go func() { time.Sleep(storageDelay) err = store.Put(ctx, dah.Hash(), eds) @@ -77,8 +79,9 @@ func TestExchange_RequestEDS(t *testing.T) { timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) - _, err := client.RequestEDS(timeoutCtx, dah.Hash(), server.host.ID()) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + _, err = client.RequestEDS(timeoutCtx, dah.Hash(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) }) diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index e8d3e439c0..8d1dad17c0 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -45,7 +45,8 @@ func TestExchange_RequestND_NotFound(t *testing.T) { t.Cleanup(cancel) eds := edstest.RandEDS(t, 4) - dah := da.NewDataAvailabilityHeader(eds) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) randNamespace := dah.RowRoots[(len(dah.RowRoots)-1)/2][:share.NamespaceSize] From 889f56953afa23e21f02c52cf4d6f4c9a8dd92a1 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 12 Jul 2023 21:04:44 +0300 Subject: [PATCH 083/388] fix(lint): fix importing (#2467) --- share/getters/testing.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/share/getters/testing.go b/share/getters/testing.go index 3a15bb5bf2..71c6231f3c 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -5,9 +5,10 @@ import ( "fmt" "testing" + "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" - "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/edstest" From e41f62662cf1844ba52d7a93d03cf79b6e363897 Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Wed, 12 Jul 2023 13:31:12 -0700 Subject: [PATCH 084/388] cmd: rpc.go fix error parsing height message (#2469) Fixing the error message returned by the rpc cli client, when an invalid height is passed to the `Get`, `GetAll` commands of the Blob module. --- cmd/celestia/rpc.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 169ca4b8d1..64b21aaf86 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -195,7 +195,7 @@ func parseParams(method string, params []string) []interface{} { // 1. Height num, err := strconv.ParseUint(params[0], 10, 64) if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") + panic("Error parsing height: uint64 could not be parsed.") } parsedParams[0] = num // 2. NamespaceID @@ -215,7 +215,7 @@ func parseParams(method string, params []string) []interface{} { // 1. Height num, err := strconv.ParseUint(params[0], 10, 64) if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") + panic("Error parsing height: uint64 could not be parsed.") } parsedParams[0] = num // 2. Namespace From 85cec8e82a53b1154034e162f16cac8441406c5c Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 13 Jul 2023 14:26:43 +0200 Subject: [PATCH 085/388] fix(nodebuilder): fix nil pointer exception and simplify synchronization (#2466) We hit an infamous use case here where nil is not nil. Also, I simplified locking here while thinking that's a race condition. --- nodebuilder/store.go | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/nodebuilder/store.go b/nodebuilder/store.go index 046dd41bbd..f0957a16b6 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -104,24 +104,18 @@ func (f *fsStore) PutConfig(cfg *Config) error { } func (f *fsStore) Keystore() (_ keystore.Keystore, err error) { - f.lock.RLock() - defer f.lock.RUnlock() if f.keys == nil { return nil, fmt.Errorf("node: no Keystore found") } return f.keys, nil } -func (f *fsStore) Datastore() (_ datastore.Batching, err error) { - f.lock.RLock() +func (f *fsStore) Datastore() (datastore.Batching, error) { + f.dataMu.Lock() + defer f.dataMu.Unlock() if f.data != nil { - f.lock.RUnlock() return f.data, nil } - f.lock.RUnlock() - - f.lock.Lock() - defer f.lock.Unlock() opts := dsbadger.DefaultOptions // this should be copied @@ -144,29 +138,31 @@ func (f *fsStore) Datastore() (_ datastore.Batching, err error) { // TODO(@Wondertan): Make configurable with more conservative defaults for Light Node opts.MaxTableSize = 64 << 20 - f.data, err = dsbadger.NewDatastore(dataPath(f.path), &opts) + ds, err := dsbadger.NewDatastore(dataPath(f.path), &opts) if err != nil { return nil, fmt.Errorf("node: can't open Badger Datastore: %w", err) } - return f.data, nil + f.data = ds + return ds, nil } func (f *fsStore) Close() (err error) { err = errors.Join(err, f.dirLock.Unlock()) + f.dataMu.Lock() if f.data != nil { err = errors.Join(err, f.data.Close()) } + f.dataMu.Unlock() return } type fsStore struct { path string - data datastore.Batching - keys keystore.Keystore - - lock sync.RWMutex // protects all the fields + dataMu sync.Mutex + data datastore.Batching + keys keystore.Keystore dirLock *fslock.Locker // protects directory } From eea06684bf5b012489c2400c305aa64793eb51b0 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 13 Jul 2023 10:04:18 -0400 Subject: [PATCH 086/388] docs: clarify usage of `len(merkleRowRoots)` (#2431) Closes https://github.com/celestiaorg/celestia-node/issues/2392 Note: this doesn't introduce a new variable name (the proposed solution in the issue). Happy to do so if reviewers request. --- share/eds/byzantine/bad_encoding.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index ae77026acf..9d6cbff229 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -143,6 +143,10 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { ) } if len(p.Shares) != len(merkleRoots) { + // Since p.Shares should contain all the shares from either a row or a + // column, it should exactly match the number of row roots. In this + // context, the number of row roots is the width of the extended data + // square. return fmt.Errorf("invalid %s proof: incorrect number of shares %d != %d", BadEncoding, len(p.Shares), len(merkleRoots), ) From 360ba6b447504b9c2f89931463b994e8b0b5b8c3 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 13 Jul 2023 16:50:48 +0200 Subject: [PATCH 087/388] chore(share/p2p/discovery): demote disc peers log to debug (#2473) self explaanory --- share/p2p/discovery/discovery.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index 96eda8bd78..9779d72195 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -251,7 +251,10 @@ func (d *Discovery) discover(ctx context.Context) bool { log.Debugw("reached soft peer limit, skipping discovery", "size", size) return true } - log.Infow("discovering peers", "want", want) + // TODO @renaynay: eventually, have a mechanism to catch if wanted amount of peers + // has not been discovered in X amount of time so that users are warned of degraded + // FN connectivity. + log.Debugw("discovering peers", "want", want) // we use errgroup as it provide limits var wg errgroup.Group From 11dced407aad141b11acab623d987620529bba09 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 13 Jul 2023 23:02:29 +0800 Subject: [PATCH 088/388] chore(deps): bump otel, go-header, and go-fraud (#2472) --- cmd/flags_misc.go | 8 +- das/metrics.go | 135 ++++++++++++++++----------------- go.mod | 34 +++++---- go.sum | 67 ++++++++-------- nodebuilder/node/metrics.go | 53 ++++++------- nodebuilder/settings.go | 12 +-- share/getters/shrex.go | 30 ++++---- share/p2p/discovery/metrics.go | 73 +++++++++--------- share/p2p/metrics.go | 25 +++--- share/p2p/peers/metrics.go | 127 ++++++++++++++++--------------- state/metrics.go | 30 ++++---- 11 files changed, 296 insertions(+), 298 deletions(-) diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index 4483e17201..26f99985b8 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -13,6 +13,7 @@ import ( flag "github.com/spf13/pflag" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" "go.opentelemetry.io/otel/sdk/resource" tracesdk "go.opentelemetry.io/otel/sdk/trace" @@ -210,15 +211,16 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e opts = append(opts, otlptracehttp.WithInsecure()) } - exp, err := otlptracehttp.New(cmd.Context(), opts...) + client := otlptracehttp.NewClient(opts...) + exporter, err := otlptrace.New(ctx, client) if err != nil { - return ctx, err + return ctx, fmt.Errorf("creating OTLP trace exporter: %w", err) } tp = tracesdk.NewTracerProvider( tracesdk.WithSampler(tracesdk.AlwaysSample()), // Always be sure to batch in production. - tracesdk.WithBatcher(exp), + tracesdk.WithBatcher(exporter), // Record information about this application in a Resource. tracesdk.WithResource(resource.NewWithAttributes( semconv.SchemaURL, diff --git a/das/metrics.go b/das/metrics.go index 1dcf5c8165..42b472d909 100644 --- a/das/metrics.go +++ b/das/metrics.go @@ -6,11 +6,9 @@ import ( "sync/atomic" "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/instrument/syncfloat64" - "go.opentelemetry.io/otel/metric/instrument/syncint64" + "go.opentelemetry.io/otel/metric" "github.com/celestiaorg/celestia-node/header" ) @@ -22,73 +20,70 @@ const ( ) var ( - meter = global.MeterProvider().Meter("das") + meter = otel.Meter("das") ) type metrics struct { - sampled syncint64.Counter - sampleTime syncfloat64.Histogram - getHeaderTime syncfloat64.Histogram - newHead syncint64.Counter + sampled metric.Int64Counter + sampleTime metric.Float64Histogram + getHeaderTime metric.Float64Histogram + newHead metric.Int64Counter lastSampledTS uint64 } func (d *DASer) InitMetrics() error { - sampled, err := meter.SyncInt64().Counter("das_sampled_headers_counter", - instrument.WithDescription("sampled headers counter")) + sampled, err := meter.Int64Counter("das_sampled_headers_counter", + metric.WithDescription("sampled headers counter")) if err != nil { return err } - sampleTime, err := meter.SyncFloat64().Histogram("das_sample_time_hist", - instrument.WithDescription("duration of sampling a single header")) + sampleTime, err := meter.Float64Histogram("das_sample_time_hist", + metric.WithDescription("duration of sampling a single header")) if err != nil { return err } - getHeaderTime, err := meter.SyncFloat64().Histogram("das_get_header_time_hist", - instrument.WithDescription("duration of getting header from header store")) + getHeaderTime, err := meter.Float64Histogram("das_get_header_time_hist", + metric.WithDescription("duration of getting header from header store")) if err != nil { return err } - newHead, err := meter.SyncInt64().Counter("das_head_updated_counter", - instrument.WithDescription("amount of times DAS'er advanced network head")) + newHead, err := meter.Int64Counter("das_head_updated_counter", + metric.WithDescription("amount of times DAS'er advanced network head")) if err != nil { return err } - lastSampledTS, err := meter.AsyncInt64().Gauge("das_latest_sampled_ts", - instrument.WithDescription("latest sampled timestamp")) + lastSampledTS, err := meter.Int64ObservableGauge("das_latest_sampled_ts", + metric.WithDescription("latest sampled timestamp")) if err != nil { return err } - busyWorkers, err := meter.AsyncInt64().Gauge("das_busy_workers_amount", - instrument.WithDescription("number of active parallel workers in DAS'er")) + busyWorkers, err := meter.Int64ObservableGauge("das_busy_workers_amount", + metric.WithDescription("number of active parallel workers in DAS'er")) if err != nil { return err } - networkHead, err := meter.AsyncInt64().Gauge("das_network_head", - instrument.WithDescription("most recent network head")) + networkHead, err := meter.Int64ObservableGauge("das_network_head", + metric.WithDescription("most recent network head")) if err != nil { return err } - sampledChainHead, err := meter.AsyncInt64().Gauge("das_sampled_chain_head", - instrument.WithDescription("height of the sampled chain - all previous headers have been successfully sampled")) + sampledChainHead, err := meter.Int64ObservableGauge("das_sampled_chain_head", + metric.WithDescription("height of the sampled chain - all previous headers have been successfully sampled")) if err != nil { return err } - totalSampled, err := meter. - AsyncInt64(). - Gauge( - "das_total_sampled_headers", - instrument.WithDescription("total sampled headers gauge"), - ) + totalSampled, err := meter.Int64ObservableGauge("das_total_sampled_headers", + metric.WithDescription("total sampled headers gauge"), + ) if err != nil { return err } @@ -100,36 +95,38 @@ func (d *DASer) InitMetrics() error { newHead: newHead, } - err = meter.RegisterCallback( - []instrument.Asynchronous{ - lastSampledTS, - busyWorkers, - networkHead, - sampledChainHead, - totalSampled, - }, - func(ctx context.Context) { - stats, err := d.sampler.stats(ctx) - if err != nil { - log.Errorf("observing stats: %s", err.Error()) - } - - for jobType, amount := range stats.workersByJobType() { - busyWorkers.Observe(ctx, amount, - attribute.String(jobTypeLabel, string(jobType))) - } - - networkHead.Observe(ctx, int64(stats.NetworkHead)) - sampledChainHead.Observe(ctx, int64(stats.SampledChainHead)) - - if ts := atomic.LoadUint64(&d.sampler.metrics.lastSampledTS); ts != 0 { - lastSampledTS.Observe(ctx, int64(ts)) - } - - totalSampled.Observe(ctx, int64(stats.totalSampled())) - }, - ) + callback := func(ctx context.Context, observer metric.Observer) error { + stats, err := d.sampler.stats(ctx) + if err != nil { + log.Errorf("observing stats: %s", err.Error()) + return err + } + + for jobType, amount := range stats.workersByJobType() { + observer.ObserveInt64(busyWorkers, amount, + metric.WithAttributes( + attribute.String(jobTypeLabel, string(jobType)), + )) + } + + observer.ObserveInt64(networkHead, int64(stats.NetworkHead)) + observer.ObserveInt64(sampledChainHead, int64(stats.SampledChainHead)) + + if ts := atomic.LoadUint64(&d.sampler.metrics.lastSampledTS); ts != 0 { + observer.ObserveInt64(lastSampledTS, int64(ts)) + } + observer.ObserveInt64(totalSampled, int64(stats.totalSampled())) + return nil + } + + _, err = meter.RegisterCallback(callback, + lastSampledTS, + busyWorkers, + networkHead, + sampledChainHead, + totalSampled, + ) if err != nil { return fmt.Errorf("registering metrics callback: %w", err) } @@ -153,16 +150,18 @@ func (m *metrics) observeSample( ctx = context.Background() } m.sampleTime.Record(ctx, sampleTime.Seconds(), - attribute.Bool(failedLabel, err != nil), - attribute.Int(headerWidthLabel, len(h.DAH.RowRoots)), - attribute.String(jobTypeLabel, string(jobType)), - ) + metric.WithAttributes( + attribute.Bool(failedLabel, err != nil), + attribute.Int(headerWidthLabel, len(h.DAH.RowRoots)), + attribute.String(jobTypeLabel, string(jobType)), + )) m.sampled.Add(ctx, 1, - attribute.Bool(failedLabel, err != nil), - attribute.Int(headerWidthLabel, len(h.DAH.RowRoots)), - attribute.String(jobTypeLabel, string(jobType)), - ) + metric.WithAttributes( + attribute.Bool(failedLabel, err != nil), + attribute.Int(headerWidthLabel, len(h.DAH.RowRoots)), + attribute.String(jobTypeLabel, string(jobType)), + )) atomic.StoreUint64(&m.lastSampledTS, uint64(time.Now().UTC().Unix())) } diff --git a/go.mod b/go.mod index f7af093ece..eae1f5842b 100644 --- a/go.mod +++ b/go.mod @@ -11,8 +11,8 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc9 - github.com/celestiaorg/go-fraud v0.1.0 - github.com/celestiaorg/go-header v0.2.11 + github.com/celestiaorg/go-fraud v0.1.2 + github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.17.0 github.com/celestiaorg/rsmt2d v0.10.0 @@ -63,13 +63,14 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 - go.opentelemetry.io/otel v1.13.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.34.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 - go.opentelemetry.io/otel/metric v0.34.0 - go.opentelemetry.io/otel/sdk v1.11.2 - go.opentelemetry.io/otel/sdk/metric v0.34.0 - go.opentelemetry.io/otel/trace v1.13.0 + go.opentelemetry.io/otel v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 + go.opentelemetry.io/otel/metric v1.16.0 + go.opentelemetry.io/otel/sdk v1.16.0 + go.opentelemetry.io/otel/sdk/metric v0.39.0 + go.opentelemetry.io/otel/trace v1.16.0 go.opentelemetry.io/proto/otlp v0.19.0 go.uber.org/fx v1.19.3 go.uber.org/zap v1.24.0 @@ -103,7 +104,7 @@ require ( github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect github.com/celestiaorg/quantum-gravity-bridge v1.3.0 // indirect - github.com/cenkalti/backoff/v4 v4.2.0 // indirect + github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v1.5.1 // indirect @@ -174,7 +175,7 @@ require ( github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect - github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect + github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c // indirect github.com/gtank/merlin v0.1.1 // indirect github.com/gtank/ristretto255 v0.1.2 // indirect @@ -304,22 +305,23 @@ require ( github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.34.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect golang.org/x/net v0.10.0 // indirect - golang.org/x/oauth2 v0.7.0 // indirect + golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.8.0 // indirect golang.org/x/term v0.8.0 // indirect golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect + google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 7d3cc73274..7ef5215bfd 100644 --- a/go.sum +++ b/go.sum @@ -352,10 +352,10 @@ github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13 h1:N1PrCWcYkaODeIQyyVBmDK github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= -github.com/celestiaorg/go-fraud v0.1.0 h1:v6mZvlmf2J5ELZfPnrtmmOvKbaYIUs/erDWPO8NbZyY= -github.com/celestiaorg/go-fraud v0.1.0/go.mod h1:yoNM35cKMAkt5Mi/Qx3Wi9bnPilLi8n6RpHZVglTUDs= -github.com/celestiaorg/go-header v0.2.11 h1:dLpuUfpGNxFfJNw3Ar3SqWc+AeyT1DlTP5mLjx9Ths8= -github.com/celestiaorg/go-header v0.2.11/go.mod h1:i9OpY70+PJ1xPw1IgMfF0Pk6vBD6VWPmjY3bgubJBcU= +github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= +github.com/celestiaorg/go-fraud v0.1.2/go.mod h1:kHZXQY+6gd1kYkoWRFFKgWyrLPWRgDN3vd1Ll9gE/oo= +github.com/celestiaorg/go-header v0.2.12 h1:3H9nir20+MTY1vXbLxOUOV05ZspotR6JOiZGKxACHCQ= +github.com/celestiaorg/go-header v0.2.12/go.mod h1:NhiWq97NtAYyRBu8quzYOUghQULjgOzO2Ql0iVEFOf0= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= @@ -371,8 +371,8 @@ github.com/celestiaorg/rsmt2d v0.10.0/go.mod h1:BiCZkCJfhDHUEOJKXUeu+CudjluecKvR github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= -github.com/cenkalti/backoff/v4 v4.2.0 h1:HN5dHm3WBOgndBH6E8V0q2jIYIR3s9yglV8k/+MN3u4= -github.com/cenkalti/backoff/v4 v4.2.0/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/cespare/cp v0.1.0 h1:SE+dxFebS7Iik5LK0tsi1k9ZCxEaFX4AjQmoyA+1dJk= github.com/cespare/cp v0.1.0/go.mod h1:SOGHArjBr4JWaSDEVpWpo/hNg6RoKrls6Oh40hiwW+s= @@ -854,8 +854,9 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0= github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= +github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c/go.mod h1:NMPJylDgVpX0MLRlPy15sqSwOFv/U1GZ2m21JhFfek0= github.com/gtank/merlin v0.1.1-0.20191105220539-8318aed1a79f/go.mod h1:T86dnYJhcGOh5BjZFCJWTDeTK7XW8uE+E21Cy/bIQ+s= @@ -2054,31 +2055,31 @@ go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= -go.opentelemetry.io/otel v1.13.0 h1:1ZAKnNQKwBBxFtww/GwxNUyTf0AxkZzrukO8MeXqe4Y= -go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2 h1:htgM8vZIF8oPSCxa341e3IZ4yr/sKxgu8KZYllByiVY= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.11.2/go.mod h1:rqbht/LlhVBgn5+k3M5QK96K5Xb0DvXpMJ5SFQpY6uw= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.34.0 h1:kpskzLZ60cJ48SJ4uxWa6waBL+4kSV6nVK8rP+QM8Wg= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.34.0/go.mod h1:4+x3i62TEegDHuzNva0bMcAN8oUi5w4liGb1d/VgPYo= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.34.0 h1:t4Ajxj8JGjxkqoBtbkCOY2cDUl9RwiNE9LPQavooi9U= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.34.0/go.mod h1:WO7omosl4P7JoanH9NgInxDxEn2F2M5YinIh8EyeT8w= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2 h1:fqR1kli93643au1RKo0Uma3d2aPQKT+WBKfTSBaKbOc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.11.2/go.mod h1:5Qn6qvgkMsLDX+sYK64rHb1FPhpn0UtxF+ouX1uhyJE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2 h1:Us8tbCmuN16zAnK5TC69AtODLycKbwnskQzaB6DfFhc= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.11.2/go.mod h1:GZWSQQky8AgdJj50r1KJm8oiQiIPaAX7uZCFQX9GzC8= +go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= +go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0 h1:IZXpCEtI7BbX01DRQEWTGDkvjMB6hEhiEZXS+eg2YqY= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0/go.mod h1:xY111jIZtWb+pUUgT4UiiSonAaY2cD2Ts5zvuKLki3o= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v0.34.0 h1:MCPoQxcg/26EuuJwpYN1mZTeCYAUGx8ABxfW07YkjP8= -go.opentelemetry.io/otel/metric v0.34.0/go.mod h1:ZFuI4yQGNCupurTXCwkeD/zHBt+C2bR7bw5JqUm/AP8= +go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= +go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.11.2 h1:GF4JoaEx7iihdMFu30sOyRx52HDHOkl9xQ8SMqNXUiU= -go.opentelemetry.io/otel/sdk v1.11.2/go.mod h1:wZ1WxImwpq+lVRo4vsmSOxdd+xwoUJ6rqyLc3SyX9aU= -go.opentelemetry.io/otel/sdk/metric v0.34.0 h1:7ElxfQpXCFZlRTvVRTkcUvK8Gt5DC8QzmzsLsO2gdzo= -go.opentelemetry.io/otel/sdk/metric v0.34.0/go.mod h1:l4r16BIqiqPy5rd14kkxllPy/fOI4tWo1jkpD9Z3ffQ= +go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= +go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= +go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= +go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= -go.opentelemetry.io/otel/trace v1.13.0 h1:CBgRZ6ntv+Amuj1jDsMhZtlAPT6gbyIRdaIzFhfBSdY= -go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= +go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= +go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= @@ -2321,8 +2322,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.7.0 h1:qe6s0zUXlPX80/dITx3440hWZ7GwMwgDDyrSGTPJG/g= -golang.org/x/oauth2 v0.7.0/go.mod h1:hPLQkd9LyjfXTiRohC/41GhcFqxisoUQ99sCUOHO9x4= +golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= +golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -2762,8 +2763,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 h1:KpwkzHKEF7B9Zxg18WzOa7djJ+Ha5DzthMyZYQfEn2A= -google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1/go.mod h1:nKE/iIaLqn2bQwXBg8f1g2Ylh6r5MN5CmZvuzZCgsCU= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= +google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= +google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= diff --git a/nodebuilder/node/metrics.go b/nodebuilder/node/metrics.go index 625e8425e8..7d722524e8 100644 --- a/nodebuilder/node/metrics.go +++ b/nodebuilder/node/metrics.go @@ -4,11 +4,11 @@ import ( "context" "time" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" ) -var meter = global.MeterProvider().Meter("node") +var meter = otel.Meter("node") var ( timeStarted time.Time @@ -17,37 +17,34 @@ var ( // WithMetrics registers node metrics. func WithMetrics() error { - nodeStartTS, err := meter. - AsyncFloat64(). - Gauge( - "node_start_ts", - instrument.WithDescription("timestamp when the node was started"), - ) + nodeStartTS, err := meter.Int64ObservableGauge( + "node_start_ts", + metric.WithDescription("timestamp when the node was started"), + ) if err != nil { return err } - totalNodeRunTime, err := meter. - AsyncFloat64(). - Counter( - "node_runtime_counter_in_seconds", - instrument.WithDescription("total time the node has been running"), - ) + totalNodeRunTime, err := meter.Float64ObservableCounter( + "node_runtime_counter_in_seconds", + metric.WithDescription("total time the node has been running"), + ) if err != nil { return err } - return meter.RegisterCallback( - []instrument.Asynchronous{nodeStartTS, totalNodeRunTime}, - func(ctx context.Context) { - if !nodeStarted { - // Observe node start timestamp - timeStarted = time.Now() - nodeStartTS.Observe(ctx, float64(timeStarted.Unix())) - nodeStarted = true - } - - totalNodeRunTime.Observe(ctx, time.Since(timeStarted).Seconds()) - }, - ) + callback := func(ctx context.Context, observer metric.Observer) error { + if !nodeStarted { + // Observe node start timestamp + timeStarted = time.Now() + observer.ObserveInt64(nodeStartTS, timeStarted.Unix()) + nodeStarted = true + } + + observer.ObserveFloat64(totalNodeRunTime, time.Since(timeStarted).Seconds()) + return nil + } + + _, err = meter.RegisterCallback(callback, nodeStartTS, totalNodeRunTime) + return err } diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index bea3c78ad2..66b02c34e4 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -7,9 +7,9 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/pyroscope-io/client/pyroscope" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/sdk/metric" + sdk "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" semconv "go.opentelemetry.io/otel/semconv/v1.11.0" "go.uber.org/fx" @@ -119,9 +119,9 @@ func initializeMetrics( return err } - provider := metric.NewMeterProvider( - metric.WithReader(metric.NewPeriodicReader(exp, metric.WithTimeout(2*time.Second))), - metric.WithResource(resource.NewWithAttributes( + provider := sdk.NewMeterProvider( + sdk.WithReader(sdk.NewPeriodicReader(exp, sdk.WithTimeout(2*time.Second))), + sdk.WithResource(resource.NewWithAttributes( semconv.SchemaURL, semconv.ServiceNamespaceKey.String(fmt.Sprintf("Celestia-%s", nodeType.String())), semconv.ServiceNameKey.String(fmt.Sprintf("semver-%s", buildInfo.SemanticVersion)), @@ -132,6 +132,6 @@ func initializeMetrics( return provider.Shutdown(ctx) }, }) - global.SetMeterProvider(provider) + otel.SetMeterProvider(provider) return nil } diff --git a/share/getters/shrex.go b/share/getters/shrex.go index bb5e7ca7a7..c754d73c1d 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -6,11 +6,9 @@ import ( "fmt" "time" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/instrument/syncint64" - "go.opentelemetry.io/otel/metric/unit" + "go.opentelemetry.io/otel/metric" "go.opentelemetry.io/otel/trace" "github.com/celestiaorg/rsmt2d" @@ -32,11 +30,11 @@ const ( defaultMinAttemptsCount = 3 ) -var meter = global.MeterProvider().Meter("shrex/getter") +var meter = otel.Meter("shrex/getter") type metrics struct { - edsAttempts syncint64.Histogram - ndAttempts syncint64.Histogram + edsAttempts metric.Int64Histogram + ndAttempts metric.Int64Histogram } func (m *metrics) recordEDSAttempt(ctx context.Context, attemptCount int, success bool) { @@ -46,7 +44,9 @@ func (m *metrics) recordEDSAttempt(ctx context.Context, attemptCount int, succes if ctx.Err() != nil { ctx = context.Background() } - m.edsAttempts.Record(ctx, int64(attemptCount), attribute.Bool("success", success)) + m.edsAttempts.Record(ctx, int64(attemptCount), + metric.WithAttributes( + attribute.Bool("success", success))) } func (m *metrics) recordNDAttempt(ctx context.Context, attemptCount int, success bool) { @@ -56,23 +56,23 @@ func (m *metrics) recordNDAttempt(ctx context.Context, attemptCount int, success if ctx.Err() != nil { ctx = context.Background() } - m.ndAttempts.Record(ctx, int64(attemptCount), attribute.Bool("success", success)) + m.ndAttempts.Record(ctx, int64(attemptCount), + metric.WithAttributes( + attribute.Bool("success", success))) } func (sg *ShrexGetter) WithMetrics() error { - edsAttemptHistogram, err := meter.SyncInt64().Histogram( + edsAttemptHistogram, err := meter.Int64Histogram( "getters_shrex_eds_attempts_per_request", - instrument.WithUnit(unit.Dimensionless), - instrument.WithDescription("Number of attempts per shrex/eds request"), + metric.WithDescription("Number of attempts per shrex/eds request"), ) if err != nil { return err } - ndAttemptHistogram, err := meter.SyncInt64().Histogram( + ndAttemptHistogram, err := meter.Int64Histogram( "getters_shrex_nd_attempts_per_request", - instrument.WithUnit(unit.Dimensionless), - instrument.WithDescription("Number of attempts per shrex/nd request"), + metric.WithDescription("Number of attempts per shrex/nd request"), ) if err != nil { return err diff --git a/share/p2p/discovery/metrics.go b/share/p2p/discovery/metrics.go index b6adbb1984..99c9bb4548 100644 --- a/share/p2p/discovery/metrics.go +++ b/share/p2p/discovery/metrics.go @@ -5,11 +5,9 @@ import ( "fmt" "github.com/libp2p/go-libp2p/core/peer" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/instrument/asyncint64" - "go.opentelemetry.io/otel/metric/instrument/syncint64" + "go.opentelemetry.io/otel/metric" ) const ( @@ -28,18 +26,18 @@ const ( ) var ( - meter = global.MeterProvider().Meter("share_discovery") + meter = otel.Meter("share_discovery") ) type handlePeerResult string type metrics struct { - peersAmount asyncint64.Gauge - discoveryResult syncint64.Counter // attributes: enough_peers[bool],is_canceled[bool] - handlePeerResult syncint64.Counter // attributes: result[string] - advertise syncint64.Counter // attributes: failed[bool] - peerAdded syncint64.Counter - peerRemoved syncint64.Counter + peersAmount metric.Int64ObservableGauge + discoveryResult metric.Int64Counter // attributes: enough_peers[bool],is_canceled[bool] + handlePeerResult metric.Int64Counter // attributes: result[string] + advertise metric.Int64Counter // attributes: failed[bool] + peerAdded metric.Int64Counter + peerRemoved metric.Int64Counter } // WithMetrics turns on metric collection in discoery. @@ -54,44 +52,44 @@ func (d *Discovery) WithMetrics() error { } func initMetrics(d *Discovery) (*metrics, error) { - peersAmount, err := meter.AsyncInt64().Gauge("discovery_amount_of_peers", - instrument.WithDescription("amount of peers in discovery set")) + peersAmount, err := meter.Int64ObservableGauge("discovery_amount_of_peers", + metric.WithDescription("amount of peers in discovery set")) if err != nil { return nil, err } - discoveryResult, err := meter.SyncInt64().Counter("discovery_find_peers_result", - instrument.WithDescription("result of find peers run")) + discoveryResult, err := meter.Int64Counter("discovery_find_peers_result", + metric.WithDescription("result of find peers run")) if err != nil { return nil, err } - handlePeerResultCounter, err := meter.SyncInt64().Counter("discovery_handler_peer_result", - instrument.WithDescription("result handling found peer")) + handlePeerResultCounter, err := meter.Int64Counter("discovery_handler_peer_result", + metric.WithDescription("result handling found peer")) if err != nil { return nil, err } - advertise, err := meter.SyncInt64().Counter("discovery_advertise_event", - instrument.WithDescription("advertise events counter")) + advertise, err := meter.Int64Counter("discovery_advertise_event", + metric.WithDescription("advertise events counter")) if err != nil { return nil, err } - peerAdded, err := meter.SyncInt64().Counter("discovery_add_peer", - instrument.WithDescription("add peer to discovery set counter")) + peerAdded, err := meter.Int64Counter("discovery_add_peer", + metric.WithDescription("add peer to discovery set counter")) if err != nil { return nil, err } - peerRemoved, err := meter.SyncInt64().Counter("discovery_remove_peer", - instrument.WithDescription("remove peer from discovery set counter")) + peerRemoved, err := meter.Int64Counter("discovery_remove_peer", + metric.WithDescription("remove peer from discovery set counter")) if err != nil { return nil, err } - backOffSize, err := meter.AsyncInt64().Gauge("discovery_backoff_amount", - instrument.WithDescription("amount of peers in backoff")) + backOffSize, err := meter.Int64ObservableGauge("discovery_backoff_amount", + metric.WithDescription("amount of peers in backoff")) if err != nil { return nil, err } @@ -105,16 +103,12 @@ func initMetrics(d *Discovery) (*metrics, error) { peerRemoved: peerRemoved, } - err = meter.RegisterCallback( - []instrument.Asynchronous{ - peersAmount, - backOffSize, - }, - func(ctx context.Context) { - peersAmount.Observe(ctx, int64(d.set.Size())) - backOffSize.Observe(ctx, int64(d.connector.Size())) - }, - ) + callback := func(ctx context.Context, observer metric.Observer) error { + observer.ObserveInt64(peersAmount, int64(d.set.Size())) + observer.ObserveInt64(backOffSize, int64(d.connector.Size())) + return nil + } + _, err = meter.RegisterCallback(callback, peersAmount, backOffSize) if err != nil { return nil, fmt.Errorf("registering metrics callback: %w", err) } @@ -130,7 +124,8 @@ func (m *metrics) observeFindPeers(ctx context.Context, isEnoughPeers bool) { } m.discoveryResult.Add(ctx, 1, - attribute.Bool(discoveryEnoughPeersKey, isEnoughPeers)) + metric.WithAttributes( + attribute.Bool(discoveryEnoughPeersKey, isEnoughPeers))) } func (m *metrics) observeHandlePeer(ctx context.Context, result handlePeerResult) { @@ -142,7 +137,8 @@ func (m *metrics) observeHandlePeer(ctx context.Context, result handlePeerResult } m.handlePeerResult.Add(ctx, 1, - attribute.String(handlePeerResultKey, string(result))) + metric.WithAttributes( + attribute.String(handlePeerResultKey, string(result)))) } func (m *metrics) observeAdvertise(ctx context.Context, err error) { @@ -154,7 +150,8 @@ func (m *metrics) observeAdvertise(ctx context.Context, err error) { } m.advertise.Add(ctx, 1, - attribute.Bool(advertiseFailedKey, err != nil)) + metric.WithAttributes( + attribute.Bool(advertiseFailedKey, err != nil))) } func (m *metrics) observeOnPeersUpdate(_ peer.ID, isAdded bool) { diff --git a/share/p2p/metrics.go b/share/p2p/metrics.go index 87c1e2eeb0..1942be5d6b 100644 --- a/share/p2p/metrics.go +++ b/share/p2p/metrics.go @@ -4,14 +4,12 @@ import ( "context" "fmt" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/instrument/syncint64" - "go.opentelemetry.io/otel/metric/unit" + "go.opentelemetry.io/otel/metric" ) -var meter = global.MeterProvider().Meter("shrex/eds") +var meter = otel.Meter("shrex/eds") type status string @@ -24,7 +22,7 @@ const ( ) type Metrics struct { - totalRequestCounter syncint64.Counter + totalRequestCounter metric.Int64Counter } // ObserveRequests increments the total number of requests sent with the given status as an @@ -36,14 +34,16 @@ func (m *Metrics) ObserveRequests(ctx context.Context, count int64, status statu if ctx.Err() != nil { ctx = context.Background() } - m.totalRequestCounter.Add(ctx, count, attribute.String("status", string(status))) + m.totalRequestCounter.Add(ctx, count, + metric.WithAttributes( + attribute.String("status", string(status)), + )) } func InitClientMetrics(protocol string) (*Metrics, error) { - totalRequestCounter, err := meter.SyncInt64().Counter( + totalRequestCounter, err := meter.Int64Counter( fmt.Sprintf("shrex_%s_client_total_requests", protocol), - instrument.WithUnit(unit.Dimensionless), - instrument.WithDescription(fmt.Sprintf("Total count of sent shrex/%s requests", protocol)), + metric.WithDescription(fmt.Sprintf("Total count of sent shrex/%s requests", protocol)), ) if err != nil { return nil, err @@ -55,10 +55,9 @@ func InitClientMetrics(protocol string) (*Metrics, error) { } func InitServerMetrics(protocol string) (*Metrics, error) { - totalRequestCounter, err := meter.SyncInt64().Counter( + totalRequestCounter, err := meter.Int64Counter( fmt.Sprintf("shrex_%s_server_total_responses", protocol), - instrument.WithUnit(unit.Dimensionless), - instrument.WithDescription(fmt.Sprintf("Total count of sent shrex/%s responses", protocol)), + metric.WithDescription(fmt.Sprintf("Total count of sent shrex/%s responses", protocol)), ) if err != nil { return nil, err diff --git a/share/p2p/peers/metrics.go b/share/p2p/peers/metrics.go index bf4d544d9f..95d1ce65d9 100644 --- a/share/p2p/peers/metrics.go +++ b/share/p2p/peers/metrics.go @@ -8,11 +8,9 @@ import ( pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/peer" + "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/instrument/asyncint64" - "go.opentelemetry.io/otel/metric/instrument/syncint64" + "go.opentelemetry.io/otel/metric" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -52,7 +50,7 @@ const ( ) var ( - meter = global.MeterProvider().Meter("shrex_peer_manager") + meter = otel.Meter("shrex_peer_manager") ) type blacklistPeerReason string @@ -64,63 +62,63 @@ type poolStatus string type peerSource string type metrics struct { - getPeer syncint64.Counter // attributes: source, is_instant - getPeerWaitTimeHistogram syncint64.Histogram // attributes: source - getPeerPoolSizeHistogram syncint64.Histogram // attributes: source - doneResult syncint64.Counter // attributes: source, done_result - validationResult syncint64.Counter // attributes: validation_result - - shrexPools asyncint64.Gauge // attributes: pool_status - fullNodesPool asyncint64.Gauge // attributes: pool_status + getPeer metric.Int64Counter // attributes: source, is_instant + getPeerWaitTimeHistogram metric.Int64Histogram // attributes: source + getPeerPoolSizeHistogram metric.Int64Histogram // attributes: source + doneResult metric.Int64Counter // attributes: source, done_result + validationResult metric.Int64Counter // attributes: validation_result + + shrexPools metric.Int64ObservableGauge // attributes: pool_status + fullNodesPool metric.Int64ObservableGauge // attributes: pool_status blacklistedPeersByReason sync.Map - blacklistedPeers asyncint64.Gauge // attributes: blacklist_reason + blacklistedPeers metric.Int64ObservableGauge // attributes: blacklist_reason } func initMetrics(manager *Manager) (*metrics, error) { - getPeer, err := meter.SyncInt64().Counter("peer_manager_get_peer_counter", - instrument.WithDescription("get peer counter")) + getPeer, err := meter.Int64Counter("peer_manager_get_peer_counter", + metric.WithDescription("get peer counter")) if err != nil { return nil, err } - getPeerWaitTimeHistogram, err := meter.SyncInt64().Histogram("peer_manager_get_peer_ms_time_hist", - instrument.WithDescription("get peer time histogram(ms), observed only for async get(is_instant = false)")) + getPeerWaitTimeHistogram, err := meter.Int64Histogram("peer_manager_get_peer_ms_time_hist", + metric.WithDescription("get peer time histogram(ms), observed only for async get(is_instant = false)")) if err != nil { return nil, err } - getPeerPoolSizeHistogram, err := meter.SyncInt64().Histogram("peer_manager_get_peer_pool_size_hist", - instrument.WithDescription("amount of available active peers in pool at time when get was called")) + getPeerPoolSizeHistogram, err := meter.Int64Histogram("peer_manager_get_peer_pool_size_hist", + metric.WithDescription("amount of available active peers in pool at time when get was called")) if err != nil { return nil, err } - doneResult, err := meter.SyncInt64().Counter("peer_manager_done_result_counter", - instrument.WithDescription("done results counter")) + doneResult, err := meter.Int64Counter("peer_manager_done_result_counter", + metric.WithDescription("done results counter")) if err != nil { return nil, err } - validationResult, err := meter.SyncInt64().Counter("peer_manager_validation_result_counter", - instrument.WithDescription("validation result counter")) + validationResult, err := meter.Int64Counter("peer_manager_validation_result_counter", + metric.WithDescription("validation result counter")) if err != nil { return nil, err } - shrexPools, err := meter.AsyncInt64().Gauge("peer_manager_pools_gauge", - instrument.WithDescription("pools amount")) + shrexPools, err := meter.Int64ObservableGauge("peer_manager_pools_gauge", + metric.WithDescription("pools amount")) if err != nil { return nil, err } - fullNodesPool, err := meter.AsyncInt64().Gauge("peer_manager_full_nodes_gauge", - instrument.WithDescription("full nodes pool peers amount")) + fullNodesPool, err := meter.Int64ObservableGauge("peer_manager_full_nodes_gauge", + metric.WithDescription("full nodes pool peers amount")) if err != nil { return nil, err } - blacklisted, err := meter.AsyncInt64().Gauge("peer_manager_blacklisted_peers", - instrument.WithDescription("blacklisted peers amount")) + blacklisted, err := meter.Int64ObservableGauge("peer_manager_blacklisted_peers", + metric.WithDescription("blacklisted peers amount")) if err != nil { return nil, err } @@ -136,33 +134,31 @@ func initMetrics(manager *Manager) (*metrics, error) { blacklistedPeers: blacklisted, } - err = meter.RegisterCallback( - []instrument.Asynchronous{ - shrexPools, - fullNodesPool, - blacklisted, - }, - func(ctx context.Context) { - for poolStatus, count := range manager.shrexPools() { - shrexPools.Observe(ctx, count, - attribute.String(poolStatusKey, string(poolStatus))) - } - - fullNodesPool.Observe(ctx, int64(manager.fullNodes.len()), - attribute.String(peerStatusKey, string(peerStatusActive))) - fullNodesPool.Observe(ctx, int64(manager.fullNodes.cooldown.len()), - attribute.String(peerStatusKey, string(peerStatusCooldown))) - - metrics.blacklistedPeersByReason.Range(func(key, value any) bool { - reason := key.(blacklistPeerReason) - amount := value.(int) - blacklisted.Observe(ctx, int64(amount), - attribute.String(blacklistPeerReasonKey, string(reason))) - return true - }) - }, - ) + callback := func(ctx context.Context, observer metric.Observer) error { + for poolStatus, count := range manager.shrexPools() { + observer.ObserveInt64(shrexPools, count, + metric.WithAttributes( + attribute.String(poolStatusKey, string(poolStatus)))) + } + observer.ObserveInt64(fullNodesPool, int64(manager.fullNodes.len()), + metric.WithAttributes( + attribute.String(peerStatusKey, string(peerStatusActive)))) + observer.ObserveInt64(fullNodesPool, int64(manager.fullNodes.cooldown.len()), + metric.WithAttributes( + attribute.String(peerStatusKey, string(peerStatusCooldown)))) + + metrics.blacklistedPeersByReason.Range(func(key, value any) bool { + reason := key.(blacklistPeerReason) + amount := value.(int) + observer.ObserveInt64(blacklisted, int64(amount), + metric.WithAttributes( + attribute.String(blacklistPeerReasonKey, string(reason)))) + return true + }) + return nil + } + _, err = meter.RegisterCallback(callback, shrexPools, fullNodesPool, blacklisted) if err != nil { return nil, fmt.Errorf("registering metrics callback: %w", err) } @@ -180,17 +176,20 @@ func (m *metrics) observeGetPeer( ctx = context.Background() } m.getPeer.Add(ctx, 1, - attribute.String(sourceKey, string(source)), - attribute.Bool(isInstantKey, waitTime == 0)) + metric.WithAttributes( + attribute.String(sourceKey, string(source)), + attribute.Bool(isInstantKey, waitTime == 0))) if source == sourceShrexSub { m.getPeerPoolSizeHistogram.Record(ctx, int64(poolSize), - attribute.String(sourceKey, string(source))) + metric.WithAttributes( + attribute.String(sourceKey, string(source)))) } // record wait time only for async gets if waitTime > 0 { m.getPeerWaitTimeHistogram.Record(ctx, waitTime.Milliseconds(), - attribute.String(sourceKey, string(source))) + metric.WithAttributes( + attribute.String(sourceKey, string(source)))) } } @@ -201,8 +200,9 @@ func (m *metrics) observeDoneResult(source peerSource, result result) { ctx := context.Background() m.doneResult.Add(ctx, 1, - attribute.String(sourceKey, string(source)), - attribute.String(doneResultKey, string(result))) + metric.WithAttributes( + attribute.String(sourceKey, string(source)), + attribute.String(doneResultKey, string(result)))) } // validationObserver is a middleware that observes validation results as metrics @@ -230,7 +230,8 @@ func (m *metrics) validationObserver(validator shrexsub.ValidatorFn) shrexsub.Va } m.validationResult.Add(ctx, 1, - attribute.String(validationResultKey, resStr)) + metric.WithAttributes( + attribute.String(validationResultKey, resStr))) return res } } diff --git a/state/metrics.go b/state/metrics.go index e465e2833d..169023b5f9 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -3,32 +3,28 @@ package state import ( "context" - "go.opentelemetry.io/otel/metric/global" - "go.opentelemetry.io/otel/metric/instrument" - "go.opentelemetry.io/otel/metric/unit" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" ) -var meter = global.MeterProvider().Meter("state") +var meter = otel.Meter("state") func WithMetrics(ca *CoreAccessor) { - pfbCounter, _ := meter.AsyncInt64().Counter( + pfbCounter, _ := meter.Int64ObservableCounter( "pfb_count", - instrument.WithUnit(unit.Dimensionless), - instrument.WithDescription("Total count of submitted PayForBlob transactions"), + metric.WithDescription("Total count of submitted PayForBlob transactions"), ) - lastPfbTimestamp, _ := meter.AsyncInt64().Counter( + lastPfbTimestamp, _ := meter.Int64ObservableCounter( "last_pfb_timestamp", - instrument.WithUnit(unit.Milliseconds), - instrument.WithDescription("Timestamp of the last submitted PayForBlob transaction"), + metric.WithDescription("Timestamp of the last submitted PayForBlob transaction"), ) - err := meter.RegisterCallback( - []instrument.Asynchronous{pfbCounter, lastPfbTimestamp}, - func(ctx context.Context) { - pfbCounter.Observe(ctx, ca.payForBlobCount) - lastPfbTimestamp.Observe(ctx, ca.lastPayForBlob) - }, - ) + callback := func(ctx context.Context, observer metric.Observer) error { + observer.ObserveInt64(pfbCounter, ca.payForBlobCount) + observer.ObserveInt64(lastPfbTimestamp, ca.lastPayForBlob) + return nil + } + _, err := meter.RegisterCallback(callback, pfbCounter, lastPfbTimestamp) if err != nil { panic(err) } From 05e00e8cfbc574d23ad586af81de14e404f123d5 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 14 Jul 2023 10:48:49 +0300 Subject: [PATCH 089/388] chore: move versioning outside the main package (#2468) --- Makefile | 3 ++- cmd/celestia/util.go | 6 ------ cmd/celestia/version.go | 21 +++++++-------------- cmd/env.go | 11 ----------- cmd/flags_misc.go | 3 ++- nodebuilder/node/buildInfo.go | 28 +++++++++++++++++++++++++++- nodebuilder/node_test.go | 2 +- nodebuilder/settings.go | 4 ++-- 8 files changed, 41 insertions(+), 37 deletions(-) diff --git a/Makefile b/Makefile index 310e65cbe0..2567b225d3 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,7 @@ SHELL=/usr/bin/env bash PROJECTNAME=$(shell basename "$(PWD)") -LDFLAGS=-ldflags="-X 'main.buildTime=$(shell date)' -X 'main.lastCommit=$(shell git rev-parse HEAD)' -X 'main.semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" +versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node" +LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" ifeq (${PREFIX},) PREFIX := /usr/local endif diff --git a/cmd/celestia/util.go b/cmd/celestia/util.go index 85505c3a60..a38860d1f7 100644 --- a/cmd/celestia/util.go +++ b/cmd/celestia/util.go @@ -26,12 +26,6 @@ func persistentPreRunEnv(cmd *cobra.Command, nodeType node.Type, _ []string) err return err } ctx = cmdnode.WithNetwork(ctx, parsedNetwork) - ctx = cmdnode.WithNodeBuildInfo(ctx, &node.BuildInfo{ - LastCommit: lastCommit, - SemanticVersion: semanticVersion, - SystemVersion: systemVersion, - GolangVersion: golangVersion, - }) // loads existing config into the environment ctx, err = cmdnode.ParseNodeFlags(ctx, cmd, cmdnode.Network(ctx)) diff --git a/cmd/celestia/version.go b/cmd/celestia/version.go index 462f17b474..f0d379e7a7 100644 --- a/cmd/celestia/version.go +++ b/cmd/celestia/version.go @@ -2,18 +2,10 @@ package main import ( "fmt" - "runtime" "github.com/spf13/cobra" -) - -var ( - buildTime string - lastCommit string - semanticVersion string - systemVersion = fmt.Sprintf("%s/%s", runtime.GOARCH, runtime.GOOS) - golangVersion = runtime.Version() + "github.com/celestiaorg/celestia-node/nodebuilder/node" ) var versionCmd = &cobra.Command{ @@ -24,9 +16,10 @@ var versionCmd = &cobra.Command{ } func printBuildInfo(_ *cobra.Command, _ []string) { - fmt.Printf("Semantic version: %s\n", semanticVersion) - fmt.Printf("Commit: %s\n", lastCommit) - fmt.Printf("Build Date: %s\n", buildTime) - fmt.Printf("System version: %s\n", systemVersion) - fmt.Printf("Golang version: %s\n", golangVersion) + buildInfo := node.GetBuildInfo() + fmt.Printf("Semantic version: %s\n", buildInfo.SemanticVersion) + fmt.Printf("Commit: %s\n", buildInfo.LastCommit) + fmt.Printf("Build Date: %s\n", buildInfo.BuildTime) + fmt.Printf("System version: %s\n", buildInfo.SystemVersion) + fmt.Printf("Golang version: %s\n", buildInfo.GolangVersion) } diff --git a/cmd/env.go b/cmd/env.go index ca915d884f..f9860a2de8 100644 --- a/cmd/env.go +++ b/cmd/env.go @@ -38,11 +38,6 @@ func NodeConfig(ctx context.Context) nodebuilder.Config { return cfg } -// NodeInfo reads the node build inforamtion from the context. -func NodeInfo(ctx context.Context) node.BuildInfo { - return ctx.Value(buildInfo{}).(node.BuildInfo) -} - // WithNodeType sets the node type in the given context. func WithNodeType(ctx context.Context, tp node.Type) context.Context { return context.WithValue(ctx, nodeTypeKey{}, tp) @@ -78,16 +73,10 @@ func WithNodeConfig(ctx context.Context, config *nodebuilder.Config) context.Con return context.WithValue(ctx, configKey{}, *config) } -// WithNodeConfig sets the node config build information. -func WithNodeBuildInfo(ctx context.Context, info *node.BuildInfo) context.Context { - return context.WithValue(ctx, buildInfo{}, *info) -} - type ( optionsKey struct{} configKey struct{} storePathKey struct{} nodeTypeKey struct{} networkKey struct{} - buildInfo struct{} ) diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index 26f99985b8..9dbb35ebf4 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -22,6 +22,7 @@ import ( "github.com/celestiaorg/celestia-node/logs" "github.com/celestiaorg/celestia-node/nodebuilder" + "github.com/celestiaorg/celestia-node/nodebuilder/node" modp2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" ) @@ -264,7 +265,7 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e opts = append(opts, otlpmetrichttp.WithInsecure()) } - ctx = WithNodeOptions(ctx, nodebuilder.WithMetrics(opts, NodeType(ctx), NodeInfo(ctx))) + ctx = WithNodeOptions(ctx, nodebuilder.WithMetrics(opts, NodeType(ctx), node.GetBuildInfo())) } ok, err = cmd.Flags().GetBool(p2pMetrics) diff --git a/nodebuilder/node/buildInfo.go b/nodebuilder/node/buildInfo.go index 5f5bdde28e..53d8554d4d 100644 --- a/nodebuilder/node/buildInfo.go +++ b/nodebuilder/node/buildInfo.go @@ -1,9 +1,35 @@ package node -// BuildInfo stores all necessary information for the current build. +import ( + "fmt" + "runtime" +) + +var ( + buildTime string + lastCommit string + semanticVersion string + + systemVersion = fmt.Sprintf("%s/%s", runtime.GOARCH, runtime.GOOS) + golangVersion = runtime.Version() +) + +// BuildInfo represents all necessary information about current build. type BuildInfo struct { + BuildTime string LastCommit string SemanticVersion string SystemVersion string GolangVersion string } + +// GetBuildInfo returns information about current build. +func GetBuildInfo() *BuildInfo { + return &BuildInfo{ + buildTime, + lastCommit, + semanticVersion, + systemVersion, + golangVersion, + } +} diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index e6775419c7..5ef86a0479 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -81,7 +81,7 @@ func TestLifecycle_WithMetrics(t *testing.T) { otlpmetrichttp.WithInsecure(), }, tt.tp, - node.BuildInfo{}, + &node.BuildInfo{}, ), ) require.NotNil(t, node) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 66b02c34e4..ace222179b 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -62,7 +62,7 @@ func WithPyroscope(endpoint string, nodeType node.Type) fx.Option { } // WithMetrics enables metrics exporting for the node. -func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type, buildInfo node.BuildInfo) fx.Option { +func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type, buildInfo *node.BuildInfo) fx.Option { baseComponents := fx.Options( fx.Supply(metricOpts), fx.Supply(buildInfo), @@ -111,7 +111,7 @@ func initializeMetrics( lc fx.Lifecycle, peerID peer.ID, nodeType node.Type, - buildInfo node.BuildInfo, + buildInfo *node.BuildInfo, opts []otlpmetrichttp.Option, ) error { exp, err := otlpmetrichttp.New(ctx, opts...) From b59c21c32275b26729208324831874befb7b3d7d Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Fri, 14 Jul 2023 23:38:20 +0800 Subject: [PATCH 090/388] refactor(share/availability): don't log error canceled operation (#2475) Small cleanup to prevent node from spamming error availability logs on shutdown. --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- share/availability/full/availability.go | 5 +++-- share/availability/light/availability.go | 6 +++--- 2 files changed, 6 insertions(+), 5 deletions(-) diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index cfa5bd0c39..4b19183be1 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -72,13 +72,14 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro _, err := fa.getter.GetEDS(ctx, root) if err != nil { + if errors.Is(err, context.Canceled) { + return err + } log.Errorw("availability validation failed", "root", root.String(), "err", err.Error()) var byzantineErr *byzantine.ErrByzantine if ipldFormat.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) { return share.ErrNotAvailable } - - return err } return err } diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index 761671b955..cc2e08129e 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -84,13 +84,13 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Roo } if err != nil { - if !errors.Is(err, context.Canceled) { - log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error()) + if errors.Is(err, context.Canceled) { + return err } + log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error()) if ipldFormat.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) { return share.ErrNotAvailable } - return err } } From 9c40a7f379ca4cefa738e265e7d318b2e5245ef2 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 14 Jul 2023 21:56:41 +0300 Subject: [PATCH 091/388] feat(nodebuilder): Invoke traces from fx (#2477) --- cmd/flags_misc.go | 35 +++--------------------- nodebuilder/node_test.go | 1 - nodebuilder/settings.go | 58 +++++++++++++++++++++++++++++++++++----- 3 files changed, 55 insertions(+), 39 deletions(-) diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index 9dbb35ebf4..4a11977b86 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -11,18 +11,11 @@ import ( otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "github.com/spf13/cobra" flag "github.com/spf13/pflag" - "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" - "go.opentelemetry.io/otel/exporters/otlp/otlptrace" "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" - "go.opentelemetry.io/otel/sdk/resource" - tracesdk "go.opentelemetry.io/otel/sdk/trace" - semconv "go.opentelemetry.io/otel/semconv/v1.11.0" - "go.opentelemetry.io/otel/trace" "github.com/celestiaorg/celestia-node/logs" "github.com/celestiaorg/celestia-node/nodebuilder" - "github.com/celestiaorg/celestia-node/nodebuilder/node" modp2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" ) @@ -201,7 +194,6 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e } if ok { - var tp trace.TracerProvider opts := []otlptracehttp.Option{ otlptracehttp.WithCompression(otlptracehttp.GzipCompression), otlptracehttp.WithEndpoint(cmd.Flag(tracingEndpointFlag).Value.String()), @@ -212,31 +204,13 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e opts = append(opts, otlptracehttp.WithInsecure()) } - client := otlptracehttp.NewClient(opts...) - exporter, err := otlptrace.New(ctx, client) - if err != nil { - return ctx, fmt.Errorf("creating OTLP trace exporter: %w", err) - } - - tp = tracesdk.NewTracerProvider( - tracesdk.WithSampler(tracesdk.AlwaysSample()), - // Always be sure to batch in production. - tracesdk.WithBatcher(exporter), - // Record information about this application in a Resource. - tracesdk.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - semconv.ServiceNameKey.String(fmt.Sprintf("Celestia-%s", NodeType(ctx).String())), - // TODO(@Wondertan): Versioning: semconv.ServiceVersionKey - )), - ) - + pyroOpts := make([]otelpyroscope.Option, 0) ok, err = cmd.Flags().GetBool(pyroscopeTracing) if err != nil { panic(err) } if ok { - tp = otelpyroscope.NewTracerProvider( - tp, + pyroOpts = append(pyroOpts, otelpyroscope.WithAppName("celestia.da-node"), otelpyroscope.WithPyroscopeURL(cmd.Flag(pyroscopeEndpoint).Value.String()), otelpyroscope.WithRootSpanOnly(true), @@ -245,8 +219,7 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e otelpyroscope.WithProfileBaselineURL(true), ) } - - otel.SetTracerProvider(tp) + ctx = WithNodeOptions(ctx, nodebuilder.WithTraces(opts, pyroOpts)) } ok, err = cmd.Flags().GetBool(metricsFlag) @@ -265,7 +238,7 @@ func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, e opts = append(opts, otlpmetrichttp.WithInsecure()) } - ctx = WithNodeOptions(ctx, nodebuilder.WithMetrics(opts, NodeType(ctx), node.GetBuildInfo())) + ctx = WithNodeOptions(ctx, nodebuilder.WithMetrics(opts, NodeType(ctx))) } ok, err = cmd.Flags().GetBool(p2pMetrics) diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index 5ef86a0479..f481162b5c 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -81,7 +81,6 @@ func TestLifecycle_WithMetrics(t *testing.T) { otlpmetrichttp.WithInsecure(), }, tt.tp, - &node.BuildInfo{}, ), ) require.NotNil(t, node) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index ace222179b..9a9f7fcf47 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -7,11 +7,16 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/pyroscope-io/client/pyroscope" + otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace" + "go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp" sdk "go.opentelemetry.io/otel/sdk/metric" "go.opentelemetry.io/otel/sdk/resource" + tracesdk "go.opentelemetry.io/otel/sdk/trace" semconv "go.opentelemetry.io/otel/semconv/v1.11.0" + "go.opentelemetry.io/otel/trace" "go.uber.org/fx" "github.com/celestiaorg/go-fraud" @@ -62,10 +67,9 @@ func WithPyroscope(endpoint string, nodeType node.Type) fx.Option { } // WithMetrics enables metrics exporting for the node. -func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type, buildInfo *node.BuildInfo) fx.Option { +func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Option { baseComponents := fx.Options( fx.Supply(metricOpts), - fx.Supply(buildInfo), fx.Invoke(initializeMetrics), fx.Invoke(state.WithMetrics), fx.Invoke(fraud.WithMetrics), @@ -105,13 +109,55 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type, buildIn return opts } +func WithTraces(opts []otlptracehttp.Option, pyroOpts []otelpyroscope.Option) fx.Option { + options := fx.Options( + fx.Supply(opts), + fx.Supply(pyroOpts), + fx.Invoke(initializeTraces), + ) + return options +} + +func initializeTraces( + ctx context.Context, + nodeType node.Type, + peerID peer.ID, + network p2p.Network, + opts []otlptracehttp.Option, + pyroOpts []otelpyroscope.Option, +) error { + client := otlptracehttp.NewClient(opts...) + exporter, err := otlptrace.New(ctx, client) + if err != nil { + return fmt.Errorf("creating OTLP trace exporter: %w", err) + } + + var tp trace.TracerProvider + tp = tracesdk.NewTracerProvider( + tracesdk.WithSampler(tracesdk.AlwaysSample()), + // Always be sure to batch in production. + tracesdk.WithBatcher(exporter), + // Record information about this application in a Resource. + tracesdk.WithResource(resource.NewWithAttributes( + semconv.SchemaURL, + semconv.ServiceNamespaceKey.String(nodeType.String()), + semconv.ServiceNameKey.String(fmt.Sprintf("%s/%s", network.String(), peerID.String()))), + )) + + if len(pyroOpts) > 0 { + tp = otelpyroscope.NewTracerProvider(tp, pyroOpts...) + } + otel.SetTracerProvider(tp) + return nil +} + // initializeMetrics initializes the global meter provider. func initializeMetrics( ctx context.Context, lc fx.Lifecycle, peerID peer.ID, nodeType node.Type, - buildInfo *node.BuildInfo, + network p2p.Network, opts []otlpmetrichttp.Option, ) error { exp, err := otlpmetrichttp.New(ctx, opts...) @@ -123,10 +169,8 @@ func initializeMetrics( sdk.WithReader(sdk.NewPeriodicReader(exp, sdk.WithTimeout(2*time.Second))), sdk.WithResource(resource.NewWithAttributes( semconv.SchemaURL, - semconv.ServiceNamespaceKey.String(fmt.Sprintf("Celestia-%s", nodeType.String())), - semconv.ServiceNameKey.String(fmt.Sprintf("semver-%s", buildInfo.SemanticVersion)), - semconv.ServiceInstanceIDKey.String(peerID.String()), - ))) + semconv.ServiceNamespaceKey.String(nodeType.String()), + semconv.ServiceNameKey.String(fmt.Sprintf("%s/%s", network.String(), peerID.String()))))) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return provider.Shutdown(ctx) From 03976abcc05925cb92df2966241688ceb31448d1 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 17 Jul 2023 17:49:14 +0300 Subject: [PATCH 092/388] fix: remove root from traces (#2483) --- share/eds/retriever.go | 2 -- share/eds/store.go | 11 +++++------ share/getters/cascade.go | 6 +----- share/getters/ipld.go | 6 +----- share/getters/shrex.go | 5 +---- share/getters/store.go | 6 +----- share/getters/tee.go | 6 +----- share/getters/utils.go | 1 - share/ipld/namespace_data.go | 1 - 9 files changed, 10 insertions(+), 34 deletions(-) diff --git a/share/eds/retriever.go b/share/eds/retriever.go index 2483a37a92..0f52a903bc 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -64,7 +64,6 @@ func (r *Retriever) Retrieve(ctx context.Context, dah *da.DataAvailabilityHeader defer span.End() span.SetAttributes( attribute.Int("size", len(dah.RowRoots)), - attribute.String("data_hash", dah.String()), ) log.Debugw("retrieving data square", "data_hash", dah.String(), "size", len(dah.RowRoots)) @@ -248,7 +247,6 @@ func (rs *retrievalSession) doRequest(ctx context.Context, q *quadrant) { nd, err := ipld.GetNode(ctx, rs.bget, root) if err != nil { rs.span.RecordError(err, trace.WithAttributes( - attribute.String("requesting-root", root.String()), attribute.Int("root-index", i), )) return diff --git a/share/eds/store.go b/share/eds/store.go index f0d02a1141..0683df0d99 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -168,7 +168,6 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext } ctx, span := tracer.Start(ctx, "store/put", trace.WithAttributes( - attribute.String("root", root.String()), attribute.Int("width", int(square.Width())), )) defer func() { @@ -214,7 +213,7 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext // The shard is cached in the Store, so subsequent calls to GetCAR with the same root will use the // same reader. The cache is responsible for closing the underlying reader. func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.Reader, error) { - ctx, span := tracer.Start(ctx, "store/get-car", trace.WithAttributes(attribute.String("root", root.String()))) + ctx, span := tracer.Start(ctx, "store/get-car") defer span.End() key := root.String() @@ -251,7 +250,7 @@ func (s *Store) CARBlockstore( // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. func (s *Store) GetDAH(ctx context.Context, root share.DataHash) (*share.Root, error) { - ctx, span := tracer.Start(ctx, "store/get-dah", trace.WithAttributes(attribute.String("root", root.String()))) + ctx, span := tracer.Start(ctx, "store/get-dah") defer span.End() key := shard.KeyFromString(root.String()) @@ -330,7 +329,7 @@ func (s *Store) getCachedAccessor(ctx context.Context, key shard.Key) (*accessor // Remove removes EDS from Store by the given share.Root hash and cleans up all // the indexing. func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) { - ctx, span := tracer.Start(ctx, "store/remove", trace.WithAttributes(attribute.String("root", root.String()))) + ctx, span := tracer.Start(ctx, "store/remove") defer func() { utils.SetStatusAndEnd(span, err) }() @@ -371,7 +370,7 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) { // It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by // recomputing it. func (s *Store) Get(ctx context.Context, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "store/get", trace.WithAttributes(attribute.String("root", root.String()))) + ctx, span := tracer.Start(ctx, "store/get") defer func() { utils.SetStatusAndEnd(span, err) }() @@ -389,7 +388,7 @@ func (s *Store) Get(ctx context.Context, root share.DataHash) (eds *rsmt2d.Exten // Has checks if EDS exists by the given share.Root hash. func (s *Store) Has(ctx context.Context, root share.DataHash) (bool, error) { - _, span := tracer.Start(ctx, "store/has", trace.WithAttributes(attribute.String("root", root.String()))) + _, span := tracer.Start(ctx, "store/has") defer span.End() key := root.String() diff --git a/share/getters/cascade.go b/share/getters/cascade.go index fe99bed4c3..ef59946666 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -33,7 +33,6 @@ func NewCascadeGetter(getters []share.Getter) *CascadeGetter { // GetShare gets a share from any of registered share.Getters in cascading order. func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, col int) (share.Share, error) { ctx, span := tracer.Start(ctx, "cascade/get-share", trace.WithAttributes( - attribute.String("root", root.String()), attribute.Int("row", row), attribute.Int("col", col), )) @@ -48,9 +47,7 @@ func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, co // GetEDS gets a full EDS from any of registered share.Getters in cascading order. func (cg *CascadeGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) { - ctx, span := tracer.Start(ctx, "cascade/get-eds", trace.WithAttributes( - attribute.String("root", root.String()), - )) + ctx, span := tracer.Start(ctx, "cascade/get-eds") defer span.End() get := func(ctx context.Context, get share.Getter) (*rsmt2d.ExtendedDataSquare, error) { @@ -68,7 +65,6 @@ func (cg *CascadeGetter) GetSharesByNamespace( namespace share.Namespace, ) (share.NamespacedShares, error) { ctx, span := tracer.Start(ctx, "cascade/get-shares-by-namespace", trace.WithAttributes( - attribute.String("root", root.String()), attribute.String("namespace", namespace.String()), )) defer span.End() diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 92d1c55678..af6e7673ba 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -41,7 +41,6 @@ func NewIPLDGetter(bServ blockservice.BlockService) *IPLDGetter { func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) { var err error ctx, span := tracer.Start(ctx, "ipld/get-share", trace.WithAttributes( - attribute.String("root", dah.String()), attribute.Int("row", row), attribute.Int("col", col), )) @@ -66,9 +65,7 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in } func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "ipld/get-eds", trace.WithAttributes( - attribute.String("root", root.String()), - )) + ctx, span := tracer.Start(ctx, "ipld/get-eds") defer func() { utils.SetStatusAndEnd(span, err) }() @@ -91,7 +88,6 @@ func (ig *IPLDGetter) GetSharesByNamespace( namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes( - attribute.String("root", root.String()), attribute.String("namespace", namespace.String()), )) defer func() { diff --git a/share/getters/shrex.go b/share/getters/shrex.go index c754d73c1d..5a2f696854 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -128,9 +128,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex attempt int err error ) - ctx, span := tracer.Start(ctx, "shrex/get-eds", trace.WithAttributes( - attribute.String("root", root.String()), - )) + ctx, span := tracer.Start(ctx, "shrex/get-eds") defer func() { utils.SetStatusAndEnd(span, err) }() @@ -198,7 +196,6 @@ func (sg *ShrexGetter) GetSharesByNamespace( err error ) ctx, span := tracer.Start(ctx, "shrex/get-shares-by-namespace", trace.WithAttributes( - attribute.String("root", root.String()), attribute.String("namespace", namespace.String()), )) defer func() { diff --git a/share/getters/store.go b/share/getters/store.go index 0b88d89155..28c6c6d021 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -36,7 +36,6 @@ func NewStoreGetter(store *eds.Store) *StoreGetter { func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) { var err error ctx, span := tracer.Start(ctx, "store/get-share", trace.WithAttributes( - attribute.String("root", dah.String()), attribute.Int("row", row), attribute.Int("col", col), )) @@ -70,9 +69,7 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i // GetEDS gets the EDS identified by the given root from the EDS store. func (sg *StoreGetter) GetEDS(ctx context.Context, root *share.Root) (data *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "store/get-eds", trace.WithAttributes( - attribute.String("root", root.String()), - )) + ctx, span := tracer.Start(ctx, "store/get-eds") defer func() { utils.SetStatusAndEnd(span, err) }() @@ -96,7 +93,6 @@ func (sg *StoreGetter) GetSharesByNamespace( namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "store/get-shares-by-namespace", trace.WithAttributes( - attribute.String("root", root.String()), attribute.String("namespace", namespace.String()), )) defer func() { diff --git a/share/getters/tee.go b/share/getters/tee.go index 213f05f90c..fe43c971b7 100644 --- a/share/getters/tee.go +++ b/share/getters/tee.go @@ -35,7 +35,6 @@ func NewTeeGetter(getter share.Getter, store *eds.Store) *TeeGetter { func (tg *TeeGetter) GetShare(ctx context.Context, root *share.Root, row, col int) (share share.Share, err error) { ctx, span := tracer.Start(ctx, "tee/get-share", trace.WithAttributes( - attribute.String("root", root.String()), attribute.Int("row", row), attribute.Int("col", col), )) @@ -47,9 +46,7 @@ func (tg *TeeGetter) GetShare(ctx context.Context, root *share.Root, row, col in } func (tg *TeeGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "tee/get-eds", trace.WithAttributes( - attribute.String("root", root.String()), - )) + ctx, span := tracer.Start(ctx, "tee/get-eds") defer func() { utils.SetStatusAndEnd(span, err) }() @@ -73,7 +70,6 @@ func (tg *TeeGetter) GetSharesByNamespace( namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "tee/get-shares-by-namespace", trace.WithAttributes( - attribute.String("root", root.String()), attribute.String("namespace", namespace.String()), )) defer func() { diff --git a/share/getters/utils.go b/share/getters/utils.go index 94b1cfe5ee..5305c5c737 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -46,7 +46,6 @@ func collectSharesByNamespace( namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( - attribute.String("root", root.String()), attribute.String("namespace", namespace.String()), )) defer func() { diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index d776da219d..38dfdb2169 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -198,7 +198,6 @@ func (n *NamespaceData) CollectLeavesByNamespace( span.SetAttributes( attribute.String("namespace", n.namespace.String()), - attribute.String("root", root.String()), ) // buffer the jobs to avoid blocking, we only need as many From 6a6c68c42ff940467476c0a6876e7d986b0e84aa Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 17 Jul 2023 18:30:59 +0300 Subject: [PATCH 093/388] feat(share/byzantine): handle additional cases in befp validation (#2383) Co-authored-by: Hlib Kanunnikov --- share/eds/byzantine/bad_encoding.go | 45 +++++++++++++++++++++++++---- 1 file changed, 39 insertions(+), 6 deletions(-) diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 9d6cbff229..95f2a7fd03 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -2,6 +2,7 @@ package byzantine import ( "bytes" + "errors" "fmt" "github.com/celestiaorg/celestia-app/pkg/wrapper" @@ -114,6 +115,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { if !ok { panic(fmt.Sprintf("invalid header type received during BEFP validation: expected %T, got %T", header, hdr)) } + if header.Height() != int64(p.BlockHeight) { return fmt.Errorf("incorrect block height during BEFP validation: expected %d, got %d", p.BlockHeight, header.Height(), @@ -137,11 +139,13 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { if p.Axis == rsmt2d.Row { merkleRoots = header.DAH.ColumnRoots } + if int(p.Index) >= len(merkleRoots) { return fmt.Errorf("invalid %s proof: index out of bounds (%d >= %d)", BadEncoding, int(p.Index), len(merkleRoots), ) } + if len(p.Shares) != len(merkleRoots) { // Since p.Shares should contain all the shares from either a row or a // column, it should exactly match the number of row roots. In this @@ -152,6 +156,22 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { ) } + odsWidth := uint64(len(merkleRoots) / 2) + amount := uint64(0) + for _, share := range p.Shares { + if share == nil { + continue + } + amount++ + if amount == odsWidth { + break + } + } + + if amount < odsWidth { + return errors.New("fraud: invalid proof: not enough shares provided to reconstruct row/col") + } + // verify that Merkle proofs correspond to particular shares. shares := make([][]byte, len(merkleRoots)) for index, shr := range p.Shares { @@ -167,17 +187,24 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { shares[index] = share.GetData(shr.Share) } - odsWidth := uint64(len(merkleRoots) / 2) codec := share.DefaultRSMT2DCodec() - // rebuild a row or col. + // We can conclude that the proof is valid in case we proved the inclusion of `Shares` but + // the row/col can't be reconstructed, or the building of NMTree fails. rebuiltShares, err := codec.Decode(shares) if err != nil { - return err + log.Infow("failed to decode shares at height", + "height", header.Height(), "err", err, + ) + return nil } + rebuiltExtendedShares, err := codec.Encode(rebuiltShares[0:odsWidth]) if err != nil { - return err + log.Infow("failed to encode shares at height", + "height", header.Height(), "err", err, + ) + return nil } copy(rebuiltShares[odsWidth:], rebuiltExtendedShares) @@ -185,13 +212,19 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { for _, share := range rebuiltShares { err = tree.Push(share) if err != nil { - return err + log.Infow("failed to build a tree from the reconstructed shares at height", + "height", header.Height(), "err", err, + ) + return nil } } expectedRoot, err := tree.Root() if err != nil { - return err + log.Infow("failed to build a tree root at height", + "height", header.Height(), "err", err, + ) + return nil } // root is a merkle root of the row/col where ErrByzantine occurred From c5565885d68f515d2c90ceee014372e1cfe3c3a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 18 Jul 2023 13:35:46 +0000 Subject: [PATCH 094/388] chore(deps): bump celestiaorg/.github from 0.2.0 to 0.2.2 (#2484) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [celestiaorg/.github](https://github.com/celestiaorg/.github) from 0.2.0 to 0.2.2.
Release notes

Sourced from celestiaorg/.github's releases.

Release v0.2.2

0.2.2 (2023-07-11)

Overview

This release fixes a bug in the docker reusable workflow that was causing it to fail on forks.

What's Changed

Full Changelog: https://github.com/celestiaorg/.github/compare/v0.2.1...v0.2.2

v0.2.1

What's Changed

Full Changelog: https://github.com/celestiaorg/.github/compare/v0.2.0...v0.2.1

Commits
  • 84d7d3c Revert "Remove failing docker publish line" (#64)
  • 911f77b feat: use context - seems like it fixes the ldflags issue (#65)
  • 5f647a9 Remove failing docker publish line (#62)
  • d3fbccf chore(deps): bump necojackarc/auto-request-review from 0.10.0 to 0.12.0 (#61)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=celestiaorg/.github&package-manager=github_actions&previous-version=0.2.0&new-version=0.2.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 653add9256..e10a032681 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -25,7 +25,7 @@ on: jobs: # Dockerfile Linting hadolint: - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.0 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.2 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v3 - - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.0 + - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.2 markdown-lint: name: Markdown Lint @@ -59,7 +59,7 @@ jobs: steps: - uses: actions/checkout@v3 - name: Version Release - uses: celestiaorg/.github/.github/actions/version-release@v0.2.0 + uses: celestiaorg/.github/.github/actions/version-release@v0.2.2 with: github-token: ${{secrets.GITHUB_TOKEN}} version-bump: ${{inputs.version}} From 4556e6f73fea197c2a472da765279164d73d0db8 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 20 Jul 2023 17:46:56 +0200 Subject: [PATCH 095/388] fix(nodebuilder/tests/swamp): Bump timeout (#2504) Necessary to get core client to produce blocks in swamp at the moment. Found by @rootulp --- nodebuilder/tests/swamp/swamp.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index eabe33dc9c..3532635883 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -164,7 +164,7 @@ func (s *Swamp) createPeer(ks keystore.Keystore) host.Host { // setupGenesis sets up genesis Header. // This is required to initialize and start correctly. func (s *Swamp) setupGenesis() { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer cancel() // ensure core has surpassed genesis block From 8a06f313afb2fcb27295ccd4fbade61daff4edce Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 24 Jul 2023 17:54:48 +0300 Subject: [PATCH 096/388] dep: update nmt version (#2509) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index eae1f5842b..4275047ede 100644 --- a/go.mod +++ b/go.mod @@ -14,7 +14,7 @@ require ( github.com/celestiaorg/go-fraud v0.1.2 github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 - github.com/celestiaorg/nmt v0.17.0 + github.com/celestiaorg/nmt v0.18.1 github.com/celestiaorg/rsmt2d v0.10.0 github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 diff --git a/go.sum b/go.sum index 7ef5215bfd..a6d7f9e346 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.17.0 h1:/k8YLwJvuHgT/jQ435zXKaDX811+sYEMXL4B/vYdSLU= -github.com/celestiaorg/nmt v0.17.0/go.mod h1:ZndCeAR4l9lxm7W51ouoyTo1cxhtFgK+4DpEIkxRA3A= +github.com/celestiaorg/nmt v0.18.1 h1:zU3apzW4y0fs0ilQA74XnEYW8FvRv0CUK2LXK66L3rA= +github.com/celestiaorg/nmt v0.18.1/go.mod h1:0l8q6UYRju1xNrxtvV6NwPdW3lfsN6KuZ0htRnModdc= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.10.0 h1:8dprr6CW5mCk5YPnbiLdirojw9YsJOE+XB+GORb8sT0= From 6a7275d137be3cdf9cdd5f17e3bb7aef6537d9d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ramon=20Ma=C3=B1es?= <32740567+jrmanes@users.noreply.github.com> Date: Wed, 26 Jul 2023 15:14:38 +0200 Subject: [PATCH 097/388] feat: new metrics - deployment dashboard (#2464) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hello team! I've added these changes to the code to expose some new metrics via Prometheus client to create the deployment dashboard in Grafana, this new dashboard will include info related to the chains, such as branches/tags, number of nodes, URLs, etc... *I'm using the values generated by the `ldflags`* The dashboard will look like this: Screenshot 2023-07-12 at 13 07 12 *Pending adding the consensus info, it will contain more info* Please, check the code and let me know if you find something to improve, I'll gladly do it 😊 Thanks in advance! 🚀 Jose Ramon Mañes closes: https://github.com/celestiaorg/devops/issues/401 --- nodebuilder/node/metrics.go | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) diff --git a/nodebuilder/node/metrics.go b/nodebuilder/node/metrics.go index 7d722524e8..07c9a5fc0f 100644 --- a/nodebuilder/node/metrics.go +++ b/nodebuilder/node/metrics.go @@ -5,6 +5,7 @@ import ( "time" "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" ) @@ -33,6 +34,14 @@ func WithMetrics() error { return err } + buildInfoGauge, err := meter.Float64ObservableGauge( + "build_info", + metric.WithDescription("Celestia Node build information"), + ) + if err != nil { + return err + } + callback := func(ctx context.Context, observer metric.Observer) error { if !nodeStarted { // Observe node start timestamp @@ -42,9 +51,22 @@ func WithMetrics() error { } observer.ObserveFloat64(totalNodeRunTime, time.Since(timeStarted).Seconds()) + + // Observe build info with labels + labels := metric.WithAttributes( + attribute.String("build_time", buildTime), + attribute.String("last_commit", lastCommit), + attribute.String("semantic_version", semanticVersion), + attribute.String("system_version", systemVersion), + attribute.String("golang_version", golangVersion), + ) + + observer.ObserveFloat64(buildInfoGauge, 1, labels) + return nil } - _, err = meter.RegisterCallback(callback, nodeStartTS, totalNodeRunTime) + _, err = meter.RegisterCallback(callback, nodeStartTS, totalNodeRunTime, buildInfoGauge) + return err } From 7d981d58a5055d0130b832e3a8a0e889635f7c7d Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 26 Jul 2023 21:26:18 +0800 Subject: [PATCH 098/388] test(share/utils): fix no deadline test for ctxWithSplitTimeout (#2502) Quick fix for one of the ctxWithSplitTimeout test cases. --- share/getters/utils_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/share/getters/utils_test.go b/share/getters/utils_test.go index db8cd0f8cc..5ffad228f9 100644 --- a/share/getters/utils_test.go +++ b/share/getters/utils_test.go @@ -186,7 +186,7 @@ func Test_ctxWithSplitTimeout(t *testing.T) { splitFactor: []int{-1, 0, 1, 2}, minTimeout: time.Minute, }, - want: 0, + want: time.Minute, }, { name: "no context timeout, minTimeout = 0", @@ -201,12 +201,18 @@ func Test_ctxWithSplitTimeout(t *testing.T) { for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { for _, sf := range tt.args.splitFactor { - ctx, cancel := context.WithTimeout(context.Background(), tt.args.ctxTimeout) + ctx, cancel := context.WithCancel(context.Background()) + // add timeout if original context should have it + if tt.args.ctxTimeout > 0 { + ctx, cancel = context.WithTimeout(ctx, tt.args.ctxTimeout) + } t.Cleanup(cancel) got, _ := ctxWithSplitTimeout(ctx, sf, tt.args.minTimeout) dl, ok := got.Deadline() - if !ok { - require.Equal(t, tt.want, 0) + // in case no deadline is found in ctx or not expected to be found, check both cases apply at the same time + if !ok || tt.want == 0 { + require.False(t, ok) + require.Equal(t, tt.want, time.Duration(0)) continue } d := time.Until(dl) From e354bb5414b5bde5b7b950f6b29a0419ca44214b Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 26 Jul 2023 21:44:59 +0800 Subject: [PATCH 099/388] fix(share/availability): update availability ErrNotAvailable handling logic (#2445) ## Overview `ipldFormat.IsNotFound` could no longer reach Availability layer. It will get converted on getter layer to `share.ErrNotFound`. Update outdated error handling. --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- share/availability/full/availability.go | 3 +-- share/availability/full/availability_test.go | 26 ++++++++++++++++++++ 2 files changed, 27 insertions(+), 2 deletions(-) diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index 4b19183be1..fb8dead839 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -4,7 +4,6 @@ import ( "context" "errors" - ipldFormat "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" "github.com/celestiaorg/celestia-node/share" @@ -77,7 +76,7 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro } log.Errorw("availability validation failed", "root", root.String(), "err", err.Error()) var byzantineErr *byzantine.ErrByzantine - if ipldFormat.IsNotFound(err) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) { + if errors.Is(err, share.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) { return share.ErrNotAvailable } } diff --git a/share/availability/full/availability_test.go b/share/availability/full/availability_test.go index 48ae0534aa..b6d217cd6b 100644 --- a/share/availability/full/availability_test.go +++ b/share/availability/full/availability_test.go @@ -4,9 +4,16 @@ import ( "context" "testing" + "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/pkg/da" + + "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/mocks" ) func TestShareAvailableOverMocknet_Full(t *testing.T) { @@ -32,3 +39,22 @@ func TestSharesAvailable_Full(t *testing.T) { err := avail.SharesAvailable(ctx, dah) assert.NoError(t, err) } + +func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) { + ctrl := gomock.NewController(t) + getter := mocks.NewMockGetter(ctrl) + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + eds := edstest.RandEDS(t, 4) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + avail := TestAvailability(getter) + + errors := []error{share.ErrNotFound, context.DeadlineExceeded} + for _, getterErr := range errors { + getter.EXPECT().GetEDS(gomock.Any(), gomock.Any()).Return(nil, getterErr) + err := avail.SharesAvailable(ctx, &dah) + require.ErrorIs(t, err, share.ErrNotAvailable) + } +} From 26d97c33fc566b740d9e19a994e13ab58d0fd848 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 11:32:46 +0000 Subject: [PATCH 100/388] chore(deps): bump cosmossdk.io/errors from 1.0.0-beta.7 to 1.0.0 (#2486) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Hlib Kanunnikov --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index 4275047ede..76baac83cc 100644 --- a/go.mod +++ b/go.mod @@ -5,7 +5,7 @@ go 1.20 replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1-lazypatch require ( - cosmossdk.io/errors v1.0.0-beta.7 + cosmossdk.io/errors v1.0.0 cosmossdk.io/math v1.0.0-rc.0 github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 @@ -74,11 +74,11 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 go.uber.org/fx v1.19.3 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.9.0 + golang.org/x/crypto v0.11.0 golang.org/x/exp v0.0.0-20230321023759-10a507213a29 golang.org/x/sync v0.2.0 - golang.org/x/text v0.9.0 - google.golang.org/grpc v1.56.1 + golang.org/x/text v0.11.0 + google.golang.org/grpc v1.56.2 google.golang.org/protobuf v1.31.0 ) @@ -311,17 +311,17 @@ require ( go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.10.0 // indirect - golang.org/x/net v0.10.0 // indirect + golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.8.0 // indirect - golang.org/x/term v0.8.0 // indirect + golang.org/x/sys v0.10.0 // indirect + golang.org/x/term v0.10.0 // indirect golang.org/x/tools v0.9.1 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e // indirect + google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a6d7f9e346..3a44e1f25b 100644 --- a/go.sum +++ b/go.sum @@ -194,8 +194,8 @@ cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuW cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= -cosmossdk.io/errors v1.0.0-beta.7 h1:gypHW76pTQGVnHKo6QBkb4yFOJjC+sUGRc5Al3Odj1w= -cosmossdk.io/errors v1.0.0-beta.7/go.mod h1:mz6FQMJRku4bY7aqS/Gwfcmr/ue91roMEKAmDUDpBfE= +cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= +cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= cosmossdk.io/math v1.0.0-rc.0 h1:ml46ukocrAAoBpYKMidF0R2tQJ1Uxfns0yH8wqgMAFc= cosmossdk.io/math v1.0.0-rc.0/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= @@ -2159,8 +2159,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.9.0 h1:LF6fAI+IutBocDJ2OT0Q1g8plpYljMZ4+lty+dsqw3g= -golang.org/x/crypto v0.9.0/go.mod h1:yrmDGqONDYtNj3tH8X9dzUun2m2lzPa9ngI6/RUPGR0= +golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= +golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2293,8 +2293,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.10.0 h1:X2//UzNDwYmtCLn7To6G58Wr6f5ahEAQgKNzv9Y951M= -golang.org/x/net v0.10.0/go.mod h1:0qNGK6F8kojg2nk9dLZ2mShWaEBan6FAoqfSigmmuDg= +golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= +golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2470,14 +2470,14 @@ golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.8.0 h1:EBmGv8NaZBZTWvrbjNoL6HVt+IVy3QDQpJs7VRIw3tU= -golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= +golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.8.0 h1:n5xxQn2i3PC0yLAbjTpNT85q/Kgzcr2gIoX9OrJUols= -golang.org/x/term v0.8.0/go.mod h1:xPskH00ivmX89bAKVGSKKtLOWNx2+17Eiy94tnKShWo= +golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= +golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2488,8 +2488,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.9.0 h1:2sjJmO8cDvYveuX97RDLsxlyUxLl+GHoLxBiRdHllBE= -golang.org/x/text v0.9.0/go.mod h1:e1OnstbJyHTd6l/uOt8jFFHp6TRDWZR/bV3emEE/zU8= +golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= +golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2763,12 +2763,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e h1:Ao9GzfUMPH3zjVfzXG5rlWlk+Q8MXWKwWpwVQE1MXfw= -google.golang.org/genproto v0.0.0-20230526203410-71b5a4ffd15e/go.mod h1:zqTuNwFlFRsw5zIts5VnzLQxSRqh+CGOTVMlYbY0Eyk= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs= +google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc h1:XSJ8Vk1SWuNr8S18z1NZSziL0CPIXLCCMDOEFtHBOFc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:66JfowdXAEgad5O9NnYcsNPLCPZJD++2L9X0PCMODrA= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2815,8 +2815,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.56.1 h1:z0dNfjIl0VpaZ9iSVjA6daGatAYwPGstTjt5vkRMFkQ= -google.golang.org/grpc v1.56.1/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= +google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 329ab208e6d1b5a09320831cc6db55fbab457af3 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Mon, 31 Jul 2023 07:53:48 -0400 Subject: [PATCH 101/388] fix: check for namespace version 0 (#2514) Closes https://github.com/celestiaorg/celestia-node/issues/2513 Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- share/namespace.go | 3 +++ share/namespace_test.go | 53 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 56 insertions(+) diff --git a/share/namespace.go b/share/namespace.go index 433529a57f..276575a309 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -113,6 +113,9 @@ func (n Namespace) ValidateForData() error { if n.Equals(ParitySharesNamespace) || n.Equals(TailPaddingNamespace) { return fmt.Errorf("invalid data namespace(%s): parity and tail padding namespace are forbidden", n) } + if n.Version() != appns.NamespaceVersionZero { + return fmt.Errorf("invalid data namespace(%s): only version 0 is supported", n) + } return nil } diff --git a/share/namespace_test.go b/share/namespace_test.go index 62746e4b4e..7c18d233c8 100644 --- a/share/namespace_test.go +++ b/share/namespace_test.go @@ -5,6 +5,7 @@ import ( "testing" "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" appns "github.com/celestiaorg/celestia-app/pkg/namespace" ) @@ -135,3 +136,55 @@ func TestFrom(t *testing.T) { }) } } + +func TestValidateForBlob(t *testing.T) { + type testCase struct { + name string + ns Namespace + wantErr bool + } + + validNamespace, err := NewBlobNamespaceV0(bytes.Repeat([]byte{0x1}, appns.NamespaceVersionZeroIDSize)) + require.NoError(t, err) + + testCases := []testCase{ + { + name: "valid blob namespace", + ns: validNamespace, + wantErr: false, + }, + { + name: "invalid blob namespace: parity shares namespace", + ns: ParitySharesNamespace, + wantErr: true, + }, + { + name: "invalid blob namespace: tail padding namespace", + ns: TailPaddingNamespace, + wantErr: true, + }, + { + name: "invalid blob namespace: tx namespace", + ns: TxNamespace, + wantErr: true, + }, + { + name: "invalid blob namespace: namespace version max", + ns: append([]byte{appns.NamespaceVersionMax}, bytes.Repeat([]byte{0x0}, appns.NamespaceIDSize)...), + wantErr: true, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + err := tc.ns.ValidateForBlob() + + if tc.wantErr { + assert.Error(t, err) + return + } + assert.NoError(t, err) + }) + } + +} From 954f748ee63ebbdd95ec706b5b82a01ef4f5b1bc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 31 Jul 2023 13:02:42 +0000 Subject: [PATCH 102/388] chore(deps): bump github.com/spf13/cobra from 1.6.1 to 1.7.0 (#2487) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.6.1 to 1.7.0.
Release notes

Sourced from github.com/spf13/cobra's releases.

v1.7.0

✨ Features

🐛 Bug fixes

🧪 Testing & CI/CD

🔧 Maintenance

✏️ Documentation

This release contains several long running fixes, improvements to powershell completions, and further optimizations for completions.

Thank you everyone who contributed to this release and all your hard work! Cobra and this community would never be possible without all of you! 🐍

Full changelog: https://github.com/spf13/cobra/compare/v1.6.1...v1.7.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/cobra&package-manager=go_modules&previous-version=1.6.1&new-version=1.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: John Adler --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 76baac83cc..875f6557e7 100644 --- a/go.mod +++ b/go.mod @@ -59,7 +59,7 @@ require ( github.com/prometheus/client_golang v1.14.0 github.com/pyroscope-io/client v0.7.1 github.com/pyroscope-io/otel-profiling-go v0.4.0 - github.com/spf13/cobra v1.6.1 + github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 @@ -193,7 +193,7 @@ require ( github.com/huin/goupnp v1.2.0 // indirect github.com/iancoleman/orderedmap v0.1.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect - github.com/inconshreveable/mousetrap v1.0.1 // indirect + github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/ipfs/bbloom v0.0.4 // indirect diff --git a/go.sum b/go.sum index 3a44e1f25b..b6a8f996e0 100644 --- a/go.sum +++ b/go.sum @@ -942,8 +942,8 @@ github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+h github.com/improbable-eng/grpc-web v0.15.0 h1:BN+7z6uNXZ1tQGcNAuaU1YjsLTApzkjt2tzCixLaUPQ= github.com/improbable-eng/grpc-web v0.15.0/go.mod h1:1sy9HKV4Jt9aEs9JSnkWlRJPuPtwNr0l57L4f878wP8= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= -github.com/inconshreveable/mousetrap v1.0.1 h1:U3uMjPSQEBMNp1lFxmllqCPM6P5u/Xq7Pgzkat/bFNc= -github.com/inconshreveable/mousetrap v1.0.1/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= +github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= +github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= github.com/influxdata/flux v0.65.1/go.mod h1:J754/zds0vvpfwuq7Gc2wRdVwEodfpCFM7mYlOw2LqY= github.com/influxdata/influxdb v1.8.3/go.mod h1:JugdFhsvvI8gadxOI6noqNeeBHvWNTbfYGtiAn+2jhI= github.com/influxdata/influxdb-client-go/v2 v2.4.0/go.mod h1:vLNHdxTJkIf2mSLvGrpj8TCcISApPoXkaxP8g9uRlW8= @@ -1900,8 +1900,8 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.6.1 h1:o94oiPyS4KD1mPy2fmcYYHHfCxLqYjJOhGsCHFZtEzA= -github.com/spf13/cobra v1.6.1/go.mod h1:IOw/AERYS7UzyrGinqmz6HLUo219MORXGxhbaJUqzrY= +github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= +github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= From 794360f2f83c7c797ea5c0be7d8c61aaed8cd0db Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 31 Jul 2023 21:21:49 +0800 Subject: [PATCH 103/388] test(share/p2p): fix namespace add operations in tests (#2422) ## Overview Fixes the test with unreliable add/sub operations. Adds AddToNamespace for tests --- share/getters/shrex_test.go | 119 ++++++++++++++++++++++++++++++++---- 1 file changed, 108 insertions(+), 11 deletions(-) diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 0dbe7e44db..d2dcfeca5e 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -2,6 +2,8 @@ package getters import ( "context" + "encoding/binary" + "errors" "testing" "time" @@ -100,18 +102,16 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - // corrupt NID - nID := make([]byte, share.NamespaceSize) - copy(nID, maxNamespace) - nID[share.NamespaceSize-1]-- + namespace, err := addToNamespace(maxNamespace, -1) + require.NoError(t, err) // check for namespace to be between max and min namespace in root - require.Len(t, filterRootsByNamespace(&dah, maxNamespace), 1) + require.Len(t, filterRootsByNamespace(&dah, namespace), 1) - emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, nID) + emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, namespace) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(&dah, nID)) + require.Nil(t, emptyShares.Verify(&dah, namespace)) }) t.Run("ND_namespace_not_in_dah", func(t *testing.T) { @@ -126,10 +126,8 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - // corrupt namespace - namespace := make([]byte, share.NamespaceSize) - copy(namespace, maxNamesapce) - namespace[share.NamespaceSize-1]++ + namespace, err := addToNamespace(maxNamesapce, 1) + require.NoError(t, err) // check for namespace to be not in root require.Len(t, filterRootsByNamespace(&dah, namespace), 0) @@ -271,3 +269,102 @@ func newEDSClientServer( require.NoError(t, err) return client, server } + +// addToNamespace adds arbitrary int value to namespace, treating namespace as big-endian implementation of int +func addToNamespace(namespace share.Namespace, val int) (share.Namespace, error) { + if val == 0 { + return namespace, nil + } + // Convert the input integer to a byte slice and add it to result slice + result := make([]byte, len(namespace)) + if val > 0 { + binary.BigEndian.PutUint64(result[len(namespace)-8:], uint64(val)) + } else { + binary.BigEndian.PutUint64(result[len(namespace)-8:], uint64(-val)) + } + + // Perform addition byte by byte + var carry int + for i := len(namespace) - 1; i >= 0; i-- { + sum := 0 + if val > 0 { + sum = int(namespace[i]) + int(result[i]) + carry + } else { + sum = int(namespace[i]) - int(result[i]) + carry + } + + switch { + case sum > 255: + carry = 1 + sum -= 256 + case sum < 0: + carry = -1 + sum += 256 + default: + carry = 0 + } + + result[i] = uint8(sum) + } + + // Handle any remaining carry + if carry != 0 { + return nil, errors.New("namespace overflow") + } + + return result, nil +} + +func TestAddToNamespace(t *testing.T) { + testCases := []struct { + name string + value int + input share.Namespace + expected share.Namespace + expectedError error + }{ + { + name: "Positive value addition", + value: 42, + input: share.Namespace{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + expected: share.Namespace{0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x1, 0x2b}, + expectedError: nil, + }, + { + name: "Negative value addition", + value: -42, + input: share.Namespace{0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01}, + expected: share.Namespace{0x1, 0x1, 0x1, 0x1, 0x1, 0x01, 0x1, 0x1, 0x1, 0x0, 0xd7}, + expectedError: nil, + }, + { + name: "Overflow error", + value: 1, + input: share.Namespace{0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF, 0xFF}, + expected: nil, + expectedError: errors.New("namespace overflow"), + }, + { + name: "Overflow error negative", + value: -1, + input: share.Namespace{0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0, 0x0}, + expected: nil, + expectedError: errors.New("namespace overflow"), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + result, err := addToNamespace(tc.input, tc.value) + if tc.expectedError == nil { + require.NoError(t, err) + require.Equal(t, tc.expected, result) + return + } + require.Error(t, err) + if err.Error() != tc.expectedError.Error() { + t.Errorf("Unexpected error message. Expected: %v, Got: %v", tc.expectedError, err) + } + }) + } +} From 208bb86598b21c6b368f86e1212cf882b8568b77 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 31 Jul 2023 22:17:09 +0800 Subject: [PATCH 104/388] fix(share/eds): remove Has() and Sync() badger calls in inverted index (#2518) Remove unnecessary `Has()` and `Sync()` badger calls in inverted index. Removal of calls greatly improve `Put()` performance on eds.Store --- share/eds/inverted_index.go | 25 +++++++------------------ share/eds/inverted_index_test.go | 4 ++-- 2 files changed, 9 insertions(+), 20 deletions(-) diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index f917619676..0e3665d973 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -40,35 +40,24 @@ func (s *simpleInvertedIndex) AddMultihashesForShard( return fmt.Errorf("failed to create ds batch: %w", err) } - if err := mhIter.ForEach(func(mh multihash.Multihash) error { + err = mhIter.ForEach(func(mh multihash.Multihash) error { key := ds.NewKey(string(mh)) - ok, err := s.ds.Has(ctx, key) + bz, err := json.Marshal(sk) if err != nil { - return fmt.Errorf("failed to check if value for multihash exists %s, err: %w", mh, err) + return fmt.Errorf("failed to marshal shard key to bytes: %w", err) } - - if !ok { - bz, err := json.Marshal(sk) - if err != nil { - return fmt.Errorf("failed to marshal shard key to bytes: %w", err) - } - if err := batch.Put(ctx, key, bz); err != nil { - return fmt.Errorf("failed to put mh=%s, err=%w", mh, err) - } + if err := batch.Put(ctx, key, bz); err != nil { + return fmt.Errorf("failed to put mh=%s, err=%w", mh, err) } - return nil - }); err != nil { + }) + if err != nil { return fmt.Errorf("failed to add index entry: %w", err) } if err := batch.Commit(ctx); err != nil { return fmt.Errorf("failed to commit batch: %w", err) } - - if err := s.ds.Sync(ctx, ds.Key{}); err != nil { - return fmt.Errorf("failed to sync puts: %w", err) - } return nil } diff --git a/share/eds/inverted_index_test.go b/share/eds/inverted_index_test.go index f228aa0d92..8fb037bb92 100644 --- a/share/eds/inverted_index_test.go +++ b/share/eds/inverted_index_test.go @@ -47,10 +47,10 @@ func TestMultihashesForShard(t *testing.T) { require.NoError(t, err) require.Equal(t, []shard.Key{shard.KeyFromString("shard1")}, shardKeys) - // 2. Add mh1 to shard2, and ensure that mh1 still points to shard1 + // 2. Add mh1 to shard2, and ensure that mh1 no longer points to shard1 err = invertedIndex.AddMultihashesForShard(ctx, &mockIterator{mhs: mhs[:1]}, shard.KeyFromString("shard2")) require.NoError(t, err) shardKeys, err = invertedIndex.GetShardsForMultihash(ctx, mhs[0]) require.NoError(t, err) - require.Equal(t, []shard.Key{shard.KeyFromString("shard1")}, shardKeys) + require.Equal(t, []shard.Key{shard.KeyFromString("shard2")}, shardKeys) } From 13c2daba63c9d999b9de8c017749187035ad34d3 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:21:43 +0800 Subject: [PATCH 105/388] fix(share/eds): turn off eds store dagstore gc by default (#2529) Dagstore GC performs garbage collection by reclaiming transient files of shards that are currently available but inactive, or errored. We don't use transient files right now, so GC is turned off by default. --- share/eds/store.go | 12 +++++++++--- 1 file changed, 9 insertions(+), 3 deletions(-) diff --git a/share/eds/store.go b/share/eds/store.go index 0683df0d99..9bc539590b 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -34,7 +34,10 @@ const ( indexPath = "/index/" transientsPath = "/transients/" - defaultGCInterval = time.Hour + // GC performs DAG store garbage collection by reclaiming transient files of + // shards that are currently available but inactive, or errored. + // We don't use transient files right now, so GC is turned off by default. + defaultGCInterval = 0 ) var ErrNotFound = errors.New("eds not found in store") @@ -117,13 +120,16 @@ func (s *Store) Start(ctx context.Context) error { return err } // start Store only if DagStore succeeds - ctx, cancel := context.WithCancel(context.Background()) + runCtx, cancel := context.WithCancel(context.Background()) s.cancel = cancel // initialize empty gc result to avoid panic on access s.lastGCResult.Store(&dagstore.GCResult{ Shards: make(map[shard.Key]error), }) - go s.gc(ctx) + + if s.gcInterval != 0 { + go s.gc(runCtx) + } return nil } From 08ffb69f0c14776f6b2acd7cdf774d92524533fd Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 1 Aug 2023 20:39:28 +0800 Subject: [PATCH 106/388] feat(share/shrex/nd): add swamp test for shrex nd (#2227) ## Overview Closes https://github.com/celestiaorg/celestia-node/issues/2119 --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- nodebuilder/tests/nd_test.go | 199 +++++++++++++++++++++++++++++++++++ share/p2p/shrexnd/server.go | 24 +++-- 2 files changed, 215 insertions(+), 8 deletions(-) create mode 100644 nodebuilder/tests/nd_test.go diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go new file mode 100644 index 0000000000..955cff2ffe --- /dev/null +++ b/nodebuilder/tests/nd_test.go @@ -0,0 +1,199 @@ +package tests + +import ( + "context" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/network" + "github.com/stretchr/testify/require" + "go.uber.org/fx" + + "github.com/celestiaorg/celestia-node/nodebuilder" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" +) + +func TestShrexNDFromLights(t *testing.T) { + const ( + blocks = 10 + btime = time.Millisecond * 300 + bsize = 16 + ) + + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) + + sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) + + bridge := sw.NewBridgeNode() + sw.SetBootstrapper(t, bridge) + + cfg := nodebuilder.DefaultConfig(node.Light) + cfg.Share.Discovery.PeersLimit = 1 + light := sw.NewNodeWithConfig(node.Light, cfg) + + err := bridge.Start(ctx) + require.NoError(t, err) + err = light.Start(ctx) + require.NoError(t, err) + + // wait for chain to be filled + require.NoError(t, <-fillDn) + + // first 2 blocks are not filled with data + for i := 3; i < blocks; i++ { + h, err := bridge.HeaderServ.GetByHeight(ctx, uint64(i)) + require.NoError(t, err) + + reqCtx, cancel := context.WithTimeout(ctx, time.Second*5) + + // ensure to fetch random namespace (not the reserved namespace) + namespace := h.DAH.RowRoots[1][:share.NamespaceSize] + + expected, err := bridge.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + require.NoError(t, err) + got, err := light.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + require.NoError(t, err) + + require.True(t, len(got[0].Shares) > 0) + require.Equal(t, expected, got) + + cancel() + } +} + +func TestShrexNDFromLightsWithBadFulls(t *testing.T) { + const ( + blocks = 10 + btime = time.Millisecond * 300 + bsize = 16 + amountOfFulls = 5 + testTimeout = time.Second * 10 + ) + + ctx, cancel := context.WithTimeout(context.Background(), testTimeout) + t.Cleanup(cancel) + + sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) + + bridge := sw.NewBridgeNode() + sw.SetBootstrapper(t, bridge) + + // create full nodes with basic stream.reset handler + ndHandler := func(stream network.Stream) { + _ = stream.Reset() + } + fulls := make([]*nodebuilder.Node, 0, amountOfFulls) + for i := 0; i < amountOfFulls; i++ { + cfg := nodebuilder.DefaultConfig(node.Full) + setTimeInterval(cfg, testTimeout) + full := sw.NewNodeWithConfig(node.Full, cfg, replaceNDServer(cfg, ndHandler), replaceShareGetter()) + fulls = append(fulls, full) + } + + lnConfig := nodebuilder.DefaultConfig(node.Light) + lnConfig.Share.Discovery.PeersLimit = uint(amountOfFulls) + light := sw.NewNodeWithConfig(node.Light, lnConfig) + + // start all nodes + require.NoError(t, bridge.Start(ctx)) + require.NoError(t, startFullNodes(ctx, fulls...)) + require.NoError(t, light.Start(ctx)) + + // wait for chain to fill up + require.NoError(t, <-fillDn) + + // first 2 blocks are not filled with data + for i := 3; i < blocks; i++ { + h, err := bridge.HeaderServ.GetByHeight(ctx, uint64(i)) + require.NoError(t, err) + + if len(h.DAH.RowRoots) != bsize*2 { + // fill blocks does not always fill every block to the given block + // size - this check prevents trying to fetch shares for the parity + // namespace. + continue + } + + reqCtx, cancel := context.WithTimeout(ctx, time.Second*5) + + // ensure to fetch random namespace (not the reserved namespace) + namespace := h.DAH.RowRoots[1][:share.NamespaceSize] + + expected, err := bridge.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + require.NoError(t, err) + require.True(t, len(expected[0].Shares) > 0) + + // choose a random full to test + gotFull, err := fulls[len(fulls)/2].ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + require.NoError(t, err) + require.True(t, len(gotFull[0].Shares) > 0) + + gotLight, err := light.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + require.NoError(t, err) + require.True(t, len(gotLight[0].Shares) > 0) + + require.Equal(t, expected, gotFull) + require.Equal(t, expected, gotLight) + + cancel() + } +} + +func startFullNodes(ctx context.Context, fulls ...*nodebuilder.Node) error { + for _, full := range fulls { + err := full.Start(ctx) + if err != nil { + return err + } + } + return nil +} + +func replaceNDServer(cfg *nodebuilder.Config, handler network.StreamHandler) fx.Option { + return fx.Decorate(fx.Annotate( + func( + host host.Host, + store *eds.Store, + getter *getters.StoreGetter, + network p2p.Network, + ) (*shrexnd.Server, error) { + cfg.Share.ShrExNDParams.WithNetworkID(network.String()) + return shrexnd.NewServer(cfg.Share.ShrExNDParams, host, store, getter) + }, + fx.OnStart(func(ctx context.Context, server *shrexnd.Server) error { + // replace handler for server + server.SetHandler(handler) + return server.Start(ctx) + }), + fx.OnStop(func(ctx context.Context, server *shrexnd.Server) error { + return server.Start(ctx) + }), + )) +} + +func replaceShareGetter() fx.Option { + return fx.Decorate(fx.Annotate( + func( + host host.Host, + store *eds.Store, + storeGetter *getters.StoreGetter, + shrexGetter *getters.ShrexGetter, + network p2p.Network, + ) share.Getter { + cascade := make([]share.Getter, 0, 2) + cascade = append(cascade, storeGetter) + cascade = append(cascade, getters.NewTeeGetter(shrexGetter, store)) + return getters.NewCascadeGetter(cascade) + }, + )) +} diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index d61e6b2e68..830a853946 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -29,8 +29,9 @@ type Server struct { host host.Host protocolID protocol.ID - getter share.Getter - store *eds.Store + handler network.StreamHandler + getter share.Getter + store *eds.Store params *Parameters middleware *p2p.Middleware @@ -52,18 +53,20 @@ func NewServer(params *Parameters, host host.Host, store *eds.Store, getter shar middleware: p2p.NewMiddleware(params.ConcurrencyLimit), } - return srv, nil -} - -// Start starts the server -func (srv *Server) Start(context.Context) error { ctx, cancel := context.WithCancel(context.Background()) srv.cancel = cancel handler := func(s network.Stream) { srv.handleNamespacedData(ctx, s) } - srv.host.SetStreamHandler(srv.protocolID, srv.middleware.RateLimitHandler(handler)) + srv.handler = srv.middleware.RateLimitHandler(handler) + + return srv, nil +} + +// Start starts the server +func (srv *Server) Start(context.Context) error { + srv.host.SetStreamHandler(srv.protocolID, srv.handler) return nil } @@ -74,6 +77,11 @@ func (srv *Server) Stop(context.Context) error { return nil } +// SetHandler sets server handler +func (srv *Server) SetHandler(handler network.StreamHandler) { + srv.handler = handler +} + func (srv *Server) observeRateLimitedRequests() { numRateLimited := srv.middleware.DrainCounter() if numRateLimited > 0 { From d0e271ebaad6af7680cba889c49c39768d50524c Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 2 Aug 2023 09:30:25 +0200 Subject: [PATCH 107/388] feat(share/eds): introduce List method for the Store (#2494) --- share/eds/store.go | 15 +++++++++++++++ share/eds/store_test.go | 17 +++++++++++++++++ 2 files changed, 32 insertions(+) diff --git a/share/eds/store.go b/share/eds/store.go index 9bc539590b..bffbed9f64 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -4,6 +4,7 @@ import ( "bufio" "bytes" "context" + "encoding/hex" "errors" "fmt" "io" @@ -409,6 +410,20 @@ func (s *Store) Has(ctx context.Context, root share.DataHash) (bool, error) { } } +// List lists all the registered EDSes. +func (s *Store) List() ([]share.DataHash, error) { + shards := s.dgstr.AllShardsInfo() + hashes := make([]share.DataHash, 0, len(shards)) + for shrd := range shards { + hash, err := hex.DecodeString(shrd.String()) + if err != nil { + return nil, err + } + hashes = append(hashes, hash) + } + return hashes, nil +} + func setupPath(basepath string) error { err := os.MkdirAll(basepath+blocksPath, os.ModePerm) if err != nil { diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 491b63c48a..4b263e7062 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -154,6 +154,23 @@ func TestEDSStore(t *testing.T) { _, err = edsStore.cache.Get(shardKey) assert.NoError(t, err, errCacheMiss) }) + + t.Run("List", func(t *testing.T) { + const amount = 10 + hashes := make([]share.DataHash, 0, amount) + for range make([]byte, amount) { + eds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), eds) + require.NoError(t, err) + hashes = append(hashes, dah.Hash()) + } + + hashesOut, err := edsStore.List() + require.NoError(t, err) + for _, hash := range hashes { + assert.Contains(t, hashesOut, hash) + } + }) } // TestEDSStore_GC verifies that unused transient shards are collected by the GC periodically. From 165f81ae9f848b383bea054c55232f2a3aeb2685 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 2 Aug 2023 18:41:46 +0800 Subject: [PATCH 108/388] fix(share/eds): print error for stuck register shard (#2516) Sometimes register shard can take longer, that provided context timeout. This PR logs errors if register shard is stuck. --- share/eds/store.go | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) diff --git a/share/eds/store.go b/share/eds/store.go index bffbed9f64..cd85380ba1 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -203,6 +203,7 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext select { case <-ctx.Done(): + go logLateResult(ch, time.Minute*5) return ctx.Err() case result := <-ch: if result.Error != nil { @@ -212,6 +213,31 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext } } +// waitForResult waits for a result from the res channel for a maximum duration specified by +// maxWait. If the result is not received within the specified duration, it logs an error +// indicating that the parent context has expired and the shard registration is stuck. If a result +// is received, it checks for any error and logs appropriate messages. +func logLateResult(res <-chan dagstore.ShardResult, maxWait time.Duration) { + tnow := time.Now() + select { + case <-time.After(maxWait): + log.Errorf("parent context is expired, while register shard is stuck for more than %v sec", time.Since(tnow)) + return + case result := <-res: + // don't log if result was received right after launch of the func + if time.Since(tnow) < time.Second { + return + } + if result.Error != nil { + log.Errorf("failed to register shard after context expired: %v ago, err: %w", time.Since(tnow), result.Error) + return + } + log.Warnf("parent context expired, but register shard finished with no error,"+ + " after context expired: %v ago", time.Since(tnow)) + return + } +} + // GetCAR takes a DataRoot and returns a buffered reader to the respective EDS serialized as a // CARv1 file. // The Reader strictly reads the CAR header and first quadrant (1/4) of the EDS, omitting all the From 110a41cf70e88a165f68c50b23e4f4789eecfcaa Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Wed, 2 Aug 2023 15:56:17 +0200 Subject: [PATCH 109/388] chore(deps): bump celestia-app to rc10 (#2508) --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 875f6557e7..ef1d8cc01a 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc9 + github.com/celestiaorg/celestia-app v1.0.0-rc10 github.com/celestiaorg/go-fraud v0.1.2 github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 @@ -332,7 +332,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app diff --git a/go.sum b/go.sum index b6a8f996e0..82e18196ba 100644 --- a/go.sum +++ b/go.sum @@ -344,12 +344,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc9 h1:6xDYE+OziXO/rLeYy/MutnJpE8M2sIPryZ/ifSWUmdc= -github.com/celestiaorg/celestia-app v1.0.0-rc9/go.mod h1:aGFnIIdA30DtFzznYbcfMdNnXiUebfEUkkrQu8imC3I= +github.com/celestiaorg/celestia-app v1.0.0-rc10 h1:wglTplF5qVVOtSYCGtNHz8qA37azD+fgcxCKAphYL6c= +github.com/celestiaorg/celestia-app v1.0.0-rc10/go.mod h1:A17xfzurB2RHpplh6+nGMeC9hCquu0aMYnanOeifDO0= github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHVxcTzhzQW44KgTcooR3OxnK4= github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= -github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13 h1:N1PrCWcYkaODeIQyyVBmDKDTwiQWZ31bgtTEYIGeby8= -github.com/celestiaorg/cosmos-sdk v1.16.0-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= +github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 h1:CxEQDQEQR1ypB+VUmCISIqFVmHfb+mx8x+zh7rHbyU8= +github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= From fb8702d5814f258926eea16e0e30c32f8b877930 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 2 Aug 2023 19:16:50 +0200 Subject: [PATCH 110/388] chore: bump app v1.0.0 rc11 (#2527) --- core/client_test.go | 4 +- core/exchange_test.go | 2 +- core/fetcher_test.go | 4 +- core/testing.go | 86 ++++----------------------- go.mod | 2 +- go.sum | 4 +- nodebuilder/tests/blob_test.go | 4 +- nodebuilder/tests/reconstruct_test.go | 10 ++++ nodebuilder/tests/swamp/config.go | 25 ++++---- nodebuilder/tests/swamp/swamp.go | 6 +- 10 files changed, 46 insertions(+), 101 deletions(-) diff --git a/core/client_test.go b/core/client_test.go index 7467b3c3a3..8ad9060555 100644 --- a/core/client_test.go +++ b/core/client_test.go @@ -10,7 +10,7 @@ import ( ) func TestRemoteClient_Status(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) client := StartTestNode(t).Client @@ -20,7 +20,7 @@ func TestRemoteClient_Status(t *testing.T) { } func TestRemoteClient_StartBlockSubscription_And_GetBlock(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) t.Cleanup(cancel) client := StartTestNode(t).Client diff --git a/core/exchange_test.go b/core/exchange_test.go index f6302b5742..579c69fba6 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -31,7 +31,7 @@ func TestCoreExchange_RequestHeaders(t *testing.T) { assert.Equal(t, 10, len(headers)) } -func createCoreFetcher(t *testing.T, cfg *TestConfig) (*BlockFetcher, testnode.Context) { +func createCoreFetcher(t *testing.T, cfg *testnode.Config) (*BlockFetcher, testnode.Context) { cctx := StartTestNodeWithConfig(t, cfg) // wait for height 2 in order to be able to start submitting txs (this prevents // flakiness with accessing account state) diff --git a/core/fetcher_test.go b/core/fetcher_test.go index de7d1ba05e..3380dbb402 100644 --- a/core/fetcher_test.go +++ b/core/fetcher_test.go @@ -11,7 +11,7 @@ import ( ) func TestBlockFetcher_GetBlock_and_SubscribeNewBlockEvent(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) t.Cleanup(cancel) client := StartTestNode(t).Client @@ -42,7 +42,7 @@ func TestBlockFetcher_GetBlock_and_SubscribeNewBlockEvent(t *testing.T) { // TestBlockFetcherHeaderValues tests that both the Commit and ValidatorSet // endpoints are working as intended. func TestBlockFetcherHeaderValues(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*3) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) t.Cleanup(cancel) client := StartTestNode(t).Client diff --git a/core/testing.go b/core/testing.go index 393ec62c09..8d29ce9bbc 100644 --- a/core/testing.go +++ b/core/testing.go @@ -1,30 +1,18 @@ package core import ( - "fmt" "net" "net/url" "testing" + "time" - appconfig "github.com/cosmos/cosmos-sdk/server/config" "github.com/stretchr/testify/require" tmconfig "github.com/tendermint/tendermint/config" tmrand "github.com/tendermint/tendermint/libs/rand" - tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/celestiaorg/celestia-app/test/util/testnode" ) -// TestConfig encompasses all the configs required to run test Tendermint + Celestia App tandem. -type TestConfig struct { - ConsensusParams *tmproto.ConsensusParams - Tendermint *tmconfig.Config - App *appconfig.Config - - Accounts []string - SuppressLogs bool -} - // DefaultTestConfig returns the default testing configuration for Tendermint + Celestia App tandem. // // It fetches free ports from OS and sets them into configs, s.t. @@ -32,17 +20,8 @@ type TestConfig struct { // multiple tests nodes in parallel. // // Additionally, it instructs Tendermint + Celestia App tandem to setup 10 funded accounts. -func DefaultTestConfig() *TestConfig { - conCfg := testnode.DefaultParams() - - tnCfg := testnode.DefaultTendermintConfig() - tnCfg.RPC.ListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", getFreePort()) - tnCfg.RPC.GRPCListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", getFreePort()) - tnCfg.P2P.ListenAddress = fmt.Sprintf("tcp://127.0.0.1:%d", getFreePort()) - - appCfg := testnode.DefaultAppConfig() - appCfg.GRPC.Address = fmt.Sprintf("127.0.0.1:%d", getFreePort()) - appCfg.API.Address = fmt.Sprintf("tcp://127.0.0.1:%d", getFreePort()) +func DefaultTestConfig() *testnode.Config { + cfg := testnode.DefaultConfig() // instructs creating funded accounts // 10 usually is enough for testing @@ -51,13 +30,13 @@ func DefaultTestConfig() *TestConfig { accounts[i] = tmrand.Str(9) } - return &TestConfig{ - ConsensusParams: conCfg, - Tendermint: tnCfg, - App: appCfg, - Accounts: accounts, - SuppressLogs: true, - } + cfg.TmConfig.Consensus.TimeoutCommit = time.Millisecond * 200 + + cfg = cfg. + WithAccounts(accounts). + WithSupressLogs(true) + + return cfg } // StartTestNode simply starts Tendermint and Celestia App tandem with default testing @@ -67,40 +46,13 @@ func StartTestNode(t *testing.T) testnode.Context { } // StartTestNodeWithConfig starts Tendermint and Celestia App tandem with custom configuration. -func StartTestNodeWithConfig(t *testing.T, cfg *TestConfig) testnode.Context { - state, kr, err := testnode.DefaultGenesisState(cfg.Accounts...) - require.NoError(t, err) - - tmNode, app, cctx, err := testnode.New( - t, - cfg.ConsensusParams, - cfg.Tendermint, - cfg.SuppressLogs, - state, - kr, - "private", - ) - require.NoError(t, err) - - cctx, cleanupCoreNode, err := testnode.StartNode(tmNode, cctx) - require.NoError(t, err) - t.Cleanup(func() { - err := cleanupCoreNode() - require.NoError(t, err) - }) - - cctx, cleanupGRPCServer, err := StartGRPCServer(app, cfg.App, cctx) - require.NoError(t, err) - t.Cleanup(func() { - err := cleanupGRPCServer() - require.NoError(t, err) - }) - +func StartTestNodeWithConfig(t *testing.T, cfg *testnode.Config) testnode.Context { + cctx, _, _ := testnode.NewNetwork(t, cfg) // we want to test over remote http client, // so we are as close to the real environment as possible // however, it might be useful to use local tendermint client // if you need to debug something inside of it - ip, port, err := getEndpoint(cfg.Tendermint) + ip, port, err := getEndpoint(cfg.TmConfig) require.NoError(t, err) client, err := NewRemote(ip, port) require.NoError(t, err) @@ -116,18 +68,6 @@ func StartTestNodeWithConfig(t *testing.T, cfg *TestConfig) testnode.Context { return cctx } -func getFreePort() int { - a, err := net.ResolveTCPAddr("tcp", "localhost:0") - if err == nil { - var l *net.TCPListener - if l, err = net.ListenTCP("tcp", a); err == nil { - defer l.Close() - return l.Addr().(*net.TCPAddr).Port - } - } - panic("while getting free port: " + err.Error()) -} - func getEndpoint(cfg *tmconfig.Config) (string, string, error) { url, err := url.Parse(cfg.RPC.ListenAddress) if err != nil { diff --git a/go.mod b/go.mod index ef1d8cc01a..c89d7d77d3 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc10 + github.com/celestiaorg/celestia-app v1.0.0-rc11 github.com/celestiaorg/go-fraud v0.1.2 github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 diff --git a/go.sum b/go.sum index 82e18196ba..f8cedecfd1 100644 --- a/go.sum +++ b/go.sum @@ -344,8 +344,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc10 h1:wglTplF5qVVOtSYCGtNHz8qA37azD+fgcxCKAphYL6c= -github.com/celestiaorg/celestia-app v1.0.0-rc10/go.mod h1:A17xfzurB2RHpplh6+nGMeC9hCquu0aMYnanOeifDO0= +github.com/celestiaorg/celestia-app v1.0.0-rc11 h1:u94drhPdvgwNvx0F2xGib+fyOyP+fIvp3aL6Avd/HT0= +github.com/celestiaorg/celestia-app v1.0.0-rc11/go.mod h1:A17xfzurB2RHpplh6+nGMeC9hCquu0aMYnanOeifDO0= github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHVxcTzhzQW44KgTcooR3OxnK4= github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 h1:CxEQDQEQR1ypB+VUmCISIqFVmHfb+mx8x+zh7rHbyU8= diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 6a7db50c7c..a9e3a0464b 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -19,9 +19,9 @@ import ( ) func TestBlobModule(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) t.Cleanup(cancel) - sw := swamp.NewSwamp(t) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second*1)) appBlobs0, err := blobtest.GenerateV0Blobs([]int{8, 4}, true) require.NoError(t, err) diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index a4e3296c2f..8b1490272c 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -53,6 +53,11 @@ func TestFullReconstructFromBridge(t *testing.T) { err := bridge.Start(ctx) require.NoError(t, err) + // TODO: This is required to avoid flakes coming from unfinished retry + // mechanism for the same peer in go-header + _, err = bridge.HeaderServ.WaitForHeight(ctx, uint64(blocks)) + require.NoError(t, err) + cfg := nodebuilder.DefaultConfig(node.Full) cfg.Share.UseShareExchange = false cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, getMultiAddr(t, bridge.Host)) @@ -125,6 +130,11 @@ func TestFullReconstructFromLights(t *testing.T) { require.NoError(t, bridge.Start(ctx)) bootstrapperAddr := host.InfoFromHost(bootstrapper.Host) + // TODO: This is required to avoid flakes coming from unfinished retry + // mechanism for the same peer in go-header + _, err = bridge.HeaderServ.WaitForHeight(ctx, uint64(blocks)) + require.NoError(t, err) + cfg = nodebuilder.DefaultConfig(node.Full) setTimeInterval(cfg, defaultTimeInterval) cfg.Share.UseShareExchange = false diff --git a/nodebuilder/tests/swamp/config.go b/nodebuilder/tests/swamp/config.go index e6c06b40cc..047baa9f59 100644 --- a/nodebuilder/tests/swamp/config.go +++ b/nodebuilder/tests/swamp/config.go @@ -3,35 +3,30 @@ package swamp import ( "time" + "github.com/celestiaorg/celestia-app/test/util/testnode" + "github.com/celestiaorg/celestia-node/core" ) -// Config struct represents a set of pre-requisite attributes from the test scenario -type Config struct { - *core.TestConfig -} - // DefaultConfig creates a celestia-app instance with a block time of around // 100ms -func DefaultConfig() *Config { +func DefaultConfig() *testnode.Config { cfg := core.DefaultTestConfig() // timeout commit lower than this tend to be flakier - cfg.Tendermint.Consensus.TimeoutCommit = 200 * time.Millisecond - return &Config{ - cfg, - } + cfg.TmConfig.Consensus.TimeoutCommit = 200 * time.Millisecond + return cfg } // Option for modifying Swamp's Config. -type Option func(*Config) +type Option func(*testnode.Config) // WithBlockTime sets a custom interval for block creation. func WithBlockTime(t time.Duration) Option { - return func(c *Config) { + return func(c *testnode.Config) { // for empty block - c.Tendermint.Consensus.CreateEmptyBlocksInterval = t + c.TmConfig.Consensus.CreateEmptyBlocksInterval = t // for filled block - c.Tendermint.Consensus.TimeoutCommit = t - c.Tendermint.Consensus.SkipTimeoutCommit = false + c.TmConfig.Consensus.TimeoutCommit = t + c.TmConfig.Consensus.SkipTimeoutCommit = false } } diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 3532635883..58584912be 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -48,7 +48,7 @@ const DefaultTestTimeout = time.Minute * 5 // - trustedHash taken from the CoreClient and shared between nodes type Swamp struct { t *testing.T - cfg *Config + cfg *testnode.Config Network mocknet.Mocknet Bootstrappers []ma.Multiaddr @@ -76,7 +76,7 @@ func NewSwamp(t *testing.T, options ...Option) *Swamp { // Now, we are making an assumption that consensus mechanism is already tested out // so, we are not creating bridge nodes with each one containing its own core client // instead we are assigning all created BNs to 1 Core from the swamp - cctx := core.StartTestNodeWithConfig(t, ic.TestConfig) + cctx := core.StartTestNodeWithConfig(t, ic) swp := &Swamp{ t: t, cfg: ic, @@ -188,7 +188,7 @@ func (s *Swamp) setupGenesis() { func (s *Swamp) DefaultTestConfig(tp node.Type) *nodebuilder.Config { cfg := nodebuilder.DefaultConfig(tp) - ip, port, err := net.SplitHostPort(s.cfg.App.GRPC.Address) + ip, port, err := net.SplitHostPort(s.cfg.AppConfig.GRPC.Address) require.NoError(s.t, err) cfg.Core.IP = ip From 135977825d57b574759331f9a71a375fec9d6c5d Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 3 Aug 2023 23:48:16 +0800 Subject: [PATCH 111/388] fix(share/p2p): fix race in shrexeds test (#2534) Quick fix for race in test. Found in CI run --- share/p2p/shrexeds/exchange_test.go | 14 ++++++++++---- 1 file changed, 10 insertions(+), 4 deletions(-) diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index b1d2e0ad18..14e220a8f9 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -47,21 +47,27 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is unavailable initially, but is found after multiple requests t.Run("EDS_AvailableAfterDelay", func(t *testing.T) { - storageDelay := time.Second eds := edstest.RandEDS(t, 4) dah, err := da.NewDataAvailabilityHeader(eds) require.NoError(t, err) + + lock := make(chan struct{}) go func() { - time.Sleep(storageDelay) + <-lock err = store.Put(ctx, dah.Hash(), eds) - // require.NoError(t, err) + require.NoError(t, err) + lock <- struct{}{} }() requestedEDS, err := client.RequestEDS(ctx, dah.Hash(), server.host.ID()) assert.ErrorIs(t, err, p2p.ErrNotFound) assert.Nil(t, requestedEDS) - time.Sleep(storageDelay * 2) + // unlock write + lock <- struct{}{} + // wait for write to finish + <-lock + requestedEDS, err = client.RequestEDS(ctx, dah.Hash(), server.host.ID()) assert.NoError(t, err) assert.Equal(t, eds.Flattened(), requestedEDS.Flattened()) From f7da123759c474826f8bd6574fc2f41859bf5bcc Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 2 Aug 2023 18:31:44 +0800 Subject: [PATCH 112/388] feat(share/eds)!: upgrade badger to v4 (#2520) Upgrades badger to latest version (v4). Also update badger config to the one shown no hangups. --- go.mod | 8 +++++--- go.sum | 16 +++++++++------- nodebuilder/store.go | 25 ++++--------------------- 3 files changed, 18 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index c89d7d77d3..50d2ddc9a9 100644 --- a/go.mod +++ b/go.mod @@ -11,6 +11,7 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc11 + github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.1.2 github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 @@ -19,7 +20,6 @@ require ( github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 - github.com/dgraph-io/badger/v2 v2.2007.4 github.com/etclabscore/go-openrpc-reflect v0.0.37 github.com/filecoin-project/dagstore v0.5.6 github.com/filecoin-project/go-jsonrpc v0.3.1 @@ -33,7 +33,6 @@ require ( github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-ds-badger2 v0.1.3 github.com/ipfs/go-ipfs-blockstore v1.2.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipfs-exchange-offline v0.3.0 @@ -131,7 +130,9 @@ require ( github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 // indirect github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect - github.com/dgraph-io/ristretto v0.1.0 // indirect + github.com/dgraph-io/badger/v2 v2.2007.4 // indirect + github.com/dgraph-io/badger/v4 v4.1.0 // indirect + github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/docker/go-units v0.5.0 // indirect github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac // indirect @@ -164,6 +165,7 @@ require ( github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/btree v1.1.2 // indirect + github.com/google/flatbuffers v1.12.1 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect diff --git a/go.sum b/go.sum index f8cedecfd1..2593760535 100644 --- a/go.sum +++ b/go.sum @@ -228,7 +228,6 @@ github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRr github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= -github.com/DataDog/zstd v1.4.1/go.mod h1:1jcaCB/ufaK+sKp1NBhlGmpz41jOoPQ35bpF36t7BBo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -352,6 +351,8 @@ github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 h1:CxEQDQEQR1ypB+VUmCISIq github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= +github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= +github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= github.com/celestiaorg/go-fraud v0.1.2/go.mod h1:kHZXQY+6gd1kYkoWRFFKgWyrLPWRgDN3vd1Ll9gE/oo= github.com/celestiaorg/go-header v0.2.12 h1:3H9nir20+MTY1vXbLxOUOV05ZspotR6JOiZGKxACHCQ= @@ -510,14 +511,15 @@ github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhY github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= -github.com/dgraph-io/badger/v2 v2.2007.3/go.mod h1:26P/7fbL4kUZVEVKLAKXkBXKOydDmM2p1e+NhhnBCAE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= +github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= -github.com/dgraph-io/ristretto v0.1.0 h1:Jv3CGQHp9OjuMBSne1485aDpUkTKEcUqF+jm/LuerPI= -github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu6v9Vcb3Q9ug= +github.com/dgraph-io/ristretto v0.1.1 h1:6CWw5tJNgpegArSHpNHJKldNeq03FQCwYvfMVWajOK8= +github.com/dgraph-io/ristretto v0.1.1/go.mod h1:S1GPSBCYCIhmVNfcth17y2zZtQT6wzkzgwUve0VDWWA= github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-bitstream v0.0.0-20180413035011-3522498ce2c8/go.mod h1:VMaSuZ+SZcx/wljOQKvp5srsbCiKDEb6K2wC4+PiBmQ= github.com/dgryski/go-farm v0.0.0-20190104051053-3adb47b1fb0f/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= @@ -740,7 +742,6 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3 h1:KhyjKVUg7Usr/dYsdSqoFveMYd5ko72D+zANwlG1mmg= github.com/golang/protobuf v1.5.3/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= -github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb h1:PBC98N2aIaM3XXiurYmW7fx4GZkL8feAMVq7nEjURHk= @@ -751,6 +752,8 @@ github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/flatbuffers v1.11.0/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= +github.com/google/flatbuffers v1.12.1 h1:MVlul7pQNoDzWRLTw5imwYsl+usrS1TXG2H4jg6ImGw= +github.com/google/flatbuffers v1.12.1/go.mod h1:1AeVuKshWv4vARoZatz6mlQ0JxURH0Kv5+zNeJKJCa8= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -999,8 +1002,6 @@ github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaH github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= -github.com/ipfs/go-ds-badger2 v0.1.3 h1:Zo9JicXJ1DmXTN4KOw7oPXkspZ0AWHcAFCP1tQKnegg= -github.com/ipfs/go-ds-badger2 v0.1.3/go.mod h1:TPhhljfrgewjbtuL/tczP8dNrBYwwk+SdPYbms/NO9w= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= @@ -2467,6 +2468,7 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= diff --git a/nodebuilder/store.go b/nodebuilder/store.go index f0957a16b6..6d313893b1 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -5,13 +5,14 @@ import ( "fmt" "path/filepath" "sync" + "time" "github.com/cosmos/cosmos-sdk/crypto/keyring" - "github.com/dgraph-io/badger/v2/options" "github.com/ipfs/go-datastore" - dsbadger "github.com/ipfs/go-ds-badger2" "github.com/mitchellh/go-homedir" + dsbadger "github.com/celestiaorg/go-ds-badger4" + "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/keystore" ) @@ -118,25 +119,7 @@ func (f *fsStore) Datastore() (datastore.Batching, error) { } opts := dsbadger.DefaultOptions // this should be copied - - // Badger sets ValueThreshold to 1K by default and this makes shares being stored in LSM tree - // instead of the value log, so we change the value to be lower than share size, - // so shares are store in value log. For value log and LSM definitions - opts.ValueThreshold = 128 - // We always write unique values to Badger transaction so there is no need to detect conflicts. - opts.DetectConflicts = false - // Use MemoryMap for better performance - opts.ValueLogLoadingMode = options.MemoryMap - opts.TableLoadingMode = options.MemoryMap - // Truncate set to true will truncate corrupted data on start if there is any. - // If we don't truncate, the node will refuse to start and will beg for recovering, etc. - // If we truncate, the node will start with any uncorrupted data and reliably sync again what was - // corrupted in most cases. - opts.Truncate = true - // MaxTableSize defines in memory and on disk size of LSM tree - // Bigger values constantly takes more RAM - // TODO(@Wondertan): Make configurable with more conservative defaults for Light Node - opts.MaxTableSize = 64 << 20 + opts.GcInterval = time.Minute * 10 ds, err := dsbadger.NewDatastore(dataPath(f.path), &opts) if err != nil { From 3c257589d6070941c94201bec97034b34a48ca34 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 2 Aug 2023 22:08:50 +0800 Subject: [PATCH 113/388] fix(share/eds)!: use separate badger for inverted index (#2517) Having same badger instance for dagstore shards and inverted index has proven to have hangup on Put operation for 1-10min. During tests it has shown, that splitting badger into 2 instances solves the hangup issue. Also once #2479 is implemented and inverted index will needs to be removed. Separate badger will be much easier for users migration. --- share/eds/inverted_index.go | 26 ++++++++++++++++++++++---- share/eds/inverted_index_test.go | 9 ++++----- share/eds/store.go | 28 +++++++++++++++++----------- 3 files changed, 43 insertions(+), 20 deletions(-) diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index 0e3665d973..d302932369 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -8,10 +8,13 @@ import ( "github.com/filecoin-project/dagstore/index" "github.com/filecoin-project/dagstore/shard" ds "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" "github.com/multiformats/go-multihash" + + dsbadger "github.com/celestiaorg/go-ds-badger4" ) +const invertedIndexPath = "/inverted_index/" + // simpleInvertedIndex is an inverted index that only stores a single shard key per multihash. Its // implementation is modified from the default upstream implementation in dagstore/index. type simpleInvertedIndex struct { @@ -21,10 +24,21 @@ type simpleInvertedIndex struct { // newSimpleInvertedIndex returns a new inverted index that only stores a single shard key per // multihash. This is because we use badger as a storage backend, so updates are expensive, and we // don't care which shard is used to serve a cid. -func newSimpleInvertedIndex(dts ds.Batching) *simpleInvertedIndex { - return &simpleInvertedIndex{ - ds: namespace.Wrap(dts, ds.NewKey("/inverted/index")), +func newSimpleInvertedIndex(storePath string) (*simpleInvertedIndex, error) { + opts := dsbadger.DefaultOptions // this should be copied + // turn off value log GC + opts.GcInterval = 0 + // 20 compactors show to have no hangups on put operation up to 40k blocks with eds size 128. + opts.NumCompactors = 20 + // use minimum amount of NumLevelZeroTables to trigger L0 compaction faster + opts.NumLevelZeroTables = 1 + + ds, err := dsbadger.NewDatastore(storePath+invertedIndexPath, &opts) + if err != nil { + return nil, fmt.Errorf("can't open Badger Datastore: %w", err) } + + return &simpleInvertedIndex{ds: ds}, nil } func (s *simpleInvertedIndex) AddMultihashesForShard( @@ -75,3 +89,7 @@ func (s *simpleInvertedIndex) GetShardsForMultihash(ctx context.Context, mh mult return []shard.Key{shardKey}, nil } + +func (s *simpleInvertedIndex) close() error { + return s.ds.Close() +} diff --git a/share/eds/inverted_index_test.go b/share/eds/inverted_index_test.go index 8fb037bb92..e83c2be267 100644 --- a/share/eds/inverted_index_test.go +++ b/share/eds/inverted_index_test.go @@ -5,8 +5,6 @@ import ( "testing" "github.com/filecoin-project/dagstore/shard" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" "github.com/multiformats/go-multihash" "github.com/stretchr/testify/require" ) @@ -37,11 +35,12 @@ func TestMultihashesForShard(t *testing.T) { } mi := &mockIterator{mhs: mhs} - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - invertedIndex := newSimpleInvertedIndex(ds) + path := t.TempDir() + invertedIndex, err := newSimpleInvertedIndex(path) + require.NoError(t, err) // 1. Add all 3 multihashes to shard1 - err := invertedIndex.AddMultihashesForShard(ctx, mi, shard.KeyFromString("shard1")) + err = invertedIndex.AddMultihashesForShard(ctx, mi, shard.KeyFromString("shard1")) require.NoError(t, err) shardKeys, err := invertedIndex.GetShardsForMultihash(ctx, mhs[0]) require.NoError(t, err) diff --git a/share/eds/store.go b/share/eds/store.go index cd85380ba1..2218e13954 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -56,8 +56,8 @@ type Store struct { cache *blockstoreCache bs bstore.Blockstore - topIdx index.Inverted - carIdx index.FullIndexRepo + carIdx index.FullIndexRepo + invertedIdx *simpleInvertedIndex basepath string gcInterval time.Duration @@ -83,14 +83,17 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { return nil, fmt.Errorf("failed to create index repository: %w", err) } - invertedRepo := newSimpleInvertedIndex(ds) + invertedIdx, err := newSimpleInvertedIndex(basepath) + if err != nil { + return nil, fmt.Errorf("failed to create index: %w", err) + } dagStore, err := dagstore.NewDAGStore( dagstore.Config{ TransientsDir: basepath + transientsPath, IndexRepo: fsRepo, Datastore: ds, MountRegistry: r, - TopLevelIndex: invertedRepo, + TopLevelIndex: invertedIdx, }, ) if err != nil { @@ -103,13 +106,13 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { } store := &Store{ - basepath: basepath, - dgstr: dagStore, - topIdx: invertedRepo, - carIdx: fsRepo, - gcInterval: defaultGCInterval, - mounts: r, - cache: cache, + basepath: basepath, + dgstr: dagStore, + carIdx: fsRepo, + invertedIdx: invertedIdx, + gcInterval: defaultGCInterval, + mounts: r, + cache: cache, } store.bs = newBlockstore(store, cache) return store, nil @@ -137,6 +140,9 @@ func (s *Store) Start(ctx context.Context) error { // Stop stops the underlying DAGStore. func (s *Store) Stop(context.Context) error { defer s.cancel() + if err := s.invertedIdx.close(); err != nil { + return err + } return s.dgstr.Close() } From 9ec085baf103397c94e6405957044928c3b0dbc1 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 2 Aug 2023 17:23:33 +0300 Subject: [PATCH 114/388] feat(share/proof)!: use nmt definition of Proof (#2510) Resolves #2512 --- Makefile | 3 +- share/eds/byzantine/bad_encoding.go | 4 +- share/eds/byzantine/pb/share.pb.go | 353 +++------------------------ share/eds/byzantine/pb/share.proto | 10 +- share/eds/byzantine/share_proof.go | 21 +- share/p2p/shrexnd/client.go | 7 +- share/p2p/shrexnd/params.go | 2 +- share/p2p/shrexnd/pb/share.pb.go | 356 +++------------------------- share/p2p/shrexnd/pb/share.proto | 10 +- share/p2p/shrexnd/server.go | 12 +- 10 files changed, 90 insertions(+), 688 deletions(-) diff --git a/Makefile b/Makefile index 2567b225d3..1044d75d3b 100644 --- a/Makefile +++ b/Makefile @@ -137,13 +137,14 @@ PB_PKGS=$(shell find . -name 'pb' -type d) PB_CORE=$(shell go list -f {{.Dir}} -m github.com/tendermint/tendermint) PB_GOGO=$(shell go list -f {{.Dir}} -m github.com/gogo/protobuf) PB_CELESTIA_APP=$(shell go list -f {{.Dir}} -m github.com/celestiaorg/celestia-app) +PB_NMT=$(shell go list -f {{.Dir}} -m github.com/celestiaorg/nmt) ## pb-gen: Generate protobuf code for all /pb/*.proto files in the project. pb-gen: @echo '--> Generating protobuf' @for dir in $(PB_PKGS); \ do for file in `find $$dir -type f -name "*.proto"`; \ - do protoc -I=. -I=${PB_CORE}/proto/ -I=${PB_GOGO} -I=${PB_CELESTIA_APP}/proto --gogofaster_out=paths=source_relative:. $$file; \ + do protoc -I=. -I=${PB_CORE}/proto/ -I=${PB_GOGO} -I=${PB_CELESTIA_APP}/proto -I=${PB_NMT} --gogofaster_out=paths=source_relative:. $$file; \ echo '-->' $$file; \ done; \ done; diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 95f2a7fd03..3c5bc6951b 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -17,7 +17,9 @@ import ( ) const ( - BadEncoding fraud.ProofType = "badencoding" + version = "v0.1" + + BadEncoding fraud.ProofType = "badencoding" + version ) func init() { diff --git a/share/eds/byzantine/pb/share.pb.go b/share/eds/byzantine/pb/share.pb.go index 33b9cdd1ab..4186eabc64 100644 --- a/share/eds/byzantine/pb/share.pb.go +++ b/share/eds/byzantine/pb/share.pb.go @@ -5,6 +5,7 @@ package share_eds_byzantine_pb import ( fmt "fmt" + pb "github.com/celestiaorg/nmt/pb" proto "github.com/gogo/protobuf/proto" io "io" math "math" @@ -47,84 +48,16 @@ func (Axis) EnumDescriptor() ([]byte, []int) { return fileDescriptor_d28ce8f160a920d1, []int{0} } -type MerkleProof struct { - Start int64 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` - End int64 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` - Nodes [][]byte `protobuf:"bytes,3,rep,name=nodes,proto3" json:"nodes,omitempty"` - LeafHash []byte `protobuf:"bytes,4,opt,name=leaf_hash,json=leafHash,proto3" json:"leaf_hash,omitempty"` -} - -func (m *MerkleProof) Reset() { *m = MerkleProof{} } -func (m *MerkleProof) String() string { return proto.CompactTextString(m) } -func (*MerkleProof) ProtoMessage() {} -func (*MerkleProof) Descriptor() ([]byte, []int) { - return fileDescriptor_d28ce8f160a920d1, []int{0} -} -func (m *MerkleProof) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *MerkleProof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_MerkleProof.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *MerkleProof) XXX_Merge(src proto.Message) { - xxx_messageInfo_MerkleProof.Merge(m, src) -} -func (m *MerkleProof) XXX_Size() int { - return m.Size() -} -func (m *MerkleProof) XXX_DiscardUnknown() { - xxx_messageInfo_MerkleProof.DiscardUnknown(m) -} - -var xxx_messageInfo_MerkleProof proto.InternalMessageInfo - -func (m *MerkleProof) GetStart() int64 { - if m != nil { - return m.Start - } - return 0 -} - -func (m *MerkleProof) GetEnd() int64 { - if m != nil { - return m.End - } - return 0 -} - -func (m *MerkleProof) GetNodes() [][]byte { - if m != nil { - return m.Nodes - } - return nil -} - -func (m *MerkleProof) GetLeafHash() []byte { - if m != nil { - return m.LeafHash - } - return nil -} - type Share struct { - Data []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"` - Proof *MerkleProof `protobuf:"bytes,2,opt,name=Proof,proto3" json:"Proof,omitempty"` + Data []byte `protobuf:"bytes,1,opt,name=Data,proto3" json:"Data,omitempty"` + Proof *pb.Proof `protobuf:"bytes,2,opt,name=Proof,proto3" json:"Proof,omitempty"` } func (m *Share) Reset() { *m = Share{} } func (m *Share) String() string { return proto.CompactTextString(m) } func (*Share) ProtoMessage() {} func (*Share) Descriptor() ([]byte, []int) { - return fileDescriptor_d28ce8f160a920d1, []int{1} + return fileDescriptor_d28ce8f160a920d1, []int{0} } func (m *Share) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -160,7 +93,7 @@ func (m *Share) GetData() []byte { return nil } -func (m *Share) GetProof() *MerkleProof { +func (m *Share) GetProof() *pb.Proof { if m != nil { return m.Proof } @@ -179,7 +112,7 @@ func (m *BadEncoding) Reset() { *m = BadEncoding{} } func (m *BadEncoding) String() string { return proto.CompactTextString(m) } func (*BadEncoding) ProtoMessage() {} func (*BadEncoding) Descriptor() ([]byte, []int) { - return fileDescriptor_d28ce8f160a920d1, []int{2} + return fileDescriptor_d28ce8f160a920d1, []int{1} } func (m *BadEncoding) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) @@ -245,7 +178,6 @@ func (m *BadEncoding) GetAxis() Axis { func init() { proto.RegisterEnum("share.eds.byzantine.pb.Axis", Axis_name, Axis_value) - proto.RegisterType((*MerkleProof)(nil), "share.eds.byzantine.pb.MerkleProof") proto.RegisterType((*Share)(nil), "share.eds.byzantine.pb.Share") proto.RegisterType((*BadEncoding)(nil), "share.eds.byzantine.pb.BadEncoding") } @@ -255,78 +187,26 @@ func init() { } var fileDescriptor_d28ce8f160a920d1 = []byte{ - // 347 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x91, 0x41, 0x4b, 0xf3, 0x30, - 0x1c, 0xc6, 0x9b, 0xb7, 0xed, 0xde, 0xf7, 0xfd, 0x77, 0xca, 0x08, 0x32, 0x02, 0x6a, 0x28, 0xf5, - 0x52, 0x3c, 0xb4, 0x32, 0xf1, 0xe0, 0xd1, 0xa9, 0x30, 0x41, 0x99, 0x44, 0xd0, 0xa3, 0xa4, 0x26, - 0x5b, 0x8b, 0x23, 0x1d, 0x4d, 0x0f, 0xd3, 0x4f, 0xe1, 0x87, 0xf2, 0xe0, 0x71, 0x47, 0x8f, 0xb2, - 0x7d, 0x11, 0x49, 0x3a, 0x64, 0x07, 0x77, 0xfb, 0x3f, 0x0f, 0x4f, 0xf2, 0xfc, 0xfe, 0x09, 0x44, - 0x3a, 0xe7, 0x95, 0x4c, 0xa5, 0xd0, 0x69, 0xf6, 0xf2, 0xca, 0x55, 0x5d, 0x28, 0x99, 0x4e, 0xb3, - 0xd4, 0xda, 0xc9, 0xb4, 0x2a, 0xeb, 0x12, 0x77, 0x1b, 0x21, 0x85, 0x4e, 0x7e, 0x32, 0xc9, 0x34, - 0x8b, 0x72, 0x08, 0x6e, 0x64, 0xf5, 0x3c, 0x91, 0xb7, 0x55, 0x59, 0x8e, 0xf0, 0x0e, 0xf8, 0xba, - 0xe6, 0x55, 0x4d, 0x50, 0x88, 0x62, 0x97, 0x35, 0x02, 0x77, 0xc0, 0x95, 0x4a, 0x90, 0x3f, 0xd6, - 0x33, 0xa3, 0xc9, 0xa9, 0x52, 0x48, 0x4d, 0xdc, 0xd0, 0x8d, 0xdb, 0xac, 0x11, 0x78, 0x17, 0xfe, - 0x4f, 0x24, 0x1f, 0x3d, 0xe6, 0x5c, 0xe7, 0xc4, 0x0b, 0x51, 0xdc, 0x66, 0xff, 0x8c, 0x31, 0xe0, - 0x3a, 0x8f, 0xee, 0xc1, 0xbf, 0x33, 0x0c, 0x18, 0x83, 0x77, 0xc1, 0x6b, 0x6e, 0x2b, 0xda, 0xcc, - 0xce, 0xf8, 0x14, 0x7c, 0x0b, 0x60, 0x3b, 0x82, 0xde, 0x41, 0xf2, 0x3b, 0x6e, 0xb2, 0xc6, 0xca, - 0x9a, 0x13, 0xd1, 0x3b, 0x82, 0xa0, 0xcf, 0xc5, 0xa5, 0x7a, 0x2a, 0x45, 0xa1, 0xc6, 0x98, 0x02, - 0x0c, 0x24, 0x17, 0xb2, 0x32, 0xad, 0xab, 0x92, 0x35, 0x07, 0x77, 0xa1, 0x35, 0x90, 0xc5, 0x38, - 0xaf, 0x6d, 0x97, 0xc7, 0x56, 0x0a, 0x9f, 0x40, 0xcb, 0xf2, 0x35, 0x3b, 0x05, 0xbd, 0xfd, 0x4d, - 0x0c, 0x36, 0xc5, 0x56, 0x61, 0xf3, 0x12, 0x57, 0x4a, 0xc8, 0x99, 0xdd, 0x77, 0x8b, 0x35, 0x02, - 0x1f, 0x81, 0x77, 0x36, 0x2b, 0x34, 0xf1, 0x43, 0x14, 0x6f, 0xf7, 0xf6, 0x36, 0x5d, 0xc5, 0x67, - 0x85, 0x66, 0x36, 0x79, 0x48, 0xc0, 0x33, 0x0a, 0xff, 0x05, 0x97, 0x0d, 0x1f, 0x3a, 0x8e, 0x19, - 0xce, 0x87, 0xd7, 0x1d, 0xd4, 0x27, 0x1f, 0x0b, 0x8a, 0xe6, 0x0b, 0x8a, 0xbe, 0x16, 0x14, 0xbd, - 0x2d, 0xa9, 0x33, 0x5f, 0x52, 0xe7, 0x73, 0x49, 0x9d, 0xac, 0x65, 0xff, 0xf6, 0xf8, 0x3b, 0x00, - 0x00, 0xff, 0xff, 0x0f, 0xa3, 0x91, 0xb4, 0x01, 0x02, 0x00, 0x00, -} - -func (m *MerkleProof) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *MerkleProof) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *MerkleProof) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.LeafHash) > 0 { - i -= len(m.LeafHash) - copy(dAtA[i:], m.LeafHash) - i = encodeVarintShare(dAtA, i, uint64(len(m.LeafHash))) - i-- - dAtA[i] = 0x22 - } - if len(m.Nodes) > 0 { - for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Nodes[iNdEx]) - copy(dAtA[i:], m.Nodes[iNdEx]) - i = encodeVarintShare(dAtA, i, uint64(len(m.Nodes[iNdEx]))) - i-- - dAtA[i] = 0x1a - } - } - if m.End != 0 { - i = encodeVarintShare(dAtA, i, uint64(m.End)) - i-- - dAtA[i] = 0x10 - } - if m.Start != 0 { - i = encodeVarintShare(dAtA, i, uint64(m.Start)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil + // 297 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x74, 0x90, 0xc1, 0x4a, 0xec, 0x30, + 0x14, 0x86, 0x9b, 0x3b, 0xed, 0x5c, 0x38, 0xa3, 0xe3, 0x10, 0x64, 0x08, 0xa2, 0xa1, 0x0c, 0x08, + 0xc5, 0x45, 0x2a, 0x15, 0x1f, 0xc0, 0xaa, 0x50, 0x41, 0x18, 0x89, 0x0b, 0xd7, 0x89, 0x89, 0xd3, + 0x6e, 0xda, 0xd2, 0x74, 0x51, 0x7d, 0x0a, 0x1f, 0xca, 0x85, 0xcb, 0x59, 0xba, 0x94, 0xf6, 0x45, + 0xa4, 0x69, 0x11, 0x17, 0xba, 0x3b, 0xdf, 0x9f, 0x2f, 0x3f, 0xc9, 0x81, 0x95, 0x49, 0x45, 0xa5, + 0x43, 0xad, 0x4c, 0x28, 0x9f, 0x5f, 0x44, 0x5e, 0x67, 0xb9, 0x0e, 0x4b, 0x19, 0xda, 0x98, 0x95, + 0x55, 0x51, 0x17, 0x78, 0x39, 0x80, 0x56, 0x86, 0x7d, 0x3b, 0xac, 0x94, 0x07, 0xf3, 0x52, 0x86, + 0x65, 0x55, 0x14, 0x4f, 0x83, 0xb7, 0x8a, 0xc1, 0xbb, 0xef, 0x4d, 0x8c, 0xc1, 0xbd, 0x12, 0xb5, + 0x20, 0xc8, 0x47, 0xc1, 0x0e, 0xb7, 0x33, 0x3e, 0x06, 0xef, 0xae, 0x77, 0xc9, 0x3f, 0x1f, 0x05, + 0xb3, 0x68, 0x8f, 0x8d, 0x37, 0x25, 0xb3, 0x31, 0x1f, 0x4e, 0x57, 0x6f, 0x08, 0x66, 0xb1, 0x50, + 0xd7, 0xf9, 0x63, 0xa1, 0xb2, 0x7c, 0x83, 0x29, 0x40, 0xa2, 0x85, 0xd2, 0x55, 0x22, 0x4c, 0x3a, + 0x16, 0xfe, 0x48, 0xf0, 0x12, 0xa6, 0x89, 0xce, 0x36, 0x69, 0x6d, 0x7b, 0x5d, 0x3e, 0x12, 0x3e, + 0x87, 0xa9, 0x7d, 0x8b, 0x21, 0x13, 0x7f, 0x12, 0xcc, 0xa2, 0x23, 0xf6, 0xfb, 0x27, 0x98, 0xb5, + 0xf8, 0x28, 0xe3, 0x7d, 0xf0, 0x6e, 0x72, 0xa5, 0x1b, 0xe2, 0xfa, 0x28, 0xd8, 0xe5, 0x03, 0xe0, + 0x53, 0x70, 0x2f, 0x9a, 0xcc, 0x10, 0xcf, 0x47, 0xc1, 0x3c, 0x3a, 0xfc, 0xab, 0x4a, 0x34, 0x99, + 0xe1, 0xd6, 0x3c, 0x21, 0xe0, 0xf6, 0x84, 0xff, 0xc3, 0x84, 0xaf, 0x1f, 0x16, 0x4e, 0x3f, 0x5c, + 0xae, 0x6f, 0x17, 0x28, 0x26, 0xef, 0x2d, 0x45, 0xdb, 0x96, 0xa2, 0xcf, 0x96, 0xa2, 0xd7, 0x8e, + 0x3a, 0xdb, 0x8e, 0x3a, 0x1f, 0x1d, 0x75, 0xe4, 0xd4, 0x6e, 0xf1, 0xec, 0x2b, 0x00, 0x00, 0xff, + 0xff, 0xb1, 0x96, 0xb9, 0xbe, 0x93, 0x01, 0x00, 0x00, } func (m *Share) Marshal() (dAtA []byte, err error) { @@ -441,31 +321,6 @@ func encodeVarintShare(dAtA []byte, offset int, v uint64) int { dAtA[offset] = uint8(v) return base } -func (m *MerkleProof) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Start != 0 { - n += 1 + sovShare(uint64(m.Start)) - } - if m.End != 0 { - n += 1 + sovShare(uint64(m.End)) - } - if len(m.Nodes) > 0 { - for _, b := range m.Nodes { - l = len(b) - n += 1 + l + sovShare(uint64(l)) - } - } - l = len(m.LeafHash) - if l > 0 { - n += 1 + l + sovShare(uint64(l)) - } - return n -} - func (m *Share) Size() (n int) { if m == nil { return 0 @@ -517,160 +372,6 @@ func sovShare(x uint64) (n int) { func sozShare(x uint64) (n int) { return sovShare(uint64((x << 1) ^ uint64((int64(x) >> 63)))) } -func (m *MerkleProof) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: MerkleProof: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: MerkleProof: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) - } - m.Start = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Start |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field End", wireType) - } - m.End = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.End |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Nodes = append(m.Nodes, make([]byte, postIndex-iNdEx)) - copy(m.Nodes[len(m.Nodes)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field LeafHash", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.LeafHash = append(m.LeafHash[:0], dAtA[iNdEx:postIndex]...) - if m.LeafHash == nil { - m.LeafHash = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipShare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func (m *Share) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 @@ -764,7 +465,7 @@ func (m *Share) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Proof == nil { - m.Proof = &MerkleProof{} + m.Proof = &pb.Proof{} } if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err diff --git a/share/eds/byzantine/pb/share.proto b/share/eds/byzantine/pb/share.proto index e08dffae45..33e3dae2c2 100644 --- a/share/eds/byzantine/pb/share.proto +++ b/share/eds/byzantine/pb/share.proto @@ -1,17 +1,11 @@ syntax = "proto3"; package share.eds.byzantine.pb; - -message MerkleProof { - int64 start = 1; - int64 end = 2; - repeated bytes nodes = 3; - bytes leaf_hash = 4; -} +import "pb/proof.proto"; message Share { bytes Data = 1; - MerkleProof Proof = 2; + proof.pb.Proof Proof = 2; } enum axis { diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index d6aa9dad51..0a79ebfe33 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -9,6 +9,7 @@ import ( "github.com/minio/sha256-simd" "github.com/celestiaorg/nmt" + nmt_pb "github.com/celestiaorg/nmt/pb" "github.com/celestiaorg/celestia-node/share" pb "github.com/celestiaorg/celestia-node/share/eds/byzantine/pb" @@ -58,11 +59,12 @@ func (s *ShareWithProof) ShareWithProofToProto() *pb.Share { return &pb.Share{ Data: s.Share, - Proof: &pb.MerkleProof{ - Start: int64(s.Proof.Start()), - End: int64(s.Proof.End()), - Nodes: s.Proof.Nodes(), - LeafHash: s.Proof.LeafHash(), + Proof: &nmt_pb.Proof{ + Start: int64(s.Proof.Start()), + End: int64(s.Proof.End()), + Nodes: s.Proof.Nodes(), + LeafHash: s.Proof.LeafHash(), + IsMaxNamespaceIgnored: s.Proof.IsMaxNamespaceIDIgnored(), }, } } @@ -122,6 +124,11 @@ func ProtoToShare(protoShares []*pb.Share) []*ShareWithProof { return shares } -func ProtoToProof(protoProof *pb.MerkleProof) nmt.Proof { - return nmt.NewInclusionProof(int(protoProof.Start), int(protoProof.End), protoProof.Nodes, ipld.NMTIgnoreMaxNamespace) +func ProtoToProof(protoProof *nmt_pb.Proof) nmt.Proof { + return nmt.NewInclusionProof( + int(protoProof.Start), + int(protoProof.End), + protoProof.Nodes, + protoProof.IsMaxNamespaceIgnored, + ) } diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index 74162f5981..bbe34ddc38 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -17,7 +17,6 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p" pb "github.com/celestiaorg/celestia-node/share/p2p/shrexnd/pb" ) @@ -135,7 +134,7 @@ func convertToNamespacedShares(rows []*pb.Row) share.NamespacedShares { int(row.Proof.Start), int(row.Proof.End), row.Proof.Nodes, - ipld.NMTIgnoreMaxNamespace, + row.Proof.IsMaxNamespaceIgnored, ) proof = &tmpProof } @@ -155,8 +154,8 @@ func convertToNonInclusionProofs(rows []*pb.Row) share.NamespacedShares { int(row.Proof.Start), int(row.Proof.End), row.Proof.Nodes, - row.Proof.Hashleaf, - ipld.NMTIgnoreMaxNamespace, + row.Proof.LeafHash, + row.Proof.IsMaxNamespaceIgnored, ) shares = append(shares, share.NamespacedRow{ Proof: &proof, diff --git a/share/p2p/shrexnd/params.go b/share/p2p/shrexnd/params.go index a645267962..8489627a07 100644 --- a/share/p2p/shrexnd/params.go +++ b/share/p2p/shrexnd/params.go @@ -8,7 +8,7 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p" ) -const protocolString = "/shrex/nd/v0.0.2" +const protocolString = "/shrex/nd/v0.0.3" var log = logging.Logger("shrex/nd") diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go index e4c7656b4e..933f5cbe54 100644 --- a/share/p2p/shrexnd/pb/share.pb.go +++ b/share/p2p/shrexnd/pb/share.pb.go @@ -5,6 +5,7 @@ package share_p2p_shrex_nd import ( fmt "fmt" + pb "github.com/celestiaorg/nmt/pb" proto "github.com/gogo/protobuf/proto" io "io" math "math" @@ -161,8 +162,8 @@ func (m *GetSharesByNamespaceResponse) GetRows() []*Row { } type Row struct { - Shares [][]byte `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` - Proof *Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` + Shares [][]byte `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` + Proof *pb.Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` } func (m *Row) Reset() { *m = Row{} } @@ -205,118 +206,47 @@ func (m *Row) GetShares() [][]byte { return nil } -func (m *Row) GetProof() *Proof { +func (m *Row) GetProof() *pb.Proof { if m != nil { return m.Proof } return nil } -type Proof struct { - Start int64 `protobuf:"varint,1,opt,name=start,proto3" json:"start,omitempty"` - End int64 `protobuf:"varint,2,opt,name=end,proto3" json:"end,omitempty"` - Nodes [][]byte `protobuf:"bytes,3,rep,name=nodes,proto3" json:"nodes,omitempty"` - Hashleaf []byte `protobuf:"bytes,4,opt,name=hashleaf,proto3" json:"hashleaf,omitempty"` -} - -func (m *Proof) Reset() { *m = Proof{} } -func (m *Proof) String() string { return proto.CompactTextString(m) } -func (*Proof) ProtoMessage() {} -func (*Proof) Descriptor() ([]byte, []int) { - return fileDescriptor_ed9f13149b0de397, []int{3} -} -func (m *Proof) XXX_Unmarshal(b []byte) error { - return m.Unmarshal(b) -} -func (m *Proof) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { - if deterministic { - return xxx_messageInfo_Proof.Marshal(b, m, deterministic) - } else { - b = b[:cap(b)] - n, err := m.MarshalToSizedBuffer(b) - if err != nil { - return nil, err - } - return b[:n], nil - } -} -func (m *Proof) XXX_Merge(src proto.Message) { - xxx_messageInfo_Proof.Merge(m, src) -} -func (m *Proof) XXX_Size() int { - return m.Size() -} -func (m *Proof) XXX_DiscardUnknown() { - xxx_messageInfo_Proof.DiscardUnknown(m) -} - -var xxx_messageInfo_Proof proto.InternalMessageInfo - -func (m *Proof) GetStart() int64 { - if m != nil { - return m.Start - } - return 0 -} - -func (m *Proof) GetEnd() int64 { - if m != nil { - return m.End - } - return 0 -} - -func (m *Proof) GetNodes() [][]byte { - if m != nil { - return m.Nodes - } - return nil -} - -func (m *Proof) GetHashleaf() []byte { - if m != nil { - return m.Hashleaf - } - return nil -} - func init() { proto.RegisterEnum("share.p2p.shrex.nd.StatusCode", StatusCode_name, StatusCode_value) proto.RegisterType((*GetSharesByNamespaceRequest)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceRequest") proto.RegisterType((*GetSharesByNamespaceResponse)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceResponse") proto.RegisterType((*Row)(nil), "share.p2p.shrex.nd.Row") - proto.RegisterType((*Proof)(nil), "share.p2p.shrex.nd.Proof") } func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } var fileDescriptor_ed9f13149b0de397 = []byte{ - // 396 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x92, 0xcf, 0x6e, 0xd3, 0x40, - 0x10, 0xc6, 0x6d, 0x6f, 0x62, 0x92, 0x49, 0x40, 0xd6, 0x80, 0xa8, 0xa1, 0x95, 0x15, 0xf9, 0x14, - 0x81, 0x64, 0x4b, 0x46, 0xe2, 0xee, 0xb6, 0x01, 0x22, 0xca, 0xa6, 0xda, 0x94, 0x3f, 0xb7, 0x68, - 0x8b, 0xb7, 0xf2, 0x01, 0xbc, 0x8b, 0x77, 0xa3, 0xc0, 0x99, 0x17, 0xe0, 0xb1, 0x38, 0xe6, 0xc8, - 0x11, 0x25, 0x2f, 0x82, 0xbc, 0x4e, 0x94, 0x03, 0xb9, 0xf9, 0x9b, 0xf9, 0xcd, 0x37, 0x9f, 0x47, - 0x0b, 0x23, 0x5d, 0xf2, 0x5a, 0xa4, 0x2a, 0x53, 0xa9, 0x2e, 0x6b, 0xf1, 0xbd, 0x2a, 0x52, 0x75, - 0x9b, 0xda, 0x62, 0xa2, 0x6a, 0x69, 0x24, 0xe2, 0x4e, 0x64, 0x2a, 0xb1, 0x44, 0x52, 0x15, 0xf1, - 0x27, 0x38, 0x7d, 0x2d, 0xcc, 0xbc, 0x69, 0xe8, 0xf3, 0x1f, 0x94, 0x7f, 0x15, 0x5a, 0xf1, 0xcf, - 0x82, 0x89, 0x6f, 0x4b, 0xa1, 0x0d, 0x9e, 0x42, 0xbf, 0x96, 0xd2, 0x2c, 0x4a, 0xae, 0xcb, 0xd0, - 0x1d, 0xb9, 0xe3, 0x21, 0xeb, 0x35, 0x85, 0x37, 0x5c, 0x97, 0x78, 0x06, 0xfd, 0x6a, 0x3f, 0x10, - 0x7a, 0xb6, 0x79, 0x28, 0xc4, 0x3f, 0x5d, 0x38, 0x3b, 0x6e, 0xad, 0x95, 0xac, 0xb4, 0xc0, 0x97, - 0xe0, 0x6b, 0xc3, 0xcd, 0x52, 0x5b, 0xe3, 0x07, 0x59, 0x94, 0xfc, 0x9f, 0x2f, 0x99, 0x5b, 0xe2, - 0x42, 0x16, 0x82, 0xed, 0x68, 0x7c, 0x0e, 0x9d, 0x5a, 0xae, 0x74, 0xe8, 0x8d, 0xc8, 0x78, 0x90, - 0x9d, 0x1c, 0x9b, 0x62, 0x72, 0xc5, 0x2c, 0x14, 0x53, 0x20, 0x4c, 0xae, 0xf0, 0x31, 0xf8, 0x16, - 0x6b, 0x76, 0x91, 0xf1, 0x90, 0xed, 0x14, 0xa6, 0xd0, 0x55, 0xb5, 0x94, 0x77, 0x36, 0xfe, 0x20, - 0x7b, 0x72, 0xcc, 0xec, 0xba, 0x01, 0x58, 0xcb, 0xc5, 0x1c, 0xba, 0x56, 0xe3, 0x23, 0xe8, 0x6a, - 0xc3, 0x6b, 0x63, 0xc3, 0x13, 0xd6, 0x0a, 0x0c, 0x80, 0x88, 0xaa, 0xb0, 0x6e, 0x84, 0x35, 0x9f, - 0x0d, 0x57, 0xc9, 0x42, 0xe8, 0x90, 0xd8, 0xc5, 0xad, 0xc0, 0xa7, 0xd0, 0x6b, 0x4e, 0xfa, 0x45, - 0xf0, 0xbb, 0xb0, 0xd3, 0x9e, 0x75, 0xaf, 0x9f, 0x7d, 0x04, 0x38, 0xfc, 0x35, 0x0e, 0xe0, 0xde, - 0x94, 0x7e, 0xc8, 0xaf, 0xa6, 0x97, 0x81, 0x83, 0x3e, 0x78, 0xb3, 0xb7, 0x81, 0x8b, 0xf7, 0xa1, - 0x4f, 0x67, 0x37, 0x8b, 0x57, 0xb3, 0xf7, 0xf4, 0x32, 0xf0, 0x70, 0x08, 0xbd, 0x29, 0xbd, 0x99, - 0x30, 0x9a, 0x5f, 0x05, 0x04, 0x4f, 0xe0, 0x21, 0xcd, 0xdf, 0x4d, 0xe6, 0xd7, 0xf9, 0xc5, 0x64, - 0x71, 0xc0, 0x3a, 0xe7, 0xe1, 0xef, 0x4d, 0xe4, 0xae, 0x37, 0x91, 0xfb, 0x77, 0x13, 0xb9, 0xbf, - 0xb6, 0x91, 0xb3, 0xde, 0x46, 0xce, 0x9f, 0x6d, 0xe4, 0xdc, 0xfa, 0xf6, 0x81, 0xbc, 0xf8, 0x17, - 0x00, 0x00, 0xff, 0xff, 0x0b, 0x93, 0xfd, 0x1b, 0x44, 0x02, 0x00, 0x00, + // 357 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x4d, 0x6b, 0xea, 0x40, + 0x14, 0x86, 0xf3, 0xe1, 0xcd, 0xd5, 0xa3, 0xd7, 0x1b, 0xe6, 0xc2, 0x35, 0x54, 0x09, 0x22, 0x14, + 0xa4, 0x85, 0x04, 0x52, 0xe8, 0x3e, 0x7e, 0xb4, 0x95, 0xda, 0x51, 0x46, 0xfb, 0xb1, 0x93, 0xa4, + 0x4e, 0xc9, 0xa6, 0x99, 0x69, 0x26, 0x62, 0xbb, 0xee, 0x1f, 0xe8, 0xcf, 0xea, 0xd2, 0x65, 0x97, + 0x45, 0xff, 0x48, 0xc9, 0x98, 0xe2, 0xa2, 0xee, 0xe6, 0xbc, 0xe7, 0x39, 0x67, 0x5e, 0xde, 0x03, + 0x4d, 0x11, 0x05, 0x09, 0x75, 0xb9, 0xc7, 0x5d, 0x11, 0x25, 0xf4, 0x39, 0x9e, 0xbb, 0x3c, 0x74, + 0xa5, 0xe8, 0xf0, 0x84, 0xa5, 0x0c, 0xa1, 0xbc, 0xf0, 0xb8, 0x23, 0x09, 0x27, 0x9e, 0x1f, 0x54, + 0x79, 0xe8, 0xf2, 0x84, 0xb1, 0x87, 0x2d, 0xd3, 0xba, 0x83, 0xfa, 0x39, 0x4d, 0x27, 0x19, 0x28, + 0x3a, 0x2f, 0x38, 0x78, 0xa4, 0x82, 0x07, 0xf7, 0x94, 0xd0, 0xa7, 0x05, 0x15, 0x29, 0xaa, 0x43, + 0x29, 0x61, 0x2c, 0x9d, 0x45, 0x81, 0x88, 0x2c, 0xb5, 0xa9, 0xb6, 0x2b, 0xa4, 0x98, 0x09, 0x17, + 0x81, 0x88, 0x50, 0x03, 0x4a, 0xf1, 0xf7, 0x80, 0xa5, 0xc9, 0xe6, 0x4e, 0x68, 0xbd, 0xaa, 0xd0, + 0xd8, 0xbf, 0x5a, 0x70, 0x16, 0x0b, 0x8a, 0x4e, 0xc1, 0x10, 0x69, 0x90, 0x2e, 0x84, 0x5c, 0x5c, + 0xf5, 0x6c, 0xe7, 0xa7, 0x5f, 0x67, 0x22, 0x89, 0x2e, 0x9b, 0x53, 0x92, 0xd3, 0xe8, 0x18, 0x0a, + 0x09, 0x5b, 0x0a, 0x4b, 0x6b, 0xea, 0xed, 0xb2, 0x57, 0xdb, 0x37, 0x45, 0xd8, 0x92, 0x48, 0xa8, + 0xd5, 0x03, 0x9d, 0xb0, 0x25, 0xfa, 0x0f, 0x86, 0xc4, 0xb2, 0xbf, 0xf4, 0x76, 0x85, 0xe4, 0x15, + 0x3a, 0x84, 0x5f, 0x32, 0x0d, 0x69, 0xbf, 0xec, 0xfd, 0x75, 0xf2, 0x6c, 0x42, 0x67, 0x9c, 0x3d, + 0xc8, 0xb6, 0x7b, 0x74, 0x0b, 0xb0, 0x33, 0x82, 0xca, 0xf0, 0x7b, 0x80, 0x6f, 0xfc, 0xe1, 0xa0, + 0x67, 0x2a, 0xc8, 0x00, 0x6d, 0x74, 0x69, 0xaa, 0xe8, 0x0f, 0x94, 0xf0, 0x68, 0x3a, 0x3b, 0x1b, + 0x5d, 0xe3, 0x9e, 0xa9, 0xa1, 0x0a, 0x14, 0x07, 0x78, 0xda, 0x27, 0xd8, 0x1f, 0x9a, 0x3a, 0xaa, + 0xc1, 0x3f, 0xec, 0x5f, 0xf5, 0x27, 0x63, 0xbf, 0xdb, 0x9f, 0xed, 0xb0, 0x42, 0xc7, 0x7a, 0x5f, + 0xdb, 0xea, 0x6a, 0x6d, 0xab, 0x9f, 0x6b, 0x5b, 0x7d, 0xdb, 0xd8, 0xca, 0x6a, 0x63, 0x2b, 0x1f, + 0x1b, 0x5b, 0x09, 0x0d, 0x79, 0x9f, 0x93, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8e, 0x8b, 0x14, + 0xc5, 0xe7, 0x01, 0x00, 0x00, } func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { @@ -442,55 +372,6 @@ func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { return len(dAtA) - i, nil } -func (m *Proof) Marshal() (dAtA []byte, err error) { - size := m.Size() - dAtA = make([]byte, size) - n, err := m.MarshalToSizedBuffer(dAtA[:size]) - if err != nil { - return nil, err - } - return dAtA[:n], nil -} - -func (m *Proof) MarshalTo(dAtA []byte) (int, error) { - size := m.Size() - return m.MarshalToSizedBuffer(dAtA[:size]) -} - -func (m *Proof) MarshalToSizedBuffer(dAtA []byte) (int, error) { - i := len(dAtA) - _ = i - var l int - _ = l - if len(m.Hashleaf) > 0 { - i -= len(m.Hashleaf) - copy(dAtA[i:], m.Hashleaf) - i = encodeVarintShare(dAtA, i, uint64(len(m.Hashleaf))) - i-- - dAtA[i] = 0x22 - } - if len(m.Nodes) > 0 { - for iNdEx := len(m.Nodes) - 1; iNdEx >= 0; iNdEx-- { - i -= len(m.Nodes[iNdEx]) - copy(dAtA[i:], m.Nodes[iNdEx]) - i = encodeVarintShare(dAtA, i, uint64(len(m.Nodes[iNdEx]))) - i-- - dAtA[i] = 0x1a - } - } - if m.End != 0 { - i = encodeVarintShare(dAtA, i, uint64(m.End)) - i-- - dAtA[i] = 0x10 - } - if m.Start != 0 { - i = encodeVarintShare(dAtA, i, uint64(m.Start)) - i-- - dAtA[i] = 0x8 - } - return len(dAtA) - i, nil -} - func encodeVarintShare(dAtA []byte, offset int, v uint64) int { offset -= sovShare(v) base := offset @@ -556,31 +437,6 @@ func (m *Row) Size() (n int) { return n } -func (m *Proof) Size() (n int) { - if m == nil { - return 0 - } - var l int - _ = l - if m.Start != 0 { - n += 1 + sovShare(uint64(m.Start)) - } - if m.End != 0 { - n += 1 + sovShare(uint64(m.End)) - } - if len(m.Nodes) > 0 { - for _, b := range m.Nodes { - l = len(b) - n += 1 + l + sovShare(uint64(l)) - } - } - l = len(m.Hashleaf) - if l > 0 { - n += 1 + l + sovShare(uint64(l)) - } - return n -} - func sovShare(x uint64) (n int) { return (math_bits.Len64(x|1) + 6) / 7 } @@ -899,7 +755,7 @@ func (m *Row) Unmarshal(dAtA []byte) error { return io.ErrUnexpectedEOF } if m.Proof == nil { - m.Proof = &Proof{} + m.Proof = &pb.Proof{} } if err := m.Proof.Unmarshal(dAtA[iNdEx:postIndex]); err != nil { return err @@ -926,160 +782,6 @@ func (m *Row) Unmarshal(dAtA []byte) error { } return nil } -func (m *Proof) Unmarshal(dAtA []byte) error { - l := len(dAtA) - iNdEx := 0 - for iNdEx < l { - preIndex := iNdEx - var wire uint64 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - wire |= uint64(b&0x7F) << shift - if b < 0x80 { - break - } - } - fieldNum := int32(wire >> 3) - wireType := int(wire & 0x7) - if wireType == 4 { - return fmt.Errorf("proto: Proof: wiretype end group for non-group") - } - if fieldNum <= 0 { - return fmt.Errorf("proto: Proof: illegal tag %d (wire type %d)", fieldNum, wire) - } - switch fieldNum { - case 1: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field Start", wireType) - } - m.Start = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.Start |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 2: - if wireType != 0 { - return fmt.Errorf("proto: wrong wireType = %d for field End", wireType) - } - m.End = 0 - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - m.End |= int64(b&0x7F) << shift - if b < 0x80 { - break - } - } - case 3: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Nodes", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Nodes = append(m.Nodes, make([]byte, postIndex-iNdEx)) - copy(m.Nodes[len(m.Nodes)-1], dAtA[iNdEx:postIndex]) - iNdEx = postIndex - case 4: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Hashleaf", wireType) - } - var byteLen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - byteLen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if byteLen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + byteLen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Hashleaf = append(m.Hashleaf[:0], dAtA[iNdEx:postIndex]...) - if m.Hashleaf == nil { - m.Hashleaf = []byte{} - } - iNdEx = postIndex - default: - iNdEx = preIndex - skippy, err := skipShare(dAtA[iNdEx:]) - if err != nil { - return err - } - if (skippy < 0) || (iNdEx+skippy) < 0 { - return ErrInvalidLengthShare - } - if (iNdEx + skippy) > l { - return io.ErrUnexpectedEOF - } - iNdEx += skippy - } - } - - if iNdEx > l { - return io.ErrUnexpectedEOF - } - return nil -} func skipShare(dAtA []byte) (n int, err error) { l := len(dAtA) iNdEx := 0 diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto index 306865d17f..931b25faa9 100644 --- a/share/p2p/shrexnd/pb/share.proto +++ b/share/p2p/shrexnd/pb/share.proto @@ -1,6 +1,7 @@ syntax = "proto3"; package share.p2p.shrex.nd; +import "pb/proof.proto"; message GetSharesByNamespaceRequest{ bytes root_hash = 1; @@ -22,12 +23,5 @@ enum StatusCode { message Row { repeated bytes shares = 1; - Proof proof = 2; -} - -message Proof { - int64 start = 1; - int64 end = 2; - repeated bytes nodes = 3; - bytes hashleaf = 4; + proof.pb.Proof proof = 2; } diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 830a853946..42810b1f48 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -14,6 +14,7 @@ import ( "go.uber.org/zap" "github.com/celestiaorg/go-libp2p-messenger/serde" + nmt_pb "github.com/celestiaorg/nmt/pb" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" @@ -188,11 +189,12 @@ func namespacedSharesToResponse(shares share.NamespacedShares) *pb.GetSharesByNa for _, row := range shares { row := &pb.Row{ Shares: row.Shares, - Proof: &pb.Proof{ - Start: int64(row.Proof.Start()), - End: int64(row.Proof.End()), - Nodes: row.Proof.Nodes(), - Hashleaf: row.Proof.LeafHash(), + Proof: &nmt_pb.Proof{ + Start: int64(row.Proof.Start()), + End: int64(row.Proof.End()), + Nodes: row.Proof.Nodes(), + LeafHash: row.Proof.LeafHash(), + IsMaxNamespaceIgnored: row.Proof.IsMaxNamespaceIDIgnored(), }, } From a9cff49083f866c03aac34004cd47a654eb3f36d Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 3 Aug 2023 00:30:19 +0800 Subject: [PATCH 115/388] feat(share/eds)!: Don't use json.Marshal for inverted index (#2410) ## Overview Inverted index stores shard keys as json objects. `json.Marshal` and `json.Unmarshal` are memory heavy operations and slows down inverted index building. Shard keys are strings and could be stored and retrieved as is, without marshalling/unmarshallng. Benchmarks shows 15% performance gain for `store.Put` operation. --- share/eds/inverted_index.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index d302932369..bb4092fa52 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -2,7 +2,6 @@ package eds import ( "context" - "encoding/json" "fmt" "github.com/filecoin-project/dagstore/index" @@ -56,11 +55,7 @@ func (s *simpleInvertedIndex) AddMultihashesForShard( err = mhIter.ForEach(func(mh multihash.Multihash) error { key := ds.NewKey(string(mh)) - bz, err := json.Marshal(sk) - if err != nil { - return fmt.Errorf("failed to marshal shard key to bytes: %w", err) - } - if err := batch.Put(ctx, key, bz); err != nil { + if err := batch.Put(ctx, key, []byte(sk.String())); err != nil { return fmt.Errorf("failed to put mh=%s, err=%w", mh, err) } return nil @@ -82,12 +77,7 @@ func (s *simpleInvertedIndex) GetShardsForMultihash(ctx context.Context, mh mult return nil, fmt.Errorf("failed to lookup index for mh %s, err: %w", mh, err) } - var shardKey shard.Key - if err := json.Unmarshal(sbz, &shardKey); err != nil { - return nil, fmt.Errorf("failed to unmarshal shard key for mh=%s, err=%w", mh, err) - } - - return []shard.Key{shardKey}, nil + return []shard.Key{shard.KeyFromString(string(sbz))}, nil } func (s *simpleInvertedIndex) close() error { From 2ed901c3c55f86cdaa8f1d386ac0d58889824467 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 3 Aug 2023 00:30:51 +0800 Subject: [PATCH 116/388] feat(share/store): store eds in memory for index builder (#2409) ## Overview Previously generated eds was first store to disk and then was read again while building car index for dagstore. This PR contains 2 changes: - writes eds into buffer to write it as 1 chunk instead of many small writes to disk to reduce disk i/o pressure. - introduces in-memory cache that will be used to read eds for building car index. Combined changes brings 3x performance improvement for eds.Put operation --- share/eds/store.go | 58 ++++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 53 insertions(+), 5 deletions(-) diff --git a/share/eds/store.go b/share/eds/store.go index 2218e13954..bf244a41d8 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -73,7 +73,10 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { } r := mount.NewRegistry() - err = r.Register("fs", &mount.FileMount{Path: basepath + blocksPath}) + err = r.Register("fs", &inMemoryOnceMount{Mount: &mount.FileMount{}}) + if err != nil { + return nil, fmt.Errorf("failed to register memory mount on the registry: %w", err) + } if err != nil { return nil, fmt.Errorf("failed to register FS mount on the registry: %w", err) } @@ -194,15 +197,24 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext } defer f.Close() - err = WriteEDS(ctx, square, f) + // save encoded eds into buffer + mount := &inMemoryOnceMount{ + // TODO: buffer could be pre-allocated with capacity calculated based on eds size. + buf: bytes.NewBuffer(nil), + Mount: &mount.FileMount{Path: s.basepath + blocksPath + key}, + } + err = WriteEDS(ctx, square, mount) if err != nil { return fmt.Errorf("failed to write EDS to file: %w", err) } + // write whole buffered mount data in one go to optimize i/o + if _, err = mount.WriteTo(f); err != nil { + return fmt.Errorf("failed to write EDS to file: %w", err) + } + ch := make(chan dagstore.ShardResult, 1) - err = s.dgstr.RegisterShard(ctx, shard.KeyFromString(key), &mount.FileMount{ - Path: s.basepath + blocksPath + key, - }, ch, dagstore.RegisterOpts{}) + err = s.dgstr.RegisterShard(ctx, shard.KeyFromString(key), mount, ch, dagstore.RegisterOpts{}) if err != nil { return fmt.Errorf("failed to initiate shard registration: %w", err) } @@ -471,3 +483,39 @@ func setupPath(basepath string) error { } return nil } + +// inMemoryOnceMount is used to allow reading once from buffer before using main mount.Reader +type inMemoryOnceMount struct { + buf *bytes.Buffer + + readOnce atomic.Bool + mount.Mount +} + +func (m *inMemoryOnceMount) Fetch(ctx context.Context) (mount.Reader, error) { + if !m.readOnce.Swap(true) { + reader := &inMemoryReader{Reader: bytes.NewReader(m.buf.Bytes())} + // release memory for gc, otherwise buffer will stick forever + m.buf = nil + return reader, nil + } + return m.Mount.Fetch(ctx) +} + +func (m *inMemoryOnceMount) Write(b []byte) (int, error) { + return m.buf.Write(b) +} + +func (m *inMemoryOnceMount) WriteTo(w io.Writer) (int64, error) { + return io.Copy(w, bytes.NewReader(m.buf.Bytes())) +} + +// inMemoryReader extends bytes.Reader to implement mount.Reader interface +type inMemoryReader struct { + *bytes.Reader +} + +// Close allows inMemoryReader to satisfy mount.Reader interface +func (r *inMemoryReader) Close() error { + return nil +} From d24e18cbd4b58ef34a73f00d31f2dc86302028b6 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 7 Aug 2023 17:09:44 +0800 Subject: [PATCH 117/388] feat(share/store): add eds store metrics (#2536) --- nodebuilder/settings.go | 2 + nodebuilder/share/opts.go | 5 + share/eds/metrics.go | 267 ++++++++++++++++++++++++++++++++++++++ share/eds/store.go | 132 +++++++++++++++---- 4 files changed, 378 insertions(+), 28 deletions(-) create mode 100644 share/eds/metrics.go diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 9a9f7fcf47..c30996d24e 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -90,6 +90,7 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti case node.Full: opts = fx.Options( baseComponents, + fx.Invoke(share.WithStoreMetrics), fx.Invoke(share.WithShrexServerMetrics), samplingMetrics, ) @@ -101,6 +102,7 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti case node.Bridge: opts = fx.Options( baseComponents, + fx.Invoke(share.WithStoreMetrics), fx.Invoke(share.WithShrexServerMetrics), ) default: diff --git a/nodebuilder/share/opts.go b/nodebuilder/share/opts.go index 20ba0ce58c..e236847f41 100644 --- a/nodebuilder/share/opts.go +++ b/nodebuilder/share/opts.go @@ -1,6 +1,7 @@ package share import ( + "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" @@ -41,3 +42,7 @@ func WithShrexServerMetrics(edsServer *shrexeds.Server, ndServer *shrexnd.Server func WithShrexGetterMetrics(sg *getters.ShrexGetter) error { return sg.WithMetrics() } + +func WithStoreMetrics(s *eds.Store) error { + return s.WithMetrics() +} diff --git a/share/eds/metrics.go b/share/eds/metrics.go new file mode 100644 index 0000000000..82adb246f1 --- /dev/null +++ b/share/eds/metrics.go @@ -0,0 +1,267 @@ +package eds + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const ( + failedKey = "failed" + sizeKey = "eds_size" + cachedKey = "cached" + + putResultKey = "result" + putOK putResult = "ok" + putExists putResult = "exists" + putFailed putResult = "failed" + + opNameKey = "op" + longOpResultKey = "result" + longOpUnresolved longOpResult = "unresolved" + longOpOK longOpResult = "ok" + longOpFailed longOpResult = "failed" +) + +var ( + meter = otel.Meter("eds_store") +) + +type putResult string + +type longOpResult string + +type metrics struct { + putTime metric.Float64Histogram + getCARTime metric.Float64Histogram + getCARBlockstoreTime metric.Float64Histogram + getDAHTime metric.Float64Histogram + removeTime metric.Float64Histogram + getTime metric.Float64Histogram + hasTime metric.Float64Histogram + listTime metric.Float64Histogram + getAccessorTime metric.Float64Histogram + + longOpTime metric.Float64Histogram + gcTime metric.Float64Histogram +} + +func (s *Store) WithMetrics() error { + putTime, err := meter.Float64Histogram("eds_store_put_time_histogram", + metric.WithDescription("eds store put time histogram(s)")) + if err != nil { + return err + } + + getCARTime, err := meter.Float64Histogram("eds_store_get_car_time_histogram", + metric.WithDescription("eds store get car time histogram(s)")) + if err != nil { + return err + } + + getCARBlockstoreTime, err := meter.Float64Histogram("eds_store_get_car_blockstore_time_histogram", + metric.WithDescription("eds store get car blockstore time histogram(s)")) + if err != nil { + return err + } + + getDAHTime, err := meter.Float64Histogram("eds_store_get_dah_time_histogram", + metric.WithDescription("eds store get dah time histogram(s)")) + if err != nil { + return err + } + + removeTime, err := meter.Float64Histogram("eds_store_remove_time_histogram", + metric.WithDescription("eds store remove time histogram(s)")) + if err != nil { + return err + } + + getTime, err := meter.Float64Histogram("eds_store_get_time_histogram", + metric.WithDescription("eds store get time histogram(s)")) + if err != nil { + return err + } + + hasTime, err := meter.Float64Histogram("eds_store_has_time_histogram", + metric.WithDescription("eds store has time histogram(s)")) + if err != nil { + return err + } + + listTime, err := meter.Float64Histogram("eds_store_list_time_histogram", + metric.WithDescription("eds store list time histogram(s)")) + if err != nil { + return err + } + + getAccessorTime, err := meter.Float64Histogram("eds_store_get_accessor_time_histogram", + metric.WithDescription("eds store get accessor time histogram(s)")) + if err != nil { + return err + } + + longOpTime, err := meter.Float64Histogram("eds_store_long_operation_time_histogram", + metric.WithDescription("eds store long operation time histogram(s)")) + if err != nil { + return err + } + + gcTime, err := meter.Float64Histogram("eds_store_gc_time", + metric.WithDescription("dagstore gc time histogram(s)")) + if err != nil { + return err + } + + s.metrics = &metrics{ + putTime: putTime, + getCARTime: getCARTime, + getCARBlockstoreTime: getCARBlockstoreTime, + getDAHTime: getDAHTime, + removeTime: removeTime, + getTime: getTime, + hasTime: hasTime, + listTime: listTime, + getAccessorTime: getAccessorTime, + longOpTime: longOpTime, + gcTime: gcTime, + } + return nil +} + +func (m *metrics) observeGCtime(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + m.gcTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observePut(ctx context.Context, dur time.Duration, result putResult, size uint) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.putTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.String(putResultKey, string(result)), + attribute.Int(sizeKey, int(size)))) +} + +func (m *metrics) observeLongOp(ctx context.Context, opName string, dur time.Duration, result longOpResult) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.longOpTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.String(opNameKey, opName), + attribute.String(longOpResultKey, string(result)))) +} + +func (m *metrics) observeGetCAR(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.getCARTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeCARBlockstore(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.getCARBlockstoreTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeGetDAH(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.getDAHTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeRemove(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.removeTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeGet(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.getTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeHas(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.hasTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeList(ctx context.Context, dur time.Duration, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.listTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeGetAccessor(ctx context.Context, dur time.Duration, cached, failed bool) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.getAccessorTime.Record(ctx, dur.Seconds(), metric.WithAttributes( + attribute.Bool(cachedKey, cached), + attribute.Bool(failedKey, failed))) +} diff --git a/share/eds/store.go b/share/eds/store.go index cd85380ba1..2b2c37b26d 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -63,6 +63,8 @@ type Store struct { gcInterval time.Duration // lastGCResult is only stored on the store for testing purposes. lastGCResult atomic.Pointer[dagstore.GCResult] + + metrics *metrics } // NewStore creates a new EDS Store under the given basepath and datastore. @@ -148,7 +150,9 @@ func (s *Store) gc(ctx context.Context) { case <-ctx.Done(): return case <-ticker.C: + tnow := time.Now() res, err := s.dgstr.GC(ctx) + s.metrics.observeGCtime(ctx, time.Since(tnow), err != nil) if err != nil { log.Errorf("garbage collecting dagstore: %v", err) return @@ -164,22 +168,30 @@ func (s *Store) gc(ctx context.Context) { // The square is verified on the Exchange level, and Put only stores the square, trusting it. // The resulting file stores all the shares and NMT Merkle Proofs of the EDS. // Additionally, the file gets indexed s.t. store.Blockstore can access them. -func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.ExtendedDataSquare) (err error) { - // if root already exists, short-circuit - has, err := s.Has(ctx, root) - if err != nil { - return fmt.Errorf("failed to check if root already exists in index: %w", err) - } - if has { - return dagstore.ErrShardExists - } - +func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.ExtendedDataSquare) error { ctx, span := tracer.Start(ctx, "store/put", trace.WithAttributes( attribute.Int("width", int(square.Width())), )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() + + tnow := time.Now() + err := s.put(ctx, root, square) + result := putOK + switch { + case errors.Is(err, dagstore.ErrShardExists): + result = putExists + case err != nil: + result = putFailed + } + utils.SetStatusAndEnd(span, err) + s.metrics.observePut(ctx, time.Since(tnow), result, square.Width()) + return err +} + +func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.ExtendedDataSquare) (err error) { + // if root already exists, short-circuit + if has, _ := s.Has(ctx, root); has { + return dagstore.ErrShardExists + } key := root.String() f, err := os.OpenFile(s.basepath+blocksPath+key, os.O_CREATE|os.O_WRONLY, 0600) @@ -203,7 +215,8 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext select { case <-ctx.Done(): - go logLateResult(ch, time.Minute*5) + // if context finished before result was received, track result in separate goroutine + go trackLateResult("put", ch, s.metrics, time.Minute*5) return ctx.Err() case result := <-ch: if result.Error != nil { @@ -217,21 +230,24 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext // maxWait. If the result is not received within the specified duration, it logs an error // indicating that the parent context has expired and the shard registration is stuck. If a result // is received, it checks for any error and logs appropriate messages. -func logLateResult(res <-chan dagstore.ShardResult, maxWait time.Duration) { +func trackLateResult(opName string, res <-chan dagstore.ShardResult, metrics *metrics, maxWait time.Duration) { tnow := time.Now() select { case <-time.After(maxWait): + metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpUnresolved) log.Errorf("parent context is expired, while register shard is stuck for more than %v sec", time.Since(tnow)) return case result := <-res: - // don't log if result was received right after launch of the func + // don't observe if result was received right after launch of the func if time.Since(tnow) < time.Second { return } if result.Error != nil { + metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpFailed) log.Errorf("failed to register shard after context expired: %v ago, err: %w", time.Since(tnow), result.Error) return } + metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpOK) log.Warnf("parent context expired, but register shard finished with no error,"+ " after context expired: %v ago", time.Since(tnow)) return @@ -247,8 +263,14 @@ func logLateResult(res <-chan dagstore.ShardResult, maxWait time.Duration) { // same reader. The cache is responsible for closing the underlying reader. func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.Reader, error) { ctx, span := tracer.Start(ctx, "store/get-car") - defer span.End() + tnow := time.Now() + r, err := s.getCAR(ctx, root) + s.metrics.observeGetCAR(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return r, err +} +func (s *Store) getCAR(ctx context.Context, root share.DataHash) (io.Reader, error) { key := root.String() accessor, err := s.getCachedAccessor(ctx, shard.KeyFromString(key)) if err != nil { @@ -272,6 +294,18 @@ func (s *Store) Blockstore() bstore.Blockstore { func (s *Store) CARBlockstore( ctx context.Context, root share.DataHash, +) (dagstore.ReadBlockstore, error) { + ctx, span := tracer.Start(ctx, "store/car-blockstore") + tnow := time.Now() + r, err := s.carBlockstore(ctx, root) + s.metrics.observeCARBlockstore(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return r, err +} + +func (s *Store) carBlockstore( + ctx context.Context, + root share.DataHash, ) (dagstore.ReadBlockstore, error) { key := shard.KeyFromString(root.String()) accessor, err := s.getCachedAccessor(ctx, key) @@ -283,9 +317,15 @@ func (s *Store) CARBlockstore( // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. func (s *Store) GetDAH(ctx context.Context, root share.DataHash) (*share.Root, error) { - ctx, span := tracer.Start(ctx, "store/get-dah") - defer span.End() + ctx, span := tracer.Start(ctx, "store/car-dah") + tnow := time.Now() + r, err := s.getDAH(ctx, root) + s.metrics.observeGetDAH(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return r, err +} +func (s *Store) getDAH(ctx context.Context, root share.DataHash) (*share.Root, error) { key := shard.KeyFromString(root.String()) accessor, err := s.getCachedAccessor(ctx, key) if err != nil { @@ -334,6 +374,7 @@ func (s *Store) getAccessor(ctx context.Context, key shard.Key) (*dagstore.Shard } return res.Accessor, nil case <-ctx.Done(): + go trackLateResult("get_shard", ch, s.metrics, time.Minute) return nil, ctx.Err() } } @@ -343,30 +384,40 @@ func (s *Store) getCachedAccessor(ctx context.Context, key shard.Key) (*accessor lk.Lock() defer lk.Unlock() + tnow := time.Now() accessor, err := s.cache.unsafeGet(key) if err != nil && err != errCacheMiss { log.Errorf("unexpected error while reading key from bs cache %s: %s", key, err) } if accessor != nil { + s.metrics.observeGetAccessor(ctx, time.Since(tnow), true, false) return accessor, nil } // wasn't found in cache, so acquire it and add to cache shardAccessor, err := s.getAccessor(ctx, key) if err != nil { + s.metrics.observeGetAccessor(ctx, time.Since(tnow), false, err != nil) return nil, err } - return s.cache.unsafeAdd(key, shardAccessor) + + a, err := s.cache.unsafeAdd(key, shardAccessor) + s.metrics.observeGetAccessor(ctx, time.Since(tnow), false, err != nil) + return a, err } // Remove removes EDS from Store by the given share.Root hash and cleans up all // the indexing. -func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) { +func (s *Store) Remove(ctx context.Context, root share.DataHash) error { ctx, span := tracer.Start(ctx, "store/remove") - defer func() { - utils.SetStatusAndEnd(span, err) - }() + tnow := time.Now() + err := s.remove(ctx, root) + s.metrics.observeRemove(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return err +} +func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { key := root.String() ch := make(chan dagstore.ShardResult, 1) err = s.dgstr.DestroyShard(ctx, shard.KeyFromString(key), ch, dagstore.DestroyOpts{}) @@ -380,6 +431,7 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) { return fmt.Errorf("failed to destroy shard: %w", result.Error) } case <-ctx.Done(): + go trackLateResult("remove", ch, s.metrics, time.Minute) return ctx.Err() } @@ -402,7 +454,16 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) (err error) { // // It reads only one quadrant(1/4) of the EDS and verifies the integrity of the stored data by // recomputing it. -func (s *Store) Get(ctx context.Context, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) { +func (s *Store) Get(ctx context.Context, root share.DataHash) (*rsmt2d.ExtendedDataSquare, error) { + ctx, span := tracer.Start(ctx, "store/get") + tnow := time.Now() + eds, err := s.get(ctx, root) + s.metrics.observeGet(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return eds, err +} + +func (s *Store) get(ctx context.Context, root share.DataHash) (eds *rsmt2d.ExtendedDataSquare, err error) { ctx, span := tracer.Start(ctx, "store/get") defer func() { utils.SetStatusAndEnd(span, err) @@ -420,10 +481,16 @@ func (s *Store) Get(ctx context.Context, root share.DataHash) (eds *rsmt2d.Exten } // Has checks if EDS exists by the given share.Root hash. -func (s *Store) Has(ctx context.Context, root share.DataHash) (bool, error) { - _, span := tracer.Start(ctx, "store/has") - defer span.End() +func (s *Store) Has(ctx context.Context, root share.DataHash) (has bool, err error) { + ctx, span := tracer.Start(ctx, "store/has") + tnow := time.Now() + eds, err := s.has(ctx, root) + s.metrics.observeHas(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return eds, err +} +func (s *Store) has(_ context.Context, root share.DataHash) (bool, error) { key := root.String() info, err := s.dgstr.GetShardInfo(shard.KeyFromString(key)) switch err { @@ -438,6 +505,15 @@ func (s *Store) Has(ctx context.Context, root share.DataHash) (bool, error) { // List lists all the registered EDSes. func (s *Store) List() ([]share.DataHash, error) { + ctx, span := tracer.Start(context.Background(), "store/list") + tnow := time.Now() + hashes, err := s.list() + s.metrics.observeList(ctx, time.Since(tnow), err != nil) + utils.SetStatusAndEnd(span, err) + return hashes, err +} + +func (s *Store) list() ([]share.DataHash, error) { shards := s.dgstr.AllShardsInfo() hashes := make([]share.DataHash, 0, len(shards)) for shrd := range shards { From cf1b117386c245e80a4015f20f69fb5d761b74c8 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 7 Aug 2023 17:09:59 +0800 Subject: [PATCH 118/388] !feat(share/p2p/shrex-nd): rework shrex-nd to serve data in multiple messages (#2444) ## Overview shrex-nd protocol is limited to send messages with max size of 1mb. PR modifies shrex-nd protocol to stream namespace data row by row. Next time we will have msg size problem after ods size > 2048. Resolves https://github.com/celestiaorg/celestia-node/issues/2419 --- share/eds/edstest/testing.go | 14 ++ share/getters/shrex_test.go | 34 ++++- share/p2p/errors.go | 2 + share/p2p/metrics.go | 4 + share/p2p/shrexnd/client.go | 105 +++++++------- share/p2p/shrexnd/exchange_test.go | 2 +- share/p2p/shrexnd/pb/share.pb.go | 221 ++++++++++------------------- share/p2p/shrexnd/pb/share.proto | 6 +- share/p2p/shrexnd/server.go | 218 +++++++++++++++------------- share/sharetest/testing.go | 20 +++ 10 files changed, 325 insertions(+), 301 deletions(-) diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go index 2b6d9ef78d..e66b42a566 100644 --- a/share/eds/edstest/testing.go +++ b/share/eds/edstest/testing.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/rsmt2d" @@ -29,3 +30,16 @@ func RandEDS(t require.TestingT, size int) *rsmt2d.ExtendedDataSquare { require.NoError(t, err, "failure to recompute the extended data square") return eds } + +func RandEDSWithNamespace( + t require.TestingT, + namespace share.Namespace, + size int, +) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader) { + shares := sharetest.RandSharesWithNamespace(t, namespace, size*size) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + require.NoError(t, err, "failure to recompute the extended data square") + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + return eds, dah +} diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index d2dcfeca5e..ef07cf6797 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -1,9 +1,12 @@ package getters import ( + "bytes" "context" "encoding/binary" "errors" + "math/rand" + "sort" "testing" "time" @@ -17,6 +20,7 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/wrapper" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -31,6 +35,7 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestShrexGetter(t *testing.T) { @@ -58,12 +63,13 @@ func TestShrexGetter(t *testing.T) { getter := NewShrexGetter(edsClient, ndClient, peerManager) require.NoError(t, getter.Start(ctx)) - t.Run("ND_Available", func(t *testing.T) { - ctx, cancel := context.WithTimeout(ctx, time.Second) + t.Run("ND_Available, total data size > 1mb", func(t *testing.T) { + ctx, cancel := context.WithTimeout(ctx, time.Second*10) t.Cleanup(cancel) // generate test data - randEDS, dah, namespace := generateTestEDS(t) + namespace := sharetest.RandV0Namespace() + randEDS, dah := singleNamespaceEds(t, namespace, 64) require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), @@ -368,3 +374,25 @@ func TestAddToNamespace(t *testing.T) { }) } } + +func singleNamespaceEds( + t require.TestingT, + namespace share.Namespace, + size int, +) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader) { + shares := make([]share.Share, size*size) + rnd := rand.New(rand.NewSource(time.Now().Unix())) + for i := range shares { + shr := make([]byte, share.Size) + copy(share.GetNamespace(shr), namespace) + _, err := rnd.Read(share.GetData(shr)) + require.NoError(t, err) + shares[i] = shr + } + sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) + eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + require.NoError(t, err, "failure to recompute the extended data square") + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + return eds, dah +} diff --git a/share/p2p/errors.go b/share/p2p/errors.go index 77f23c554e..cb7b596f47 100644 --- a/share/p2p/errors.go +++ b/share/p2p/errors.go @@ -9,6 +9,8 @@ import ( // available at the moment. The request may be retried later, but it's unlikely to succeed. var ErrNotFound = errors.New("the requested data or resource could not be found") +var ErrRateLimited = errors.New("server is overloaded and rate limited the request") + // ErrInvalidResponse is returned when a peer returns an invalid response or caused an internal // error. It is used to signal that the peer couldn't serve the data successfully, and should not be // retried. diff --git a/share/p2p/metrics.go b/share/p2p/metrics.go index 1942be5d6b..fee3b12413 100644 --- a/share/p2p/metrics.go +++ b/share/p2p/metrics.go @@ -14,6 +14,10 @@ var meter = otel.Meter("shrex/eds") type status string const ( + StatusBadRequest status = "bad_request" + StatusSendRespErr status = "send_resp_err" + StatusSendReqErr status = "send_req_err" + StatusReadRespErr status = "read_resp_err" StatusInternalErr status = "internal_err" StatusNotFound status = "not_found" StatusTimeout status = "timeout" diff --git a/share/p2p/shrexnd/client.go b/share/p2p/shrexnd/client.go index bbe34ddc38..86c5150095 100644 --- a/share/p2p/shrexnd/client.go +++ b/share/p2p/shrexnd/client.go @@ -45,7 +45,7 @@ func NewClient(params *Parameters, host host.Host) (*Client, error) { } // RequestND requests namespaced data from the given peer. -// Returns shares with unverified inclusion proofs against the share.Root. +// Returns NamespacedShares with unverified inclusion proofs against the share.Root. func (c *Client) RequestND( ctx context.Context, root *share.Root, @@ -73,7 +73,7 @@ func (c *Client) RequestND( return nil, context.DeadlineExceeded } } - if err != p2p.ErrNotFound { + if err != p2p.ErrNotFound && err != p2p.ErrRateLimited { log.Warnw("client-nd: peer returned err", "err", err) } return nil, err @@ -100,6 +100,7 @@ func (c *Client) doRequest( _, err = serde.Write(stream, req) if err != nil { + c.metrics.ObserveRequests(ctx, 1, p2p.StatusSendReqErr) stream.Reset() //nolint:errcheck return nil, fmt.Errorf("client-nd: writing request: %w", err) } @@ -109,59 +110,70 @@ func (c *Client) doRequest( log.Debugw("client-nd: closing write side of the stream", "err", err) } - var resp pb.GetSharesByNamespaceResponse - _, err = serde.Read(stream, &resp) + if err := c.readStatus(ctx, stream); err != nil { + return nil, err + } + return c.readNamespacedShares(ctx, stream) +} + +func (c *Client) readStatus(ctx context.Context, stream network.Stream) error { + var resp pb.GetSharesByNamespaceStatusResponse + _, err := serde.Read(stream, &resp) if err != nil { // server is overloaded and closed the stream if errors.Is(err, io.EOF) { c.metrics.ObserveRequests(ctx, 1, p2p.StatusRateLimited) - return nil, p2p.ErrNotFound + return p2p.ErrRateLimited } + c.metrics.ObserveRequests(ctx, 1, p2p.StatusReadRespErr) stream.Reset() //nolint:errcheck - return nil, fmt.Errorf("client-nd: reading response: %w", err) + return fmt.Errorf("client-nd: reading status response: %w", err) } - return c.convertResponse(ctx, resp) + return c.convertStatusToErr(ctx, resp.Status) } -// convertToNamespacedShares converts proto Rows to share.NamespacedShares -func convertToNamespacedShares(rows []*pb.Row) share.NamespacedShares { - shares := make([]share.NamespacedRow, 0, len(rows)) - for _, row := range rows { - var proof *nmt.Proof +// readNamespacedShares converts proto Rows to share.NamespacedShares +func (c *Client) readNamespacedShares( + ctx context.Context, + stream network.Stream, +) (share.NamespacedShares, error) { + var shares share.NamespacedShares + for { + var row pb.NamespaceRowResponse + _, err := serde.Read(stream, &row) + if err != nil { + if errors.Is(err, io.EOF) { + // all data is received and steam is closed by server + return shares, nil + } + c.metrics.ObserveRequests(ctx, 1, p2p.StatusReadRespErr) + return nil, err + } + var proof nmt.Proof if row.Proof != nil { - tmpProof := nmt.NewInclusionProof( - int(row.Proof.Start), - int(row.Proof.End), - row.Proof.Nodes, - row.Proof.IsMaxNamespaceIgnored, - ) - proof = &tmpProof + if len(row.Shares) != 0 { + proof = nmt.NewInclusionProof( + int(row.Proof.Start), + int(row.Proof.End), + row.Proof.Nodes, + row.Proof.IsMaxNamespaceIgnored, + ) + } else { + proof = nmt.NewAbsenceProof( + int(row.Proof.Start), + int(row.Proof.End), + row.Proof.Nodes, + row.Proof.LeafHash, + row.Proof.IsMaxNamespaceIgnored, + ) + } } - shares = append(shares, share.NamespacedRow{ Shares: row.Shares, - Proof: proof, - }) - } - return shares -} - -func convertToNonInclusionProofs(rows []*pb.Row) share.NamespacedShares { - shares := make([]share.NamespacedRow, 0, len(rows)) - for _, row := range rows { - proof := nmt.NewAbsenceProof( - int(row.Proof.Start), - int(row.Proof.End), - row.Proof.Nodes, - row.Proof.LeafHash, - row.Proof.IsMaxNamespaceIgnored, - ) - shares = append(shares, share.NamespacedRow{ - Proof: &proof, + Proof: &proof, }) } - return shares } func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) { @@ -192,23 +204,20 @@ func (c *Client) setStreamDeadlines(ctx context.Context, stream network.Stream) } } -func (c *Client) convertResponse( - ctx context.Context, resp pb.GetSharesByNamespaceResponse) (share.NamespacedShares, error) { - switch resp.Status { +func (c *Client) convertStatusToErr(ctx context.Context, status pb.StatusCode) error { + switch status { case pb.StatusCode_OK: c.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) - return convertToNamespacedShares(resp.Rows), nil - case pb.StatusCode_NAMESPACE_NOT_FOUND: - return convertToNonInclusionProofs(resp.Rows), nil + return nil case pb.StatusCode_NOT_FOUND: c.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - return nil, p2p.ErrNotFound + return p2p.ErrNotFound case pb.StatusCode_INVALID: - log.Debug("client-nd: invalid request") + log.Warn("client-nd: invalid request") fallthrough case pb.StatusCode_INTERNAL: fallthrough default: - return nil, p2p.ErrInvalidResponse + return p2p.ErrInvalidResponse } } diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 8d1dad17c0..1ca0736cd6 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -100,7 +100,7 @@ func TestExchange_RequestND(t *testing.T) { // wait until all server slots are taken wg.Wait() _, err = client.RequestND(ctx, nil, sharetest.RandV0Namespace(), server.host.ID()) - require.ErrorIs(t, err, p2p.ErrNotFound) + require.ErrorIs(t, err, p2p.ErrRateLimited) }) } diff --git a/share/p2p/shrexnd/pb/share.pb.go b/share/p2p/shrexnd/pb/share.pb.go index 933f5cbe54..7e3c11416f 100644 --- a/share/p2p/shrexnd/pb/share.pb.go +++ b/share/p2p/shrexnd/pb/share.pb.go @@ -26,11 +26,10 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package type StatusCode int32 const ( - StatusCode_INVALID StatusCode = 0 - StatusCode_OK StatusCode = 1 - StatusCode_NOT_FOUND StatusCode = 2 - StatusCode_INTERNAL StatusCode = 3 - StatusCode_NAMESPACE_NOT_FOUND StatusCode = 4 + StatusCode_INVALID StatusCode = 0 + StatusCode_OK StatusCode = 1 + StatusCode_NOT_FOUND StatusCode = 2 + StatusCode_INTERNAL StatusCode = 3 ) var StatusCode_name = map[int32]string{ @@ -38,15 +37,13 @@ var StatusCode_name = map[int32]string{ 1: "OK", 2: "NOT_FOUND", 3: "INTERNAL", - 4: "NAMESPACE_NOT_FOUND", } var StatusCode_value = map[string]int32{ - "INVALID": 0, - "OK": 1, - "NOT_FOUND": 2, - "INTERNAL": 3, - "NAMESPACE_NOT_FOUND": 4, + "INVALID": 0, + "OK": 1, + "NOT_FOUND": 2, + "INTERNAL": 3, } func (x StatusCode) String() string { @@ -109,23 +106,22 @@ func (m *GetSharesByNamespaceRequest) GetNamespace() []byte { return nil } -type GetSharesByNamespaceResponse struct { +type GetSharesByNamespaceStatusResponse struct { Status StatusCode `protobuf:"varint,1,opt,name=status,proto3,enum=share.p2p.shrex.nd.StatusCode" json:"status,omitempty"` - Rows []*Row `protobuf:"bytes,2,rep,name=rows,proto3" json:"rows,omitempty"` } -func (m *GetSharesByNamespaceResponse) Reset() { *m = GetSharesByNamespaceResponse{} } -func (m *GetSharesByNamespaceResponse) String() string { return proto.CompactTextString(m) } -func (*GetSharesByNamespaceResponse) ProtoMessage() {} -func (*GetSharesByNamespaceResponse) Descriptor() ([]byte, []int) { +func (m *GetSharesByNamespaceStatusResponse) Reset() { *m = GetSharesByNamespaceStatusResponse{} } +func (m *GetSharesByNamespaceStatusResponse) String() string { return proto.CompactTextString(m) } +func (*GetSharesByNamespaceStatusResponse) ProtoMessage() {} +func (*GetSharesByNamespaceStatusResponse) Descriptor() ([]byte, []int) { return fileDescriptor_ed9f13149b0de397, []int{1} } -func (m *GetSharesByNamespaceResponse) XXX_Unmarshal(b []byte) error { +func (m *GetSharesByNamespaceStatusResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *GetSharesByNamespaceResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *GetSharesByNamespaceStatusResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_GetSharesByNamespaceResponse.Marshal(b, m, deterministic) + return xxx_messageInfo_GetSharesByNamespaceStatusResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -135,49 +131,42 @@ func (m *GetSharesByNamespaceResponse) XXX_Marshal(b []byte, deterministic bool) return b[:n], nil } } -func (m *GetSharesByNamespaceResponse) XXX_Merge(src proto.Message) { - xxx_messageInfo_GetSharesByNamespaceResponse.Merge(m, src) +func (m *GetSharesByNamespaceStatusResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_GetSharesByNamespaceStatusResponse.Merge(m, src) } -func (m *GetSharesByNamespaceResponse) XXX_Size() int { +func (m *GetSharesByNamespaceStatusResponse) XXX_Size() int { return m.Size() } -func (m *GetSharesByNamespaceResponse) XXX_DiscardUnknown() { - xxx_messageInfo_GetSharesByNamespaceResponse.DiscardUnknown(m) +func (m *GetSharesByNamespaceStatusResponse) XXX_DiscardUnknown() { + xxx_messageInfo_GetSharesByNamespaceStatusResponse.DiscardUnknown(m) } -var xxx_messageInfo_GetSharesByNamespaceResponse proto.InternalMessageInfo +var xxx_messageInfo_GetSharesByNamespaceStatusResponse proto.InternalMessageInfo -func (m *GetSharesByNamespaceResponse) GetStatus() StatusCode { +func (m *GetSharesByNamespaceStatusResponse) GetStatus() StatusCode { if m != nil { return m.Status } return StatusCode_INVALID } -func (m *GetSharesByNamespaceResponse) GetRows() []*Row { - if m != nil { - return m.Rows - } - return nil -} - -type Row struct { +type NamespaceRowResponse struct { Shares [][]byte `protobuf:"bytes,1,rep,name=shares,proto3" json:"shares,omitempty"` Proof *pb.Proof `protobuf:"bytes,2,opt,name=proof,proto3" json:"proof,omitempty"` } -func (m *Row) Reset() { *m = Row{} } -func (m *Row) String() string { return proto.CompactTextString(m) } -func (*Row) ProtoMessage() {} -func (*Row) Descriptor() ([]byte, []int) { +func (m *NamespaceRowResponse) Reset() { *m = NamespaceRowResponse{} } +func (m *NamespaceRowResponse) String() string { return proto.CompactTextString(m) } +func (*NamespaceRowResponse) ProtoMessage() {} +func (*NamespaceRowResponse) Descriptor() ([]byte, []int) { return fileDescriptor_ed9f13149b0de397, []int{2} } -func (m *Row) XXX_Unmarshal(b []byte) error { +func (m *NamespaceRowResponse) XXX_Unmarshal(b []byte) error { return m.Unmarshal(b) } -func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { +func (m *NamespaceRowResponse) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { if deterministic { - return xxx_messageInfo_Row.Marshal(b, m, deterministic) + return xxx_messageInfo_NamespaceRowResponse.Marshal(b, m, deterministic) } else { b = b[:cap(b)] n, err := m.MarshalToSizedBuffer(b) @@ -187,26 +176,26 @@ func (m *Row) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { return b[:n], nil } } -func (m *Row) XXX_Merge(src proto.Message) { - xxx_messageInfo_Row.Merge(m, src) +func (m *NamespaceRowResponse) XXX_Merge(src proto.Message) { + xxx_messageInfo_NamespaceRowResponse.Merge(m, src) } -func (m *Row) XXX_Size() int { +func (m *NamespaceRowResponse) XXX_Size() int { return m.Size() } -func (m *Row) XXX_DiscardUnknown() { - xxx_messageInfo_Row.DiscardUnknown(m) +func (m *NamespaceRowResponse) XXX_DiscardUnknown() { + xxx_messageInfo_NamespaceRowResponse.DiscardUnknown(m) } -var xxx_messageInfo_Row proto.InternalMessageInfo +var xxx_messageInfo_NamespaceRowResponse proto.InternalMessageInfo -func (m *Row) GetShares() [][]byte { +func (m *NamespaceRowResponse) GetShares() [][]byte { if m != nil { return m.Shares } return nil } -func (m *Row) GetProof() *pb.Proof { +func (m *NamespaceRowResponse) GetProof() *pb.Proof { if m != nil { return m.Proof } @@ -216,37 +205,35 @@ func (m *Row) GetProof() *pb.Proof { func init() { proto.RegisterEnum("share.p2p.shrex.nd.StatusCode", StatusCode_name, StatusCode_value) proto.RegisterType((*GetSharesByNamespaceRequest)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceRequest") - proto.RegisterType((*GetSharesByNamespaceResponse)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceResponse") - proto.RegisterType((*Row)(nil), "share.p2p.shrex.nd.Row") + proto.RegisterType((*GetSharesByNamespaceStatusResponse)(nil), "share.p2p.shrex.nd.GetSharesByNamespaceStatusResponse") + proto.RegisterType((*NamespaceRowResponse)(nil), "share.p2p.shrex.nd.NamespaceRowResponse") } func init() { proto.RegisterFile("share/p2p/shrexnd/pb/share.proto", fileDescriptor_ed9f13149b0de397) } var fileDescriptor_ed9f13149b0de397 = []byte{ - // 357 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x91, 0x4d, 0x6b, 0xea, 0x40, - 0x14, 0x86, 0xf3, 0xe1, 0xcd, 0xd5, 0xa3, 0xd7, 0x1b, 0xe6, 0xc2, 0x35, 0x54, 0x09, 0x22, 0x14, - 0xa4, 0x85, 0x04, 0x52, 0xe8, 0x3e, 0x7e, 0xb4, 0x95, 0xda, 0x51, 0x46, 0xfb, 0xb1, 0x93, 0xa4, - 0x4e, 0xc9, 0xa6, 0x99, 0x69, 0x26, 0x62, 0xbb, 0xee, 0x1f, 0xe8, 0xcf, 0xea, 0xd2, 0x65, 0x97, - 0x45, 0xff, 0x48, 0xc9, 0x98, 0xe2, 0xa2, 0xee, 0xe6, 0xbc, 0xe7, 0x39, 0x67, 0x5e, 0xde, 0x03, - 0x4d, 0x11, 0x05, 0x09, 0x75, 0xb9, 0xc7, 0x5d, 0x11, 0x25, 0xf4, 0x39, 0x9e, 0xbb, 0x3c, 0x74, - 0xa5, 0xe8, 0xf0, 0x84, 0xa5, 0x0c, 0xa1, 0xbc, 0xf0, 0xb8, 0x23, 0x09, 0x27, 0x9e, 0x1f, 0x54, - 0x79, 0xe8, 0xf2, 0x84, 0xb1, 0x87, 0x2d, 0xd3, 0xba, 0x83, 0xfa, 0x39, 0x4d, 0x27, 0x19, 0x28, - 0x3a, 0x2f, 0x38, 0x78, 0xa4, 0x82, 0x07, 0xf7, 0x94, 0xd0, 0xa7, 0x05, 0x15, 0x29, 0xaa, 0x43, - 0x29, 0x61, 0x2c, 0x9d, 0x45, 0x81, 0x88, 0x2c, 0xb5, 0xa9, 0xb6, 0x2b, 0xa4, 0x98, 0x09, 0x17, - 0x81, 0x88, 0x50, 0x03, 0x4a, 0xf1, 0xf7, 0x80, 0xa5, 0xc9, 0xe6, 0x4e, 0x68, 0xbd, 0xaa, 0xd0, - 0xd8, 0xbf, 0x5a, 0x70, 0x16, 0x0b, 0x8a, 0x4e, 0xc1, 0x10, 0x69, 0x90, 0x2e, 0x84, 0x5c, 0x5c, - 0xf5, 0x6c, 0xe7, 0xa7, 0x5f, 0x67, 0x22, 0x89, 0x2e, 0x9b, 0x53, 0x92, 0xd3, 0xe8, 0x18, 0x0a, - 0x09, 0x5b, 0x0a, 0x4b, 0x6b, 0xea, 0xed, 0xb2, 0x57, 0xdb, 0x37, 0x45, 0xd8, 0x92, 0x48, 0xa8, - 0xd5, 0x03, 0x9d, 0xb0, 0x25, 0xfa, 0x0f, 0x86, 0xc4, 0xb2, 0xbf, 0xf4, 0x76, 0x85, 0xe4, 0x15, - 0x3a, 0x84, 0x5f, 0x32, 0x0d, 0x69, 0xbf, 0xec, 0xfd, 0x75, 0xf2, 0x6c, 0x42, 0x67, 0x9c, 0x3d, - 0xc8, 0xb6, 0x7b, 0x74, 0x0b, 0xb0, 0x33, 0x82, 0xca, 0xf0, 0x7b, 0x80, 0x6f, 0xfc, 0xe1, 0xa0, - 0x67, 0x2a, 0xc8, 0x00, 0x6d, 0x74, 0x69, 0xaa, 0xe8, 0x0f, 0x94, 0xf0, 0x68, 0x3a, 0x3b, 0x1b, - 0x5d, 0xe3, 0x9e, 0xa9, 0xa1, 0x0a, 0x14, 0x07, 0x78, 0xda, 0x27, 0xd8, 0x1f, 0x9a, 0x3a, 0xaa, - 0xc1, 0x3f, 0xec, 0x5f, 0xf5, 0x27, 0x63, 0xbf, 0xdb, 0x9f, 0xed, 0xb0, 0x42, 0xc7, 0x7a, 0x5f, - 0xdb, 0xea, 0x6a, 0x6d, 0xab, 0x9f, 0x6b, 0x5b, 0x7d, 0xdb, 0xd8, 0xca, 0x6a, 0x63, 0x2b, 0x1f, - 0x1b, 0x5b, 0x09, 0x0d, 0x79, 0x9f, 0x93, 0xaf, 0x00, 0x00, 0x00, 0xff, 0xff, 0x8e, 0x8b, 0x14, - 0xc5, 0xe7, 0x01, 0x00, 0x00, + // 326 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x6c, 0x90, 0x4f, 0x4b, 0xf3, 0x40, + 0x10, 0xc6, 0x93, 0x96, 0x37, 0x6f, 0x3b, 0xad, 0x35, 0x2c, 0x22, 0xc5, 0xca, 0x52, 0x02, 0x42, + 0xf1, 0xb0, 0x81, 0x08, 0x1e, 0x85, 0xd6, 0xfa, 0xa7, 0x58, 0x52, 0xd9, 0xb6, 0xe2, 0x41, 0x28, + 0x1b, 0xbb, 0x92, 0x8b, 0xd9, 0x35, 0xbb, 0x45, 0xfd, 0x16, 0x7e, 0x2c, 0x8f, 0x3d, 0x7a, 0x94, + 0xf6, 0x8b, 0x48, 0xb6, 0xd1, 0x1c, 0xf4, 0xb6, 0xf3, 0xcc, 0x33, 0xbf, 0x7d, 0x66, 0xa0, 0xad, + 0x62, 0x96, 0x72, 0x5f, 0x06, 0xd2, 0x57, 0x71, 0xca, 0x5f, 0x92, 0xb9, 0x2f, 0x23, 0xdf, 0x88, + 0x44, 0xa6, 0x42, 0x0b, 0x84, 0xf2, 0x22, 0x90, 0xc4, 0x38, 0x48, 0x32, 0xdf, 0x6b, 0xc8, 0xc8, + 0x97, 0xa9, 0x10, 0x0f, 0x1b, 0x8f, 0x77, 0x0b, 0xad, 0x0b, 0xae, 0xc7, 0x99, 0x51, 0xf5, 0x5e, + 0x43, 0xf6, 0xc8, 0x95, 0x64, 0xf7, 0x9c, 0xf2, 0xa7, 0x05, 0x57, 0x1a, 0xb5, 0xa0, 0x9a, 0x0a, + 0xa1, 0x67, 0x31, 0x53, 0x71, 0xd3, 0x6e, 0xdb, 0x9d, 0x3a, 0xad, 0x64, 0xc2, 0x25, 0x53, 0x31, + 0xda, 0x87, 0x6a, 0xf2, 0x3d, 0xd0, 0x2c, 0x99, 0x66, 0x21, 0x78, 0x77, 0xe0, 0xfd, 0x45, 0x1e, + 0x6b, 0xa6, 0x17, 0x8a, 0x72, 0x25, 0x45, 0xa2, 0x38, 0x3a, 0x06, 0x47, 0x19, 0xc5, 0xd0, 0x1b, + 0x01, 0x26, 0xbf, 0x43, 0x93, 0xcd, 0xcc, 0xa9, 0x98, 0x73, 0x9a, 0xbb, 0xbd, 0x29, 0xec, 0x14, + 0x61, 0xc5, 0xf3, 0x0f, 0x6f, 0x17, 0x1c, 0x03, 0xc8, 0x78, 0xe5, 0x4e, 0x9d, 0xe6, 0x15, 0x3a, + 0x80, 0x7f, 0x66, 0x6d, 0x93, 0xb3, 0x16, 0x6c, 0x93, 0xfc, 0x08, 0x11, 0xb9, 0xce, 0x1e, 0x74, + 0xd3, 0x3d, 0x3c, 0x01, 0x28, 0x3e, 0x43, 0x35, 0xf8, 0x3f, 0x08, 0x6f, 0xba, 0xc3, 0x41, 0xdf, + 0xb5, 0x90, 0x03, 0xa5, 0xd1, 0x95, 0x6b, 0xa3, 0x2d, 0xa8, 0x86, 0xa3, 0xc9, 0xec, 0x7c, 0x34, + 0x0d, 0xfb, 0x6e, 0x09, 0xd5, 0xa1, 0x32, 0x08, 0x27, 0x67, 0x34, 0xec, 0x0e, 0xdd, 0x72, 0xaf, + 0xf9, 0xbe, 0xc2, 0xf6, 0x72, 0x85, 0xed, 0xcf, 0x15, 0xb6, 0xdf, 0xd6, 0xd8, 0x5a, 0xae, 0xb1, + 0xf5, 0xb1, 0xc6, 0x56, 0xe4, 0x98, 0x7b, 0x1f, 0x7d, 0x05, 0x00, 0x00, 0xff, 0xff, 0x1a, 0x53, + 0xb4, 0x86, 0xb7, 0x01, 0x00, 0x00, } func (m *GetSharesByNamespaceRequest) Marshal() (dAtA []byte, err error) { @@ -286,7 +273,7 @@ func (m *GetSharesByNamespaceRequest) MarshalToSizedBuffer(dAtA []byte) (int, er return len(dAtA) - i, nil } -func (m *GetSharesByNamespaceResponse) Marshal() (dAtA []byte, err error) { +func (m *GetSharesByNamespaceStatusResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -296,30 +283,16 @@ func (m *GetSharesByNamespaceResponse) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *GetSharesByNamespaceResponse) MarshalTo(dAtA []byte) (int, error) { +func (m *GetSharesByNamespaceStatusResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *GetSharesByNamespaceResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *GetSharesByNamespaceStatusResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int _ = l - if len(m.Rows) > 0 { - for iNdEx := len(m.Rows) - 1; iNdEx >= 0; iNdEx-- { - { - size, err := m.Rows[iNdEx].MarshalToSizedBuffer(dAtA[:i]) - if err != nil { - return 0, err - } - i -= size - i = encodeVarintShare(dAtA, i, uint64(size)) - } - i-- - dAtA[i] = 0x12 - } - } if m.Status != 0 { i = encodeVarintShare(dAtA, i, uint64(m.Status)) i-- @@ -328,7 +301,7 @@ func (m *GetSharesByNamespaceResponse) MarshalToSizedBuffer(dAtA []byte) (int, e return len(dAtA) - i, nil } -func (m *Row) Marshal() (dAtA []byte, err error) { +func (m *NamespaceRowResponse) Marshal() (dAtA []byte, err error) { size := m.Size() dAtA = make([]byte, size) n, err := m.MarshalToSizedBuffer(dAtA[:size]) @@ -338,12 +311,12 @@ func (m *Row) Marshal() (dAtA []byte, err error) { return dAtA[:n], nil } -func (m *Row) MarshalTo(dAtA []byte) (int, error) { +func (m *NamespaceRowResponse) MarshalTo(dAtA []byte) (int, error) { size := m.Size() return m.MarshalToSizedBuffer(dAtA[:size]) } -func (m *Row) MarshalToSizedBuffer(dAtA []byte) (int, error) { +func (m *NamespaceRowResponse) MarshalToSizedBuffer(dAtA []byte) (int, error) { i := len(dAtA) _ = i var l int @@ -400,7 +373,7 @@ func (m *GetSharesByNamespaceRequest) Size() (n int) { return n } -func (m *GetSharesByNamespaceResponse) Size() (n int) { +func (m *GetSharesByNamespaceStatusResponse) Size() (n int) { if m == nil { return 0 } @@ -409,16 +382,10 @@ func (m *GetSharesByNamespaceResponse) Size() (n int) { if m.Status != 0 { n += 1 + sovShare(uint64(m.Status)) } - if len(m.Rows) > 0 { - for _, e := range m.Rows { - l = e.Size() - n += 1 + l + sovShare(uint64(l)) - } - } return n } -func (m *Row) Size() (n int) { +func (m *NamespaceRowResponse) Size() (n int) { if m == nil { return 0 } @@ -561,7 +528,7 @@ func (m *GetSharesByNamespaceRequest) Unmarshal(dAtA []byte) error { } return nil } -func (m *GetSharesByNamespaceResponse) Unmarshal(dAtA []byte) error { +func (m *GetSharesByNamespaceStatusResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -584,10 +551,10 @@ func (m *GetSharesByNamespaceResponse) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: GetSharesByNamespaceResponse: wiretype end group for non-group") + return fmt.Errorf("proto: GetSharesByNamespaceStatusResponse: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: GetSharesByNamespaceResponse: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: GetSharesByNamespaceStatusResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: @@ -609,40 +576,6 @@ func (m *GetSharesByNamespaceResponse) Unmarshal(dAtA []byte) error { break } } - case 2: - if wireType != 2 { - return fmt.Errorf("proto: wrong wireType = %d for field Rows", wireType) - } - var msglen int - for shift := uint(0); ; shift += 7 { - if shift >= 64 { - return ErrIntOverflowShare - } - if iNdEx >= l { - return io.ErrUnexpectedEOF - } - b := dAtA[iNdEx] - iNdEx++ - msglen |= int(b&0x7F) << shift - if b < 0x80 { - break - } - } - if msglen < 0 { - return ErrInvalidLengthShare - } - postIndex := iNdEx + msglen - if postIndex < 0 { - return ErrInvalidLengthShare - } - if postIndex > l { - return io.ErrUnexpectedEOF - } - m.Rows = append(m.Rows, &Row{}) - if err := m.Rows[len(m.Rows)-1].Unmarshal(dAtA[iNdEx:postIndex]); err != nil { - return err - } - iNdEx = postIndex default: iNdEx = preIndex skippy, err := skipShare(dAtA[iNdEx:]) @@ -664,7 +597,7 @@ func (m *GetSharesByNamespaceResponse) Unmarshal(dAtA []byte) error { } return nil } -func (m *Row) Unmarshal(dAtA []byte) error { +func (m *NamespaceRowResponse) Unmarshal(dAtA []byte) error { l := len(dAtA) iNdEx := 0 for iNdEx < l { @@ -687,10 +620,10 @@ func (m *Row) Unmarshal(dAtA []byte) error { fieldNum := int32(wire >> 3) wireType := int(wire & 0x7) if wireType == 4 { - return fmt.Errorf("proto: Row: wiretype end group for non-group") + return fmt.Errorf("proto: NamespaceRowResponse: wiretype end group for non-group") } if fieldNum <= 0 { - return fmt.Errorf("proto: Row: illegal tag %d (wire type %d)", fieldNum, wire) + return fmt.Errorf("proto: NamespaceRowResponse: illegal tag %d (wire type %d)", fieldNum, wire) } switch fieldNum { case 1: diff --git a/share/p2p/shrexnd/pb/share.proto b/share/p2p/shrexnd/pb/share.proto index 931b25faa9..a5bdbfa071 100644 --- a/share/p2p/shrexnd/pb/share.proto +++ b/share/p2p/shrexnd/pb/share.proto @@ -8,9 +8,8 @@ message GetSharesByNamespaceRequest{ bytes namespace = 2; } -message GetSharesByNamespaceResponse{ +message GetSharesByNamespaceStatusResponse{ StatusCode status = 1; - repeated Row rows = 2; } enum StatusCode { @@ -18,10 +17,9 @@ enum StatusCode { OK = 1; NOT_FOUND = 2; INTERNAL = 3; - NAMESPACE_NOT_FOUND = 4; }; -message Row { +message NamespaceRowResponse { repeated bytes shares = 1; proof.pb.Proof proof = 2; } diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 42810b1f48..b391a3b2ce 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -2,7 +2,6 @@ package shrexnd import ( "context" - "encoding/hex" "errors" "fmt" "time" @@ -57,11 +56,7 @@ func NewServer(params *Parameters, host host.Host, store *eds.Store, getter shar ctx, cancel := context.WithCancel(context.Background()) srv.cancel = cancel - handler := func(s network.Stream) { - srv.handleNamespacedData(ctx, s) - } - srv.handler = srv.middleware.RateLimitHandler(handler) - + srv.handler = srv.middleware.RateLimitHandler(srv.streamHandler(ctx)) return srv, nil } @@ -78,6 +73,19 @@ func (srv *Server) Stop(context.Context) error { return nil } +func (srv *Server) streamHandler(ctx context.Context) network.StreamHandler { + return func(s network.Stream) { + err := srv.handleNamespacedData(ctx, s) + if err != nil { + s.Reset() //nolint:errcheck + return + } + if err = s.Close(); err != nil { + log.Debugw("server: closing stream", "err", err) + } + } +} + // SetHandler sets server handler func (srv *Server) SetHandler(handler network.StreamHandler) { srv.handler = handler @@ -90,104 +98,124 @@ func (srv *Server) observeRateLimitedRequests() { } } -func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stream) { - logger := log.With("peer", stream.Conn().RemotePeer().String()) - logger.Debug("server: handling nd request") +func (srv *Server) handleNamespacedData(ctx context.Context, stream network.Stream) error { + logger := log.With("source", "server", "peer", stream.Conn().RemotePeer().String()) + logger.Debug("handling nd request") srv.observeRateLimitedRequests() + req, err := srv.readRequest(logger, stream) + if err != nil { + logger.Warnw("read request", "err", err) + srv.metrics.ObserveRequests(ctx, 1, p2p.StatusBadRequest) + return err + } + + logger = logger.With("namespace", share.Namespace(req.Namespace).String(), + "hash", share.DataHash(req.RootHash).String()) + + ctx, cancel := context.WithTimeout(ctx, srv.params.HandleRequestTimeout) + defer cancel() + + shares, status, err := srv.getNamespaceData(ctx, req.RootHash, req.Namespace) + if err != nil { + // server should respond with status regardless if there was an error getting data + sendErr := srv.respondStatus(ctx, logger, stream, status) + if sendErr != nil { + logger.Errorw("sending response", "err", sendErr) + srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSendRespErr) + } + logger.Errorw("handling request", "err", err) + return errors.Join(err, sendErr) + } + + err = srv.respondStatus(ctx, logger, stream, status) + if err != nil { + logger.Errorw("sending response", "err", err) + srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSendRespErr) + return err + } + + err = srv.sendNamespacedShares(shares, stream) + if err != nil { + logger.Errorw("send nd data", "err", err) + srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSendRespErr) + return err + } + return nil +} +func (srv *Server) readRequest( + logger *zap.SugaredLogger, + stream network.Stream, +) (*pb.GetSharesByNamespaceRequest, error) { err := stream.SetReadDeadline(time.Now().Add(srv.params.ServerReadTimeout)) if err != nil { - logger.Debugw("server: setting read deadline", "err", err) + logger.Debugw("setting read deadline", "err", err) } var req pb.GetSharesByNamespaceRequest _, err = serde.Read(stream, &req) if err != nil { - logger.Warnw("server: reading request", "err", err) - stream.Reset() //nolint:errcheck - return + return nil, fmt.Errorf("reading request: %w", err) + } - logger = logger.With("namespace", hex.EncodeToString(req.Namespace), "hash", share.DataHash(req.RootHash).String()) - logger.Debugw("server: new request") + logger.Debugw("new request") err = stream.CloseRead() if err != nil { - logger.Debugw("server: closing read side of the stream", "err", err) + logger.Debugw("closing read side of the stream", "err", err) } err = validateRequest(req) if err != nil { - logger.Warnw("server: invalid request", "err", err) - stream.Reset() //nolint:errcheck - return + return nil, fmt.Errorf("invalid request: %w", err) } + return &req, nil +} - ctx, cancel := context.WithTimeout(ctx, srv.params.HandleRequestTimeout) - defer cancel() - - dah, err := srv.store.GetDAH(ctx, req.RootHash) +func (srv *Server) getNamespaceData(ctx context.Context, + hash share.DataHash, namespace share.Namespace) (share.NamespacedShares, pb.StatusCode, error) { + dah, err := srv.store.GetDAH(ctx, hash) if err != nil { if errors.Is(err, eds.ErrNotFound) { - logger.Warn("server: DAH not found") - srv.respondNotFoundError(ctx, logger, stream) - return + return nil, pb.StatusCode_NOT_FOUND, nil } - logger.Errorw("server: retrieving DAH", "err", err) - srv.respondInternalError(ctx, logger, stream) - return + return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) } - shares, err := srv.getter.GetSharesByNamespace(ctx, dah, req.Namespace) - switch { - case errors.Is(err, share.ErrNotFound): - logger.Warn("server: nd not found") - srv.respondNotFoundError(ctx, logger, stream) - return - case err != nil: - logger.Errorw("server: retrieving shares", "err", err) - srv.respondInternalError(ctx, logger, stream) - return - } - - resp := namespacedSharesToResponse(shares) - srv.respond(ctx, logger, stream, resp) -} - -// validateRequest checks correctness of the request -func validateRequest(req pb.GetSharesByNamespaceRequest) error { - if err := share.Namespace(req.Namespace).ValidateForData(); err != nil { - return err - } - if len(req.RootHash) != sha256.Size { - return fmt.Errorf("incorrect root hash length: %v", len(req.RootHash)) + shares, err := srv.getter.GetSharesByNamespace(ctx, dah, namespace) + if err != nil { + return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving shares: %w", err) } - return nil + + return shares, pb.StatusCode_OK, nil } -// respondNotFoundError sends a not found response to client -func (srv *Server) respondNotFoundError(ctx context.Context, - logger *zap.SugaredLogger, stream network.Stream) { - resp := &pb.GetSharesByNamespaceResponse{ - Status: pb.StatusCode_NOT_FOUND, +func (srv *Server) respondStatus( + ctx context.Context, + logger *zap.SugaredLogger, + stream network.Stream, + status pb.StatusCode, +) error { + srv.observeStatus(ctx, status) + + err := stream.SetWriteDeadline(time.Now().Add(srv.params.ServerWriteTimeout)) + if err != nil { + logger.Debugw("setting write deadline", "err", err) } - srv.respond(ctx, logger, stream, resp) -} -// respondInternalError sends internal error response to client -func (srv *Server) respondInternalError(ctx context.Context, - logger *zap.SugaredLogger, stream network.Stream) { - resp := &pb.GetSharesByNamespaceResponse{ - Status: pb.StatusCode_INTERNAL, + _, err = serde.Write(stream, &pb.GetSharesByNamespaceStatusResponse{Status: status}) + if err != nil { + return fmt.Errorf("writing response: %w", err) } - srv.respond(ctx, logger, stream, resp) + + return nil } -// namespacedSharesToResponse encodes shares into proto and sends it to client with OK status code -func namespacedSharesToResponse(shares share.NamespacedShares) *pb.GetSharesByNamespaceResponse { - rows := make([]*pb.Row, 0, len(shares)) +// sendNamespacedShares encodes shares into proto messages and sends it to client +func (srv *Server) sendNamespacedShares(shares share.NamespacedShares, stream network.Stream) error { for _, row := range shares { - row := &pb.Row{ + row := &pb.NamespaceRowResponse{ Shares: row.Shares, Proof: &nmt_pb.Proof{ Start: int64(row.Proof.Start()), @@ -197,44 +225,32 @@ func namespacedSharesToResponse(shares share.NamespacedShares) *pb.GetSharesByNa IsMaxNamespaceIgnored: row.Proof.IsMaxNamespaceIDIgnored(), }, } - - rows = append(rows, row) - } - - status := pb.StatusCode_OK - if len(shares) == 0 || (len(shares) == 1 && len(shares[0].Shares) == 0) { - status = pb.StatusCode_NAMESPACE_NOT_FOUND - } - - return &pb.GetSharesByNamespaceResponse{ - Status: status, - Rows: rows, + _, err := serde.Write(stream, row) + if err != nil { + return fmt.Errorf("writing nd data to stream: %w", err) + } } + return nil } -func (srv *Server) respond(ctx context.Context, - logger *zap.SugaredLogger, stream network.Stream, resp *pb.GetSharesByNamespaceResponse) { - err := stream.SetWriteDeadline(time.Now().Add(srv.params.ServerWriteTimeout)) - if err != nil { - logger.Debugw("server: setting write deadline", "err", err) - } - - _, err = serde.Write(stream, resp) - if err != nil { - logger.Warnw("server: writing response", "err", err) - stream.Reset() //nolint:errcheck - return - } - +func (srv *Server) observeStatus(ctx context.Context, status pb.StatusCode) { switch { - case resp.Status == pb.StatusCode_OK: + case status == pb.StatusCode_OK: srv.metrics.ObserveRequests(ctx, 1, p2p.StatusSuccess) - case resp.Status == pb.StatusCode_NOT_FOUND: + case status == pb.StatusCode_NOT_FOUND: srv.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) - case resp.Status == pb.StatusCode_INTERNAL: + case status == pb.StatusCode_INTERNAL: srv.metrics.ObserveRequests(ctx, 1, p2p.StatusInternalErr) } - if err = stream.Close(); err != nil { - logger.Debugw("server: closing stream", "err", err) +} + +// validateRequest checks correctness of the request +func validateRequest(req pb.GetSharesByNamespaceRequest) error { + if err := share.Namespace(req.Namespace).ValidateForData(); err != nil { + return err } + if len(req.RootHash) != sha256.Size { + return fmt.Errorf("incorrect root hash length: %v", len(req.RootHash)) + } + return nil } diff --git a/share/sharetest/testing.go b/share/sharetest/testing.go index 50eb0eb003..3889260393 100644 --- a/share/sharetest/testing.go +++ b/share/sharetest/testing.go @@ -37,6 +37,26 @@ func RandShares(t require.TestingT, total int) []share.Share { return shares } +// RandSharesWithNamespace is same the as RandShares, but sets same namespace for all shares. +func RandSharesWithNamespace(t require.TestingT, namespace share.Namespace, total int) []share.Share { + if total&(total-1) != 0 { + t.Errorf("total must be power of 2: %d", total) + t.FailNow() + } + + shares := make([]share.Share, total) + rnd := rand.New(rand.NewSource(time.Now().Unix())) //nolint:gosec + for i := range shares { + shr := make([]byte, share.Size) + copy(share.GetNamespace(shr), namespace) + _, err := rnd.Read(share.GetData(shr)) + require.NoError(t, err) + shares[i] = shr + } + sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) + return shares +} + // RandV0Namespace generates random valid data namespace for testing purposes. func RandV0Namespace() share.Namespace { rb := make([]byte, namespace.NamespaceVersionZeroIDSize) From f00f8dad3e002484a0e96a860f69074deaff70c0 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Mon, 7 Aug 2023 14:15:53 +0200 Subject: [PATCH 119/388] chore: bump libp2p (#2533) --- go.mod | 28 ++++++++--------- go.sum | 58 ++++++++++++++++++------------------ nodebuilder/p2p/metrics.go | 3 -- nodebuilder/p2p/resources.go | 3 +- 4 files changed, 44 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index c89d7d77d3..0a02fe7c15 100644 --- a/go.mod +++ b/go.mod @@ -44,7 +44,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.10.0 github.com/ipld/go-car v0.6.0 - github.com/libp2p/go-libp2p v0.28.1 + github.com/libp2p/go-libp2p v0.29.0 github.com/libp2p/go-libp2p-kad-dht v0.21.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 @@ -52,9 +52,9 @@ require ( github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.9.0 + github.com/multiformats/go-multiaddr v0.10.1 github.com/multiformats/go-multiaddr-dns v0.3.1 - github.com/multiformats/go-multihash v0.2.2 + github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.14.0 github.com/pyroscope-io/client v0.7.1 @@ -72,11 +72,11 @@ require ( go.opentelemetry.io/otel/sdk/metric v0.39.0 go.opentelemetry.io/otel/trace v1.16.0 go.opentelemetry.io/proto/otlp v0.19.0 - go.uber.org/fx v1.19.3 + go.uber.org/fx v1.20.0 go.uber.org/zap v1.24.0 golang.org/x/crypto v0.11.0 - golang.org/x/exp v0.0.0-20230321023759-10a507213a29 - golang.org/x/sync v0.2.0 + golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/sync v0.3.0 golang.org/x/text v0.11.0 google.golang.org/grpc v1.56.2 google.golang.org/protobuf v1.31.0 @@ -167,7 +167,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect - github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 // indirect + github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.7.1 // indirect @@ -218,7 +218,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.5 // indirect + github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/reedsolomon v1.11.1 // indirect github.com/koron/go-ssdp v0.0.4 // indirect @@ -232,7 +232,7 @@ require ( github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-reuseport v0.3.0 // indirect - github.com/libp2p/go-yamux/v4 v4.0.0 // indirect + github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/magiconair/properties v1.8.6 // indirect github.com/mailru/easyjson v0.7.6 // indirect github.com/manifoldco/promptui v0.9.0 // indirect @@ -240,7 +240,7 @@ require ( github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.54 // indirect + github.com/miekg/dns v1.1.55 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect @@ -255,7 +255,7 @@ require ( github.com/multiformats/go-multicodec v0.9.0 // indirect github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.9.7 // indirect + github.com/onsi/ginkgo/v2 v2.11.0 // indirect github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -273,7 +273,7 @@ require ( github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.3.2 // indirect github.com/quic-go/qtls-go1-20 v0.2.2 // indirect - github.com/quic-go/quic-go v0.33.0 // indirect + github.com/quic-go/quic-go v0.36.2 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect @@ -310,12 +310,12 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.10.0 // indirect + golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.10.0 // indirect golang.org/x/term v0.10.0 // indirect - golang.org/x/tools v0.9.1 // indirect + golang.org/x/tools v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index f8cedecfd1..813ec946d7 100644 --- a/go.sum +++ b/go.sum @@ -799,8 +799,8 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230602150820-91b7bce49751 h1:hR7/MlvK23p6+lIw9SN1TigNLn9ZnF3W4SYRKq2gAHs= -github.com/google/pprof v0.0.0-20230602150820-91b7bce49751/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= +github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1148,8 +1148,8 @@ github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8 github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.16.5 h1:IFV2oUNUzZaz+XyusxpLzpzS8Pt5rh0Z16For/djlyI= -github.com/klauspost/compress v1.16.5/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= +github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1214,8 +1214,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= -github.com/libp2p/go-libp2p v0.28.1 h1:YurK+ZAI6cKfASLJBVFkpVBdl3wGhFi6fusOt725ii8= -github.com/libp2p/go-libp2p v0.28.1/go.mod h1:s3Xabc9LSwOcnv9UD4nORnXKTsWkPMkIMB/JIGXVnzk= +github.com/libp2p/go-libp2p v0.29.0 h1:QduJ2XQr/Crg4EnloueWDL0Jj86N3Ezhyyj7XH+XwHI= +github.com/libp2p/go-libp2p v0.29.0/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= @@ -1422,8 +1422,8 @@ github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZj github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= github.com/libp2p/go-yamux/v3 v3.0.2/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= github.com/libp2p/go-yamux/v3 v3.1.1/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= -github.com/libp2p/go-yamux/v4 v4.0.0 h1:+Y80dV2Yx/kv7Y7JKu0LECyVdMXm1VUoko+VQ9rBfZQ= -github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= +github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= +github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/libp2p/zeroconf/v2 v2.1.1/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= @@ -1497,8 +1497,8 @@ github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7 github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.54 h1:5jon9mWcb0sFJGpnI99tOMhCPyJ+RPVz5b63MQG0VWI= -github.com/miekg/dns v1.1.54/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= +github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1574,8 +1574,8 @@ github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9x github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= -github.com/multiformats/go-multiaddr v0.9.0 h1:3h4V1LHIk5w4hJHekMKWALPXErDfz/sggzwC/NcqbDQ= -github.com/multiformats/go-multiaddr v0.9.0/go.mod h1:mI67Lb1EeTOYb8GQfL/7wpIZwc46ElrvzhYnoJOmTT0= +github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= +github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1609,8 +1609,8 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= -github.com/multiformats/go-multihash v0.2.2 h1:Uu7LWs/PmWby1gkj1S1DXx3zyd3aVabA4FiMKn/2tAc= -github.com/multiformats/go-multihash v0.2.2/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= +github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= +github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= @@ -1662,8 +1662,8 @@ github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvw github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= -github.com/onsi/ginkgo/v2 v2.9.7 h1:06xGQy5www2oN160RtEZoTvnP2sPhEfePYmCDc2szss= -github.com/onsi/ginkgo/v2 v2.9.7/go.mod h1:cxrmXWykAwTwhQsJOPfdIDiJ+l2RYq7U8hFU+M/1uw0= +github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= +github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1671,7 +1671,7 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= -github.com/onsi/gomega v1.27.7 h1:fVih9JD6ogIiHUN6ePK7HJidyEDpWGVB5mzM7cWNXoU= +github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= @@ -1795,8 +1795,8 @@ github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc8 github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.33.0 h1:ItNoTDN/Fm/zBlq769lLJc8ECe9gYaW40veHCCco7y0= -github.com/quic-go/quic-go v0.33.0/go.mod h1:YMuhaAV9/jIu0XclDXwZPAsP/2Kgr5yMYhe9oxhhOFA= +github.com/quic-go/quic-go v0.36.2 h1:ZX/UNQ4gvpCv2RmwdbA6lrRjF6EBm5yZ7TMoT4NQVrA= +github.com/quic-go/quic-go v0.36.2/go.mod h1:zPetvwDlILVxt15n3hr3Gf/I3mDf7LpLKPhR4Ez0AZQ= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= @@ -2093,8 +2093,8 @@ go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= -go.uber.org/fx v1.19.3 h1:YqMRE4+2IepTYCMOvXqQpRa+QAVdiSTnsHU4XNWBceA= -go.uber.org/fx v1.19.3/go.mod h1:w2HrQg26ql9fLK7hlBiZ6JsRUKV+Lj/atT1KCjT8YhM= +go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ= +go.uber.org/fx v1.20.0/go.mod h1:qCUj0btiR3/JnanEr1TYEePfSw6o/4qYJscgvzQ5Ub0= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= @@ -2178,8 +2178,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29 h1:ooxPy7fPvB4kwsA2h+iBNHkAbp/4JxTSwCmvdjEYmug= -golang.org/x/exp v0.0.0-20230321023759-10a507213a29/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= +golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2211,8 +2211,8 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= -golang.org/x/mod v0.10.0 h1:lFO9qtOdlre5W1jxS3r/4szv2/6iXxScdzjoBMXNhYk= -golang.org/x/mod v0.10.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= +golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2339,8 +2339,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.2.0 h1:PUR+T4wwASmuSTYdKjYHI5TD22Wy5ogLU5qZCOLxBrI= -golang.org/x/sync v0.2.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= +golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2569,8 +2569,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.9.1 h1:8WMNJAz3zrtPmnYC7ISf5dEn3MT0gY7jBJfw27yrrLo= -golang.org/x/tools v0.9.1/go.mod h1:owI94Op576fPu3cIGQeHs3joujW/2Oc6MtlxbF5dfNc= +golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= +golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/nodebuilder/p2p/metrics.go b/nodebuilder/p2p/metrics.go index 3e6caf08ca..095c30d9b7 100644 --- a/nodebuilder/p2p/metrics.go +++ b/nodebuilder/p2p/metrics.go @@ -6,7 +6,6 @@ import ( "net/http" "time" - rcmgrObs "github.com/libp2p/go-libp2p/p2p/host/resource-manager/obs" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.uber.org/fx" @@ -28,8 +27,6 @@ const ( // prometheusMetrics option sets up native libp2p metrics up func prometheusMetrics(lifecycle fx.Lifecycle, registerer prometheus.Registerer) error { - rcmgrObs.MustRegisterWith(registerer) - registry := registerer.(*prometheus.Registry) mux := http.NewServeMux() diff --git a/nodebuilder/p2p/resources.go b/nodebuilder/p2p/resources.go index 371747463a..6e24e1e542 100644 --- a/nodebuilder/p2p/resources.go +++ b/nodebuilder/p2p/resources.go @@ -6,7 +6,6 @@ import ( "github.com/libp2p/go-libp2p" "github.com/libp2p/go-libp2p/core/network" rcmgr "github.com/libp2p/go-libp2p/p2p/host/resource-manager" - rcmgrObs "github.com/libp2p/go-libp2p/p2p/host/resource-manager/obs" ma "github.com/multiformats/go-multiaddr" madns "github.com/multiformats/go-multiaddr-dns" "go.uber.org/fx" @@ -60,7 +59,7 @@ func allowList(ctx context.Context, cfg Config, bootstrappers Bootstrappers) (rc } func traceReporter() rcmgr.Option { - str, err := rcmgrObs.NewStatsTraceReporter() + str, err := rcmgr.NewStatsTraceReporter() if err != nil { panic(err) // err is always nil as per sources } From a86ddca1773f86e3427fa48669bb219b99c0d765 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 7 Aug 2023 16:15:35 +0300 Subject: [PATCH 120/388] feat(cmd): use rpc client instead of `http.Request` (#2521) --- cmd/celestia/blob.go | 229 +++++++++++++++++++++++++++++++++++++++++++ cmd/celestia/rpc.go | 155 ++++++++--------------------- 2 files changed, 270 insertions(+), 114 deletions(-) create mode 100644 cmd/celestia/blob.go diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go new file mode 100644 index 0000000000..9dcd72dcb7 --- /dev/null +++ b/cmd/celestia/blob.go @@ -0,0 +1,229 @@ +package main + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "os" + "reflect" + "strconv" + + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/share" +) + +var plaintext bool + +func init() { + blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) + + getCmd.PersistentFlags().BoolVar( + &plaintext, + "plaintext", + false, + "printed blob's data as a plain text", + ) + getAllCmd.PersistentFlags().BoolVar( + &plaintext, + "plaintext", + false, + "printed blob's data as a plain text", + ) +} + +var blobCmd = &cobra.Command{ + Use: "blob [command]", + Short: "Allows to interact with the Blob Service via JSON-RPC", + Args: cobra.NoArgs, +} + +var getCmd = &cobra.Command{ + Use: "get [height, namespace, commitment]", + Args: cobra.ExactArgs(3), + Short: "Returns the blob for the given namespace by commitment at a particular height.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + height, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a height:%v", err) + } + + namespace, err := parseV0Namespace(args[1]) + if err != nil { + return fmt.Errorf("error parsing a namespace:%v", err) + } + + commitment, err := base64.StdEncoding.DecodeString(args[2]) + if err != nil { + return fmt.Errorf("error parsing a commitment:%v", err) + } + + blob, err := client.Blob.Get(cmd.Context(), height, namespace, commitment) + + printOutput(blob, err) + return nil + }, +} + +var getAllCmd = &cobra.Command{ + Use: "get-all [height, namespace]", + Args: cobra.ExactArgs(2), + Short: "Returns all blobs for the given namespace at a particular height.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + height, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a height:%v", err) + } + + namespace, err := parseV0Namespace(args[1]) + if err != nil { + return fmt.Errorf("error parsing a namespace:%v", err) + } + + blobs, err := client.Blob.GetAll(cmd.Context(), height, []share.Namespace{namespace}) + + printOutput(blobs, err) + return nil + }, +} + +var submitCmd = &cobra.Command{ + Use: "submit [namespace, blobData]", + Args: cobra.ExactArgs(2), + Short: "Submit the blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + namespace, err := parseV0Namespace(args[0]) + if err != nil { + return fmt.Errorf("error parsing a namespace:%v", err) + } + + parsedBlob, err := blob.NewBlobV0(namespace, []byte(args[1])) + if err != nil { + return fmt.Errorf("error creating a blob:%v", err) + } + + height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) + + response := struct { + Height uint64 `json:"uint64"` + Commitment blob.Commitment `json:"commitment"` + }{ + Height: height, + Commitment: parsedBlob.Commitment, + } + + printOutput(response, err) + return nil + }, +} + +var getProofCmd = &cobra.Command{ + Use: "get-proof [height, namespace, commitment]", + Args: cobra.ExactArgs(3), + Short: "Retrieves the blob in the given namespaces at the given height by commitment and returns its Proof.", + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + height, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a height:%v", err) + } + + namespace, err := parseV0Namespace(args[1]) + if err != nil { + return fmt.Errorf("error parsing a namespace:%v", err) + } + + commitment, err := base64.StdEncoding.DecodeString(args[2]) + if err != nil { + return fmt.Errorf("error parsing a commitment:%v", err) + } + + proof, err := client.Blob.GetProof(cmd.Context(), height, namespace, commitment) + + printOutput(proof, err) + return nil + }, +} + +func printOutput(data interface{}, err error) { + if err != nil { + data = err + } + + if plaintext && err == nil { + data = formatData(data) + } + + resp := struct { + Result interface{} `json:"result"` + }{ + Result: data, + } + + bytes, err := json.MarshalIndent(resp, "", " ") + if err != nil { + fmt.Fprintln(os.Stderr, err) + os.Exit(1) + } + fmt.Fprintln(os.Stdout, string(bytes)) +} + +func formatData(data interface{}) interface{} { + type tempBlob struct { + Namespace []byte `json:"namespace"` + Data string `json:"data"` + ShareVersion uint32 `json:"share_version"` + Commitment []byte `json:"commitment"` + } + + if reflect.TypeOf(data).Kind() == reflect.Slice { + blobs, ok := data.([]*blob.Blob) + if !ok { + fmt.Fprintln(os.Stderr, "could not cast to []blob.Blob") + os.Exit(1) + } + + result := make([]tempBlob, len(blobs)) + for i, b := range blobs { + result[i] = tempBlob{ + Namespace: b.Namespace(), + Data: string(b.Data), + ShareVersion: b.ShareVersion, + Commitment: b.Commitment, + } + } + return result + } + + b, ok := data.(*blob.Blob) + if !ok { + fmt.Fprintln(os.Stderr, "could not cast to blob.Blob") + os.Exit(1) + } + return tempBlob{ + Namespace: b.Namespace(), + Data: string(b.Data), + ShareVersion: b.ShareVersion, + Commitment: b.Commitment, + } +} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 64b21aaf86..05439ad7b4 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -2,9 +2,11 @@ package main import ( "bytes" + "context" "encoding/base64" "encoding/hex" "encoding/json" + "errors" "fmt" "io" "log" @@ -17,14 +19,11 @@ import ( "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/api/rpc/client" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/state" ) -const ( - authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" -) +const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" var requestURL string var authTokenFlag string @@ -61,6 +60,7 @@ func init() { false, "Print JSON-RPC request along with the response", ) + rpcCmd.AddCommand(blobCmd) rootCmd.AddCommand(rpcCmd) } @@ -68,6 +68,25 @@ var rpcCmd = &cobra.Command{ Use: "rpc [namespace] [method] [params...]", Short: "Send JSON-RPC request", Args: cobra.MinimumNArgs(2), + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + rpcClient, err := newRPCClient(cmd.Context()) + if err != nil { + return err + } + + ctx := context.WithValue(cmd.Context(), rpcClientKey{}, rpcClient) + cmd.SetContext(ctx) + return nil + }, + PersistentPostRunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + client.Close() + return nil + }, ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { modules := client.Modules if len(args) == 0 { @@ -115,116 +134,6 @@ func parseParams(method string, params []string) []interface{} { panic(fmt.Sprintf("Error parsing namespace: %v", err)) } parsedParams[1] = namespace - case "Submit": - // 1. Namespace - var err error - namespace, err := parseV0Namespace(params[0]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - // 2. Blob data - var blobData []byte - switch { - case strings.HasPrefix(params[1], "0x"): - decoded, err := hex.DecodeString(params[1][2:]) - if err != nil { - panic("Error decoding blob: hex string could not be decoded.") - } - blobData = decoded - case strings.HasPrefix(params[1], "\""): - // user input an utf string that needs to be encoded to base64 - src := []byte(params[1]) - blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) - base64.StdEncoding.Encode(blobData, []byte(params[1])) - default: - // otherwise, we assume the user has already encoded their input to base64 - blobData, err = base64.StdEncoding.DecodeString(params[1]) - if err != nil { - panic("Error decoding blob data: base64 string could not be decoded.") - } - } - parsedBlob, err := blob.NewBlobV0(namespace, blobData) - if err != nil { - panic(fmt.Sprintf("Error creating blob: %v", err)) - } - parsedParams[0] = []*blob.Blob{parsedBlob} - // param count doesn't match input length, so cut off nil values - return parsedParams[:1] - case "SubmitPayForBlob": - // 1. Fee (state.Int is a string) - parsedParams[0] = params[0] - // 2. GasLimit (uint64) - num, err := strconv.ParseUint(params[1], 10, 64) - if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") - } - parsedParams[1] = num - // 3. Namespace - namespace, err := parseV0Namespace(params[2]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - // 4. Blob data - var blobData []byte - switch { - case strings.HasPrefix(params[3], "0x"): - decoded, err := hex.DecodeString(params[3][2:]) - if err != nil { - panic("Error decoding blob: hex string could not be decoded.") - } - blobData = decoded - case strings.HasPrefix(params[3], "\""): - // user input an utf string that needs to be encoded to base64 - src := []byte(params[1]) - blobData = make([]byte, base64.StdEncoding.EncodedLen(len(src))) - base64.StdEncoding.Encode(blobData, []byte(params[3])) - default: - // otherwise, we assume the user has already encoded their input to base64 - blobData, err = base64.StdEncoding.DecodeString(params[3]) - if err != nil { - panic("Error decoding blob: base64 string could not be decoded.") - } - } - parsedBlob, err := blob.NewBlobV0(namespace, blobData) - if err != nil { - panic(fmt.Sprintf("Error creating blob: %v", err)) - } - parsedParams[2] = []*blob.Blob{parsedBlob} - return parsedParams[:3] - case "Get": - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - panic("Error parsing height: uint64 could not be parsed.") - } - parsedParams[0] = num - // 2. NamespaceID - namespace, err := parseV0Namespace(params[1]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - parsedParams[1] = namespace - // 3: Commitment - commitment, err := base64.StdEncoding.DecodeString(params[2]) - if err != nil { - panic("Error decoding commitment: base64 string could not be decoded.") - } - parsedParams[2] = commitment - return parsedParams - case "GetAll": // NOTE: Over the cli, you can only pass one namespace - // 1. Height - num, err := strconv.ParseUint(params[0], 10, 64) - if err != nil { - panic("Error parsing height: uint64 could not be parsed.") - } - parsedParams[0] = num - // 2. Namespace - namespace, err := parseV0Namespace(params[1]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - parsedParams[1] = []share.Namespace{namespace} - return parsedParams case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": var err error parsedParams[0], err = parseAddressFromString(params[0]) @@ -445,8 +354,26 @@ func decodeToBytes(param string) ([]byte, error) { } return decoded, nil } + func parseJSON(param string) (json.RawMessage, error) { var raw json.RawMessage err := json.Unmarshal([]byte(param), &raw) return raw, err } + +func newRPCClient(ctx context.Context) (*client.Client, error) { + if authTokenFlag == "" { + authTokenFlag = os.Getenv(authEnvKey) + } + return client.NewClient(ctx, requestURL, authTokenFlag) +} + +type rpcClientKey struct{} + +func rpcClient(ctx context.Context) (*client.Client, error) { + client, ok := ctx.Value(rpcClientKey{}).(*client.Client) + if !ok { + return nil, errors.New("rpc client was not set") + } + return client, nil +} From 65462525ea6afb819d8189c6d21cec0146ae55c5 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 7 Aug 2023 21:25:12 +0800 Subject: [PATCH 121/388] feat(eds/store): cache proofs after first eds recompute on sampling (#2429) ## Overview Before this change eds was recomputed twice on sampling: 1. inside sampling client to verify data integrity 2. On storing EDS to storage to calculate merkle tree proof nodes. This PR adds `proofsAdder` cache, that will store proof nodes for later use with key features: - cache could be optionally added to context and will be respected by ipld/shrex getter to populate on eds recompute. - cache is safe from second recompute, since it will only allow single recompute write and all consequent recomputes over same cache will be noop. Also this PR reworks writeEDS to use use `proofsAdder` instead of creating new blockstore and iterating over all keys in it. This approach speeds up writeEDS even without pre collected cache. Since there are no more need for blockstore being created, I have simplified writeEDS and removed a lot of redundant code including whole struct of `writeSession`. The PR also adds benchmark that uses disk for badger store (existing benchmark uses in-memory datastore instead of badger). This PR brings significant performance increase. On disk store badger results are: - before PR store.Put operation: **~1.5s** - after PR store.Put with cached proofs: **800ms** - after PR store.Put without cached proofs: **900ms** --- core/eds.go | 29 ++++++- core/exchange.go | 12 ++- core/listener.go | 6 +- nodebuilder/store_test.go | 83 +++++++++++++++++++ share/eds/eds.go | 170 ++++++++++++++------------------------ share/eds/eds_test.go | 29 ------- share/eds/retriever.go | 10 ++- share/getters/tee.go | 2 + share/ipld/nmt_adder.go | 101 +++++++++++++++++----- 9 files changed, 280 insertions(+), 162 deletions(-) diff --git a/core/eds.go b/core/eds.go index c87d947e35..eb93c249ba 100644 --- a/core/eds.go +++ b/core/eds.go @@ -3,11 +3,17 @@ package core import ( "context" "errors" + "fmt" "github.com/filecoin-project/dagstore" "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/shares" + "github.com/celestiaorg/celestia-app/pkg/square" + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -17,12 +23,31 @@ import ( // extendBlock extends the given block data, returning the resulting // ExtendedDataSquare (EDS). If there are no transactions in the block, // nil is returned in place of the eds. -func extendBlock(data types.Data, appVersion uint64) (*rsmt2d.ExtendedDataSquare, error) { +func extendBlock(data types.Data, appVersion uint64, options ...nmt.Option) (*rsmt2d.ExtendedDataSquare, error) { if app.IsEmptyBlock(data, appVersion) { return nil, nil } - return app.ExtendBlock(data, appVersion) + // Construct the data square from the block's transactions + dataSquare, err := square.Construct(data.Txs.ToSliceOfBytes(), appVersion, appconsts.SquareSizeUpperBound(appVersion)) + if err != nil { + return nil, err + } + return extendShares(shares.ToBytes(dataSquare), options...) +} + +func extendShares(s [][]byte, options ...nmt.Option) (*rsmt2d.ExtendedDataSquare, error) { + // Check that the length of the square is a power of 2. + if !shares.IsPowerOfTwo(len(s)) { + return nil, fmt.Errorf("number of shares is not a power of 2: got %d", len(s)) + } + // here we construct a tree + // Note: uses the nmt wrapper to construct the tree. + squareSize := square.Size(len(s)) + return rsmt2d.ComputeExtendedDataSquare(s, + appconsts.DefaultCodec(), + wrapper.NewConstructor(uint64(squareSize), + options...)) } // storeEDS will only store extended block if it is not empty and doesn't already exist. diff --git a/core/exchange.go b/core/exchange.go index a3e98c0307..e041ff980f 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -9,9 +9,11 @@ import ( "golang.org/x/sync/errgroup" libhead "github.com/celestiaorg/go-header" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/ipld" ) const concurrencyLimit = 4 @@ -105,7 +107,8 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende } // extend block data - eds, err := extendBlock(block.Data, block.Header.Version.App) + adder := ipld.NewProofsAdder(int(block.Data.SquareSize)) + eds, err := extendBlock(block.Data, block.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", &block.Height, err) } @@ -119,6 +122,8 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende return nil, fmt.Errorf("incorrect hash in header at height %d: expected %x, got %x", &block.Height, hash, eh.Hash()) } + + ctx = ipld.CtxWithProofsAdder(ctx, adder) err = storeEDS(ctx, eh.DAH.Hash(), eds, ce.store) if err != nil { return nil, fmt.Errorf("storing EDS to eds.Store for height %d: %w", &block.Height, err) @@ -142,7 +147,8 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 log.Debugw("fetched signed block from core", "height", b.Header.Height) // extend block data - eds, err := extendBlock(b.Data, b.Header.Version.App) + adder := ipld.NewProofsAdder(int(b.Data.SquareSize)) + eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", b.Header.Height, err) } @@ -151,6 +157,8 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 if err != nil { return nil, fmt.Errorf("constructing extended header for height %d: %w", b.Header.Height, err) } + + ctx = ipld.CtxWithProofsAdder(ctx, adder) err = storeEDS(ctx, eh.DAH.Hash(), eds, ce.store) if err != nil { return nil, fmt.Errorf("storing EDS to eds.Store for block height %d: %w", b.Header.Height, err) diff --git a/core/listener.go b/core/listener.go index 7ca408e4fb..334aa9293a 100644 --- a/core/listener.go +++ b/core/listener.go @@ -12,9 +12,11 @@ import ( "go.opentelemetry.io/otel/attribute" libhead "github.com/celestiaorg/go-header" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -150,7 +152,8 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS attribute.Int64("height", b.Header.Height), ) // extend block data - eds, err := extendBlock(b.Data, b.Header.Version.App) + adder := ipld.NewProofsAdder(int(b.Data.SquareSize)) + eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) if err != nil { return fmt.Errorf("extending block data: %w", err) } @@ -161,6 +164,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS } // attempt to store block data if not empty + ctx = ipld.CtxWithProofsAdder(ctx, adder) err = storeEDS(ctx, b.Header.DataHash.Bytes(), eds, cl.store) if err != nil { return fmt.Errorf("storing EDS: %w", err) diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index 512d45bb70..5f2d1bed83 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -1,13 +1,26 @@ package nodebuilder import ( + "context" "strconv" "testing" + "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" + "github.com/celestiaorg/celestia-node/share/sharetest" ) func TestRepo(t *testing.T) { @@ -50,3 +63,73 @@ func TestRepo(t *testing.T) { }) } } + +func BenchmarkStore(b *testing.B) { + ctx, cancel := context.WithCancel(context.Background()) + b.Cleanup(cancel) + + tmpDir := b.TempDir() + ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) + edsStore, err := eds.NewStore(tmpDir, ds) + require.NoError(b, err) + err = edsStore.Start(ctx) + require.NoError(b, err) + + // BenchmarkStore/bench_read_128-10 14 78970661 ns/op (~70ms) + b.Run("bench put 128", func(b *testing.B) { + b.ResetTimer() + dir := b.TempDir() + + err := Init(*DefaultConfig(node.Full), dir, node.Full) + require.NoError(b, err) + + store, err := OpenStore(dir, nil) + require.NoError(b, err) + ds, err := store.Datastore() + require.NoError(b, err) + edsStore, err := eds.NewStore(dir, ds) + require.NoError(b, err) + err = edsStore.Start(ctx) + require.NoError(b, err) + + size := 128 + b.Run("enabled eds proof caching", func(b *testing.B) { + b.StopTimer() + b.ResetTimer() + for i := 0; i < b.N; i++ { + adder := ipld.NewProofsAdder(size * 2) + shares := sharetest.RandShares(b, size*size) + eds, err := rsmt2d.ComputeExtendedDataSquare( + shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(size), + nmt.NodeVisitor(adder.VisitFn())), + ) + require.NoError(b, err) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) + ctx := ipld.CtxWithProofsAdder(ctx, adder) + + b.StartTimer() + err = edsStore.Put(ctx, dah.Hash(), eds) + b.StopTimer() + require.NoError(b, err) + } + }) + + b.Run("disabled eds proof caching", func(b *testing.B) { + b.ResetTimer() + b.StopTimer() + for i := 0; i < b.N; i++ { + eds := edstest.RandEDS(b, size) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) + + b.StartTimer() + err = edsStore.Put(ctx, dah.Hash(), eds) + b.StopTimer() + require.NoError(b, err) + } + }) + }) +} diff --git a/share/eds/eds.go b/share/eds/eds.go index cc775491c9..f1efd3b433 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -8,12 +8,7 @@ import ( "io" "math" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - bstore "github.com/ipfs/go-ipfs-blockstore" - format "github.com/ipfs/go-ipld-format" "github.com/ipld/go-car" "github.com/ipld/go-car/util" "github.com/minio/sha256-simd" @@ -30,15 +25,6 @@ import ( var ErrEmptySquare = errors.New("share: importing empty data") -// writingSession contains the components needed to write an EDS to a CARv1 file with our custom -// node order. -type writingSession struct { - eds *rsmt2d.ExtendedDataSquare - store bstore.Blockstore // caches inner nodes (proofs) while we walk the nmt tree. - hasher *nmt.NmtHasher - w io.Writer -} - // WriteEDS writes the entire EDS into the given io.Writer as CARv1 file. // This includes all shares in quadrant order, followed by all inner nodes of the NMT tree. // Order: [ Carv1Header | Q1 | Q2 | Q3 | Q4 | inner nodes ] @@ -49,80 +35,28 @@ func WriteEDS(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) utils.SetStatusAndEnd(span, err) }() - // 1. Reimport EDS. This is needed to traverse the NMT tree and cache the inner nodes (proofs) - writer, err := initializeWriter(ctx, eds, w) - if err != nil { - return fmt.Errorf("share: creating eds writer: %w", err) - } - - // 2. Creates and writes Carv1Header - // - Roots are the eds Row + Col roots - err = writer.writeHeader() + // Creates and writes Carv1Header. Roots are the eds Row + Col roots + err = writeHeader(eds, w) if err != nil { return fmt.Errorf("share: writing carv1 header: %w", err) } - - // 3. Iterates over shares in quadrant order via eds.GetCell - err = writer.writeQuadrants() + // Iterates over shares in quadrant order via eds.GetCell + err = writeQuadrants(eds, w) if err != nil { return fmt.Errorf("share: writing shares: %w", err) } - // 4. Iterates over in-memory blockstore and writes proofs to the CAR - err = writer.writeProofs(ctx) + // Iterates over proofs and writes them to the CAR + err = writeProofs(ctx, eds, w) if err != nil { return fmt.Errorf("share: writing proofs: %w", err) } return nil } -// initializeWriter reimports the EDS into an in-memory blockstore in order to cache the proofs. -func initializeWriter(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) (*writingSession, error) { - // we use an in-memory blockstore and an offline exchange - store := bstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) - bs := blockservice.New(store, nil) - // shares are extracted from the eds so that we can reimport them to traverse - shares := share.ExtractEDS(eds) - shareCount := len(shares) - if shareCount == 0 { - return nil, ErrEmptySquare - } - odsWidth := int(math.Sqrt(float64(shareCount)) / 2) - // (shareCount*2) - (odsWidth*4) is the amount of inner nodes visited - batchAdder := ipld.NewNmtNodeAdder(ctx, bs, format.MaxSizeBatchOption(innerNodeBatchSize(shareCount, odsWidth))) - // this adder ignores leaves, so that they are not added to the store we iterate through in - // writeProofs - eds, err := rsmt2d.ImportExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(odsWidth), - nmt.NodeVisitor(batchAdder.VisitInnerNodes)), - ) - if err != nil { - return nil, fmt.Errorf("recomputing data square: %w", err) - } - // compute roots - _, err = eds.RowRoots() - if err != nil { - return nil, fmt.Errorf("computing row roots: %w", err) - } - // commit the batch to DAG - err = batchAdder.Commit() - if err != nil { - return nil, fmt.Errorf("committing inner nodes to the dag: %w", err) - } - - return &writingSession{ - eds: eds, - store: store, - hasher: nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace), - w: w, - }, nil -} - // writeHeader creates a CarV1 header using the EDS's Row and Column roots as the list of DAG roots. -func (w *writingSession) writeHeader() error { - rootCids, err := rootsToCids(w.eds) +func writeHeader(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { + rootCids, err := rootsToCids(eds) if err != nil { return fmt.Errorf("getting root cids: %w", err) } @@ -130,14 +64,15 @@ func (w *writingSession) writeHeader() error { return car.WriteHeader(&car.CarHeader{ Roots: rootCids, Version: 1, - }, w.w) + }, w) } // writeQuadrants reorders the shares to quadrant order and writes them to the CARv1 file. -func (w *writingSession) writeQuadrants() error { - shares := quadrantOrder(w.eds) +func writeQuadrants(eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { + hasher := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, ipld.NMTIgnoreMaxNamespace) + shares := quadrantOrder(eds) for _, share := range shares { - leaf, err := w.hasher.HashLeaf(share) + leaf, err := hasher.HashLeaf(share) if err != nil { return fmt.Errorf("hashing share: %w", err) } @@ -145,7 +80,7 @@ func (w *writingSession) writeQuadrants() error { if err != nil { return fmt.Errorf("getting cid from share: %w", err) } - err = util.LdWrite(w.w, cid.Bytes(), share) + err = util.LdWrite(w, cid.Bytes(), share) if err != nil { return fmt.Errorf("writing share to file: %w", err) } @@ -155,30 +90,15 @@ func (w *writingSession) writeQuadrants() error { // writeProofs iterates over the in-memory blockstore's keys and writes all inner nodes to the // CARv1 file. -func (w *writingSession) writeProofs(ctx context.Context) error { - // we only stored proofs to the store, so we can just iterate over them here without getting any - // leaves - proofs, err := w.store.AllKeysChan(ctx) +func writeProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Writer) error { + // check if proofs are collected by ipld.ProofsAdder in previous reconstructions of eds + proofs, err := getProofs(ctx, eds) if err != nil { - return fmt.Errorf("getting all keys from the blockstore: %w", err) + return fmt.Errorf("recomputing proofs: %w", err) } - for proofCid := range proofs { - block, err := w.store.Get(ctx, proofCid) - if err != nil { - return fmt.Errorf("getting proof from the blockstore: %w", err) - } - node := block.RawData() - left, right := node[:ipld.NmtHashSize], node[ipld.NmtHashSize:] - hash, err := w.hasher.HashNode(left, right) - if err != nil { - return fmt.Errorf("hashing node: %w", err) - } - cid, err := ipld.CidFromNamespacedSha256(hash) - if err != nil { - return fmt.Errorf("getting cid: %w", err) - } - err = util.LdWrite(w.w, cid.Bytes(), node) + for id, proof := range proofs { + err := util.LdWrite(w, id.Bytes(), proof) if err != nil { return fmt.Errorf("writing proof to the car: %w", err) } @@ -186,6 +106,41 @@ func (w *writingSession) writeProofs(ctx context.Context) error { return nil } +func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid][]byte, error) { + // check if there are proofs collected by ipld.ProofsAdder in previous reconstruction of eds + proofs := ipld.ProofsAdderFromCtx(ctx).Proofs() + if proofs != nil { + return proofs, nil + } + + // recompute proofs from eds + shares := share.ExtractEDS(eds) + shareCount := len(shares) + if shareCount == 0 { + return nil, ErrEmptySquare + } + odsWidth := int(math.Sqrt(float64(shareCount)) / 2) + + // this adder ignores leaves, so that they are not added to the store we iterate through in + // writeProofs + adder := ipld.NewProofsAdder(odsWidth * 2) + eds, err := rsmt2d.ImportExtendedDataSquare( + shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(odsWidth), + nmt.NodeVisitor(adder.VisitFn())), + ) + if err != nil { + return nil, fmt.Errorf("recomputing data square: %w", err) + } + // compute roots + if _, err = eds.RowRoots(); err != nil { + return nil, fmt.Errorf("computing row roots: %w", err) + } + + return adder.Proofs(), nil +} + // quadrantOrder reorders the shares in the EDS to quadrant row-by-row order, prepending the // respective namespace to the shares. // e.g. [ Q1 R1 | Q1 R2 | Q1 R3 | Q1 R4 | Q2 R1 | Q2 R2 .... ] @@ -287,10 +242,17 @@ func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d shares[i] = share.GetData(block.RawData()) } + // use proofs adder if provided, to cache collected proofs while recomputing the eds + var opts []nmt.Option + visitor := ipld.ProofsAdderFromCtx(ctx).VisitFn() + if visitor != nil { + opts = append(opts, nmt.NodeVisitor(visitor)) + } + eds, err = rsmt2d.ComputeExtendedDataSquare( shares, share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(odsWidth)), + wrapper.NewConstructor(uint64(odsWidth), opts...), ) if err != nil { return nil, fmt.Errorf("share: computing eds: %w", err) @@ -309,9 +271,3 @@ func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d } return eds, nil } - -// innerNodeBatchSize calculates the total number of inner nodes in an EDS, -// to be flushed to the dagstore in a single write. -func innerNodeBatchSize(shareCount int, odsWidth int) int { - return (shareCount * 2) - (odsWidth * 4) -} diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index af870f60b3..70ce7a0e8e 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -22,7 +22,6 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" ) //go:embed "testdata/example-root.json" @@ -137,34 +136,6 @@ func TestWriteEDSInQuadrantOrder(t *testing.T) { } } -// TestInnerNodeBatchSize verifies that the number of unique inner nodes is equal to ipld.BatchSize -// - shareCount. -func TestInnerNodeBatchSize(t *testing.T) { - tests := []struct { - name string - origWidth int - }{ - {"2", 2}, - {"4", 4}, - {"8", 8}, - {"16", 16}, - {"32", 32}, - // {"64", 64}, // test case too large for CI with race detector - } - for _, tt := range tests { - t.Run(tt.name, func(t *testing.T) { - extendedWidth := tt.origWidth * 2 - shareCount := extendedWidth * extendedWidth - assert.Equalf( - t, - innerNodeBatchSize(shareCount, tt.origWidth), - ipld.BatchSize(extendedWidth)-shareCount, - "batchSize(%v)", extendedWidth, - ) - }) - } -} - func TestReadWriteRoundtrip(t *testing.T) { eds := writeRandomEDS(t) dah, err := da.NewDataAvailabilityHeader(eds) diff --git a/share/eds/retriever.go b/share/eds/retriever.go index 0f52a903bc..a870e07e22 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -17,6 +17,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -122,7 +123,14 @@ type retrievalSession struct { func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHeader) (*retrievalSession, error) { size := len(dah.RowRoots) treeFn := func(_ rsmt2d.Axis, index uint) rsmt2d.Tree { - tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(size)/2, index) + // use proofs adder if provided, to cache collected proofs while recomputing the eds + var opts []nmt.Option + visitor := ipld.ProofsAdderFromCtx(ctx).VisitFn() + if visitor != nil { + opts = append(opts, nmt.NodeVisitor(visitor)) + } + + tree := wrapper.NewErasuredNamespacedMerkleTree(uint64(size)/2, index, opts...) return &tree } diff --git a/share/getters/tee.go b/share/getters/tee.go index fe43c971b7..8ee1e91390 100644 --- a/share/getters/tee.go +++ b/share/getters/tee.go @@ -14,6 +14,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/ipld" ) var _ share.Getter = (*TeeGetter)(nil) @@ -51,6 +52,7 @@ func (tg *TeeGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d. utils.SetStatusAndEnd(span, err) }() + ctx = ipld.CtxWithProofsAdder(ctx, ipld.NewProofsAdder(len(root.RowRoots))) eds, err = tg.getter.GetEDS(ctx, root) if err != nil { return nil, err diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go index 386a0083a3..0b4c521a80 100644 --- a/share/ipld/nmt_adder.go +++ b/share/ipld/nmt_adder.go @@ -9,6 +9,14 @@ import ( "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "github.com/ipfs/go-merkledag" + + "github.com/celestiaorg/nmt" +) + +type ctxKey int + +const ( + proofsAdderKey ctxKey = iota ) // NmtNodeAdder adds ipld.Nodes to the underlying ipld.Batch if it is inserted @@ -55,26 +63,6 @@ func (n *NmtNodeAdder) Visit(hash []byte, children ...[]byte) { } } -// VisitInnerNodes is a NodeVisitor that does not store leaf nodes to the blockservice. -func (n *NmtNodeAdder) VisitInnerNodes(hash []byte, children ...[]byte) { - n.lock.Lock() - defer n.lock.Unlock() - - if n.err != nil { - return // protect from further visits if there is an error - } - - id := MustCidFromNamespacedSha256(hash) - switch len(children) { - case 1: - break - case 2: - n.err = n.add.Add(n.ctx, newNMTNode(id, append(children[0], children[1]...))) - default: - panic("expected a binary tree") - } -} - // Commit checks for errors happened during Visit and if absent commits data to inner Batch. func (n *NmtNodeAdder) Commit() error { n.lock.Lock() @@ -112,3 +100,76 @@ func BatchSize(squareSize int) int { // and for the last two layers as well: return (squareSize*2-1)*squareSize*2 - (squareSize * squareSize) } + +type ProofsAdder struct { + lock sync.RWMutex + proofs map[cid.Cid][]byte +} + +func NewProofsAdder(squareSize int) *ProofsAdder { + return &ProofsAdder{ + // preallocate map to fit all inner nodes for given square size + proofs: make(map[cid.Cid][]byte, innerNodesAmount(squareSize)), + } +} + +func CtxWithProofsAdder(ctx context.Context, adder *ProofsAdder) context.Context { + return context.WithValue(ctx, proofsAdderKey, adder) +} + +func ProofsAdderFromCtx(ctx context.Context) *ProofsAdder { + val := ctx.Value(proofsAdderKey) + adder, ok := val.(*ProofsAdder) + if !ok || adder == nil { + return nil + } + return adder +} + +func (a *ProofsAdder) Proofs() map[cid.Cid][]byte { + if a == nil { + return nil + } + + a.lock.RLock() + defer a.lock.RUnlock() + return a.proofs +} + +func (a *ProofsAdder) VisitFn() nmt.NodeVisitorFn { + if a == nil { + return nil + } + + a.lock.RLock() + defer a.lock.RUnlock() + + // proofs are already collected, don't collect second time + if len(a.proofs) > 0 { + return nil + } + return a.visitInnerNodes +} + +func (a *ProofsAdder) visitInnerNodes(hash []byte, children ...[]byte) { + switch len(children) { + case 1: + break + case 2: + id := MustCidFromNamespacedSha256(hash) + a.addProof(id, append(children[0], children[1]...)) + default: + panic("expected a binary tree") + } +} + +func (a *ProofsAdder) addProof(id cid.Cid, proof []byte) { + a.lock.Lock() + defer a.lock.Unlock() + a.proofs[id] = proof +} + +// innerNodesAmount return amount of inner nodes in eds with given size +func innerNodesAmount(squareSize int) int { + return 2 * (squareSize - 1) * squareSize +} From ec36e7ee6eeb2ff7beb7137c5d8b20a102b610a8 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 8 Aug 2023 18:24:00 +0800 Subject: [PATCH 122/388] misc(metrics)!: split global attributes into two (#2546) ## Overview Currently there is only one global attribute, that build by concatenating both exported attributes: ``` "job" = "Bridge/robusta-11rc9/12D3KooWAEQUSJh7aJFq7Cw1mP4VWnqfHtHwwwnsXHFhpQ2DMn7j" ``` `Breaking`, because this PR is breaking metrics compatibility. --- nodebuilder/settings.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index c30996d24e..44508ba5e1 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -171,8 +171,13 @@ func initializeMetrics( sdk.WithReader(sdk.NewPeriodicReader(exp, sdk.WithTimeout(2*time.Second))), sdk.WithResource(resource.NewWithAttributes( semconv.SchemaURL, - semconv.ServiceNamespaceKey.String(nodeType.String()), - semconv.ServiceNameKey.String(fmt.Sprintf("%s/%s", network.String(), peerID.String()))))) + // ServiceNamespaceKey and ServiceNameKey will be concatenated into single attribute with key: + // "job" and value: "%service.namespace%/%service.name%" + semconv.ServiceNamespaceKey.String(network.String()), + semconv.ServiceNameKey.String(nodeType.String()), + // ServiceInstanceIDKey will be exported with key: "instance" + semconv.ServiceInstanceIDKey.String(peerID.String()), + ))) lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return provider.Shutdown(ctx) From 3a3473df66796bb92fe17e6925eb9e1bfba4ac96 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 9 Aug 2023 11:02:34 +0300 Subject: [PATCH 123/388] fix(blob): ensure that the share sequence is not empty (#2547) --- blob/helper.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/blob/helper.go b/blob/helper.go index 35f0abc540..e87055ae07 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -32,6 +32,11 @@ func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { return nil, err } + // ensure that sequence length is not 0 + if len(shareSequences) == 0 { + return nil, ErrBlobNotFound + } + blobs := make([]*Blob, len(shareSequences)) for i, sequence := range shareSequences { data, err := sequence.RawData() From 94a6b63ea3e28b00e53537406ac409881f681a7f Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 9 Aug 2023 11:03:24 +0300 Subject: [PATCH 124/388] fix(rpc): fix commands handling (#2515) --- cmd/celestia/blob.go | 16 ++++++++-------- cmd/celestia/rpc.go | 26 ++++++++++++++++++++++++-- share/getters/cascade.go | 8 +++++++- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 9dcd72dcb7..0b94fdcfdc 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -14,22 +14,22 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -var plaintext bool +var base64Flag bool func init() { blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) getCmd.PersistentFlags().BoolVar( - &plaintext, - "plaintext", + &base64Flag, + "base64", false, - "printed blob's data as a plain text", + "printed blob's data a base64 string", ) getAllCmd.PersistentFlags().BoolVar( - &plaintext, - "plaintext", + &base64Flag, + "base64", false, - "printed blob's data as a plain text", + "printed blob's data as a base64 string", ) } @@ -170,7 +170,7 @@ func printOutput(data interface{}, err error) { data = err } - if plaintext && err == nil { + if !base64Flag && err == nil { data = formatData(data) } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 05439ad7b4..dd50f07fc2 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -119,9 +119,17 @@ var rpcCmd = &cobra.Command{ func parseParams(method string, params []string) []interface{} { parsedParams := make([]interface{}, len(params)) - + validateParamsFn := func(has, want int) error { + if has != want { + return fmt.Errorf("rpc: invalid amount of params. has=%d, want=%d", has, want) + } + return nil + } switch method { case "GetSharesByNamespace": + if err := validateParamsFn(len(params), 2); err != nil { + panic(err) + } // 1. Share Root root, err := parseJSON(params[0]) if err != nil { @@ -134,8 +142,12 @@ func parseParams(method string, params []string) []interface{} { panic(fmt.Sprintf("Error parsing namespace: %v", err)) } parsedParams[1] = namespace + return parsedParams case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": var err error + if err = validateParamsFn(len(params), 2); err != nil { + panic(err) + } parsedParams[0], err = parseAddressFromString(params[0]) if err != nil { panic(fmt.Errorf("error parsing address: %w", err)) @@ -155,6 +167,9 @@ func parseParams(method string, params []string) []interface{} { case "Transfer", "Delegate", "Undelegate": // 1. Address var err error + if err = validateParamsFn(len(params), 4); err != nil { + panic(err) + } parsedParams[0], err = parseAddressFromString(params[0]) if err != nil { panic(fmt.Errorf("error parsing address: %w", err)) @@ -172,6 +187,9 @@ func parseParams(method string, params []string) []interface{} { case "CancelUnbondingDelegation": // 1. Validator Address var err error + if err = validateParamsFn(len(params), 5); err != nil { + panic(err) + } parsedParams[0], err = parseAddressFromString(params[0]) if err != nil { panic(fmt.Errorf("error parsing address: %w", err)) @@ -186,9 +204,13 @@ func parseParams(method string, params []string) []interface{} { panic("Error parsing gas limit: uint64 could not be parsed.") } parsedParams[4] = num + return parsedParams case "BeginRedelegate": // 1. Source Validator Address var err error + if err = validateParamsFn(len(params), 5); err != nil { + panic(err) + } parsedParams[0], err = parseAddressFromString(params[0]) if err != nil { panic(fmt.Errorf("error parsing address: %w", err)) @@ -207,6 +229,7 @@ func parseParams(method string, params []string) []interface{} { panic("Error parsing gas limit: uint64 could not be parsed.") } parsedParams[4] = num + return parsedParams default: } @@ -228,7 +251,6 @@ func parseParams(method string, params []string) []interface{} { parsedParams[i] = param } } - return parsedParams } diff --git a/share/getters/cascade.go b/share/getters/cascade.go index ef59946666..98f29c87ca 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -3,6 +3,7 @@ package getters import ( "context" "errors" + "fmt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -37,7 +38,12 @@ func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, co attribute.Int("col", col), )) defer span.End() - + if row >= len(root.RowRoots) || col >= len(root.ColumnRoots) { + err := fmt.Errorf("cascade/get-share: invalid indexes were provided:rowIndex=%d, colIndex=%d."+ + "squarewidth=%d", row, col, len(root.RowRoots)) + span.RecordError(err) + return nil, err + } get := func(ctx context.Context, get share.Getter) (share.Share, error) { return get.GetShare(ctx, root, row, col) } From 426c337368b9ed18b41586469ac24c6b52da15c3 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 10 Aug 2023 11:14:38 +0300 Subject: [PATCH 125/388] fix(cmd): fix formatting during submit (#2551) --- cmd/celestia/blob.go | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 0b94fdcfdc..28d05b3b42 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -199,8 +199,7 @@ func formatData(data interface{}) interface{} { if reflect.TypeOf(data).Kind() == reflect.Slice { blobs, ok := data.([]*blob.Blob) if !ok { - fmt.Fprintln(os.Stderr, "could not cast to []blob.Blob") - os.Exit(1) + return data } result := make([]tempBlob, len(blobs)) @@ -217,8 +216,7 @@ func formatData(data interface{}) interface{} { b, ok := data.(*blob.Blob) if !ok { - fmt.Fprintln(os.Stderr, "could not cast to blob.Blob") - os.Exit(1) + return data } return tempBlob{ Namespace: b.Namespace(), From 32834eb6722ec34450e7253cd164b705a81eb6d5 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 10 Aug 2023 20:24:52 +0800 Subject: [PATCH 126/388] fix(share/eds): don't use in-memory buffer for mount after shard recover (#2554) When shard recovered in dagstore, it was read from in-memory mount, that could have been read 0 times and had no in-memory buffer. --- share/eds/store.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/share/eds/store.go b/share/eds/store.go index 12993e8069..69e3c6b4a4 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -569,7 +569,7 @@ type inMemoryOnceMount struct { } func (m *inMemoryOnceMount) Fetch(ctx context.Context) (mount.Reader, error) { - if !m.readOnce.Swap(true) { + if m.buf != nil && !m.readOnce.Swap(true) { reader := &inMemoryReader{Reader: bytes.NewReader(m.buf.Bytes())} // release memory for gc, otherwise buffer will stick forever m.buf = nil From 87e9500120c27450e264ab4ffd59c87e92186194 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 11 Aug 2023 13:09:53 +0200 Subject: [PATCH 127/388] =?UTF-8?q?fix(header):=20Only=20bridge=20node=20s?= =?UTF-8?q?hould=20panic=20on=20data=20root=20mismatch=20in=20e=E2=80=A6?= =?UTF-8?q?=20(#2558)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This PR fixes a DoS first discovered by @Wondertan and then secondarily by @vgonkivs 🤠 Only bridge nodes should panic on receiving a header where the computed data root does not match the DataHash in the RawHeader on ExtendedHeader validation. Resolves #2555 --- core/exchange.go | 4 ++-- core/header_test.go | 8 +++----- core/listener.go | 2 +- header/header.go | 4 ++-- 4 files changed, 8 insertions(+), 10 deletions(-) diff --git a/core/exchange.go b/core/exchange.go index e041ff980f..a9b3a532d5 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -115,7 +115,7 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende // construct extended header eh, err := ce.construct(ctx, &block.Header, comm, vals, eds) if err != nil { - return nil, fmt.Errorf("constructing extended header for height %d: %w", &block.Height, err) + panic(fmt.Errorf("constructing extended header for height %d: %w", &block.Height, err)) } // verify hashes match if !bytes.Equal(hash, eh.Hash()) { @@ -155,7 +155,7 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 // create extended header eh, err := ce.construct(ctx, &b.Header, &b.Commit, &b.ValidatorSet, eds) if err != nil { - return nil, fmt.Errorf("constructing extended header for height %d: %w", b.Header.Height, err) + panic(fmt.Errorf("constructing extended header for height %d: %w", b.Header.Height, err)) } ctx = ipld.CtxWithProofsAdder(ctx, adder) diff --git a/core/header_test.go b/core/header_test.go index 6315fbc143..1c89db9d6b 100644 --- a/core/header_test.go +++ b/core/header_test.go @@ -41,11 +41,9 @@ func TestMakeExtendedHeaderForEmptyBlock(t *testing.T) { func TestMismatchedDataHash_ComputedRoot(t *testing.T) { header := headertest.RandExtendedHeader(t) - header.DataHash = rand.Bytes(32) - panicFn := func() { - header.Validate() //nolint:errcheck - } - assert.Panics(t, panicFn) + err := header.Validate() + assert.Contains(t, err.Error(), "mismatch between data hash commitment from"+ + " core header and computed data root") } diff --git a/core/listener.go b/core/listener.go index 334aa9293a..17be12a780 100644 --- a/core/listener.go +++ b/core/listener.go @@ -160,7 +160,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS // generate extended header eh, err := cl.construct(ctx, &b.Header, &b.Commit, &b.ValidatorSet, eds) if err != nil { - return fmt.Errorf("making extended header: %w", err) + panic(fmt.Errorf("making extended header: %w", err)) } // attempt to store block data if not empty diff --git a/header/header.go b/header/header.go index 0e386be8b5..d69b11d998 100644 --- a/header/header.go +++ b/header/header.go @@ -146,8 +146,8 @@ func (eh *ExtendedHeader) Validate() error { // ensure data root from raw header matches computed root if !bytes.Equal(eh.DAH.Hash(), eh.DataHash) { - panic(fmt.Sprintf("mismatch between data hash commitment from core header and computed data root "+ - "at height %d: data hash: %X, computed root: %X", eh.Height(), eh.DataHash, eh.DAH.Hash())) + return fmt.Errorf("mismatch between data hash commitment from core header and computed data root "+ + "at height %d: data hash: %X, computed root: %X", eh.Height(), eh.DataHash, eh.DAH.Hash()) } // Make sure the header is consistent with the commit. From 003c2c4ab4e75406216982b01af7418182753340 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 14 Aug 2023 20:32:19 +0700 Subject: [PATCH 128/388] fix(share/eds): dagstore shard restore reflection workaround (#2559) Turns out `dagstore` has bug in its restore shard code. For each shard it attempts to create deep copy of registered type, but copies only basic struct fields of the mount type. In out case it means, that it copies a pointer. That causes all shards to point to the same mount (same pointer). Before `dagstore` is fixed to take into account possibility of pointers/interface (and any other reference type) in mount type struct fields, we will use direct copy of `mount.FileMount` instead of `mount.Mount` interface. --- nodebuilder/store_test.go | 92 +++++++++++++++++++++++++++++---------- share/eds/store.go | 10 ++--- 2 files changed, 75 insertions(+), 27 deletions(-) diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index 5f2d1bed83..8a39849060 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -4,9 +4,8 @@ import ( "context" "strconv" "testing" + "time" - "github.com/ipfs/go-datastore" - ds_sync "github.com/ipfs/go-datastore/sync" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -68,30 +67,13 @@ func BenchmarkStore(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) - tmpDir := b.TempDir() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(tmpDir, ds) - require.NoError(b, err) - err = edsStore.Start(ctx) - require.NoError(b, err) - // BenchmarkStore/bench_read_128-10 14 78970661 ns/op (~70ms) b.Run("bench put 128", func(b *testing.B) { - b.ResetTimer() dir := b.TempDir() - err := Init(*DefaultConfig(node.Full), dir, node.Full) require.NoError(b, err) - store, err := OpenStore(dir, nil) - require.NoError(b, err) - ds, err := store.Datastore() - require.NoError(b, err) - edsStore, err := eds.NewStore(dir, ds) - require.NoError(b, err) - err = edsStore.Start(ctx) - require.NoError(b, err) - + store := newStore(ctx, b, dir) size := 128 b.Run("enabled eds proof caching", func(b *testing.B) { b.StopTimer() @@ -111,7 +93,7 @@ func BenchmarkStore(b *testing.B) { ctx := ipld.CtxWithProofsAdder(ctx, adder) b.StartTimer() - err = edsStore.Put(ctx, dah.Hash(), eds) + err = store.edsStore.Put(ctx, dah.Hash(), eds) b.StopTimer() require.NoError(b, err) } @@ -126,10 +108,76 @@ func BenchmarkStore(b *testing.B) { require.NoError(b, err) b.StartTimer() - err = edsStore.Put(ctx, dah.Hash(), eds) + err = store.edsStore.Put(ctx, dah.Hash(), eds) b.StopTimer() require.NoError(b, err) } }) }) } + +func TestStoreRestart(t *testing.T) { + const ( + blocks = 5 + size = 32 + ) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + dir := t.TempDir() + err := Init(*DefaultConfig(node.Full), dir, node.Full) + require.NoError(t, err) + + store := newStore(ctx, t, dir) + + hashes := make([][]byte, blocks) + for i := range hashes { + edss := edstest.RandEDS(t, size) + require.NoError(t, err) + dah, err := da.NewDataAvailabilityHeader(edss) + require.NoError(t, err) + err = store.edsStore.Put(ctx, dah.Hash(), edss) + require.NoError(t, err) + + // store hashes for read loop later + hashes[i] = dah.Hash() + } + + // restart store + store.stop(ctx, t) + store = newStore(ctx, t, dir) + + for _, h := range hashes { + edsReader, err := store.edsStore.GetCAR(ctx, h) + require.NoError(t, err) + odsReader, err := eds.ODSReader(edsReader) + require.NoError(t, err) + _, err = eds.ReadEDS(ctx, odsReader, h) + require.NoError(t, err) + } +} + +type store struct { + s Store + edsStore *eds.Store +} + +func newStore(ctx context.Context, t require.TestingT, dir string) store { + s, err := OpenStore(dir, nil) + require.NoError(t, err) + ds, err := s.Datastore() + require.NoError(t, err) + edsStore, err := eds.NewStore(dir, ds) + require.NoError(t, err) + err = edsStore.Start(ctx) + require.NoError(t, err) + return store{ + s: s, + edsStore: edsStore, + } +} + +func (s *store) stop(ctx context.Context, t *testing.T) { + require.NoError(t, s.edsStore.Stop(ctx)) + require.NoError(t, s.s.Close()) +} diff --git a/share/eds/store.go b/share/eds/store.go index 69e3c6b4a4..5a85b9ab55 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -75,7 +75,7 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { } r := mount.NewRegistry() - err = r.Register("fs", &inMemoryOnceMount{Mount: &mount.FileMount{}}) + err = r.Register("fs", &inMemoryOnceMount{}) if err != nil { return nil, fmt.Errorf("failed to register memory mount on the registry: %w", err) } @@ -212,8 +212,8 @@ func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext // save encoded eds into buffer mount := &inMemoryOnceMount{ // TODO: buffer could be pre-allocated with capacity calculated based on eds size. - buf: bytes.NewBuffer(nil), - Mount: &mount.FileMount{Path: s.basepath + blocksPath + key}, + buf: bytes.NewBuffer(nil), + FileMount: mount.FileMount{Path: s.basepath + blocksPath + key}, } err = WriteEDS(ctx, square, mount) if err != nil { @@ -565,7 +565,7 @@ type inMemoryOnceMount struct { buf *bytes.Buffer readOnce atomic.Bool - mount.Mount + mount.FileMount } func (m *inMemoryOnceMount) Fetch(ctx context.Context) (mount.Reader, error) { @@ -575,7 +575,7 @@ func (m *inMemoryOnceMount) Fetch(ctx context.Context) (mount.Reader, error) { m.buf = nil return reader, nil } - return m.Mount.Fetch(ctx) + return m.FileMount.Fetch(ctx) } func (m *inMemoryOnceMount) Write(b []byte) (int, error) { From 79a195cf615dac4f93e4a05c8fdbd467355700ae Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 16 Aug 2023 18:27:05 +0700 Subject: [PATCH 129/388] metrics(runtime): add runtime metrics (#2570) Resolves https://github.com/celestiaorg/celestia-node/issues/2465 --- go.mod | 1 + go.sum | 2 ++ nodebuilder/settings.go | 35 +++++++++++++++++++++++++---------- 3 files changed, 28 insertions(+), 10 deletions(-) diff --git a/go.mod b/go.mod index 01a7325793..873c5fc2b7 100644 --- a/go.mod +++ b/go.mod @@ -62,6 +62,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 + go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 go.opentelemetry.io/otel v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 diff --git a/go.sum b/go.sum index ed814e6ede..e3e90e746b 100644 --- a/go.sum +++ b/go.sum @@ -2054,6 +2054,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= +go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 h1:EbmAUG9hEAMXyfWEasIt2kmh/WmXUznUksChApTgBGc= +go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0/go.mod h1:rD9feqRYP24P14t5kmhNMqsqm1jvKmpx2H2rKVw52V8= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 44508ba5e1..b2a29723db 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -8,6 +8,7 @@ import ( "github.com/libp2p/go-libp2p/core/peer" "github.com/pyroscope-io/client/pyroscope" otelpyroscope "github.com/pyroscope-io/otel-profiling-go" + "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" "go.opentelemetry.io/otel/exporters/otlp/otlptrace" @@ -29,6 +30,8 @@ import ( "github.com/celestiaorg/celestia-node/state" ) +const defaultMetricsCollectInterval = 10 * time.Second + // WithNetwork specifies the Network to which the Node should connect to. // WARNING: Use this option with caution and never run the Node with different networks over the // same persisted Store. @@ -168,16 +171,28 @@ func initializeMetrics( } provider := sdk.NewMeterProvider( - sdk.WithReader(sdk.NewPeriodicReader(exp, sdk.WithTimeout(2*time.Second))), - sdk.WithResource(resource.NewWithAttributes( - semconv.SchemaURL, - // ServiceNamespaceKey and ServiceNameKey will be concatenated into single attribute with key: - // "job" and value: "%service.namespace%/%service.name%" - semconv.ServiceNamespaceKey.String(network.String()), - semconv.ServiceNameKey.String(nodeType.String()), - // ServiceInstanceIDKey will be exported with key: "instance" - semconv.ServiceInstanceIDKey.String(peerID.String()), - ))) + sdk.WithReader( + sdk.NewPeriodicReader(exp, + sdk.WithTimeout(defaultMetricsCollectInterval), + sdk.WithInterval(defaultMetricsCollectInterval))), + sdk.WithResource( + resource.NewWithAttributes( + semconv.SchemaURL, + // ServiceNamespaceKey and ServiceNameKey will be concatenated into single attribute with key: + // "job" and value: "%service.namespace%/%service.name%" + semconv.ServiceNamespaceKey.String(network.String()), + semconv.ServiceNameKey.String(nodeType.String()), + // ServiceInstanceIDKey will be exported with key: "instance" + semconv.ServiceInstanceIDKey.String(peerID.String()), + ))) + + err = runtime.Start( + runtime.WithMinimumReadMemStatsInterval(defaultMetricsCollectInterval), + runtime.WithMeterProvider(provider)) + if err != nil { + return fmt.Errorf("start runtime metrics: %w", err) + } + lc.Append(fx.Hook{ OnStop: func(ctx context.Context) error { return provider.Shutdown(ctx) From e6347514c107b1518499c80eecf71b36ba9cbd93 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 17 Aug 2023 21:41:35 +0800 Subject: [PATCH 130/388] fix(nodebuilder): set default log level for bitswap-client (#2576) `bitswap-client` logger generates 10k logs per sec in INFO level. This PR sets default log level to WARN --- logs/logs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/logs/logs.go b/logs/logs.go index 15c26888c5..23d0683996 100644 --- a/logs/logs.go +++ b/logs/logs.go @@ -11,6 +11,7 @@ func SetAllLoggers(level logging.LogLevel) { _ = logging.SetLogLevel("dht", "ERROR") _ = logging.SetLogLevel("swarm2", "WARN") _ = logging.SetLogLevel("bitswap", "WARN") + _ = logging.SetLogLevel("bitswap-client", "WARN") _ = logging.SetLogLevel("connmgr", "WARN") _ = logging.SetLogLevel("nat", "INFO") _ = logging.SetLogLevel("dht/RtRefreshManager", "FATAL") From 736763e6e689f9d604f3b12a2d941328f5819f89 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 17 Aug 2023 15:43:05 +0200 Subject: [PATCH 131/388] fix: TestShareAvailable_DisconnectedFullNodes (#2560) TestShareAvailable_DisconnectedFullNodes had multiple bugs which caused it to be a false positive. The following issues were fixed: 1. The parameters used now don't allow for reconstruction from a single subnetwork, as they did before. This has been verified with a simulation - for `s = 20`, and `k = 16`, `c` set to 32 gives a 99.5% chance of reconstruction, and 16 (single subnetwork) gives a 0.5% chance. Before, both values of `c` gave a near 100% chance for reconstruction. 2. Increased the timeout for the first reconstruction. There was a check that verified that reconstruction was not possible from a single subnetwork, but it only failed to reconstruct because the timeout was too short. Increasing the timeout with the original parameters verified this. 3. Shape the network topology before the first reconstruction (for example, separating the LNs into subnets) 4. Calling final reconstruction in an error group. The full nodes are codependent on each other for either one to reconstruct. Calling them sequentially fails, because the first full node needs the second full node to have sampled in order to get shares from it. This was another signal that the full nodes could reconstruct from a single subnetwork, and it has now been fixed by allowing both FNs to sample concurrently. --- .../availability/full/reconstruction_test.go | 56 ++++++++++--------- 1 file changed, 30 insertions(+), 26 deletions(-) diff --git a/share/availability/full/reconstruction_test.go b/share/availability/full/reconstruction_test.go index 03b998d49e..f3b6ce91bd 100644 --- a/share/availability/full/reconstruction_test.go +++ b/share/availability/full/reconstruction_test.go @@ -181,35 +181,15 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { light.DefaultSampleAmount = 20 // s const ( origSquareSize = 16 // k - lightNodes = 60 // c - total number of nodes on two subnetworks + lightNodes = 32 // c - total number of nodes on two subnetworks ) - ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*60) defer cancel() net := availability_test.NewTestDAGNet(ctx, t) source, root := RandNode(net, origSquareSize) - // create two full nodes and ensure they are disconnected - full1 := Node(net) - full2 := Node(net) - net.Disconnect(full1.ID(), full2.ID()) - - // ensure fulls and source are not connected - // so that fulls take data from light nodes only - net.Disconnect(full1.ID(), source.ID()) - net.Disconnect(full2.ID(), source.ID()) - - // start reconstruction for fulls that should fail - ctxErr, cancelErr := context.WithTimeout(ctx, eds.RetrieveQuadrantTimeout*8) - errg, errCtx := errgroup.WithContext(ctxErr) - errg.Go(func() error { - return full1.SharesAvailable(errCtx, root) - }) - errg.Go(func() error { - return full2.SharesAvailable(errCtx, root) - }) - // create light nodes and start sampling for them immediately lights1, lights2 := make( []*availability_test.TestNode, lightNodes/2), @@ -237,6 +217,16 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { }(i) } + // create two full nodes and ensure they are disconnected + full1 := Node(net) + full2 := Node(net) + net.Disconnect(full1.ID(), full2.ID()) + + // ensure fulls and source are not connected + // so that fulls take data from light nodes only + net.Disconnect(full1.ID(), source.ID()) + net.Disconnect(full2.ID(), source.ID()) + // shape topology for i := 0; i < len(lights1); i++ { // ensure lights1 are only connected to source and full1 @@ -249,6 +239,16 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { net.Disconnect(lights2[i].ID(), full1.ID()) } + // start reconstruction for fulls that should fail + ctxErr, cancelErr := context.WithTimeout(ctx, time.Second*5) + errg, errCtx := errgroup.WithContext(ctxErr) + errg.Go(func() error { + return full1.SharesAvailable(errCtx, root) + }) + errg.Go(func() error { + return full2.SharesAvailable(errCtx, root) + }) + // check that any of the fulls cannot reconstruct on their own err := errg.Wait() require.ErrorIs(t, err, share.ErrNotAvailable) @@ -262,10 +262,14 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { full2.ClearStorage() // they both should be able to reconstruct the block - err = full1.SharesAvailable(ctx, root) - require.NoError(t, err, share.ErrNotAvailable) - err = full2.SharesAvailable(ctx, root) - require.NoError(t, err, share.ErrNotAvailable) + errg, bctx := errgroup.WithContext(ctx) + errg.Go(func() error { + return full1.SharesAvailable(bctx, root) + }) + errg.Go(func() error { + return full2.SharesAvailable(bctx, root) + }) + require.NoError(t, errg.Wait()) // wait for all routines to finish before exit, in case there are any errors to log wg.Wait() } From 8eef855a1eaefd9d6eb8e10dcfebb48265b225c9 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Fri, 18 Aug 2023 16:44:01 +0800 Subject: [PATCH 132/388] chore(deps): bump ipfs networking deps (#2478) ## Overview Main change is update of github.com/ipfs/go-libipfs , that is now located in github.com/ipfs/boxo Updates libs and their dependancies: - github.com/libp2p/go-libp2p-kad-dht - github.com/libp2p/go-libp2p-routing-helpers - github.com/ipfs/go-block-format - github.com/ipfs/go-blockservice - github.com/ipfs/go-ipld-format - github.com/ipfs/go-merkledag - github.com/ipld/go-car - github.com/multiformats/go-multihash Supersedes: https://github.com/celestiaorg/celestia-node/pull/2268 https://github.com/celestiaorg/celestia-node/pull/2378 https://github.com/celestiaorg/celestia-node/pull/2379 https://github.com/celestiaorg/celestia-node/pull/1957 --------- Co-authored-by: Ryan Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- go.mod | 41 ++++++----- go.sum | 93 ++++++++++++------------- nodebuilder/node.go | 2 +- nodebuilder/p2p/bitswap.go | 12 ++-- nodebuilder/p2p/ipld.go | 4 +- share/availability/test/corrupt_data.go | 4 +- share/availability/test/testing.go | 6 +- share/eds/adapters.go | 2 +- share/eds/adapters_test.go | 2 +- share/eds/blockstore.go | 4 +- share/eds/eds_test.go | 2 +- share/eds/store.go | 2 +- share/ipld/get_shares_test.go | 2 +- share/ipld/nmt.go | 2 +- 14 files changed, 88 insertions(+), 90 deletions(-) diff --git a/go.mod b/go.mod index 873c5fc2b7..032a9a2a58 100644 --- a/go.mod +++ b/go.mod @@ -30,24 +30,23 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.2 github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d github.com/imdario/mergo v0.3.16 - github.com/ipfs/go-blockservice v0.5.0 + github.com/ipfs/boxo v0.11.0 + github.com/ipfs/go-block-format v0.1.2 + github.com/ipfs/go-blockservice v0.5.1 // down 1 version, 0.5.2 is marked as deprecated and raises alerts github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-ipfs-blockstore v1.2.0 - github.com/ipfs/go-ipfs-exchange-interface v0.2.0 github.com/ipfs/go-ipfs-exchange-offline v0.3.0 github.com/ipfs/go-ipfs-routing v0.3.0 github.com/ipfs/go-ipld-cbor v0.0.6 - github.com/ipfs/go-ipld-format v0.4.0 - github.com/ipfs/go-libipfs v0.6.0 + github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-merkledag v0.10.0 - github.com/ipld/go-car v0.6.0 + github.com/ipfs/go-merkledag v0.11.0 + github.com/ipld/go-car v0.6.2 github.com/libp2p/go-libp2p v0.29.0 - github.com/libp2p/go-libp2p-kad-dht v0.21.1 + github.com/libp2p/go-libp2p-kad-dht v0.24.2 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 - github.com/libp2p/go-libp2p-routing-helpers v0.6.1 + github.com/libp2p/go-libp2p-routing-helpers v0.7.0 github.com/minio/sha256-simd v1.0.1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 @@ -74,10 +73,10 @@ require ( go.opentelemetry.io/proto/otlp v0.19.0 go.uber.org/fx v1.20.0 go.uber.org/zap v1.24.0 - golang.org/x/crypto v0.11.0 + golang.org/x/crypto v0.12.0 golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 golang.org/x/sync v0.3.0 - golang.org/x/text v0.11.0 + golang.org/x/text v0.12.0 google.golang.org/grpc v1.56.2 google.golang.org/protobuf v1.31.0 ) @@ -200,19 +199,18 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-bitswap v0.12.0 // indirect - github.com/ipfs/go-block-format v0.1.1 // indirect + github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect + github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect - github.com/ipfs/go-ipfs-util v0.0.2 // indirect - github.com/ipfs/go-ipld-legacy v0.1.1 // indirect - github.com/ipfs/go-ipns v0.3.0 // indirect + github.com/ipfs/go-ipfs-util v0.0.3 // indirect + github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/ipld/go-car/v2 v2.5.1 // indirect + github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.20.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect @@ -230,7 +228,7 @@ require ( github.com/libp2p/go-cidranger v1.1.0 // indirect github.com/libp2p/go-flow-metrics v0.1.0 // indirect github.com/libp2p/go-libp2p-asn-util v0.3.0 // indirect - github.com/libp2p/go-libp2p-kbucket v0.5.0 // indirect + github.com/libp2p/go-libp2p-kbucket v0.6.3 // indirect github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect @@ -282,7 +280,7 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect - github.com/rivo/uniseg v0.4.2 // indirect + github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/cors v1.8.2 // indirect github.com/rs/zerolog v1.29.1 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect @@ -316,10 +314,11 @@ require ( golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.12.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.10.0 // indirect - golang.org/x/term v0.10.0 // indirect + golang.org/x/sys v0.11.0 // indirect + golang.org/x/term v0.11.0 // indirect golang.org/x/tools v0.11.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect + gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.114.0 // indirect google.golang.org/appengine v1.6.7 // indirect google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect diff --git a/go.sum b/go.sum index e3e90e746b..75924752f5 100644 --- a/go.sum +++ b/go.sum @@ -831,8 +831,8 @@ github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6c github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= @@ -964,17 +964,18 @@ github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mq github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/go-bitfield v1.0.0 h1:y/XHm2GEmD9wKngheWNNCNL0pzrWXZwCdQGv1ikXknQ= +github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= +github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= -github.com/ipfs/go-bitswap v0.12.0 h1:ClbLaufwv8SRQK0sBhl4wDVqJoZGAGMVxdjQy5CTt6c= -github.com/ipfs/go-bitswap v0.12.0/go.mod h1:Iwjkd6+vaDjVIa6b6ogmZgs+b5U3EkIFEX79kQ4DjnI= +github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= -github.com/ipfs/go-block-format v0.1.1 h1:129vSO3zwbsYADcyQWcOYiuCpAqt462SFfqFHdFJhhI= -github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= +github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= -github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= -github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= +github.com/ipfs/go-blockservice v0.5.1 h1:9pAtkyKAz/skdHTh0kH8VulzWp+qmSDD0aI17TYP/s0= +github.com/ipfs/go-blockservice v0.5.1/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -1009,11 +1010,11 @@ github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUN github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= -github.com/ipfs/go-ipfs-blockstore v1.2.0 h1:n3WTeJ4LdICWs/0VSfjHrlqpPpl6MZ+ySd3j8qz0ykw= -github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= +github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= -github.com/ipfs/go-ipfs-chunker v0.0.1 h1:cHUUxKFQ99pozdahi+uSC/3Y6HeRpi9oTeUHbE27SEw= +github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -1021,8 +1022,8 @@ github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzO github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= -github.com/ipfs/go-ipfs-exchange-interface v0.2.0 h1:8lMSJmKogZYNo2jjhUs0izT+dck05pqUw4mWNW9Pw6Y= -github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= +github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s= +github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= @@ -1033,23 +1034,19 @@ github.com/ipfs/go-ipfs-routing v0.2.1/go.mod h1:xiNNiwgjmLqPS1cimvAw6EyB9rkVDbi github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyBCNzQxlJBc= -github.com/ipfs/go-ipfs-util v0.0.2 h1:59Sswnk1MFaiq+VcaknX7aYEyGyGDAA73ilhEK2POp8= github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= +github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= +github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= -github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-format v0.4.0 h1:yqJSaJftjmjc9jEOFYlpkwOLVKv68OD27jFLlSghBlQ= -github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= +github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= -github.com/ipfs/go-ipld-legacy v0.1.1 h1:BvD8PEuqwBHLTKqlGFTHSwrwFOMkVESEvwIYwR2cdcc= -github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= -github.com/ipfs/go-ipns v0.3.0 h1:ai791nTgVo+zTuq2bLvEGmWP1M0A6kGTXUsgv/Yq67A= -github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= -github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= -github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= +github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= +github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -1066,19 +1063,19 @@ github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOL github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= -github.com/ipfs/go-merkledag v0.10.0 h1:IUQhj/kzTZfam4e+LnaEpoiZ9vZF6ldimVlby+6OXL4= -github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2mCwxkQ6GIj8= +github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= +github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= -github.com/ipfs/go-unixfsnode v1.5.1 h1:JcR3t5C2nM1V7PMzhJ/Qmo19NkoFIKweDSZyDx+CjkI= -github.com/ipld/go-car v0.6.0 h1:d5QrGLnHAxiNLHor+DKGrLdqnM0dQJh2whfSXRDq6J0= -github.com/ipld/go-car v0.6.0/go.mod h1:tBrW1XZ3L2XipLxA69RnTVGW3rve6VX4TbaTYkq8aEA= +github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= +github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= +github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= -github.com/ipld/go-car/v2 v2.5.1 h1:U2ux9JS23upEgrJScW8VQuxmE94560kYxj9CQUpcfmk= -github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= +github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s= +github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= @@ -1087,8 +1084,8 @@ github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHt github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= -github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73 h1:TsyATB2ZRRQGTwafJdgEUQkmjOExRV0DNokcihZxbnQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1264,10 +1261,10 @@ github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxn github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.21.1 h1:xpfp8/t9+X2ip1l8Umap1/UGNnJ3RHJgKGAEsnRAlTo= -github.com/libp2p/go-libp2p-kad-dht v0.21.1/go.mod h1:Oy8wvbdjpB70eS5AaFaI68tOtrdo3KylTvXDjikxqFo= -github.com/libp2p/go-libp2p-kbucket v0.5.0 h1:g/7tVm8ACHDxH29BGrpsQlnNeu+6OF1A9bno/4/U1oA= -github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= +github.com/libp2p/go-libp2p-kad-dht v0.24.2 h1:zd7myKBKCmtZBhI3I0zm8xBkb28v3gmSEtQfBdAdFwc= +github.com/libp2p/go-libp2p-kad-dht v0.24.2/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= +github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= github.com/libp2p/go-libp2p-mplex v0.2.0/go.mod h1:Ejl9IyjvXJ0T9iqUTE1jpYATQ9NM3g+OtR+EMMODbKo= github.com/libp2p/go-libp2p-mplex v0.2.1/go.mod h1:SC99Rxs8Vuzrf/6WhmH41kNn13TiYdAWNYHrwImKLnE= @@ -1303,8 +1300,8 @@ github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= -github.com/libp2p/go-libp2p-routing-helpers v0.6.1 h1:tI3rHOf/FDQsxC2pHBaOZiqPJ0MZYyzGAf4V45xla4U= -github.com/libp2p/go-libp2p-routing-helpers v0.6.1/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8= +github.com/libp2p/go-libp2p-routing-helpers v0.7.0 h1:sirOYVD0wGWjkDwHZvinunIpaqPLBXkcnXApVHwZFGA= +github.com/libp2p/go-libp2p-routing-helpers v0.7.0/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -1814,8 +1811,8 @@ github.com/regen-network/cosmos-proto v0.3.1/go.mod h1:jO0sVX6a1B36nmE8C9xBFXpNw github.com/regen-network/protobuf v1.3.3-alpha.regen.1 h1:OHEc+q5iIAXpqiqFKeLpu5NwTIkVXUs48vFMwzqpqY4= github.com/regen-network/protobuf v1.3.3-alpha.regen.1/go.mod h1:2DjTFR1HhMQhiWC5sZ4OhQ3+NtdbZ6oBDKQwq5Ou+FI= github.com/retailnext/hllpp v1.0.1-0.20180308014038-101a6d2f8b52/go.mod h1:RDpi1RftBQPUCDRw6SmxeaREsAaRKnOclghuzp/WRzc= -github.com/rivo/uniseg v0.4.2 h1:YwD0ulJSJytLpiaWua0sBDusfsCZohxjxzVTYjwxfV8= -github.com/rivo/uniseg v0.4.2/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= +github.com/rivo/uniseg v0.4.4 h1:8TfxU8dW6PdqD27gjM8MVNuicgxIjxpm4K7x4jp8sis= +github.com/rivo/uniseg v0.4.4/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88= github.com/rjeczalik/notify v0.9.1/go.mod h1:rKwnCoCGeuQnwBtTSPL9Dad03Vh2n40ePRrjvIXnJho= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= @@ -2162,8 +2159,8 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.11.0 h1:6Ewdq3tDic1mg5xRO4milcWCfMVQhI4NkqWWvqejpuA= -golang.org/x/crypto v0.11.0/go.mod h1:xgJhtzW8F9jGdVFWZESrid1U1bjeNy4zgy5cRr/CIio= +golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= +golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2474,14 +2471,14 @@ golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.10.0 h1:SqMFp9UcQJZa+pmYuAKjd9xq1f0j5rLcDIk0mj4qAsA= -golang.org/x/sys v0.10.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.10.0 h1:3R7pNqamzBraeqj/Tj8qt1aQ2HpmlC+Cx/qL/7hn4/c= -golang.org/x/term v0.10.0/go.mod h1:lpqdcUyK/oCiQxvxVrppt5ggO2KCZ5QblwqPnfZ6d5o= +golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= +golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2492,8 +2489,8 @@ golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.11.0 h1:LAntKIrcmeSKERyiOh0XMV39LXS8IE9UL2yP7+f5ij4= -golang.org/x/text v0.11.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= +golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2587,6 +2584,8 @@ golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2/go.mod h1:K8+ghG5WaK9qNq gonum.org/v1/gonum v0.0.0-20180816165407-929014505bf4/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.0.0-20181121035319-3f7ecaa7e8ca/go.mod h1:Y+Yx5eoAFn32cQvJDxZx5Dpnq+c3wtXuadVZAcxbbBo= gonum.org/v1/gonum v0.6.0/go.mod h1:9mxDZsDKxgMAuccQkewq682L+0eCu4dCN2yonUJTCLU= +gonum.org/v1/gonum v0.13.0 h1:a0T3bh+7fhRyqeNbiC3qVHYmkiQgit3wnNan/2c0HMM= +gonum.org/v1/gonum v0.13.0/go.mod h1:/WPYRckkfWrhWefxyYTfrTtQR0KH4iyHNuzxqXAKyAU= gonum.org/v1/netlib v0.0.0-20181029234149-ec6d1f5cefe6/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/netlib v0.0.0-20190313105609-8cb42192e0e0/go.mod h1:wa6Ws7BG/ESfp6dHfk7C6KdzKA7wR7u/rKwOGE66zvw= gonum.org/v1/plot v0.0.0-20190515093506-e2840ee46a6b/go.mod h1:Wt8AAjI+ypCyYX3nZBvf6cAIx93T+c/OS2HFAYskSZc= diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 33132dc5e4..6d83e6c4c3 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -6,8 +6,8 @@ import ( "fmt" "strings" + "github.com/ipfs/boxo/exchange" "github.com/ipfs/go-blockservice" - exchange "github.com/ipfs/go-ipfs-exchange-interface" logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" diff --git a/nodebuilder/p2p/bitswap.go b/nodebuilder/p2p/bitswap.go index 3a99dd2c7d..19f98609fc 100644 --- a/nodebuilder/p2p/bitswap.go +++ b/nodebuilder/p2p/bitswap.go @@ -4,11 +4,11 @@ import ( "context" "fmt" + "github.com/ipfs/boxo/bitswap" + "github.com/ipfs/boxo/bitswap/network" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" "github.com/ipfs/go-datastore" - blockstore "github.com/ipfs/go-ipfs-blockstore" - exchange "github.com/ipfs/go-ipfs-exchange-interface" - "github.com/ipfs/go-libipfs/bitswap" - "github.com/ipfs/go-libipfs/bitswap/network" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" hst "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/protocol" @@ -48,7 +48,7 @@ func blockstoreFromDatastore(ctx context.Context, ds datastore.Batching) (blocks blockstore.CacheOpts{ HasBloomFilterSize: defaultBloomFilterSize, HasBloomFilterHashes: defaultBloomFilterHashes, - HasARCCacheSize: defaultARCCacheSize, + HasTwoQueueCacheSize: defaultARCCacheSize, }, ) } @@ -58,7 +58,7 @@ func blockstoreFromEDSStore(ctx context.Context, store *eds.Store) (blockstore.B ctx, store.Blockstore(), blockstore.CacheOpts{ - HasARCCacheSize: defaultARCCacheSize, + HasTwoQueueCacheSize: defaultARCCacheSize, }, ) } diff --git a/nodebuilder/p2p/ipld.go b/nodebuilder/p2p/ipld.go index 6278538825..ad32cc39f5 100644 --- a/nodebuilder/p2p/ipld.go +++ b/nodebuilder/p2p/ipld.go @@ -1,9 +1,9 @@ package p2p import ( + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" "github.com/ipfs/go-blockservice" - blockstore "github.com/ipfs/go-ipfs-blockstore" - exchange "github.com/ipfs/go-ipfs-exchange-interface" ) // blockService constructs IPFS's BlockService for fetching arbitrary Merkle structures. diff --git a/share/availability/test/corrupt_data.go b/share/availability/test/corrupt_data.go index 1bb8bd243a..f0bd8fbbc5 100644 --- a/share/availability/test/corrupt_data.go +++ b/share/availability/test/corrupt_data.go @@ -7,11 +7,11 @@ import ( mrand "math/rand" "testing" + "github.com/ipfs/boxo/blockstore" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" - blocks "github.com/ipfs/go-libipfs/blocks" ) var _ blockstore.Blockstore = (*FraudulentBlockstore)(nil) diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 27d6669061..7427699f95 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -4,13 +4,13 @@ import ( "context" "testing" + "github.com/ipfs/boxo/bitswap" + "github.com/ipfs/boxo/bitswap/network" + "github.com/ipfs/boxo/blockstore" "github.com/ipfs/go-blockservice" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" "github.com/ipfs/go-ipfs-routing/offline" - "github.com/ipfs/go-libipfs/bitswap" - "github.com/ipfs/go-libipfs/bitswap/network" record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" diff --git a/share/eds/adapters.go b/share/eds/adapters.go index 8d98f092d7..fe498400b3 100644 --- a/share/eds/adapters.go +++ b/share/eds/adapters.go @@ -5,9 +5,9 @@ import ( "sync" "github.com/filecoin-project/dagstore" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/blocks" ) var _ blockservice.BlockGetter = (*BlockGetter)(nil) diff --git a/share/eds/adapters_test.go b/share/eds/adapters_test.go index 6ccd29e99d..70165b81c8 100644 --- a/share/eds/adapters_test.go +++ b/share/eds/adapters_test.go @@ -8,8 +8,8 @@ import ( "testing" "time" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" - "github.com/ipfs/go-libipfs/blocks" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share/ipld" diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go index 3c63fa3945..9fc9d7f2f8 100644 --- a/share/eds/blockstore.go +++ b/share/eds/blockstore.go @@ -6,11 +6,11 @@ import ( "fmt" "github.com/filecoin-project/dagstore" + bstore "github.com/ipfs/boxo/blockstore" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" ipld "github.com/ipfs/go-ipld-format" - blocks "github.com/ipfs/go-libipfs/blocks" ) var _ bstore.Blockstore = (*blockstore)(nil) diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index 70ce7a0e8e..6ebca8b779 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -9,8 +9,8 @@ import ( "os" "testing" + bstore "github.com/ipfs/boxo/blockstore" ds "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" carv1 "github.com/ipld/go-car" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/share/eds/store.go b/share/eds/store.go index 5a85b9ab55..fd679e2591 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -16,8 +16,8 @@ import ( "github.com/filecoin-project/dagstore/index" "github.com/filecoin-project/dagstore/mount" "github.com/filecoin-project/dagstore/shard" + bstore "github.com/ipfs/boxo/blockstore" "github.com/ipfs/go-datastore" - bstore "github.com/ipfs/go-ipfs-blockstore" carv1 "github.com/ipld/go-car" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index 7ee7704fc0..3a79fea238 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -10,11 +10,11 @@ import ( "testing" "time" + "github.com/ipfs/boxo/blockstore" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - blockstore "github.com/ipfs/go-ipfs-blockstore" offline "github.com/ipfs/go-ipfs-exchange-offline" mdutils "github.com/ipfs/go-merkledag/test" "github.com/minio/sha256-simd" diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index 485642dde4..e7d6b4b513 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -8,10 +8,10 @@ import ( "hash" "math/rand" + blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-libipfs/blocks" logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" mhcore "github.com/multiformats/go-multihash/core" From c2c0827e48dd578fc4e13e9dc1650d853ea0a778 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Fri, 18 Aug 2023 18:00:18 +0800 Subject: [PATCH 133/388] fix(share/discovery): discovery should try to connect to empty hosts peers (#2575) libp2p routed host will attempt to find address, if none are provided. First in `Peerstore`, then by calling `FindPeer`: ``` // if we were given some addresses, keep + use them. if len(pi.Addrs) > 0 { rh.Peerstore().AddAddrs(pi.ID, pi.Addrs, peerstore.TempAddrTTL) } // Check if we have some addresses in our recent memory. addrs := rh.Peerstore().Addrs(pi.ID) if len(addrs) < 1 { // no addrs? find some with the routing system. var err error addrs, err = rh.findPeerAddrs(ctx, pi.ID) if err != nil { return err } } ``` It is worth to try to connect in such case. --- share/p2p/discovery/discovery.go | 4 ---- 1 file changed, 4 deletions(-) diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index 9779d72195..af6fef1f89 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -325,10 +325,6 @@ func (d *Discovery) handleDiscoveredPeer(ctx context.Context, peer peer.AddrInfo d.metrics.observeHandlePeer(ctx, handlePeerSkipSelf) logger.Debug("skip handle: self discovery") return false - case len(peer.Addrs) == 0: - d.metrics.observeHandlePeer(ctx, handlePeerEmptyAddrs) - logger.Debug("skip handle: empty address list") - return false case d.set.Size() >= d.set.Limit(): d.metrics.observeHandlePeer(ctx, handlePeerEnoughPeers) logger.Debug("skip handle: enough peers found") From 323f603b622438cabd626caeffd24b6c9017d523 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Fri, 18 Aug 2023 19:05:54 +0800 Subject: [PATCH 134/388] feat(eds/store): add eds blockstore cache metrics (#2567) Add metrics allowing to see blockstore cache size and eviction pressure --- share/eds/accessor_cache.go | 63 +++++++++++++++++++++++++++++++++---- share/eds/metrics.go | 3 ++ 2 files changed, 60 insertions(+), 6 deletions(-) diff --git a/share/eds/accessor_cache.go b/share/eds/accessor_cache.go index eac79b946f..cd0f0537fa 100644 --- a/share/eds/accessor_cache.go +++ b/share/eds/accessor_cache.go @@ -1,6 +1,7 @@ package eds import ( + "context" "errors" "fmt" "reflect" @@ -9,6 +10,8 @@ import ( "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" lru "github.com/hashicorp/golang-lru" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" ) var ( @@ -32,11 +35,23 @@ type blockstoreCache struct { // caches the blockstore for a given shard for shard read affinity i.e. // further reads will likely be from the same shard. Maps (shard key -> blockstore). cache *lru.Cache + + metrics *cacheMetrics } func newBlockstoreCache(cacheSize int) (*blockstoreCache, error) { + bc := &blockstoreCache{} // instantiate the blockstore cache - bslru, err := lru.NewWithEvict(cacheSize, func(_ interface{}, val interface{}) { + bslru, err := lru.NewWithEvict(cacheSize, bc.evictFn()) + if err != nil { + return nil, fmt.Errorf("failed to instantiate blockstore cache: %w", err) + } + bc.cache = bslru + return bc, nil +} + +func (bc *blockstoreCache) evictFn() func(_ interface{}, val interface{}) { + return func(_ interface{}, val interface{}) { // ensure we close the blockstore for a shard when it's evicted so dagstore can gc it. abs, ok := val.(*accessorWithBlockstore) if !ok { @@ -46,14 +61,12 @@ func newBlockstoreCache(cacheSize int) (*blockstoreCache, error) { )) } - if err := abs.sa.Close(); err != nil { + err := abs.sa.Close() + if err != nil { log.Errorf("couldn't close accessor after cache eviction: %s", err) } - }) - if err != nil { - return nil, fmt.Errorf("failed to instantiate blockstore cache: %w", err) + bc.metrics.observeEvicted(err != nil) } - return &blockstoreCache{cache: bslru}, nil } // Get retrieves the blockstore for a given shard key from the cache. If the blockstore is not in @@ -117,3 +130,41 @@ func (bc *blockstoreCache) unsafeAdd( func shardKeyToStriped(sk shard.Key) byte { return sk.String()[len(sk.String())-1] } + +type cacheMetrics struct { + evictedCounter metric.Int64Counter +} + +func (bc *blockstoreCache) withMetrics() error { + evictedCounter, err := meter.Int64Counter("eds_blockstore_cache_evicted_counter", + metric.WithDescription("eds blockstore cache evicted event counter")) + if err != nil { + return err + } + + cacheSize, err := meter.Int64ObservableGauge("eds_blockstore_cache_size", + metric.WithDescription("total amount of items in blockstore cache"), + ) + if err != nil { + return err + } + + callback := func(ctx context.Context, observer metric.Observer) error { + observer.ObserveInt64(cacheSize, int64(bc.cache.Len())) + return nil + } + _, err = meter.RegisterCallback(callback, cacheSize) + if err != nil { + return err + } + bc.metrics = &cacheMetrics{evictedCounter: evictedCounter} + return nil +} + +func (m *cacheMetrics) observeEvicted(failed bool) { + if m == nil { + return + } + m.evictedCounter.Add(context.Background(), 1, metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 82adb246f1..6547e239cd 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -116,6 +116,9 @@ func (s *Store) WithMetrics() error { return err } + if err = s.cache.withMetrics(); err != nil { + return err + } s.metrics = &metrics{ putTime: putTime, getCARTime: getCARTime, From 1cc1da6b353d67f27ac9be98ee614f9076a1a787 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Fri, 18 Aug 2023 10:06:01 -0400 Subject: [PATCH 135/388] chore: bump to rsmt2d v0.11.0 (#2580) --- go.mod | 4 +-- go.sum | 8 ++--- share/eds/eds.go | 2 +- share/eds/edstest/testing.go | 2 +- share/eds/retriever_test.go | 4 +-- share/getters/getter_test.go | 12 ++++---- share/helpers.go | 58 ----------------------------------- share/ipld/add.go | 2 +- share/ipld/get_shares_test.go | 8 ++--- 9 files changed, 21 insertions(+), 79 deletions(-) delete mode 100644 share/helpers.go diff --git a/go.mod b/go.mod index 032a9a2a58..8310e845f2 100644 --- a/go.mod +++ b/go.mod @@ -16,7 +16,7 @@ require ( github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.18.1 - github.com/celestiaorg/rsmt2d v0.10.0 + github.com/celestiaorg/rsmt2d v0.11.0 github.com/cosmos/cosmos-sdk v0.46.13 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 @@ -221,7 +221,7 @@ require ( github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.16.7 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect - github.com/klauspost/reedsolomon v1.11.1 // indirect + github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect diff --git a/go.sum b/go.sum index 75924752f5..68837e301b 100644 --- a/go.sum +++ b/go.sum @@ -367,8 +367,8 @@ github.com/celestiaorg/nmt v0.18.1 h1:zU3apzW4y0fs0ilQA74XnEYW8FvRv0CUK2LXK66L3r github.com/celestiaorg/nmt v0.18.1/go.mod h1:0l8q6UYRju1xNrxtvV6NwPdW3lfsN6KuZ0htRnModdc= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= -github.com/celestiaorg/rsmt2d v0.10.0 h1:8dprr6CW5mCk5YPnbiLdirojw9YsJOE+XB+GORb8sT0= -github.com/celestiaorg/rsmt2d v0.10.0/go.mod h1:BiCZkCJfhDHUEOJKXUeu+CudjluecKvRTqHcuxKvodc= +github.com/celestiaorg/rsmt2d v0.11.0 h1:lcto/637WyTEZR3dLRoNvyuExfnUbxvdvKi3qz/2V4k= +github.com/celestiaorg/rsmt2d v0.11.0/go.mod h1:6Y580I3gVr0+OVFfW6m2JTwnCCmvW3WfbwSLfuT+HCA= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/cenkalti/backoff/v4 v4.1.1/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= @@ -1157,8 +1157,8 @@ github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/q github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= -github.com/klauspost/reedsolomon v1.11.1 h1:0gCWQXOB8pVe1Y5SGozDA5t2qoVxX3prsV+qHgI/Fik= -github.com/klauspost/reedsolomon v1.11.1/go.mod h1:FXLZzlJIdfqEnQLdUKWNRuMZg747hZ4oYp2Ml60Lb/k= +github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= +github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= diff --git a/share/eds/eds.go b/share/eds/eds.go index f1efd3b433..bc35f59b56 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -114,7 +114,7 @@ func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid } // recompute proofs from eds - shares := share.ExtractEDS(eds) + shares := eds.Flattened() shareCount := len(shares) if shareCount == 0 { return nil, ErrEmptySquare diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go index e66b42a566..ddca285f0c 100644 --- a/share/eds/edstest/testing.go +++ b/share/eds/edstest/testing.go @@ -15,7 +15,7 @@ import ( func RandByzantineEDS(t *testing.T, size int) *rsmt2d.ExtendedDataSquare { eds := RandEDS(t, size) - shares := share.ExtractEDS(eds) + shares := eds.Flattened() copy(share.GetData(shares[0]), share.GetData(shares[1])) // corrupting eds eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) require.NoError(t, err, "failure to recompute the extended data square") diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index 2277f894a1..ebccf0e384 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -63,7 +63,7 @@ func TestRetriever_Retrieve(t *testing.T) { require.NoError(t, err) out, err := r.Retrieve(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(in, out)) + assert.True(t, in.Equals(out)) }) } } @@ -74,7 +74,7 @@ func TestRetriever_ByzantineError(t *testing.T) { defer cancel() bserv := mdutils.Bserv() - shares := share.ExtractEDS(edstest.RandEDS(t, width)) + shares := edstest.RandEDS(t, width).Flattened() _, err := ipld.ImportShares(ctx, shares, bserv) require.NoError(t, err) diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 571129f029..b86b3b98df 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -52,7 +52,7 @@ func TestTeeGetter(t *testing.T) { retrievedEDS, err := tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(randEds, retrievedEDS)) + require.True(t, randEds.Equals(retrievedEDS)) // eds store now has the EDS and it can be retrieved ok, err = edsStore.Has(ctx, dah.Hash()) @@ -60,7 +60,7 @@ func TestTeeGetter(t *testing.T) { assert.NoError(t, err) finalEDS, err := edsStore.Get(ctx, dah.Hash()) assert.NoError(t, err) - require.True(t, share.EqualEDS(randEds, finalEDS)) + require.True(t, randEds.Equals(finalEDS)) }) t.Run("ShardAlreadyExistsDoesntError", func(t *testing.T) { @@ -70,12 +70,12 @@ func TestTeeGetter(t *testing.T) { retrievedEDS, err := tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(randEds, retrievedEDS)) + require.True(t, randEds.Equals(retrievedEDS)) // no error should be returned, even though the EDS identified by the DAH already exists retrievedEDS, err = tg.GetEDS(ctx, &dah) require.NoError(t, err) - require.True(t, share.EqualEDS(randEds, retrievedEDS)) + require.True(t, randEds.Equals(retrievedEDS)) }) } @@ -120,7 +120,7 @@ func TestStoreGetter(t *testing.T) { retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(randEds, retrievedEDS)) + assert.True(t, randEds.Equals(retrievedEDS)) // root not found root := share.Root{} @@ -199,7 +199,7 @@ func TestIPLDGetter(t *testing.T) { retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) - assert.True(t, share.EqualEDS(randEds, retrievedEDS)) + assert.True(t, randEds.Equals(retrievedEDS)) }) t.Run("GetSharesByNamespace", func(t *testing.T) { diff --git a/share/helpers.go b/share/helpers.go deleted file mode 100644 index 0d729fd9b2..0000000000 --- a/share/helpers.go +++ /dev/null @@ -1,58 +0,0 @@ -package share - -import ( - "bytes" - - "github.com/celestiaorg/rsmt2d" -) - -// TODO(Wondertan): All these helpers should be methods on rsmt2d.EDS - -// ExtractODS returns the original shares of the given ExtendedDataSquare. This -// is a helper function for circumstances where AddShares must be used after the EDS has already -// been generated. -func ExtractODS(eds *rsmt2d.ExtendedDataSquare) []Share { - origWidth := eds.Width() / 2 - origShares := make([][]byte, origWidth*origWidth) - for i := uint(0); i < origWidth; i++ { - row := eds.Row(i) - for j := uint(0); j < origWidth; j++ { - origShares[(i*origWidth)+j] = row[j] - } - } - return origShares -} - -// ExtractEDS takes an EDS and extracts all shares from it in a flattened slice(row by row). -func ExtractEDS(eds *rsmt2d.ExtendedDataSquare) []Share { - flattenedEDSSize := eds.Width() * eds.Width() - out := make([][]byte, flattenedEDSSize) - count := 0 - for i := uint(0); i < eds.Width(); i++ { - for _, share := range eds.Row(i) { - out[count] = share - count++ - } - } - return out -} - -// EqualEDS check whether two given EDSes are equal. -// TODO(Wondertan): Propose use of int by default instead of uint for the sake convenience and -// Golang practices -func EqualEDS(a *rsmt2d.ExtendedDataSquare, b *rsmt2d.ExtendedDataSquare) bool { - if a.Width() != b.Width() { - return false - } - - for i := uint(0); i < a.Width(); i++ { - ar, br := a.Row(i), b.Row(i) - for j := 0; j < len(ar); j++ { - if !bytes.Equal(ar[j], br[j]) { - return false - } - } - } - - return true -} diff --git a/share/ipld/add.go b/share/ipld/add.go index 5807a320fd..7e5909669d 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -79,7 +79,7 @@ func ImportShares( } func ImportEDS(ctx context.Context, square *rsmt2d.ExtendedDataSquare, adder blockservice.BlockService) error { - shares := share.ExtractEDS(square) + shares := square.Flattened() _, err := ImportShares(ctx, shares, adder) return err } diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index 3a79fea238..11639cbec7 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -94,7 +94,7 @@ func TestBlockRecovery(t *testing.T) { colRoots, err := testEds.ColRoots() require.NoError(t, err) - flat := share.ExtractEDS(testEds) + flat := testEds.Flattened() // recover a partially complete square rdata := removeRandShares(flat, tc.d) @@ -116,7 +116,7 @@ func TestBlockRecovery(t *testing.T) { reds, err := rsmt2d.ImportExtendedDataSquare(rdata, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(squareSize)) require.NoError(t, err) // check that the squares are equal - assert.Equal(t, share.ExtractEDS(testEds), share.ExtractEDS(reds)) + assert.Equal(t, testEds.Flattened(), reds.Flattened()) }) } } @@ -133,7 +133,7 @@ func Test_ConvertEDStoShares(t *testing.T) { ) require.NoError(t, err) - resshares := share.ExtractODS(testEds) + resshares := testEds.FlattenedODS() require.Equal(t, shares, resshares) } @@ -434,7 +434,7 @@ func TestBatchSize(t *testing.T) { bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) randEds := edstest.RandEDS(t, tt.origWidth) - _, err := AddShares(ctx, share.ExtractODS(randEds), blockservice.New(bs, offline.Exchange(bs))) + _, err := AddShares(ctx, randEds.FlattenedODS(), blockservice.New(bs, offline.Exchange(bs))) require.NoError(t, err) out, err := bs.AllKeysChan(ctx) From 04e29282b8f861d693ae2fdb91f9e445002e3063 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Sat, 19 Aug 2023 03:08:16 +0800 Subject: [PATCH 136/388] fix(share/ipld): Remove proofs from leaked ctx (#2574) --- core/exchange.go | 4 ++++ core/listener.go | 2 ++ share/eds/eds.go | 8 +++++--- share/getters/tee.go | 5 ++++- share/ipld/nmt_adder.go | 20 ++++++++++++++++++++ 5 files changed, 35 insertions(+), 4 deletions(-) diff --git a/core/exchange.go b/core/exchange.go index a9b3a532d5..bed2404195 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -108,6 +108,8 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende // extend block data adder := ipld.NewProofsAdder(int(block.Data.SquareSize)) + defer adder.Purge() + eds, err := extendBlock(block.Data, block.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", &block.Height, err) @@ -148,6 +150,8 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 // extend block data adder := ipld.NewProofsAdder(int(b.Data.SquareSize)) + defer adder.Purge() + eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) if err != nil { return nil, fmt.Errorf("extending block data for height %d: %w", b.Header.Height, err) diff --git a/core/listener.go b/core/listener.go index 17be12a780..565fc62032 100644 --- a/core/listener.go +++ b/core/listener.go @@ -153,6 +153,8 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS ) // extend block data adder := ipld.NewProofsAdder(int(b.Data.SquareSize)) + defer adder.Purge() + eds, err := extendBlock(b.Data, b.Header.Version.App, nmt.NodeVisitor(adder.VisitFn())) if err != nil { return fmt.Errorf("extending block data: %w", err) diff --git a/share/eds/eds.go b/share/eds/eds.go index bc35f59b56..cf38a14cbc 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -108,9 +108,9 @@ func writeProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare, w io.Write func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid][]byte, error) { // check if there are proofs collected by ipld.ProofsAdder in previous reconstruction of eds - proofs := ipld.ProofsAdderFromCtx(ctx).Proofs() - if proofs != nil { - return proofs, nil + if adder := ipld.ProofsAdderFromCtx(ctx); adder != nil { + defer adder.Purge() + return adder.Proofs(), nil } // recompute proofs from eds @@ -124,6 +124,8 @@ func getProofs(ctx context.Context, eds *rsmt2d.ExtendedDataSquare) (map[cid.Cid // this adder ignores leaves, so that they are not added to the store we iterate through in // writeProofs adder := ipld.NewProofsAdder(odsWidth * 2) + defer adder.Purge() + eds, err := rsmt2d.ImportExtendedDataSquare( shares, share.DefaultRSMT2DCodec(), diff --git a/share/getters/tee.go b/share/getters/tee.go index 8ee1e91390..9c89b2dec5 100644 --- a/share/getters/tee.go +++ b/share/getters/tee.go @@ -52,7 +52,10 @@ func (tg *TeeGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d. utils.SetStatusAndEnd(span, err) }() - ctx = ipld.CtxWithProofsAdder(ctx, ipld.NewProofsAdder(len(root.RowRoots))) + adder := ipld.NewProofsAdder(len(root.RowRoots)) + ctx = ipld.CtxWithProofsAdder(ctx, adder) + defer adder.Purge() + eds, err = tg.getter.GetEDS(ctx, root) if err != nil { return nil, err diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go index 0b4c521a80..d090c679d9 100644 --- a/share/ipld/nmt_adder.go +++ b/share/ipld/nmt_adder.go @@ -101,11 +101,13 @@ func BatchSize(squareSize int) int { return (squareSize*2-1)*squareSize*2 - (squareSize * squareSize) } +// ProofsAdder is used to collect proof nodes, while traversing merkle tree type ProofsAdder struct { lock sync.RWMutex proofs map[cid.Cid][]byte } +// NewProofsAdder creates new instance of ProofsAdder. func NewProofsAdder(squareSize int) *ProofsAdder { return &ProofsAdder{ // preallocate map to fit all inner nodes for given square size @@ -113,10 +115,14 @@ func NewProofsAdder(squareSize int) *ProofsAdder { } } +// CtxWithProofsAdder creates context, that will contain ProofsAdder. If context is leaked to +// another go-routine, proofs will be not collected by gc. To prevent it, use Purge after Proofs +// are collected from adder, to preemptively release memory allocated for proofs. func CtxWithProofsAdder(ctx context.Context, adder *ProofsAdder) context.Context { return context.WithValue(ctx, proofsAdderKey, adder) } +// ProofsAdderFromCtx extracts ProofsAdder from context func ProofsAdderFromCtx(ctx context.Context) *ProofsAdder { val := ctx.Value(proofsAdderKey) adder, ok := val.(*ProofsAdder) @@ -126,6 +132,7 @@ func ProofsAdderFromCtx(ctx context.Context) *ProofsAdder { return adder } +// Proofs returns proofs collected by ProofsAdder func (a *ProofsAdder) Proofs() map[cid.Cid][]byte { if a == nil { return nil @@ -136,6 +143,7 @@ func (a *ProofsAdder) Proofs() map[cid.Cid][]byte { return a.proofs } +// VisitFn returns NodeVisitorFn, that will collect proof nodes while traversing merkle tree. func (a *ProofsAdder) VisitFn() nmt.NodeVisitorFn { if a == nil { return nil @@ -151,6 +159,18 @@ func (a *ProofsAdder) VisitFn() nmt.NodeVisitorFn { return a.visitInnerNodes } +// Purge removed proofs from ProofsAdder allowing GC to collect the memory +func (a *ProofsAdder) Purge() { + if a == nil { + return + } + + a.lock.Lock() + defer a.lock.Unlock() + + a.proofs = nil +} + func (a *ProofsAdder) visitInnerNodes(hash []byte, children ...[]byte) { switch len(children) { case 1: From 13e9b1f99aa5c8a528e4419f82fbddc37ef688b6 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 21 Aug 2023 17:26:11 +0200 Subject: [PATCH 137/388] feat(nodebuilder/state): Provide stubbed state module if a core endpoint not provided (#2577) This is a pre-requisite for #2511 It makes a check in CoreAccessor constructor to see if a core endpoint was provided, and returns nil CoreAccessor if not. A stubbed state module will then be provided if no core endpoint was provided so that errors are more readable. Previously, node start logic relied on the fact that the grpc Dial inside CoreAccessor.Start was non-blocking so it could silently fail under the hood and any calls made on state Module would return errors from the inability to reach the address that is the default for the core config (which was confusing). --- nodebuilder/core/config.go | 16 ++++- nodebuilder/fraud/lifecycle.go | 8 +++ nodebuilder/module.go | 2 +- nodebuilder/node_light_test.go | 10 +++ nodebuilder/node_test.go | 12 ---- nodebuilder/settings.go | 7 +- nodebuilder/state/core.go | 4 +- nodebuilder/state/module.go | 11 ++-- nodebuilder/state/stub.go | 116 +++++++++++++++++++++++++++++++++ state/core_access.go | 6 +- 10 files changed, 167 insertions(+), 25 deletions(-) create mode 100644 nodebuilder/state/stub.go diff --git a/nodebuilder/core/config.go b/nodebuilder/core/config.go index c0261c7c86..d0455be86c 100644 --- a/nodebuilder/core/config.go +++ b/nodebuilder/core/config.go @@ -18,14 +18,18 @@ type Config struct { // node's connection to a Celestia-Core endpoint. func DefaultConfig() Config { return Config{ - IP: "0.0.0.0", - RPCPort: "0", - GRPCPort: "0", + IP: "", + RPCPort: "", + GRPCPort: "", } } // Validate performs basic validation of the config. func (cfg *Config) Validate() error { + if !cfg.IsEndpointConfigured() { + return nil + } + ip, err := utils.ValidateAddr(cfg.IP) if err != nil { return err @@ -41,3 +45,9 @@ func (cfg *Config) Validate() error { } return nil } + +// IsEndpointConfigured returns whether a core endpoint has been set +// on the config (true if set). +func (cfg *Config) IsEndpointConfigured() bool { + return cfg.IP != "" +} diff --git a/nodebuilder/fraud/lifecycle.go b/nodebuilder/fraud/lifecycle.go index 24ed402f5d..cffa4d0f56 100644 --- a/nodebuilder/fraud/lifecycle.go +++ b/nodebuilder/fraud/lifecycle.go @@ -31,6 +31,10 @@ type ServiceBreaker[S service] struct { // Start starts the inner service if there are no fraud proofs stored. // Subscribes for fraud and stops the service whenever necessary. func (breaker *ServiceBreaker[S]) Start(ctx context.Context) error { + if breaker == nil { + return nil + } + proofs, err := breaker.FraudServ.Get(ctx, breaker.FraudType) switch err { default: @@ -57,6 +61,10 @@ func (breaker *ServiceBreaker[S]) Start(ctx context.Context) error { // Stop stops the service and cancels subscription. func (breaker *ServiceBreaker[S]) Stop(ctx context.Context) error { + if breaker == nil { + return nil + } + if breaker.ctx.Err() != nil { // short circuit if the service was already stopped return nil diff --git a/nodebuilder/module.go b/nodebuilder/module.go index 51edc26c72..719705e35c 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -45,7 +45,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store fx.Supply(signer), // modules provided by the node p2p.ConstructModule(tp, &cfg.P2P), - state.ConstructModule(tp, &cfg.State), + state.ConstructModule(tp, &cfg.State, &cfg.Core), header.ConstructModule(tp, &cfg.Header), share.ConstructModule(tp, &cfg.Share), rpc.ConstructModule(tp, &cfg.RPC), diff --git a/nodebuilder/node_light_test.go b/nodebuilder/node_light_test.go index a7a70d0622..7138a23c9e 100644 --- a/nodebuilder/node_light_test.go +++ b/nodebuilder/node_light_test.go @@ -1,6 +1,7 @@ package nodebuilder import ( + "context" "crypto/rand" "testing" @@ -11,6 +12,7 @@ import ( nodebuilder "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/state" ) func TestNewLightWithP2PKey(t *testing.T) { @@ -44,3 +46,11 @@ func TestLight_WithNetwork(t *testing.T) { require.NotNil(t, node) assert.Equal(t, p2p.Private, node.Network) } + +// TestLight_WithStubbedCoreAccessor ensures that a node started without +// a core connection will return a stubbed StateModule. +func TestLight_WithStubbedCoreAccessor(t *testing.T) { + node := TestNode(t, nodebuilder.Light) + _, err := node.StateServ.Balance(context.Background()) + assert.ErrorIs(t, state.ErrNoStateAccess, err) +} diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index f481162b5c..3fc3f4f02a 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -43,14 +43,8 @@ func TestLifecycle(t *testing.T) { err := node.Start(ctx) require.NoError(t, err) - // ensure the state service is running - require.False(t, node.StateServ.IsStopped(ctx)) - err = node.Stop(ctx) require.NoError(t, err) - - // ensure the state service is stopped - require.True(t, node.StateServ.IsStopped(ctx)) }) } } @@ -96,14 +90,8 @@ func TestLifecycle_WithMetrics(t *testing.T) { err := node.Start(ctx) require.NoError(t, err) - // ensure the state service is running - require.False(t, node.StateServ.IsStopped(ctx)) - err = node.Stop(ctx) require.NoError(t, err) - - // ensure the state service is stopped - require.True(t, node.StateServ.IsStopped(ctx)) }) } } diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index b2a29723db..97440fa7dc 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -74,7 +74,12 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti baseComponents := fx.Options( fx.Supply(metricOpts), fx.Invoke(initializeMetrics), - fx.Invoke(state.WithMetrics), + fx.Invoke(func(ca *state.CoreAccessor) { + if ca == nil { + return + } + state.WithMetrics(ca) + }), fx.Invoke(fraud.WithMetrics), fx.Invoke(node.WithMetrics), fx.Invoke(modheader.WithMetrics), diff --git a/nodebuilder/state/core.go b/nodebuilder/state/core.go index a3eb7f7b6d..4636e0f099 100644 --- a/nodebuilder/state/core.go +++ b/nodebuilder/state/core.go @@ -19,10 +19,10 @@ func coreAccessor( signer *apptypes.KeyringSigner, sync *sync.Syncer[*header.ExtendedHeader], fraudServ libfraud.Service, -) (*state.CoreAccessor, *modfraud.ServiceBreaker[*state.CoreAccessor]) { +) (*state.CoreAccessor, Module, *modfraud.ServiceBreaker[*state.CoreAccessor]) { ca := state.NewCoreAccessor(signer, sync, corecfg.IP, corecfg.RPCPort, corecfg.GRPCPort) - return ca, &modfraud.ServiceBreaker[*state.CoreAccessor]{ + return ca, ca, &modfraud.ServiceBreaker[*state.CoreAccessor]{ Service: ca, FraudType: byzantine.BadEncoding, FraudServ: fraudServ, diff --git a/nodebuilder/state/module.go b/nodebuilder/state/module.go index 24305dabe1..fe90d023eb 100644 --- a/nodebuilder/state/module.go +++ b/nodebuilder/state/module.go @@ -6,6 +6,8 @@ import ( logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" + "github.com/celestiaorg/celestia-node/libs/fxutil" + "github.com/celestiaorg/celestia-node/nodebuilder/core" modfraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/state" @@ -15,14 +17,14 @@ var log = logging.Logger("module/state") // ConstructModule provides all components necessary to construct the // state service. -func ConstructModule(tp node.Type, cfg *Config) fx.Option { +func ConstructModule(tp node.Type, cfg *Config, coreCfg *core.Config) fx.Option { // sanitize config values before constructing module cfgErr := cfg.Validate() baseComponents := fx.Options( fx.Supply(*cfg), fx.Error(cfgErr), - fx.Provide(fx.Annotate( + fxutil.ProvideIf(coreCfg.IsEndpointConfigured(), fx.Annotate( coreAccessor, fx.OnStart(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*state.CoreAccessor]) error { return breaker.Start(ctx) @@ -31,9 +33,8 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { return breaker.Stop(ctx) }), )), - // the module is needed for the handler - fx.Provide(func(ca *state.CoreAccessor) Module { - return ca + fxutil.ProvideIf(!coreCfg.IsEndpointConfigured(), func() (*state.CoreAccessor, Module) { + return nil, &stubbedStateModule{} }), ) diff --git a/nodebuilder/state/stub.go b/nodebuilder/state/stub.go new file mode 100644 index 0000000000..94326fed5e --- /dev/null +++ b/nodebuilder/state/stub.go @@ -0,0 +1,116 @@ +package state + +import ( + "context" + "errors" + + "github.com/cosmos/cosmos-sdk/x/staking/types" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/state" +) + +var ErrNoStateAccess = errors.New("node is running without state access") + +// stubbedStateModule provides a stub for the state module to return +// errors when state endpoints are accessed without a running connection +// to a core endpoint. +type stubbedStateModule struct{} + +func (s stubbedStateModule) IsStopped(context.Context) bool { + return true +} + +func (s stubbedStateModule) AccountAddress(context.Context) (state.Address, error) { + return state.Address{}, ErrNoStateAccess +} + +func (s stubbedStateModule) Balance(context.Context) (*state.Balance, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) BalanceForAddress( + context.Context, + state.Address, +) (*state.Balance, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) Transfer( + _ context.Context, + _ state.AccAddress, + _, _ state.Int, + _ uint64, +) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) SubmitTx(context.Context, state.Tx) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) SubmitPayForBlob( + context.Context, + state.Int, + uint64, + []*blob.Blob, +) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) CancelUnbondingDelegation( + _ context.Context, + _ state.ValAddress, + _, _, _ state.Int, + _ uint64, +) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) BeginRedelegate( + _ context.Context, + _, _ state.ValAddress, + _, _ state.Int, + _ uint64, +) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) Undelegate( + _ context.Context, + _ state.ValAddress, + _, _ state.Int, + _ uint64, +) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) Delegate( + _ context.Context, + _ state.ValAddress, + _, _ state.Int, + _ uint64, +) (*state.TxResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) QueryDelegation( + context.Context, + state.ValAddress, +) (*types.QueryDelegationResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) QueryUnbonding( + context.Context, + state.ValAddress, +) (*types.QueryUnbondingDelegationResponse, error) { + return nil, ErrNoStateAccess +} + +func (s stubbedStateModule) QueryRedelegations( + _ context.Context, + _, _ state.ValAddress, +) (*types.QueryRedelegationsResponse, error) { + return nil, ErrNoStateAccess +} diff --git a/state/core_access.go b/state/core_access.go index 3fefaa3ed9..1a50b15b72 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -90,7 +90,11 @@ func (ca *CoreAccessor) Start(ctx context.Context) error { // dial given celestia-core endpoint endpoint := fmt.Sprintf("%s:%s", ca.coreIP, ca.grpcPort) - client, err := grpc.DialContext(ctx, endpoint, grpc.WithTransportCredentials(insecure.NewCredentials())) + client, err := grpc.DialContext( + ctx, + endpoint, + grpc.WithTransportCredentials(insecure.NewCredentials()), + ) if err != nil { return err } From f839ee3c444b79620e5fc4b393f98347044ca4f2 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Tue, 22 Aug 2023 04:45:52 -0500 Subject: [PATCH 138/388] chore!: update celestia-app to v1.0.0-rc12 (#2581) ## Overview bumps app to v1.0.0-rc12 ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- blob/service_test.go | 9 +++++---- go.mod | 10 +++++----- go.sum | 16 ++++++++-------- nodebuilder/tests/nd_test.go | 8 ++++++-- share/namespace.go | 25 +++++++++++++++++-------- state/core_access.go | 1 + 6 files changed, 42 insertions(+), 27 deletions(-) diff --git a/blob/service_test.go b/blob/service_test.go index 51475178d2..1dcabe7129 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -14,6 +14,7 @@ import ( "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/go-header/store" @@ -309,9 +310,9 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { ns1, ns2 := blobs[0].Namespace().ToAppNamespace(), blobs[1].Namespace().ToAppNamespace() - padding0, err := shares.NamespacePaddingShare(ns1) + padding0, err := shares.NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) - padding1, err := shares.NamespacePaddingShare(ns2) + padding1, err := shares.NamespacePaddingShare(ns2, appconsts.ShareVersionZero) require.NoError(t, err) rawShares0, err := BlobsToShares(blobs[0]) require.NoError(t, err) @@ -354,9 +355,9 @@ func TestService_GetAllWithoutPadding(t *testing.T) { ns1, ns2 := blobs[0].Namespace().ToAppNamespace(), blobs[1].Namespace().ToAppNamespace() - padding0, err := shares.NamespacePaddingShare(ns1) + padding0, err := shares.NamespacePaddingShare(ns1, appconsts.ShareVersionZero) require.NoError(t, err) - padding1, err := shares.NamespacePaddingShare(ns2) + padding1, err := shares.NamespacePaddingShare(ns2, appconsts.ShareVersionZero) require.NoError(t, err) rawShares0, err := BlobsToShares(blobs[0]) require.NoError(t, err) diff --git a/go.mod b/go.mod index 8310e845f2..70b246143c 100644 --- a/go.mod +++ b/go.mod @@ -10,14 +10,14 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc11 + github.com/celestiaorg/celestia-app v1.0.0-rc12 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.1.2 github.com/celestiaorg/go-header v0.2.12 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.18.1 github.com/celestiaorg/rsmt2d v0.11.0 - github.com/cosmos/cosmos-sdk v0.46.13 + github.com/cosmos/cosmos-sdk v0.46.14 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 github.com/etclabscore/go-openrpc-reflect v0.0.37 @@ -55,7 +55,7 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.14.0 - github.com/pyroscope-io/client v0.7.1 + github.com/pyroscope-io/client v0.7.2 github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.7.0 github.com/spf13/pflag v1.0.5 @@ -270,7 +270,7 @@ require ( github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect github.com/prometheus/procfs v0.9.0 // indirect - github.com/pyroscope-io/godeltaprof v0.1.0 // indirect + github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-19 v0.3.2 // indirect github.com/quic-go/qtls-go1-20 v0.2.2 // indirect @@ -334,7 +334,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app diff --git a/go.sum b/go.sum index 68837e301b..95422004c7 100644 --- a/go.sum +++ b/go.sum @@ -343,12 +343,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc11 h1:u94drhPdvgwNvx0F2xGib+fyOyP+fIvp3aL6Avd/HT0= -github.com/celestiaorg/celestia-app v1.0.0-rc11/go.mod h1:A17xfzurB2RHpplh6+nGMeC9hCquu0aMYnanOeifDO0= +github.com/celestiaorg/celestia-app v1.0.0-rc12 h1:ko9hPD4oz1UTS4ZqzikGVQ0wXi5+4kEhDb7decx5Ehs= +github.com/celestiaorg/celestia-app v1.0.0-rc12/go.mod h1:vXvKEudUpdJCvUr79qVKKJ0Xo7ofsuU80+Hs9aKGjvU= github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHVxcTzhzQW44KgTcooR3OxnK4= github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= -github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13 h1:CxEQDQEQR1ypB+VUmCISIqFVmHfb+mx8x+zh7rHbyU8= -github.com/celestiaorg/cosmos-sdk v1.16.1-sdk-v0.46.13/go.mod h1:xpBZc/OYZ736hp0IZlBGNUhEgCD9C+bKs8yNLZibyv0= +github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 h1:PckXGxLJjXv97VO3xS8NPHN5oO83X5nvJLbc/4s8jUM= +github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14/go.mod h1:70Go8qNy7YAb1PUcHCChRHNX2ke7c9jgUIEklUX+Mac= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= @@ -1781,10 +1781,10 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/pyroscope-io/client v0.7.1 h1:yFRhj3vbgjBxehvxQmedmUWJQ4CAfCHhn+itPsuWsHw= -github.com/pyroscope-io/client v0.7.1/go.mod h1:4h21iOU4pUOq0prKyDlvYRL+SCKsBc5wKiEtV+rJGqU= -github.com/pyroscope-io/godeltaprof v0.1.0 h1:UBqtjt0yZi4jTxqZmLAs34XG6ycS3vUTlhEUSq4NHLE= -github.com/pyroscope-io/godeltaprof v0.1.0/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= +github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= +github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= +github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= +github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic= github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index 955cff2ffe..cb02e2a178 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -48,8 +48,12 @@ func TestShrexNDFromLights(t *testing.T) { // wait for chain to be filled require.NoError(t, <-fillDn) - // first 2 blocks are not filled with data - for i := 3; i < blocks; i++ { + // first 15 blocks are not filled with data + // + // TODO: we need to stop guessing + // the block that actually has transactions. We can get this data from the + // response returned by FillBlock. + for i := 16; i < blocks; i++ { h, err := bridge.HeaderServ.GetByHeight(ctx, uint64(i)) require.NoError(t, err) diff --git a/share/namespace.go b/share/namespace.go index 276575a309..e7ed7dc0d6 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -14,13 +14,19 @@ const NamespaceSize = appns.NamespaceSize // Various reserved namespaces. var ( - MaxReservedNamespace = Namespace(appns.MaxReservedNamespace.Bytes()) - ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) - TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) - ReservedPaddingNamespace = Namespace(appns.ReservedPaddingNamespace.Bytes()) - TxNamespace = Namespace(appns.TxNamespace.Bytes()) - PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) - ISRNamespace = Namespace(appns.IntermediateStateRootsNamespace.Bytes()) + // MaxPrimaryReservedNamespace is the highest primary reserved namespace. + // Namespaces lower than this are reserved for protocol use. + MaxPrimaryReservedNamespace = Namespace(appns.MaxPrimaryReservedNamespace.Bytes()) + // MinSecondaryReservedNamespace is the lowest secondary reserved namespace + // reserved for protocol use. Namespaces higher than this are reserved for + // protocol use. + MinSecondaryReservedNamespace = Namespace(appns.MinSecondaryReservedNamespace.Bytes()) + ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) + TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) + ReservedPaddingNamespace = Namespace(appns.PrimaryReservedPaddingNamespace.Bytes()) + TxNamespace = Namespace(appns.TxNamespace.Bytes()) + PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) + ISRNamespace = Namespace(appns.IntermediateStateRootsNamespace.Bytes()) ) // Namespace represents namespace of a Share. @@ -124,7 +130,10 @@ func (n Namespace) ValidateForBlob() error { if err := n.ValidateForData(); err != nil { return err } - if bytes.Compare(n, MaxReservedNamespace) < 1 { + if bytes.Compare(n, MaxPrimaryReservedNamespace) < 1 { + return fmt.Errorf("invalid blob namespace(%s): reserved namespaces are forbidden", n) + } + if bytes.Compare(n, MinSecondaryReservedNamespace) > -1 { return fmt.Errorf("invalid blob namespace(%s): reserved namespaces are forbidden", n) } return nil diff --git a/state/core_access.go b/state/core_access.go index 1a50b15b72..73d525347f 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -182,6 +182,7 @@ func (ca *CoreAccessor) SubmitPayForBlob( ctx, ca.signer, ca.coreConn, + sdktx.BroadcastMode_BROADCAST_MODE_BLOCK, appblobs, apptypes.SetGasLimit(gasLim), withFee(fee), From 44a0b374ab82556e02f29f81bed7efb1932b3871 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Tue, 22 Aug 2023 12:23:10 +0200 Subject: [PATCH 139/388] chore: bump deps + go1.21 (#2589) Update to go1.21 was forced by `go mod tidy`. Theoretically, we could still live on go1.20 just supporting builds for people who run go1.21, but for some uninvestigated reason, we are forced to. Closes #2566 Closes #2247 --- .github/workflows/go-ci.yml | 2 +- Dockerfile | 2 +- Makefile | 6 +- README.md | 2 +- go.mod | 80 +++++------ go.sum | 212 +++++++++++++++++++---------- share/availability/test/testing.go | 2 +- share/eds/byzantine/share_proof.go | 2 +- share/eds/eds.go | 2 +- share/getter.go | 3 +- share/getters/getter_test.go | 2 +- share/getters/shrex_test.go | 3 +- share/getters/utils_test.go | 3 +- share/ipld/get_shares_test.go | 4 +- share/p2p/shrexnd/server.go | 2 +- 15 files changed, 196 insertions(+), 131 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index e6c4e5881b..7435b7912f 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -4,7 +4,7 @@ on: workflow_call: env: - GO_VERSION: '1.20' + GO_VERSION: '1.21' concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} diff --git a/Dockerfile b/Dockerfile index 2d07babc94..c8e9bb2e1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.20-alpine3.18 as builder +FROM docker.io/golang:1.21-alpine3.18 as builder # hadolint ignore=DL3018 RUN apk update && apk add --no-cache \ diff --git a/Makefile b/Makefile index 1044d75d3b..feef6172aa 100644 --- a/Makefile +++ b/Makefile @@ -82,7 +82,7 @@ install-key: fmt: sort-imports @find . -name '*.go' -type f -not -path "*.git*" -not -name '*.pb.go' -not -name '*pb_test.go' | xargs gofmt -w -s @find . -name '*.go' -type f -not -path "*.git*" -not -name '*.pb.go' -not -name '*pb_test.go' | xargs goimports -w -local github.com/celestiaorg - @go mod tidy -compat=1.17 + @go mod tidy -compat=1.20 @cfmt -w -m=100 ./... @markdownlint --fix --quiet --config .markdownlint.yaml . .PHONY: fmt @@ -161,14 +161,14 @@ openrpc-gen: lint-imports: @echo "--> Running imports linter" @for file in `find . -type f -name '*.go'`; \ - do goimports-reviser -list-diff -set-exit-status -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/celestia-node" -output stdout $$file \ + do goimports-reviser -list-diff -set-exit-status -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" -output stdout $$file \ || exit 1; \ done; .PHONY: lint-imports ## sort-imports: Sort Go imports. sort-imports: - @goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/celestia-node" -output stdout ./... + @goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" -output stdout . .PHONY: sort-imports ## adr-gen: Generate ADR from template. Must set NUM and TITLE parameters. diff --git a/README.md b/README.md index 76c722275d..0711c2d223 100644 --- a/README.md +++ b/README.md @@ -32,7 +32,7 @@ Continue reading [here](https://blog.celestia.org/celestia-mvp-release-data-avai | Requirement | Notes | | ----------- |----------------| -| Go version | 1.20 or higher | +| Go version | 1.21 or higher | ## System Requirements diff --git a/go.mod b/go.mod index 70b246143c..91b55ad256 100644 --- a/go.mod +++ b/go.mod @@ -1,19 +1,21 @@ module github.com/celestiaorg/celestia-node -go 1.20 +go 1.21 + +toolchain go1.21.0 replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1-lazypatch require ( cosmossdk.io/errors v1.0.0 - cosmossdk.io/math v1.0.0-rc.0 + cosmossdk.io/math v1.1.1 github.com/BurntSushi/toml v1.3.2 - github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 + github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc12 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.1.2 - github.com/celestiaorg/go-header v0.2.12 + github.com/celestiaorg/go-header v0.2.13 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.18.1 github.com/celestiaorg/rsmt2d v0.11.0 @@ -27,34 +29,31 @@ require ( github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.0 - github.com/hashicorp/go-retryablehttp v0.7.2 - github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d + github.com/hashicorp/go-retryablehttp v0.7.4 + github.com/hashicorp/golang-lru v1.0.2 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.11.0 github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-blockservice v0.5.1 // down 1 version, 0.5.2 is marked as deprecated and raises alerts github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-ipfs-exchange-offline v0.3.0 - github.com/ipfs/go-ipfs-routing v0.3.0 github.com/ipfs/go-ipld-cbor v0.0.6 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.11.0 github.com/ipld/go-car v0.6.2 - github.com/libp2p/go-libp2p v0.29.0 - github.com/libp2p/go-libp2p-kad-dht v0.24.2 + github.com/libp2p/go-libp2p v0.30.0 + github.com/libp2p/go-libp2p-kad-dht v0.25.0 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 - github.com/libp2p/go-libp2p-routing-helpers v0.7.0 - github.com/minio/sha256-simd v1.0.1 + github.com/libp2p/go-libp2p-routing-helpers v0.7.1 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.10.1 + github.com/multiformats/go-multiaddr v0.11.0 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 - github.com/prometheus/client_golang v1.14.0 + github.com/prometheus/client_golang v1.16.0 github.com/pyroscope-io/client v0.7.2 github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.7.0 @@ -70,27 +69,28 @@ require ( go.opentelemetry.io/otel/sdk v1.16.0 go.opentelemetry.io/otel/sdk/metric v0.39.0 go.opentelemetry.io/otel/trace v1.16.0 - go.opentelemetry.io/proto/otlp v0.19.0 + go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 - go.uber.org/zap v1.24.0 + go.uber.org/zap v1.25.0 golang.org/x/crypto v0.12.0 - golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 + golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 golang.org/x/text v0.12.0 - google.golang.org/grpc v1.56.2 + google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 ) require ( - cloud.google.com/go v0.110.0 // indirect - cloud.google.com/go/compute v1.19.1 // indirect + cloud.google.com/go v0.110.6 // indirect + cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v0.13.0 // indirect - cloud.google.com/go/storage v1.28.1 // indirect + cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/storage v1.30.1 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect + github.com/Jorropo/jsync v1.0.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/StackExchange/wmi v1.2.1 // indirect @@ -116,7 +116,7 @@ require ( github.com/cosmos/btcutil v1.0.5 // indirect github.com/cosmos/cosmos-proto v1.0.0-alpha8 // indirect github.com/cosmos/go-bip39 v1.0.0 // indirect - github.com/cosmos/gogoproto v1.4.10 // indirect + github.com/cosmos/gogoproto v1.4.11 // indirect github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.6 // indirect github.com/cosmos/ibc-go/v6 v6.2.0 // indirect @@ -169,10 +169,11 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect - github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 // indirect + github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.7.1 // indirect + github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect @@ -188,7 +189,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.2 // indirect + github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect @@ -203,6 +204,7 @@ require ( github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect + github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect @@ -232,10 +234,10 @@ require ( github.com/libp2p/go-msgio v0.3.0 // indirect github.com/libp2p/go-nat v0.2.0 // indirect github.com/libp2p/go-netroute v0.2.1 // indirect - github.com/libp2p/go-reuseport v0.3.0 // indirect + github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect github.com/magiconair/properties v1.8.6 // indirect - github.com/mailru/easyjson v0.7.6 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect @@ -246,6 +248,7 @@ require ( github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect github.com/minio/highwayhash v1.0.2 // indirect + github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect @@ -257,7 +260,7 @@ require ( github.com/multiformats/go-multistream v0.4.1 // indirect github.com/multiformats/go-varint v0.0.7 // indirect github.com/onsi/ginkgo/v2 v2.11.0 // indirect - github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 // indirect + github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pelletier/go-toml v1.9.5 // indirect @@ -269,12 +272,11 @@ require ( github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.9.0 // indirect + github.com/prometheus/procfs v0.10.1 // indirect github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-19 v0.3.2 // indirect - github.com/quic-go/qtls-go1-20 v0.2.2 // indirect - github.com/quic-go/quic-go v0.36.2 // indirect + github.com/quic-go/qtls-go1-20 v0.3.2 // indirect + github.com/quic-go/quic-go v0.37.6 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect @@ -312,18 +314,18 @@ require ( go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.12.0 // indirect + golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect golang.org/x/sys v0.11.0 // indirect golang.org/x/term v0.11.0 // indirect - golang.org/x/tools v0.11.0 // indirect + golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect - google.golang.org/api v0.114.0 // indirect + google.golang.org/api v0.126.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 // indirect + google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index 95422004c7..84e5eb5991 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w9 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.110.0 h1:Zc8gqp3+a9/Eyph2KDmcGaPtbKRIoqq4YTlL4NMD0Ys= -cloud.google.com/go v0.110.0/go.mod h1:SJnCLqQ0FCFGSZMUNUf84MV3Aia54kn7pi8st7tMzaY= +cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= +cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -75,8 +75,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.19.1 h1:am86mquDUgjGNWxiGn+5PGLbmgiWXlE/yNWpIpNvuXY= -cloud.google.com/go/compute v1.19.1/go.mod h1:6ylj3a05WF8leseCdIf77NK0g1ey+nj5IKd5/kvShxE= +cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= +cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -116,13 +116,12 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v0.13.0 h1:+CmB+K0J/33d0zSQ9SlFWUeCCEn5XJA0ZMZ3pHE9u8k= -cloud.google.com/go/iam v0.13.0/go.mod h1:ljOg+rcNfzZ5d6f1nAUJ8ZIxOaZUVoS14bKCtaLZ/D0= +cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= +cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= cloud.google.com/go/lifesciences v0.6.0/go.mod h1:ddj6tSX/7BOnhxCSd3ZcETvtNr8NZ6t/iPhY2Tyfu08= -cloud.google.com/go/longrunning v0.4.1 h1:v+yFJOfKC3yZdY6ZUI933pIYdhyhV8S3NpWrXWmg7jM= cloud.google.com/go/mediatranslation v0.5.0/go.mod h1:jGPUhGTybqsPQn91pNXw0xVHfuJ3leR1wj37oU3y1f4= cloud.google.com/go/mediatranslation v0.6.0/go.mod h1:hHdBCTYNigsBxshbznuIMFNe5QXEowAuNmmC7h8pu5w= cloud.google.com/go/memcache v1.4.0/go.mod h1:rTOfiGZtJX1AaFUrOgsMHX5kAzaTQ8azHiuDoTPzNsE= @@ -180,8 +179,8 @@ cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3f cloud.google.com/go/storage v1.22.1/go.mod h1:S8N1cAStu7BOeFfE8KAQzmyyLkK8p/vmRq6kuBTW58Y= cloud.google.com/go/storage v1.23.0/go.mod h1:vOEEDNFnciUMhBeT6hsJIn3ieU5cFRmzeLgDvXzfIXc= cloud.google.com/go/storage v1.27.0/go.mod h1:x9DOL8TK/ygDUMieqwfhdpQryTeEkhGKMi80i/iqR2s= -cloud.google.com/go/storage v1.28.1 h1:F5QDG5ChchaAVQhINh24U99OWHURqrW8OmQcGKXcbgI= -cloud.google.com/go/storage v1.28.1/go.mod h1:Qnisd4CqDdo6BGs2AD5LLnEsmSQ80wQ5ogcBBKhU86Y= +cloud.google.com/go/storage v1.30.1 h1:uOdMxAs8HExqBlnLtnQyP0YkvbiDpdGShGKtx6U/oNM= +cloud.google.com/go/storage v1.30.1/go.mod h1:NfxhC0UJE1aXSx7CIIbCf7y9HKT7BiccwkR7+P7gN8E= cloud.google.com/go/talent v1.1.0/go.mod h1:Vl4pt9jiHKvOgF9KoZo6Kob9oV4lwd/ZD5Cto54zDRw= cloud.google.com/go/talent v1.2.0/go.mod h1:MoNF9bhFQbiJ6eFD3uSsg0uBALw4n4gaCaEjBw9zo8g= cloud.google.com/go/videointelligence v1.6.0/go.mod h1:w0DIDlVRKtwPCn/C4iwZIJdvC69yInhW0cfi+p546uU= @@ -196,8 +195,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= -cosmossdk.io/math v1.0.0-rc.0 h1:ml46ukocrAAoBpYKMidF0R2tQJ1Uxfns0yH8wqgMAFc= -cosmossdk.io/math v1.0.0-rc.0/go.mod h1:Ygz4wBHrgc7g0N+8+MrnTfS9LLn9aaTGa9hKopuym5k= +cosmossdk.io/math v1.1.1 h1:Eqx44E6fSvG055Z6VNiCLWA9fra0JSyP0kQX7VvNNfk= +cosmossdk.io/math v1.1.1/go.mod h1:uFRkSZDz38KjWjm6jN+/sI8tJWQxbGwxcjOTzapWSpE= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= @@ -220,6 +219,7 @@ github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3 github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= +github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -230,13 +230,18 @@ github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= +github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= +github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= +github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETFUpAzWW2ep1Y= github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= +github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= +github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI= @@ -256,11 +261,13 @@ github.com/Workiva/go-datastructures v1.0.53 h1:J6Y/52yX10Xc5JjXmGtWoSSxs3mZnGSa github.com/Workiva/go-datastructures v1.0.53/go.mod h1:1yZL+zfsztete+ePzZz/Zb1/t5BnDuE2Ya2MMGhzP6A= github.com/Zilliqa/gozilliqa-sdk v1.2.1-0.20201201074141-dd0ecada1be6/go.mod h1:eSYp2T6f0apnuW8TzhV3f6Aff2SE8Dwio++U4ha4yEM= github.com/adlio/schema v1.3.3 h1:oBJn8I02PyTB466pZO1UZEn1TV5XLlifBSyMrmHl/1I= +github.com/adlio/schema v1.3.3/go.mod h1:1EsRssiv9/Ce2CMzq5DoL7RiMshhuigQxrR4DMV9fHg= github.com/aead/siphash v1.0.1/go.mod h1:Nywa3cDsYNNK3gaciGTWPwHt0wlpNV15vwmswBAUSII= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921 h1:T3+cD5fYvuH36h7EZq+TDpm+d8a6FSD4pQsbmuGGQ8o= github.com/alecthomas/jsonschema v0.0.0-20200530073317-71f438968921/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= +github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b h1:doCpXjVwui6HUN+xgNsNS3SZ0/jUZ68Eb+mJRNOZfog= +github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b/go.mod h1:/n6+1/DWPltRLWL/VKyUxg6tzsl5kHUCcraimt4vr60= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -323,6 +330,7 @@ github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJ github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= +github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -339,6 +347,7 @@ github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= github.com/bufbuild/protocompile v0.1.0 h1:HjgJBI85hY/qmW5tw/66sNDZ7z0UDdVSi/5r40WHw4s= +github.com/bufbuild/protocompile v0.1.0/go.mod h1:ix/MMMdsT3fzxfw91dvbfzKW3fRRnuPCP47kpAm5m/4= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= @@ -355,8 +364,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= github.com/celestiaorg/go-fraud v0.1.2/go.mod h1:kHZXQY+6gd1kYkoWRFFKgWyrLPWRgDN3vd1Ll9gE/oo= -github.com/celestiaorg/go-header v0.2.12 h1:3H9nir20+MTY1vXbLxOUOV05ZspotR6JOiZGKxACHCQ= -github.com/celestiaorg/go-header v0.2.12/go.mod h1:NhiWq97NtAYyRBu8quzYOUghQULjgOzO2Ql0iVEFOf0= +github.com/celestiaorg/go-header v0.2.13 h1:sUJLXYs8ViPpxLXyIIaW3h4tPFgtVYMhzsLC4GHfS8I= +github.com/celestiaorg/go-header v0.2.13/go.mod h1:NhiWq97NtAYyRBu8quzYOUghQULjgOzO2Ql0iVEFOf0= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= @@ -414,9 +423,13 @@ github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b80 github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= +github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= +github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= +github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= @@ -435,6 +448,7 @@ github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJ github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= +github.com/containerd/continuity v0.3.0/go.mod h1:wJEAIwKOm/pBZuBd0JmeTvnLquTB1Ag8espWhkykbPM= github.com/coreos/etcd v3.3.10+incompatible/go.mod h1:uF7uidLiAD3TWHmW31ZFd/JWoc32PjwdhPthX9715RE= github.com/coreos/go-etcd v2.0.0+incompatible/go.mod h1:Jez6KQU2B/sWsbdaef3ED8NzMklzPG4d5KIOhIy30Tk= github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk= @@ -457,8 +471,8 @@ github.com/cosmos/cosmos-sdk/api v0.1.0/go.mod h1:CupqQBskAOiTXO1XDZ/wrtWzN/wTxU github.com/cosmos/go-bip39 v0.0.0-20180819234021-555e2067c45d/go.mod h1:tSxLoYXyBmiFeKpvmq4dzayMdCjCnu8uqmCysIGBT2Y= github.com/cosmos/go-bip39 v1.0.0 h1:pcomnQdrdH22njcAatO0yWojsUnCO3y2tNoV1cb6hHY= github.com/cosmos/go-bip39 v1.0.0/go.mod h1:RNJv0H/pOIVgxw6KS7QeX2a0Uo0aKUlfhZ4xuwvCdJw= -github.com/cosmos/gogoproto v1.4.10 h1:QH/yT8X+c0F4ZDacDv3z+xE3WU1P1Z3wQoLMBRJoKuI= -github.com/cosmos/gogoproto v1.4.10/go.mod h1:3aAZzeRWpAwr+SS/LLkICX2/kDFyaYVzckBDzygIxek= +github.com/cosmos/gogoproto v1.4.11 h1:LZcMHrx4FjUgrqQSWeaGC1v/TeuVFqSLa43CC6aWR2g= +github.com/cosmos/gogoproto v1.4.11/go.mod h1:/g39Mh8m17X8Q/GDEs5zYTSNaNnInBSohtaxzQnYq1Y= github.com/cosmos/gorocksdb v1.2.0 h1:d0l3jJG8M4hBouIZq0mDUHZ+zjOx044J3nGRskwTb4Y= github.com/cosmos/gorocksdb v1.2.0/go.mod h1:aaKvKItm514hKfNJpUJXnnOWeBnk2GL4+Qw9NHizILw= github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU= @@ -497,6 +511,7 @@ github.com/deckarep/golang-set/v2 v2.1.0 h1:g47V4Or+DUdzbs8FxCCmgb6VYd+ptPAngjM6 github.com/deckarep/golang-set/v2 v2.1.0/go.mod h1:VAky9rY/yGXJOLEDv3OMci+7wtDpOF4IN+y82NBOac4= github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn47hh8kt6rqSlvmrXFAc= github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= +github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= @@ -532,6 +547,7 @@ github.com/dnaeon/go-vcr v1.1.0/go.mod h1:M7tiix8f0r6mKKJ3Yq/kqU1OYf3MnfmBWVbPx/ github.com/dnaeon/go-vcr v1.2.0/go.mod h1:R4UdLID7HZT3taECzJs4YgbbH6PIGXB6W/sc5OLb6RQ= github.com/docker/docker v1.4.2-0.20180625184442-8e610b2b55bf/go.mod h1:eEKB0N0r5NX/I1kEveEz05bcu8tLC/8azJZsviup8Sk= github.com/docker/go-connections v0.4.0 h1:El9xVISelRB7BuFusrZozjnkIM5YnzCViNKohAFqRJQ= +github.com/docker/go-connections v0.4.0/go.mod h1:Gbd7IOopHjR8Iph03tsViu4nIes5XhDvyHbTtUxmeec= github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= @@ -570,8 +586,11 @@ github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= +github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= +github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= +github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= @@ -586,6 +605,7 @@ github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= github.com/fortytw2/leaktest v1.3.0 h1:u8491cBMTQ8ft8aeV+adlcytMZylmA5nnwwkRZjI8vw= +github.com/fortytw2/leaktest v1.3.0/go.mod h1:jDsjWgpAGjm2CA7WthBh/CdZYEPF31XHquHwclZch5g= github.com/francoispqt/gojay v1.2.13 h1:d2m3sFjloqoIUQU3TsHBgj6qg/BVGlTBeHDUmyJnXKk= github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= @@ -593,6 +613,7 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= @@ -607,11 +628,13 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/gin-gonic/gin v1.7.0 h1:jGB9xAJQ12AIGNB4HguylppmDK1Am9ppF7XnGXXJuoU= +github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= @@ -668,6 +691,7 @@ github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD87 github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= +github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -678,12 +702,15 @@ github.com/go-task/slim-sprig v0.0.0-20210107165309-348f09dbbbc0/go.mod h1:fyg78 github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 h1:tfuBGBXKqDEevZMzYi5KSi8KkcZtzBcTgAUUtapy0OI= github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572/go.mod h1:9Pwr4B2jHnOSGXyyzV8ROjYa2ojvAY6HCGYYfMoC3Ls= github.com/go-yaml/yaml v2.1.0+incompatible/go.mod h1:w2MrLa16VYP0jy6N7M5kHaCkaLENm+P+Tv+MfurjSw0= -github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee h1:s+21KNqlpePfkah2I+gwHF8xmJWRjooY+5248k6m4A0= github.com/gobwas/httphead v0.0.0-20180130184737-2c6c146eadee/go.mod h1:L0fX3K22YWvt/FAX9NnzrNzcI4wNYi9Yku4O0LKYflo= -github.com/gobwas/pool v0.2.0 h1:QEmUOlnSjWtnpRGHF3SauEiOsy82Cup83Vf2LcMlnc8= +github.com/gobwas/httphead v0.1.0 h1:exrUm0f4YX0L7EBwZHuCF4GDp8aJfVeBrlLQrs6NqWU= +github.com/gobwas/httphead v0.1.0/go.mod h1:O/RXo79gxV8G+RqlR/otEwx4Q36zl9rqC5u12GKvMCM= github.com/gobwas/pool v0.2.0/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= -github.com/gobwas/ws v1.0.2 h1:CoAavW/wd/kulfZmSIBt6p24n4j7tHgNVCjsfHVNUbo= +github.com/gobwas/pool v0.2.1 h1:xfeeEhW7pwmX8nuLVlqbzVc7udMDrwetjEv+TZIz1og= +github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6WezmKEw= github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= +github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= +github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -691,6 +718,7 @@ github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5x github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/flock v0.8.1 h1:+gYjHKf32LDeiEEFhQaotPbLuUXjY5ZqxKgXy7n59aw= +github.com/gofrs/flock v0.8.1/go.mod h1:F1TvTiK9OcQqauNUHlbJvyl9Qa1QvF/gOUDKA14jxHU= github.com/gofrs/uuid v3.3.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/gateway v1.1.0 h1:u0SuhL9+Il+UbjM9VIE3ntfRujKbvVpFvNB4HbjeVQ0= github.com/gogo/gateway v1.1.0/go.mod h1:S7rR8FRQyG3QFESeSv4l2WnsyzlCLG0CzBbUUo/mbic= @@ -700,7 +728,6 @@ github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4= github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -776,6 +803,7 @@ github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSN github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.1.1-0.20200604201612-c04b05f3adfa/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gofuzz v1.2.0 h1:xRy4A+RhZaiKjJ1bPfwQ8sedCA+YS2YcCHW6ec7JMi0= +github.com/google/gofuzz v1.2.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg= github.com/google/gopacket v1.1.17/go.mod h1:UdDNZ1OO62aGYVnPhxT1U6aI7ukYtA/kB8vaU0diBUM= github.com/google/gopacket v1.1.19 h1:ves8RnFZPGiFnTS0uPQStjwru6uO6h+nlr9j6fL7kF8= github.com/google/gopacket v1.1.19/go.mod h1:iJ8V8n6KS+z2U1A8pUwu8bW5SyEMkXJB8Yo/Vo+TKTo= @@ -785,6 +813,7 @@ github.com/google/martian/v3 v3.0.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIG github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0= github.com/google/martian/v3 v3.2.1/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/martian/v3 v3.3.2 h1:IqNFLAmvJOgVlpdEBiQbDc2EwKW77amAycfTuWKdfvw= +github.com/google/martian/v3 v3.3.2/go.mod h1:oBOf6HBosgwRXnUGWUB05QECsc6uvmMiJ3+6W4l/CUk= github.com/google/orderedcode v0.0.1 h1:UzfcAexk9Vhv8+9pNOgRu41f16lHq725vPwnSeiG/Us= github.com/google/orderedcode v0.0.1/go.mod h1:iVyU4/qPKHY5h/wSd6rZZCDcLJNxiWO6dvsYES2Sb20= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= @@ -802,9 +831,11 @@ github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8 h1:n6vlPhxsA+BW/XsS5+uqi7GyzaLa5MH7qlSLBZtRdiA= -github.com/google/pprof v0.0.0-20230705174524-200ffdc848b8/go.mod h1:Jh3hGz2jkYak8qXPD19ryItVnUgpgeqzdkY/D0EaeuA= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= +github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= +github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= +github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -827,12 +858,13 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.7.1 h1:gF4c0zjUP2H/s/hEGyLA3I0fA2ZWjzYiONAD6cvPr8A= -github.com/googleapis/gax-go/v2 v2.7.1/go.mod h1:4orTrqY6hXxxaUL4LHIPl6lGo8vAE38/qKbhSAKP6QI= +github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= +github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c h1:7lF+Vz0LqiRidnzC1Oq86fpX1q/iEv2KJdrCtttYjT4= +github.com/gopherjs/gopherjs v0.0.0-20190430165422-3e4dfb77656c/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH4= github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= @@ -857,7 +889,6 @@ github.com/grpc-ecosystem/grpc-gateway v1.8.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t github.com/grpc-ecosystem/grpc-gateway v1.9.5/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo= github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw= -github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 h1:YBftPWNWd4WwGqtY2yeZL2ef8rHAxPBD8KFhJpmcqms= github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0/go.mod h1:YN5jB8ie0yfIUg6VvR9Kz84aCaG7AsGZnLjhHbUqwPg= github.com/gsterjov/go-libsecret v0.0.0-20161001094733-a6f4afe4910c h1:6rhixN/i8ZofjG1Y75iExal34USq5p+wiN1tpie8IrU= @@ -884,6 +915,7 @@ github.com/hashicorp/go-getter v1.7.0 h1:bzrYP+qu/gMrL1au7/aDvkoOVGUJpeKBgbqRHAC github.com/hashicorp/go-getter v1.7.0/go.mod h1:W7TalhMmbPmsSMdNjD0ZskARur/9GJ17cfHTRtXV744= github.com/hashicorp/go-hclog v0.9.2/go.mod h1:5CU+agLiy3J7N7QjHK5d05KxGsuXiQLrjA0H7acj2lQ= github.com/hashicorp/go-hclog v1.2.0 h1:La19f8d7WIlm4ogzNHB0JGqs5AUDAZ2UfCY4sJXcJdM= +github.com/hashicorp/go-hclog v1.2.0/go.mod h1:whpDNt7SSdeAju8AWKIWsul05p54N/39EeqMAyrmvFQ= github.com/hashicorp/go-immutable-radix v1.0.0/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= github.com/hashicorp/go-immutable-radix v1.3.1 h1:DKHmCUm2hRBK510BaiZlwvpD40f8bJFeZnpfm2KLowc= github.com/hashicorp/go-immutable-radix v1.3.1/go.mod h1:0y9vanUI8NX6FsYoO3zeMjhV/C5i9g4Q3DwcSNZ4P60= @@ -892,8 +924,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.7.2 h1:AcYqCvkpalPnPF2pn0KamgwamS42TqUDDYFRKq/RAd0= -github.com/hashicorp/go-retryablehttp v0.7.2/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= +github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= @@ -909,10 +941,11 @@ github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d h1:dg1dEPuWpEqDnvIw251EVy4zlP8gWbsGj4BsUKCRpYs= github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= -github.com/hashicorp/golang-lru/v2 v2.0.2 h1:Dwmkdr5Nc/oBiXgJS3CDHNhJtIHkuZ3DZF5twqnfBdU= -github.com/hashicorp/golang-lru/v2 v2.0.2/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= +github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= +github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= @@ -967,8 +1000,10 @@ github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyq github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= +github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= +github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niYQH9a1Tmk= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= @@ -1015,6 +1050,7 @@ github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHv github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= +github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= @@ -1071,6 +1107,7 @@ github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68 github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= +github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= @@ -1086,6 +1123,7 @@ github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3u github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= +github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= github.com/jackpal/gateway v1.0.5/go.mod h1:lTpwd4ACLXmpyiCTRtfiNyVnUmqT9RivzCDQetPfnjA= github.com/jackpal/go-nat-pmp v1.0.1/go.mod h1:QPH045xvCAeXUZOxsnwmrtiCoxIr9eob+4orBN1SBKc= github.com/jackpal/go-nat-pmp v1.0.2 h1:KzKSgb7qkJvOUTqYl9/Hg/me3pWgBmERKrTGD7BdWus= @@ -1104,6 +1142,7 @@ github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0 github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= +github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/jmespath/go-jmespath v0.4.0 h1:BEgLn5cpjn8UN1mAw4NjwDrS35OdebyEtFe+9YPoQUg= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= @@ -1172,6 +1211,7 @@ github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfn github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pretty v0.3.1 h1:flRD4NNwYAUpkphVc1HcthR4KEIFJ65n8Mw5qdRn3LE= +github.com/kr/pretty v0.3.1/go.mod h1:hoEshYVHaxMs3cyo3Yncou5ZscifuDolrwPKZanG3xk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= @@ -1212,8 +1252,8 @@ github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniV github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= -github.com/libp2p/go-libp2p v0.29.0 h1:QduJ2XQr/Crg4EnloueWDL0Jj86N3Ezhyyj7XH+XwHI= -github.com/libp2p/go-libp2p v0.29.0/go.mod h1:iNKL7mEnZ9wAss+03IjAwM9ZAQXfVUAPUUmOACQfQ/g= +github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= +github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= @@ -1261,8 +1301,8 @@ github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxn github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= -github.com/libp2p/go-libp2p-kad-dht v0.24.2 h1:zd7myKBKCmtZBhI3I0zm8xBkb28v3gmSEtQfBdAdFwc= -github.com/libp2p/go-libp2p-kad-dht v0.24.2/go.mod h1:BShPzRbK6+fN3hk8a0WGAYKpb8m4k+DtchkqouGTrSg= +github.com/libp2p/go-libp2p-kad-dht v0.25.0 h1:T2SXQ/VlXTQVLChWY/+OyOsmGMRJvB5kiR+eJt7jtvI= +github.com/libp2p/go-libp2p-kad-dht v0.25.0/go.mod h1:P6fz+J+u4tPigvS5J0kxQ1isksqAhmXiS/pNaEw/nFI= github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= @@ -1300,8 +1340,8 @@ github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= -github.com/libp2p/go-libp2p-routing-helpers v0.7.0 h1:sirOYVD0wGWjkDwHZvinunIpaqPLBXkcnXApVHwZFGA= -github.com/libp2p/go-libp2p-routing-helpers v0.7.0/go.mod h1:R289GUxUMzRXIbWGSuUUTPrlVJZ3Y/pPz495+qgXJX8= +github.com/libp2p/go-libp2p-routing-helpers v0.7.1 h1:kc0kWCZecbBPAiFEHhxfGJZPqjg1g9zV+X+ovR4Tmnc= +github.com/libp2p/go-libp2p-routing-helpers v0.7.1/go.mod h1:cHStPSRC/wgbfpb5jYdMP7zaSmc2wWcb1mkzNr6AR8o= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -1328,6 +1368,7 @@ github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2C github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= +github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= github.com/libp2p/go-libp2p-tls v0.3.0/go.mod h1:fwF5X6PWGxm6IDRwF3V8AVCCj/hOd5oFlg+wo2FxJDY= github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EXNjTr0UQDnrIw= @@ -1385,8 +1426,8 @@ github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/libp2p/go-reuseport v0.1.0/go.mod h1:bQVn9hmfcTaoo0c9v5pBhOarsU1eNOBZdaAd2hzXRKU= -github.com/libp2p/go-reuseport v0.3.0 h1:iiZslO5byUYZEg9iCwJGf5h+sf1Agmqx2V2FDjPyvUw= -github.com/libp2p/go-reuseport v0.3.0/go.mod h1:laea40AimhtfEqysZ71UpYj4S+R9VpH8PgqLo7L+SwI= +github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= +github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= github.com/libp2p/go-reuseport-transport v0.0.3/go.mod h1:Spv+MPft1exxARzP2Sruj2Wb5JSyHNncjf1Oi2dEbzM= github.com/libp2p/go-reuseport-transport v0.0.4/go.mod h1:trPa7r/7TJK/d+0hdBLOCGvpQQVOU74OXbNCIMkufGw= @@ -1440,8 +1481,9 @@ github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.1/go.mod h1:KAzv3t3aY1NaHWoQz1+4F1ccyAH66Jk7yos7ldAVICs= -github.com/mailru/easyjson v0.7.6 h1:8yTIVnZgCoiM1TgqoeTl+LfU5Jg6/xL3QhGQnimLYnA= github.com/mailru/easyjson v0.7.6/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/manifoldco/promptui v0.9.0 h1:3V4HzJk1TtXW1MTZMP7mdlwbBpIinw3HztaIlYthEiA= github.com/manifoldco/promptui v0.9.0/go.mod h1:ka04sppxSGFAtxX0qhlYQjISsg9mR4GWtQEhdbn6Pgg= github.com/marten-seemann/qpack v0.2.1/go.mod h1:F7Gl5L1jIgN1D11ucXefiuJS9UMVP2opoCp2jDKb7wc= @@ -1482,6 +1524,7 @@ github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-runewidth v0.0.13 h1:lTGmDsbAYt5DmK6OnoV7EuIF1wEIFAcxld6ypU4OSgU= +github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh153qUoGf23w= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= @@ -1572,8 +1615,8 @@ github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9x github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= -github.com/multiformats/go-multiaddr v0.10.1 h1:HghtFrWyZEPrpTvgAMFJi6gFdgHfs2cb0pyfDsk+lqU= -github.com/multiformats/go-multiaddr v0.10.1/go.mod h1:jLEZsA61rwWNZQTHHnqq2HNa+4os/Hz54eqiRnsRqYQ= +github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= +github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -1670,15 +1713,19 @@ github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoT github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= +github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= github.com/opencontainers/go-digest v1.0.0 h1:apOUWs51W5PlhuyGyz9FCeeBIOUDA/6nW8Oi/yOhh5U= +github.com/opencontainers/go-digest v1.0.0/go.mod h1:0JzlMkj0TRzQZfJkVvzbP0HBR3IKzErnv2BNG4W4MAM= github.com/opencontainers/image-spec v1.1.0-rc2 h1:2zx/Stx4Wc5pIPDvIxHXvXtQFW/7XWJGmnM7r3wg034= +github.com/opencontainers/image-spec v1.1.0-rc2/go.mod h1:3OVijpioIKYWTqjiG0zfF6wvoJ4fAXGbjdZuI2NgsRQ= github.com/opencontainers/runc v1.1.3 h1:vIXrkId+0/J2Ymu2m7VjGvbSlAId9XNRPhn2p4b+d8w= +github.com/opencontainers/runc v1.1.3/go.mod h1:1J5XiS+vdZ3wCyZybsuxXZWGrgSr8fFJHLXuG2PsnNg= github.com/opencontainers/runtime-spec v1.0.2/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417 h1:3snG66yBm59tKhhSPQrQ/0bCrv1LQbKt40LnUPiUxdc= -github.com/opencontainers/runtime-spec v1.0.3-0.20210326190908-1c3f411f0417/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= +github.com/opencontainers/runtime-spec v1.1.0 h1:HHUyrt9mwHUjtasSbXSMvs4cyFxh+Bll4AjJ9odEGpg= +github.com/opencontainers/runtime-spec v1.1.0/go.mod h1:jwyrGlmzljRJv/Fgzds9SsS/C5hL+LL3ko9hs6T5lQ0= github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis= github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74= github.com/opentracing/opentracing-go v1.0.2/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFStlNbqXla1AfSYxPUl2o= @@ -1692,6 +1739,7 @@ github.com/openzipkin/zipkin-go v0.1.6/go.mod h1:QgAqvLzwWbR/WpD4A3cGpPtJrZXNIiJ github.com/openzipkin/zipkin-go v0.2.1/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/openzipkin/zipkin-go v0.2.2/go.mod h1:NaW6tEwdmWMaCDZzg8sh+IBNOxHMPnhQw8ySjnjRyN4= github.com/ory/dockertest v3.3.5+incompatible h1:iLLK6SQwIhcbrG783Dghaaa3WPzGc+4Emza6EbVUUGA= +github.com/ory/dockertest v3.3.5+incompatible/go.mod h1:1vX4m9wsvi00u5bseYwXaSnhNrne+V0E6LAcBILJdPs= github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0 h1:cBOtyMzM9HTpWjXfbbunk26uA6nG3a8n06Wieeh0MwY= @@ -1743,8 +1791,8 @@ github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66Id github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= -github.com/prometheus/client_golang v1.14.0 h1:nJdhIvne2eSX/XRAFV9PcvFFRbrjbcTUj0VP62TMhnw= -github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= +github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= +github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -1778,8 +1826,8 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.9.0 h1:wzCHvIvM5SxWqYvwgVL7yJY8Lz3PKn49KQtpgMYJfhI= -github.com/prometheus/procfs v0.9.0/go.mod h1:+pB4zwohETzFnmlpe6yd2lSc+0/46IYZRB/chUwxUZY= +github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= +github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= @@ -1789,12 +1837,10 @@ github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKA github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= -github.com/quic-go/qtls-go1-19 v0.3.2 h1:tFxjCFcTQzK+oMxG6Zcvp4Dq8dx4yD3dDiIiyc86Z5U= -github.com/quic-go/qtls-go1-19 v0.3.2/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= -github.com/quic-go/qtls-go1-20 v0.2.2 h1:WLOPx6OY/hxtTxKV1Zrq20FtXtDEkeY00CGQm8GEa3E= -github.com/quic-go/qtls-go1-20 v0.2.2/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/quic-go v0.36.2 h1:ZX/UNQ4gvpCv2RmwdbA6lrRjF6EBm5yZ7TMoT4NQVrA= -github.com/quic-go/quic-go v0.36.2/go.mod h1:zPetvwDlILVxt15n3hr3Gf/I3mDf7LpLKPhR4Ez0AZQ= +github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= +github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.37.6 h1:2IIUmQzT5YNxAiaPGjs++Z4hGOtIR0q79uS5qE9ccfY= +github.com/quic-go/quic-go v0.37.6/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= @@ -1819,6 +1865,7 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= +github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -1873,6 +1920,7 @@ github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrf github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= +github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= @@ -1913,6 +1961,7 @@ github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+eg github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= +github.com/status-im/keycard-go v0.2.0/go.mod h1:wlp8ZLbsmrF6g6WjugPAx+IzoLrkdf9+mHxBEeo3Hbg= github.com/streadway/amqp v0.0.0-20190404075320-75d898a42a94/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/amqp v0.0.0-20190827072141-edfb9018d271/go.mod h1:AZpEONHx3DKn8O/DFsRAY58/XVQiIPMTMB1SddzLXVw= github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5JnDBl6z3cMAg/SywNDC5ABu5ApDIw6lUbRmI= @@ -1951,12 +2000,14 @@ github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJH github.com/tidwall/gjson v1.12.1/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.0/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/gjson v1.14.4 h1:uo0p8EbA09J7RQaflQ1aBRffTR7xedD2bcIVSYxLnkM= +github.com/tidwall/gjson v1.14.4/go.mod h1:/wbyibRr2FHMks5tjHJ5F8dMZh3AcwJEMf5vlfC0lxk= github.com/tidwall/match v1.0.1/go.mod h1:LujAq0jyVjBy028G1WhWfIzbpQfMO8bBZ6Tyb0+pL9E= github.com/tidwall/match v1.1.1 h1:+Ho715JplO36QYgwN9PGYNhgZvoUSc9X2c80KVTi+GA= github.com/tidwall/match v1.1.1/go.mod h1:eRSPERbgtNPcGhD8UCthc6PmLEQXEWd3PRB5JTxsfmM= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= github.com/tidwall/pretty v1.2.0/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/pretty v1.2.1 h1:qjsOFOWWQl+N3RsoF5/ssm1pHmJJwhjlSbZ51I6wMl4= +github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhsoaGGjNU= github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= @@ -1972,6 +2023,7 @@ github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqri github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= +github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -1986,6 +2038,7 @@ github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= +github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -1995,6 +2048,7 @@ github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.11.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= @@ -2005,6 +2059,7 @@ github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:X github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= +github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= @@ -2018,6 +2073,7 @@ github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q github.com/xlab/treeprint v0.0.0-20180616005107-d6fb6747feb6/go.mod h1:ce1O1j6UtZfjr22oyGxGLbauSBp2YVXpARAosm7dHBg= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673 h1:bAn7/zixMGCfxrRTfdpNzjtPYqr8smhKouy9mxVdGPU= +github.com/xrash/smetrics v0.0.0-20201216005158-039620a65673/go.mod h1:N3UwUGtsrSj3ccvlPHLoLsHnpR27oXr4ZE984MbSER8= github.com/ybbus/jsonrpc v2.1.2+incompatible/go.mod h1:XJrh1eMSzdIYFbM08flv0wp5G35eRniyeGut1z+LSiE= github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= @@ -2032,6 +2088,7 @@ github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN github.com/zondax/ledger-go v0.14.1/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= +gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40/go.mod h1:rOnSnoRyxMI3fe/7KIbVcsHRGxe30OONv8dEgo+vCfA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= @@ -2081,8 +2138,8 @@ go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwY go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= -go.opentelemetry.io/proto/otlp v0.19.0 h1:IVN6GR+mhC4s5yfcTbmzHYODqvWAp3ZedA2SJPI1Nnw= -go.opentelemetry.io/proto/otlp v0.19.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U= +go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= +go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -2099,8 +2156,9 @@ go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= -go.uber.org/goleak v1.1.12 h1:gZAh5/EyT/HQwlpkCy6wTpqfH9H8Lz8zbm3dZh+OyzA= go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= +go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= +go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -2118,8 +2176,8 @@ go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= -go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60= -go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= +go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= +go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= @@ -2158,6 +2216,7 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= +golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= @@ -2178,8 +2237,8 @@ golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMk golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 h1:MGwJjxBy0HJshjDNfLsYO8xppfqWlA5ZT9OhtUUhTNw= -golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1/go.mod h1:FXUEEKJgO7OQYeo8N01OfiKP8RXMtf6e8aTskBGqWdc= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= +golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2293,8 +2352,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= -golang.org/x/net v0.12.0 h1:cfawfvKITfUsFCeJIHJrbSxpeu/E81khclypR0GVT50= -golang.org/x/net v0.12.0/go.mod h1:zEVYFnQC7m/vmpQFELhcD1EWkZlX69l4oqgmer6hfKA= +golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= +golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2488,6 +2547,7 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= +golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= @@ -2498,6 +2558,7 @@ golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= +golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -2570,8 +2631,8 @@ golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= -golang.org/x/tools v0.11.0 h1:EMCa6U9S2LtZXLAMoWiR/R8dAQFRqbAitmbJ2UKhoi8= -golang.org/x/tools v0.11.0/go.mod h1:anzJrxPjNtfgiYQYirP2CPGzGLxrH2u2QBhn6Bf3qY8= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= +golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2641,8 +2702,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.114.0 h1:1xQPji6cO2E2vLiI+C/XiFAnsn1WV3mjaEwGLhi3grE= -google.golang.org/api v0.114.0/go.mod h1:ifYI2ZsFK6/uGddGfAD5BMxlnkBqCmqHSDUVi45N5Yg= +google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= +google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -2766,12 +2827,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230629202037-9506855d4529 h1:9JucMWR7sPvCxUFd6UsOUNmA5kCcWOfORaT3tpAsKQs= -google.golang.org/genproto v0.0.0-20230629202037-9506855d4529/go.mod h1:xZnkP7mREFX5MORlOPEzLMr+90PPZQ2QWzrVTWfAq64= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc h1:kVKPf/IiYSBWEWtkIn6wZXwWGCnLKcC8oWfZvXjsGnM= -google.golang.org/genproto/googleapis/api v0.0.0-20230530153820-e85fd2cbaebc/go.mod h1:vHYtlOoi6TsQ3Uk2yxR7NI5z8uoV+3pZtR4jmHIkRig= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130 h1:2FZP5XuJY9zQyGM5N0rtovnoXjiMUEIUMvw0m9wlpLc= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230706204954-ccb25ca9f130/go.mod h1:8mL13HKkDa+IuJ8yruA3ci0q+0vsUz4m//+ottjwS5o= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= +google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= +google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= +google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -2808,7 +2869,6 @@ google.golang.org/grpc v1.39.0/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnD google.golang.org/grpc v1.39.1/go.mod h1:PImNr+rS9TWYb2O4/emRugxiyHZ5JyHW5F+RPnDzfrE= google.golang.org/grpc v1.40.0/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= google.golang.org/grpc v1.40.1/go.mod h1:ogyxbiOoUXAkP+4+xa6PZSE9DZgIHtSpzjDTB9KAK34= -google.golang.org/grpc v1.42.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.44.0/go.mod h1:k+4IHHFw41K8+bbowsex27ge2rCb65oeWqe4jJ590SU= google.golang.org/grpc v1.45.0/go.mod h1:lN7owxKUQEqMfSyQikvvk5tf/6zMPsrK+ONuO11+0rQ= google.golang.org/grpc v1.46.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACuMGWk= @@ -2818,8 +2878,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.56.2 h1:fVRFRnXvU+x6C4IlHZewvJOVHoOv1TUuQyoRsYnB4bI= -google.golang.org/grpc v1.56.2/go.mod h1:I9bI3vqKfayGqPUAwGdOSu7kt6oIJLixfffKrpXqQ9s= +google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= +google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= @@ -2854,6 +2914,7 @@ gopkg.in/inf.v0 v0.9.1/go.mod h1:cWUDdTG/fYaXco+Dcufb5Vnc6Gp2YChqWtbxRZE0mXw= gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= +gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= @@ -2897,6 +2958,7 @@ nhooyr.io/websocket v1.8.6/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0 nhooyr.io/websocket v1.8.7 h1:usjR2uOr/zjjkVMy0lW+PPohFok7PCow5sDjLgX4P4g= nhooyr.io/websocket v1.8.7/go.mod h1:B70DZP8IakI65RVQ51MsWP/8jndNma26DVA/nFSCgW0= pgregory.net/rapid v0.5.3 h1:163N50IHFqr1phZens4FQOdPgfJscR7a562mjQqeo4M= +pgregory.net/rapid v0.5.3/go.mod h1:PY5XlDGj0+V1FCq0o192FdRhpKHGTRIWBgqjDBTrq04= rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8= rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 7427699f95..8977678198 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -7,10 +7,10 @@ import ( "github.com/ipfs/boxo/bitswap" "github.com/ipfs/boxo/bitswap/network" "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/routing/offline" "github.com/ipfs/go-blockservice" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - "github.com/ipfs/go-ipfs-routing/offline" record "github.com/libp2p/go-libp2p-record" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index 0a79ebfe33..411fd85818 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -2,11 +2,11 @@ package byzantine import ( "context" + "crypto/sha256" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" - "github.com/minio/sha256-simd" "github.com/celestiaorg/nmt" nmt_pb "github.com/celestiaorg/nmt/pb" diff --git a/share/eds/eds.go b/share/eds/eds.go index cf38a14cbc..e689aec31c 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -3,6 +3,7 @@ package eds import ( "bytes" "context" + "crypto/sha256" "errors" "fmt" "io" @@ -11,7 +12,6 @@ import ( "github.com/ipfs/go-cid" "github.com/ipld/go-car" "github.com/ipld/go-car/util" - "github.com/minio/sha256-simd" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" diff --git a/share/getter.go b/share/getter.go index 7caa954a24..a33ea7aa87 100644 --- a/share/getter.go +++ b/share/getter.go @@ -2,11 +2,10 @@ package share import ( "context" + "crypto/sha256" "errors" "fmt" - "github.com/minio/sha256-simd" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" ) diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index b86b3b98df..5cc73aa7cb 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -5,10 +5,10 @@ import ( "testing" "time" + "github.com/ipfs/boxo/exchange/offline" bsrv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - offline "github.com/ipfs/go-ipfs-exchange-offline" mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index ef07cf6797..0ca807d0d4 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -276,7 +276,8 @@ func newEDSClientServer( return client, server } -// addToNamespace adds arbitrary int value to namespace, treating namespace as big-endian implementation of int +// addToNamespace adds arbitrary int value to namespace, treating namespace as big-endian +// implementation of int func addToNamespace(namespace share.Namespace, val int) (share.Namespace, error) { if val == 0 { return namespace, nil diff --git a/share/getters/utils_test.go b/share/getters/utils_test.go index 5ffad228f9..65de9d47f2 100644 --- a/share/getters/utils_test.go +++ b/share/getters/utils_test.go @@ -209,7 +209,8 @@ func Test_ctxWithSplitTimeout(t *testing.T) { t.Cleanup(cancel) got, _ := ctxWithSplitTimeout(ctx, sf, tt.args.minTimeout) dl, ok := got.Deadline() - // in case no deadline is found in ctx or not expected to be found, check both cases apply at the same time + // in case no deadline is found in ctx or not expected to be found, check both cases apply at the + // same time if !ok || tt.want == 0 { require.False(t, ok) require.Equal(t, tt.want, time.Duration(0)) diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index 11639cbec7..dd6c3a6636 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -3,6 +3,7 @@ package ipld import ( "bytes" "context" + "crypto/sha256" "errors" mrand "math/rand" "sort" @@ -11,13 +12,12 @@ import ( "time" "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - offline "github.com/ipfs/go-ipfs-exchange-offline" mdutils "github.com/ipfs/go-merkledag/test" - "github.com/minio/sha256-simd" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index b391a3b2ce..153bbbb1ba 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -2,6 +2,7 @@ package shrexnd import ( "context" + "crypto/sha256" "errors" "fmt" "time" @@ -9,7 +10,6 @@ import ( "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/protocol" - "github.com/minio/sha256-simd" "go.uber.org/zap" "github.com/celestiaorg/go-libp2p-messenger/serde" From 10351be4c26431f58bfef829b5e02065d20e515f Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 22 Aug 2023 22:54:21 +0800 Subject: [PATCH 140/388] fix(share/eds): scale up inverted_index badger levels (#2591) Current instance of badger uses default value (7) for amount of used levels, that allows badger to grow up to 1.1TiB. This limit could be reaching within 1 week of 128 ods size blocks, so we need to bump the value to at least 8 levels (11 TiB). --- share/eds/inverted_index.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index bb4092fa52..81df5d92ef 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -31,6 +31,8 @@ func newSimpleInvertedIndex(storePath string) (*simpleInvertedIndex, error) { opts.NumCompactors = 20 // use minimum amount of NumLevelZeroTables to trigger L0 compaction faster opts.NumLevelZeroTables = 1 + // MaxLevels = 8 will allow the db to grow to ~11.1 TiB + opts.MaxLevels = 8 ds, err := dsbadger.NewDatastore(storePath+invertedIndexPath, &opts) if err != nil { From bb9b4d45a9958f38b66ccb9b425c63fd38e30b20 Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Tue, 22 Aug 2023 17:38:19 +0200 Subject: [PATCH 141/388] feat(blob): improve gas estimation and track min gas price (#2511) ## Overview This PR incorporates the more recent gas estimation method in celestia-app. It also makes use of a new gRPC endpoint for consensus nodes that reveals the nodes min gas price. This enables the blob module to submit transactions with the correct gas price. If the node changes that price, the blob module is able to parse the error and retry using the new gas price. Additionally, a stubbed CoreAccessor was introduced for when a core endpoint is not provided in order to return a better/more readable error, whereas previously, the CoreAccessor depended on the grpc dial to silently fail during start and any call to the state module would return a "cannot dial address" error. It now looks like this: ``` { "jsonrpc": "2.0", "id": 1, "error": { "code": 1, "message": "node is running without state access" } } ``` ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- blob/helper.go | 17 ----- blob/service.go | 11 +-- state/core_access.go | 146 +++++++++++++++++++++++++++++++++----- state/core_access_test.go | 97 ++++++++++++++++++++----- state/metrics.go | 4 +- 5 files changed, 214 insertions(+), 61 deletions(-) diff --git a/blob/helper.go b/blob/helper.go index e87055ae07..5627fac998 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -6,7 +6,6 @@ import ( "github.com/tendermint/tendermint/types" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-node/share" @@ -86,22 +85,6 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { return shares.ToBytes(rawShares), nil } -const ( - perByteGasTolerance = 2 - pfbGasFixedCost = 80000 -) - -// estimateGas estimates the gas required to pay for a set of blobs in a PFB. -func estimateGas(blobs ...*Blob) uint64 { - totalByteCount := 0 - for _, blob := range blobs { - totalByteCount += len(blob.Data) + appconsts.NamespaceSize - } - variableGasAmount := (appconsts.DefaultGasPerBlobByte + perByteGasTolerance) * totalByteCount - - return uint64(variableGasAmount + pfbGasFixedCost) -} - // constructAndVerifyBlob reconstruct a Blob from the passed shares and compares commitments. func constructAndVerifyBlob(sh []share.Share, commitment Commitment) (*Blob, bool, error) { blob, err := SharesToBlobs(sh) diff --git a/blob/service.go b/blob/service.go index 960f8c6c01..da14fc06c7 100644 --- a/blob/service.go +++ b/blob/service.go @@ -10,7 +10,6 @@ import ( "github.com/cosmos/cosmos-sdk/types" logging "github.com/ipfs/go-log/v2" - "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-node/header" @@ -36,7 +35,7 @@ type Service struct { blobSumitter Submitter // shareGetter retrieves the EDS to fetch all shares from the requested header. shareGetter share.Getter - // headerGetter fetches header by the provided height + // headerGetter fetches header by the provided height headerGetter func(context.Context, uint64) (*header.ExtendedHeader, error) } @@ -55,15 +54,11 @@ func NewService( // Submit sends PFB transaction and reports the height in which it was included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. +// Handles gas estimation and fee calculation. func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { log.Debugw("submitting blobs", "amount", len(blobs)) - var ( - gasLimit = estimateGas(blobs...) - fee = int64(appconsts.DefaultMinGasPrice * float64(gasLimit)) - ) - - resp, err := s.blobSumitter.SubmitPayForBlob(ctx, types.NewInt(fee), gasLimit, blobs) + resp, err := s.blobSumitter.SubmitPayForBlob(ctx, types.OneInt().Neg(), 0, blobs) if err != nil { return 0, err } diff --git a/state/core_access.go b/state/core_access.go index 73d525347f..43f5313c5f 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -4,13 +4,17 @@ import ( "context" "errors" "fmt" + "math" + "sync" "time" sdkErrors "cosmossdk.io/errors" "github.com/cosmos/cosmos-sdk/api/tendermint/abci" + nodeservice "github.com/cosmos/cosmos-sdk/client/grpc/node" storetypes "github.com/cosmos/cosmos-sdk/store/types" sdktypes "github.com/cosmos/cosmos-sdk/types" sdktx "github.com/cosmos/cosmos-sdk/types/tx" + auth "github.com/cosmos/cosmos-sdk/x/auth/types" banktypes "github.com/cosmos/cosmos-sdk/x/bank/types" stakingtypes "github.com/cosmos/cosmos-sdk/x/staking/types" logging "github.com/ipfs/go-log/v2" @@ -21,6 +25,8 @@ import ( "google.golang.org/grpc/credentials/insecure" "github.com/celestiaorg/celestia-app/app" + apperrors "github.com/celestiaorg/celestia-app/app/errors" + "github.com/celestiaorg/celestia-app/pkg/appconsts" appblob "github.com/celestiaorg/celestia-app/x/blob" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" libhead "github.com/celestiaorg/go-header" @@ -34,6 +40,8 @@ var ( ErrInvalidAmount = errors.New("state: amount must be greater than zero") ) +const maxRetries = 5 + // CoreAccessor implements service over a gRPC connection // with a celestia-core node. type CoreAccessor struct { @@ -54,8 +62,15 @@ type CoreAccessor struct { rpcPort string grpcPort string + // these fields are mutatable and thus need to be protected by a mutex + lock sync.Mutex lastPayForBlob int64 payForBlobCount int64 + // minGasPrice is the minimum gas price that the node will accept. + // NOTE: just because the first node accepts the transaction, does not mean it + // will find a proposer that does accept the transaction. Better would be + // to set a global min gas price that correct processes conform to. + minGasPrice float64 } // NewCoreAccessor dials the given celestia-core endpoint and @@ -112,6 +127,11 @@ func (ca *CoreAccessor) Start(ctx context.Context) error { } ca.rpcCli = cli + ca.minGasPrice, err = ca.queryMinimumGasPrice(ctx) + if err != nil { + return fmt.Errorf("querying minimum gas price: %w", err) + } + return nil } @@ -160,6 +180,9 @@ func (ca *CoreAccessor) constructSignedTx( return ca.signer.EncodeTx(tx) } +// SubmitPayForBlob builds, signs, and synchronously submits a MsgPayForBlob. It blocks until the transaction +// is committed and returns the TxReponse. If gasLim is set to 0, the method will automatically estimate the +// gas limit. If the fee is negative, the method will use the nodes min gas price multiplied by the gas limit. func (ca *CoreAccessor) SubmitPayForBlob( ctx context.Context, fee Int, @@ -178,25 +201,67 @@ func (ca *CoreAccessor) SubmitPayForBlob( appblobs[i] = &b.Blob } - response, err := appblob.SubmitPayForBlob( - ctx, - ca.signer, - ca.coreConn, - sdktx.BroadcastMode_BROADCAST_MODE_BLOCK, - appblobs, - apptypes.SetGasLimit(gasLim), - withFee(fee), - ) - // metrics should only be counted on a successful PFD tx - if err == nil && response.Code == 0 { - ca.lastPayForBlob = time.Now().UnixMilli() - ca.payForBlobCount++ - } + // we only estimate gas if the user wants us to (by setting the gasLim to 0). In the future we may want + // to make these arguments optional. + if gasLim == 0 { + blobSizes := make([]uint32, len(blobs)) + for i, blob := range blobs { + blobSizes[i] = uint32(len(blob.Data)) + } - if response != nil && response.Code != 0 { - err = errors.Join(err, sdkErrors.ABCIError(response.Codespace, response.Code, response.Logs.String())) + // TODO (@cmwaters): the default gas per byte and the default tx size cost per byte could be changed + // through governance. This section could be more robust by tracking these values and adjusting the + // gas limit accordingly (as is done for the gas price) + gasLim = apptypes.EstimateGas(blobSizes, appconsts.DefaultGasPerBlobByte, auth.DefaultTxSizeCostPerByte) + } + + minGasPrice := ca.getMinGasPrice() + + // set the fee for the user as the minimum gas price multiplied by the gas limit + estimatedFee := false + if fee.IsNegative() { + estimatedFee = true + fee = sdktypes.NewInt(int64(math.Ceil(minGasPrice * float64(gasLim)))) + } + + var lastErr error + for attempt := 0; attempt < maxRetries; attempt++ { + response, err := appblob.SubmitPayForBlob( + ctx, + ca.signer, + ca.coreConn, + sdktx.BroadcastMode_BROADCAST_MODE_BLOCK, + appblobs, + apptypes.SetGasLimit(gasLim), + withFee(fee), + ) + + // the node is capable of changing the min gas price at any time so we must be able to detect it and + // update our version accordingly + if apperrors.IsInsufficientMinGasPrice(err) && estimatedFee { + // The error message contains enough information to parse the new min gas price + minGasPrice, err = apperrors.ParseInsufficientMinGasPrice(err, minGasPrice, gasLim) + if err != nil { + return nil, fmt.Errorf("parsing insufficient min gas price error: %w", err) + } + ca.setMinGasPrice(minGasPrice) + lastErr = err + // update the fee to retry again + fee = sdktypes.NewInt(int64(math.Ceil(minGasPrice * float64(gasLim)))) + continue + } + + // metrics should only be counted on a successful PFD tx + if err == nil && response.Code == 0 { + ca.markSuccessfulPFB() + } + + if response != nil && response.Code != 0 { + err = errors.Join(err, sdkErrors.ABCIError(response.Codespace, response.Code, response.Logs.String())) + } + return response, err } - return response, err + return nil, fmt.Errorf("failed to submit blobs after %d attempts: %w", maxRetries, lastErr) } func (ca *CoreAccessor) AccountAddress(context.Context) (Address, error) { @@ -460,6 +525,53 @@ func (ca *CoreAccessor) QueryRedelegations( }) } +func (ca *CoreAccessor) LastPayForBlob() int64 { + ca.lock.Lock() + defer ca.lock.Unlock() + return ca.lastPayForBlob +} + +func (ca *CoreAccessor) PayForBlobCount() int64 { + ca.lock.Lock() + defer ca.lock.Unlock() + return ca.payForBlobCount +} + +func (ca *CoreAccessor) markSuccessfulPFB() { + ca.lock.Lock() + defer ca.lock.Unlock() + ca.lastPayForBlob = time.Now().UnixMilli() + ca.payForBlobCount++ +} + +func (ca *CoreAccessor) setMinGasPrice(minGasPrice float64) { + ca.lock.Lock() + defer ca.lock.Unlock() + ca.minGasPrice = minGasPrice +} + +func (ca *CoreAccessor) getMinGasPrice() float64 { + ca.lock.Lock() + defer ca.lock.Unlock() + return ca.minGasPrice +} + +// QueryMinimumGasPrice returns the minimum gas price required by the node. +func (ca *CoreAccessor) queryMinimumGasPrice( + ctx context.Context, +) (float64, error) { + rsp, err := nodeservice.NewServiceClient(ca.coreConn).Config(ctx, &nodeservice.ConfigRequest{}) + if err != nil { + return 0, err + } + + coins, err := sdktypes.ParseDecCoins(rsp.MinimumGasPrice) + if err != nil { + return 0, err + } + return coins.AmountOf(app.BondDenom).MustFloat64(), nil +} + func (ca *CoreAccessor) IsStopped(context.Context) bool { return ca.ctx.Err() != nil } diff --git a/state/core_access_test.go b/state/core_access_test.go index 5018cee4f2..69e9f251c0 100644 --- a/state/core_access_test.go +++ b/state/core_access_test.go @@ -2,32 +2,95 @@ package state import ( "context" + "errors" + "fmt" + "strings" "testing" + "time" + "cosmossdk.io/math" + sdktypes "github.com/cosmos/cosmos-sdk/types" "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/app" + "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/test/util/testnode" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/share" ) -func TestLifecycle(t *testing.T) { - ca := NewCoreAccessor(nil, nil, "", "", "") +func TestSubmitPayForBlob(t *testing.T) { + accounts := []string{"jimy", "rob"} + tmCfg := testnode.DefaultTendermintConfig() + tmCfg.Consensus.TimeoutCommit = time.Millisecond * 1 + appConf := testnode.DefaultAppConfig() + appConf.API.Enable = true + appConf.MinGasPrices = fmt.Sprintf("0.1%s", app.BondDenom) + + config := testnode.DefaultConfig().WithTendermintConfig(tmCfg).WithAppConfig(appConf).WithAccounts(accounts) + cctx, rpcAddr, grpcAddr := testnode.NewNetwork(t, config) ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + signer := blobtypes.NewKeyringSigner(cctx.Keyring, accounts[0], cctx.ChainID) + ca := NewCoreAccessor(signer, nil, "127.0.0.1", extractPort(rpcAddr), extractPort(grpcAddr)) // start the accessor err := ca.Start(ctx) require.NoError(t, err) - // ensure accessor isn't stopped - require.False(t, ca.IsStopped(ctx)) - // cancel the top level context (this should not affect the lifecycle of the - // accessor as it should manage its own internal context) - cancel() - // ensure accessor was unaffected by top-level context cancellation - require.False(t, ca.IsStopped(ctx)) - // stop the accessor - stopCtx, stopCancel := context.WithCancel(context.Background()) - t.Cleanup(stopCancel) - err = ca.Stop(stopCtx) + t.Cleanup(func() { + _ = ca.Stop(ctx) + }) + + ns, err := share.NewBlobNamespaceV0([]byte("namespace")) + require.NoError(t, err) + blobbyTheBlob, err := blob.NewBlobV0(ns, []byte("data")) require.NoError(t, err) - // ensure accessor is stopped - require.True(t, ca.IsStopped(ctx)) - // ensure that stopping the accessor again does not return an error - err = ca.Stop(stopCtx) + + minGas, err := ca.queryMinimumGasPrice(ctx) require.NoError(t, err) + require.Equal(t, appconsts.DefaultMinGasPrice, minGas) + + testcases := []struct { + name string + blobs []*blob.Blob + fee math.Int + gasLim uint64 + expErr error + }{ + { + name: "empty blobs", + blobs: []*blob.Blob{}, + fee: sdktypes.ZeroInt(), + gasLim: 0, + expErr: errors.New("state: no blobs provided"), + }, + { + name: "good blob with user provided gas and fees", + blobs: []*blob.Blob{blobbyTheBlob}, + fee: sdktypes.NewInt(10_000), // roughly 0.12 utia per gas (should be good) + gasLim: blobtypes.DefaultEstimateGas([]uint32{uint32(len(blobbyTheBlob.Data))}), + expErr: nil, + }, + // TODO: add more test cases. The problem right now is that the celestia-app doesn't + // correctly construct the node (doesn't pass the min gas price) hence the price on + // everything is zero and we can't actually test the correct behavior + } + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + resp, err := ca.SubmitPayForBlob(ctx, tc.fee, tc.gasLim, tc.blobs) + require.Equal(t, tc.expErr, err) + if err == nil { + require.EqualValues(t, 0, resp.Code) + } + }) + } + +} + +func extractPort(addr string) string { + splitStr := strings.Split(addr, ":") + return splitStr[len(splitStr)-1] } diff --git a/state/metrics.go b/state/metrics.go index 169023b5f9..aa166e901d 100644 --- a/state/metrics.go +++ b/state/metrics.go @@ -20,8 +20,8 @@ func WithMetrics(ca *CoreAccessor) { ) callback := func(ctx context.Context, observer metric.Observer) error { - observer.ObserveInt64(pfbCounter, ca.payForBlobCount) - observer.ObserveInt64(lastPfbTimestamp, ca.lastPayForBlob) + observer.ObserveInt64(pfbCounter, ca.PayForBlobCount()) + observer.ObserveInt64(lastPfbTimestamp, ca.LastPayForBlob()) return nil } _, err := meter.RegisterCallback(callback, pfbCounter, lastPfbTimestamp) From 7682b003b2c462447a2680da9f55068d0b0327fc Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 23 Aug 2023 14:36:30 +0300 Subject: [PATCH 142/388] fix(cmd/blob): fix response (#2600) --- cmd/celestia/blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 28d05b3b42..8ab130c24d 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -121,7 +121,7 @@ var submitCmd = &cobra.Command{ height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) response := struct { - Height uint64 `json:"uint64"` + Height uint64 `json:"height"` Commitment blob.Commitment `json:"commitment"` }{ Height: height, From d5b1e199615a790083acfcdfc7b5e31c5a99827e Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 23 Aug 2023 20:12:53 +0800 Subject: [PATCH 143/388] deps: bump dagstore (#2598) Update dagstore to latest fork version --- go.mod | 8 +- go.sum | 322 +++++++++++++++++++++++++++++++++++++++++++++++++++++++-- 2 files changed, 318 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 91b55ad256..cf2329de01 100644 --- a/go.mod +++ b/go.mod @@ -171,7 +171,7 @@ require ( github.com/google/orderedcode v0.0.1 // indirect github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect github.com/google/s2a-go v0.1.4 // indirect - github.com/google/uuid v1.3.0 // indirect + github.com/google/uuid v1.3.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect github.com/googleapis/gax-go/v2 v2.11.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect @@ -214,7 +214,7 @@ require ( github.com/ipfs/go-verifcid v0.0.2 // indirect github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect - github.com/ipld/go-ipld-prime v0.20.0 // indirect + github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect github.com/jbenet/go-temp-err-catcher v0.1.0 // indirect github.com/jbenet/goprocess v0.1.4 // indirect @@ -302,7 +302,7 @@ require ( github.com/tklauser/numcpus v0.4.0 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect - github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa // indirect + github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/zondax/hid v0.9.1 // indirect github.com/zondax/ledger-go v0.14.1 // indirect @@ -337,7 +337,7 @@ require ( replace ( github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 - github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 + github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230823095320-1d66549bbeb3 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 diff --git a/go.sum b/go.sum index 84e5eb5991..90c6e89ec9 100644 --- a/go.sum +++ b/go.sum @@ -253,6 +253,7 @@ github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMx github.com/StackExchange/wmi v0.0.0-20180116203802-5d049714c4a6/go.mod h1:3eOhrUMpNV+6aFIbp5/iudMxNCF27Vw2OZgy4xEx0Fg= github.com/StackExchange/wmi v1.2.1 h1:VIkavFPXSjcnS+O8yTq7NI32k0R5Aj+v39y29VYDOSA= github.com/StackExchange/wmi v1.2.1/go.mod h1:rcmrprowKIVzvc+NUiLncP2uuArMWLCbu9SBzvHz7e8= +github.com/Stebalien/go-bitfield v0.0.1/go.mod h1:GNjFpasyUVkHMsfEOk8EFLJ9syQ6SI+XWrX9Wf2XH0s= github.com/VictoriaMetrics/fastcache v1.6.0 h1:C/3Oi3EiBCqufydp1neRZkqcwmEiuRT9c3fqvvgKm5o= github.com/VictoriaMetrics/fastcache v1.6.0/go.mod h1:0qHz5QP0GMX4pfmMA/zt5RgfNuXJrTP0zS7DqpHGGTw= github.com/VividCortex/gohistogram v1.0.0 h1:6+hBz+qvs0JOrrNhhmR7lFxo5sINxBCGXrdtl/UvroE= @@ -273,6 +274,7 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= +github.com/alecthomas/units v0.0.0-20210927113745-59d0afb8317a/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/allegro/bigcache v1.2.1-0.20190218064605-e24eb225f156/go.mod h1:Cb/ax3seSYIx7SuZdm2G2xzfwmv3TPSk2ucNfQESPXM= github.com/andreyvit/diff v0.0.0-20170406064948-c7f18ee00883/go.mod h1:rCTlJbsFo29Kk6CurOXKm700vrz8f0KW0JNfpkRJY/8= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= @@ -319,6 +321,7 @@ github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBT github.com/btcsuite/btcd v0.0.0-20190213025234-306aecffea32/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190315201642-aa6e0f35703c/go.mod h1:DrZx5ec/dmnfpw9KyYoQyYo7d0KEvTkk/5M/vbZjAr8= github.com/btcsuite/btcd v0.0.0-20190523000118-16327141da8c/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= +github.com/btcsuite/btcd v0.0.0-20190605094302-a0d1e3e36d50/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.0.0-20190824003749-130ea5bddde3/go.mod h1:3J08xEfcugPacsc34/LKRU2yO7YmuT8yt28J8k2+rrI= github.com/btcsuite/btcd v0.20.1-beta/go.mod h1:wVuoA8VJLEcwgqHBwHmzLRazpKxTv13Px/pDuV7OomQ= github.com/btcsuite/btcd v0.21.0-beta/go.mod h1:ZSWyehm27aAuS9bvkATT+Xte3hjHZ+MRgMY/8NJ7K94= @@ -327,6 +330,7 @@ github.com/btcsuite/btcd v0.22.0-beta/go.mod h1:9n5ntfhhHQBIhUvlhDvD3Qg6fRUj4jkN github.com/btcsuite/btcd v0.22.1 h1:CnwP9LM/M9xuRrGSCGeMVs9iv09uMqwsVX7EeIpgV2c= github.com/btcsuite/btcd v0.22.1/go.mod h1:wqgTSL29+50LRkmOVknEdmt8ZojIzhuWvgu/iptuN7Y= github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= +github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= @@ -358,8 +362,8 @@ github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHV github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 h1:PckXGxLJjXv97VO3xS8NPHN5oO83X5nvJLbc/4s8jUM= github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14/go.mod h1:70Go8qNy7YAb1PUcHCChRHNX2ke7c9jgUIEklUX+Mac= -github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6 h1:/yCwMCoOPcYCiG18u8/1pv5eXF04xczoQO3sR0bKsgM= -github.com/celestiaorg/dagstore v0.0.0-20230413141458-735ab09a15d6/go.mod h1:ta/DlqIH10bvhwqJIw51Nq3QU4XVMp6pz3f0Deve9fM= +github.com/celestiaorg/dagstore v0.0.0-20230823095320-1d66549bbeb3 h1:2/UcuBL0HQ8Y7QXhBjaNMI2LT8or1F67wArTJIs3Vos= +github.com/celestiaorg/dagstore v0.0.0-20230823095320-1d66549bbeb3/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= @@ -395,12 +399,15 @@ github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cheekybits/genny v1.0.0/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/cheggaaa/pb v1.0.27/go.mod h1:pQciLPpbU0oxA0h+VJYYLxO+XeDQb5pZijXscXHm81s= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= +github.com/chzyer/logex v1.2.0/go.mod h1:9+9sk7u7pGNWYMkh0hdiL++6OeibzJccyQU4p4MedaY= github.com/chzyer/logex v1.2.1 h1:XHDu3E6q+gdHgsdTPH6ImJMIp436vR6MPtH8gP05QzM= github.com/chzyer/logex v1.2.1/go.mod h1:JLbx6lG2kDbNRFnfkgvh4eRJRPX1QCoOIWomwysCBrQ= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= +github.com/chzyer/readline v1.5.0/go.mod h1:x22KAscuvRqlLoK9CsoYsmxoXZMMFVyOl86cAH8qUic= github.com/chzyer/readline v1.5.1 h1:upd/6fQk4src78LMRzh5vItIt361/o4uq553V8B5sGI= github.com/chzyer/readline v1.5.1/go.mod h1:Eh+b79XXUwfKfcPLepksvw2tcLE/Ct21YObkaSkeBlk= github.com/chzyer/test v0.0.0-20180213035817-a1ea475d72b1/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= +github.com/chzyer/test v0.0.0-20210722231415-061457976a23/go.mod h1:Q3SI9o4m/ZMnBNeIyt5eFwwo7qiLfzFZmjNmxjkiQlU= github.com/chzyer/test v1.0.0 h1:p3BQDXSxOhOG0P9z6/hGnII4LGiEPOYBhs8asl/fC04= github.com/chzyer/test v1.0.0/go.mod h1:2JlltgoNkt4TW/z9V/IzDdFaMTM2JPIi26O1pF38GC8= github.com/cilium/ebpf v0.2.0/go.mod h1:To2CFviqOWL/M0gIMsvSMlqe7em/l1ALkX1PyjrX2Qs= @@ -445,6 +452,7 @@ github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1 github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= +github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM= github.com/containerd/cgroups v1.1.0/go.mod h1:6ppBcbh/NOOUU+dMKrykgaBnK9lCIBxHqJDGwsa1mIw= github.com/containerd/continuity v0.3.0 h1:nisirsYROK15TAMVukJOUyGJjz4BNQJBVsNvAXZJ/eg= @@ -458,6 +466,7 @@ github.com/coreos/go-systemd v0.0.0-20181012123002-c6f51f82210d/go.mod h1:F5haX7 github.com/coreos/go-systemd v0.0.0-20190321100706-95778dfbb74e/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4= github.com/coreos/go-systemd/v22 v22.1.0/go.mod h1:xO0FLkIi5MaZafQlIrOotqXZ90ih+1atmu1JpKERPPk= github.com/coreos/go-systemd/v22 v22.3.2/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= +github.com/coreos/go-systemd/v22 v22.4.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/go-systemd/v22 v22.5.0 h1:RrqgGjYQKalulkV8NGVIfkXQf6YYmOyiJKk8iXXhfZs= github.com/coreos/go-systemd/v22 v22.5.0/go.mod h1:Y58oyj3AT4RCenI/lSvhwexgC+NSVTIJ3seZv2GcEnc= github.com/coreos/pkg v0.0.0-20160727233714-3ac0863d7acf/go.mod h1:E3G3o1h8I7cfcXa63jLwjI0eiQQMgzzUDFVpN/nH/eA= @@ -487,6 +496,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= +github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/creachadair/taskgroup v0.3.2 h1:zlfutDS+5XG40AOxcHDSThxKzns8Tnr9jnr6VqkYlkM= github.com/creachadair/taskgroup v0.3.2/go.mod h1:wieWwecHVzsidg2CsUnFinW1faVN4+kq+TDlRJQ0Wbk= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -513,6 +524,7 @@ github.com/decred/dcrd/crypto/blake256 v1.0.0/go.mod h1:sQl2p6Y26YV+ZOcSTP6thNdn github.com/decred/dcrd/crypto/blake256 v1.0.1 h1:7PltbUIQB7u/FfZ39+DGa/ShuMyJ5ilcvdfma9wOH6Y= github.com/decred/dcrd/crypto/blake256 v1.0.1/go.mod h1:2OfgNZ5wDpcsFmHmCK5gZTPcCXqlm2ArzUIkw9czNJo= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.0.1/go.mod h1:hyedUtir6IdtD/7lIxGeCxkaw7y45JueMRL4DIyJDKs= +github.com/decred/dcrd/dcrec/secp256k1/v4 v4.1.0/go.mod h1:DZGJHZMqrU4JJqFAWUS2UO1+lbSKsdiOoYi9Zzey7Fc= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0 h1:8UrgZ3GkP4i/CLijOJx79Yu+etlyjdBU4sfcs2WYQMs= github.com/decred/dcrd/dcrec/secp256k1/v4 v4.2.0/go.mod h1:v57UDF4pDQJcEfFUCRop3lJL149eHGSe9Jvczhzjo/0= github.com/decred/dcrd/lru v1.0.0/go.mod h1:mxKOwFd7lFjN2GZYsiz/ecgqR6kkYAl+0pz0tEMk218= @@ -601,6 +613,7 @@ github.com/filecoin-project/go-jsonrpc v0.3.1/go.mod h1:jBSvPTl8V1N7gSTuCR4bis8w github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5 h1:FtmdgXiUlNeRsoNMFlKLDt+S+6hbjVMEW6RGQ7aUf7c= github.com/fjl/memsize v0.0.0-20190710130421-bcb5799ab5e5/go.mod h1:VvhXpOYNQvB+uIk2RvXzuaQtkQJzzIx6lSBe1xv7hi0= github.com/flynn/go-shlex v0.0.0-20150515145356-3f9db97f8568/go.mod h1:xEzjJPgXI435gkrCt3MPfRiAkVrwSbHsst4LCFVfpJc= +github.com/flynn/noise v0.0.0-20180327030543-2492fe189ae6/go.mod h1:1i71OnUq3iUe1ma7Lr6yG6/rjvM3emb6yoL7xLFzcVQ= github.com/flynn/noise v1.0.0 h1:DlTHqmzmvcEiKj+4RYo/imoswx/4r6iBlCMfVtrMXpQ= github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwUTag= github.com/fogleman/gg v1.2.1-0.20190220221249-0403632d5b90/go.mod h1:R/bRT+9gY/C5z7JzPU0zXsXHKM4/ayA+zqcVNZzPa1k= @@ -612,13 +625,18 @@ github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVB github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= github.com/frankban/quicktest v1.11.3/go.mod h1:wRf/ReqHper53s+kmmSZizM8NamnL3IM0I9ntUbOk+k= github.com/frankban/quicktest v1.14.0/go.mod h1:NeW+ay9A/U67EYXNFA1nPE8e/tnQv/09mUdL/ijj8og= -github.com/frankban/quicktest v1.14.4 h1:g2rn0vABPOOXmZUj+vbmUp0lPoXEMuhTpIluN0XL9UY= +github.com/frankban/quicktest v1.14.2/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= +github.com/frankban/quicktest v1.14.3/go.mod h1:mgiwOwqx65TmIk1wJ6Q7wvnVMocbUorkibMOrVTHZps= github.com/frankban/quicktest v1.14.4/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= +github.com/frankban/quicktest v1.14.6 h1:7Xjx+VpznH+oBnejlPUj8oUpdxnVs4f8XU8WnHkI4W8= +github.com/frankban/quicktest v1.14.6/go.mod h1:4ptaffx2x8+WTWXmUCuVU6aPUX1/Mz7zb5vbUoiM6w0= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.1/go.mod h1:T3375wBYaZdLLcVNkcVbzGHY7f1l/uK5T5Ai1i3InKU= +github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4HY= github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= +github.com/gabriel-vasile/mimetype v1.4.1/go.mod h1:05Vi0w3Y9c/lNvJOdmIwvrrAhX3rYhfQQCaf9VJcv7M= github.com/gammazero/deque v0.2.0 h1:SkieyNB4bg2/uZZLxvya0Pq6diUlwx7m2TeT7GAIWaA= github.com/gammazero/deque v0.2.0/go.mod h1:LFroj8x4cMYCukHJDbxFCkT+r9AndaJnFMuZDV34tuU= github.com/gammazero/workerpool v1.1.3 h1:WixN4xzukFoN0XSeXF6puqEqFTl2mECI9S6W44HWy9Q= @@ -660,6 +678,7 @@ github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNV github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= @@ -828,9 +847,11 @@ github.com/google/pprof v0.0.0-20201203190320-1bf35d6f28c2/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20201218002935-b9804c9f04c2/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210122040257-d980be63207e/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210226084205-cbba55b83ad5/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20210407192527-94a9f03dee38/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= +github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= @@ -840,8 +861,9 @@ github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.0 h1:t6JiXgmwXMjEs8VusXIJk2BXHsn+wx8BZdTaoZ5fu7I= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= +github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= @@ -873,6 +895,7 @@ github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2z github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= +github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= @@ -944,6 +967,7 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= @@ -972,6 +996,7 @@ github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3 github.com/iancoleman/orderedmap v0.1.0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20220319035150-800ac71e25c2/go.mod h1:aYm2/VgdVmcIU8iMfdMvDMsRAQjcfZSKFby6HOFvi/w= github.com/icrowley/fake v0.0.0-20180203215853-4178557ae428/go.mod h1:uhpZMVGznybq1itEKXj6RYw9I71qK4kH+OGMjRC4KEo= github.com/imdario/mergo v0.3.16 h1:wwQJbIsHYGMUyLSPrEq1CT16AhnhNJQ51+4fdHUnCl4= github.com/imdario/mergo v0.3.16/go.mod h1:WBLT9ZmE3lPoWsEzCh9LPo3TiwVN+ZKEjmz+hD27ysY= @@ -995,20 +1020,33 @@ github.com/influxdata/promql/v2 v2.12.0/go.mod h1:fxOPu+DY0bqCTCECchSRtWfc+0X19y github.com/influxdata/roaring v0.4.13-0.20180809181101-fc520f41fab6/go.mod h1:bSgUQ7q5ZLSO+bKBGqJiCBGAl+9DxyW63zLTujjUlOE= github.com/influxdata/tdigest v0.0.0-20181121200506-bf2b5ad3c0a9/go.mod h1:Js0mqiSBE6Ffsg94weZZ2c+v/ciT8QRHFOap7EKDrR0= github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1:Wbbw6tYNvwa5dlB6304Sd+82Z3f7PmVZHVKU637d4po= +github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= +github.com/ipfs/go-bitswap v0.1.0/go.mod h1:FFJEf18E9izuCqUtHxbWEvq+reg7o4CW5wSAE1wsxj0= +github.com/ipfs/go-bitswap v0.1.2/go.mod h1:qxSWS4NXGs7jQ6zQvoPY3+NmOfHHG47mhkiLzBpJQIs= +github.com/ipfs/go-bitswap v0.1.8/go.mod h1:TOWoxllhccevbWFUR2N7B1MTSVVge1s6XSMiCSA4MzM= +github.com/ipfs/go-bitswap v0.3.4/go.mod h1:4T7fvNv/LmOys+21tnLzGKncMeeXUYUd1nUiJ2teMvI= github.com/ipfs/go-bitswap v0.5.1/go.mod h1:P+ckC87ri1xFLvk74NlXdP0Kj9RmWAh4+H78sC6Qopo= +github.com/ipfs/go-bitswap v0.6.0/go.mod h1:Hj3ZXdOC5wBJvENtdqsixmzzRukqd8EHLxZLZc3mzRA= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= github.com/ipfs/go-bitswap v0.11.0/go.mod h1:05aE8H3XOU+LXpTedeAS0OZpcO1WFsj5niYQH9a1Tmk= +github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/baNLgayt4pFc= github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= +github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= +github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= +github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= +github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= +github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= github.com/ipfs/go-blockservice v0.5.1 h1:9pAtkyKAz/skdHTh0kH8VulzWp+qmSDD0aI17TYP/s0= github.com/ipfs/go-blockservice v0.5.1/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -1019,10 +1057,18 @@ github.com/ipfs/go-cid v0.0.5/go.mod h1:plgt+Y5MnOey4vO4UlUazGqdbEXuFYitED67Fexh github.com/ipfs/go-cid v0.0.6/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.0.7/go.mod h1:6Ux9z5e+HpkQdckYoX1PG/6xqKspzlEIR5SDmgqgC/I= github.com/ipfs/go-cid v0.1.0/go.mod h1:rH5/Xv83Rfy8Rw6xG+id3DYAMUVmem1MowoKwdXmN2o= +github.com/ipfs/go-cid v0.2.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.0/go.mod h1:P+HXFDF4CVhaVayiEb4wkAy7zBHxBwsJyt0Y5U6MLro= +github.com/ipfs/go-cid v0.3.2/go.mod h1:gQ8pKqT/sUxGY+tIwy1RPpAojYu7jAyCp5Tz1svoupw= github.com/ipfs/go-cid v0.4.1 h1:A/T3qGvxi4kpKWWcPC/PgbvDA2bjVLO7n4UeVwnbs/s= github.com/ipfs/go-cid v0.4.1/go.mod h1:uQHwDeX4c6CtyrFwdqyhpNcxVewur1M7l7fNU7LKwZk= +github.com/ipfs/go-cidutil v0.1.0 h1:RW5hO7Vcf16dplUU60Hs0AKDkQAVPVplr7lk97CFL+Q= +github.com/ipfs/go-cidutil v0.1.0/go.mod h1:e7OEVBMIv9JaOxt9zaGEmAoSlXW9jdFZ5lP/0PwcfpA= github.com/ipfs/go-datastore v0.0.1/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.0.5/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= +github.com/ipfs/go-datastore v0.1.0/go.mod h1:d4KVXhMt913cLBEI/PXAy6ko+W7e9AhyAKBGh803qeE= github.com/ipfs/go-datastore v0.1.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= +github.com/ipfs/go-datastore v0.3.1/go.mod h1:w38XXW9kVFNp57Zj5knbKWM2T+KOZCGDRVNdgPHtbHw= github.com/ipfs/go-datastore v0.4.0/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.1/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= github.com/ipfs/go-datastore v0.4.4/go.mod h1:SX/xMIKoCszPqp+z9JhPYCmoOoXTvaa13XEbGtsFUhA= @@ -1035,37 +1081,62 @@ github.com/ipfs/go-detect-race v0.0.1 h1:qX/xay2W3E4Q1U7d9lNs1sU9nvguX0a7319XbyQ github.com/ipfs/go-detect-race v0.0.1/go.mod h1:8BNT7shDZPo99Q74BpGMK+4D8Mn4j46UU0LZ723meps= github.com/ipfs/go-ds-badger v0.0.2/go.mod h1:Y3QpeSFWQf6MopLTiZD+VT6IC1yZqaGmjvRcKeSGij8= github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaHzfGTzuE3s= +github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= +github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.4.2/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= github.com/ipfs/go-ds-leveldb v0.5.0 h1:s++MEBbD3ZKc9/8/njrn4flZLnCuY9I79v94gBUNumo= github.com/ipfs/go-ds-leveldb v0.5.0/go.mod h1:d3XG9RUDzQ6V4SHi8+Xgj9j1XuEk1z82lquxrVbml/Q= +github.com/ipfs/go-fetcher v1.5.0/go.mod h1:5pDZ0393oRF/fHiLmtFZtpMNBQfHOYNPtryWedVuSWE= +github.com/ipfs/go-fetcher v1.6.1/go.mod h1:27d/xMV8bodjVs9pugh/RCjjK2OZ68UgAMspMdingNo= +github.com/ipfs/go-ipfs-blockstore v0.0.1/go.mod h1:d3WClOmRQKFnJ0Jz/jj/zmksX0ma1gROTlovZKBmN08= +github.com/ipfs/go-ipfs-blockstore v0.1.0/go.mod h1:5aD0AvHPi7mZc6Ci1WCAhiBQu2IsfTduLl+422H6Rqw= +github.com/ipfs/go-ipfs-blockstore v0.1.4/go.mod h1:Jxm3XMVjh6R17WvxFEiyKBLUGr86HgIYJW/D/MwqeYQ= github.com/ipfs/go-ipfs-blockstore v0.2.1/go.mod h1:jGesd8EtCM3/zPgx+qr0/feTXGUeRai6adgwC+Q+JvE= github.com/ipfs/go-ipfs-blockstore v1.1.2/go.mod h1:w51tNR9y5+QXB0wkNcHt4O2aSZjTdqaEWaQdSxEyUOY= +github.com/ipfs/go-ipfs-blockstore v1.2.0/go.mod h1:eh8eTFLiINYNSNawfZOC7HOxNTxpB1PFuA5E1m/7exE= +github.com/ipfs/go-ipfs-blockstore v1.3.0/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blockstore v1.3.1 h1:cEI9ci7V0sRNivqaOr0elDsamxXFxJMMMy7PTTDQNsQ= github.com/ipfs/go-ipfs-blockstore v1.3.1/go.mod h1:KgtZyc9fq+P2xJUiCAzbRdhhqJHvsw8u2Dlqy2MyRTE= github.com/ipfs/go-ipfs-blocksutil v0.0.1 h1:Eh/H4pc1hsvhzsQoMEP3Bke/aW5P5rVM1IWFJMcGIPQ= github.com/ipfs/go-ipfs-blocksutil v0.0.1/go.mod h1:Yq4M86uIOmxmGPUHv/uI7uKqZNtLb449gwKqXjIsnRk= +github.com/ipfs/go-ipfs-chunker v0.0.1/go.mod h1:tWewYK0we3+rMbOh7pPFGDyypCtvGcBFymgY4rSDLAw= github.com/ipfs/go-ipfs-chunker v0.0.5 h1:ojCf7HV/m+uS2vhUGWcogIIxiO5ubl5O57Q7NapWLY8= github.com/ipfs/go-ipfs-chunker v0.0.5/go.mod h1:jhgdF8vxRHycr00k13FM8Y0E+6BoalYeobXmUyTreP8= github.com/ipfs/go-ipfs-delay v0.0.0-20181109222059-70721b86a9a8/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1IeRQ= github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= +github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= +github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= +github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= github.com/ipfs/go-ipfs-exchange-interface v0.2.1 h1:jMzo2VhLKSHbVe+mHNzYgs95n0+t0Q69GQ5WhRDZV/s= github.com/ipfs/go-ipfs-exchange-interface v0.2.1/go.mod h1:MUsYn6rKbG6CTtsDp+lKJPmVt3ZrCViNyH3rfPGsZ2E= +github.com/ipfs/go-ipfs-exchange-offline v0.0.1/go.mod h1:WhHSFCVYX36H/anEKQboAzpUws3x7UeEGkzQc3iNkM0= github.com/ipfs/go-ipfs-exchange-offline v0.1.1/go.mod h1:vTiBRIbzSwDD0OWm+i3xeT0mO7jG2cbJYatp3HPk5XY= +github.com/ipfs/go-ipfs-exchange-offline v0.2.0/go.mod h1:HjwBeW0dvZvfOMwDP0TSKXIHf2s+ksdP4E3MLDRtLKY= github.com/ipfs/go-ipfs-exchange-offline v0.3.0 h1:c/Dg8GDPzixGd0MC8Jh6mjOwU57uYokgWRFidfvEkuA= github.com/ipfs/go-ipfs-exchange-offline v0.3.0/go.mod h1:MOdJ9DChbb5u37M1IcbrRB02e++Z7521fMxqCNRrz9s= +github.com/ipfs/go-ipfs-files v0.0.3/go.mod h1:INEFm0LL2LWXBhNJ2PMIIb2w45hpXgPjNoE7yA8Y1d4= +github.com/ipfs/go-ipfs-files v0.0.8/go.mod h1:wiN/jSG8FKyk7N0WyctKSvq3ljIa2NNTiZB55kpTdOs= +github.com/ipfs/go-ipfs-files v0.3.0 h1:fallckyc5PYjuMEitPNrjRfpwl7YFt69heCOUhsbGxQ= +github.com/ipfs/go-ipfs-files v0.3.0/go.mod h1:xAUtYMwB+iu/dtf6+muHNSFQCJG2dSiStR2P6sn9tIM= +github.com/ipfs/go-ipfs-keystore v0.1.0/go.mod h1:LvLw7Qhnb0RlMOfCzK6OmyWxICip6lQ06CCmdbee75U= +github.com/ipfs/go-ipfs-posinfo v0.0.1 h1:Esoxj+1JgSjX0+ylc0hUmJCOv6V2vFoZiETLR6OtpRs= +github.com/ipfs/go-ipfs-posinfo v0.0.1/go.mod h1:SwyeVP+jCwiDu0C313l/8jg6ZxM0qqtlt2a0vILTc1A= +github.com/ipfs/go-ipfs-pq v0.0.1/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.2/go.mod h1:LWIqQpqfRG3fNc5XsnIhz/wQ2XXGyugQwls7BgUmUfY= github.com/ipfs/go-ipfs-pq v0.0.3 h1:YpoHVJB+jzK15mr/xsWC574tyDLkezVrDNeaalQBsTE= github.com/ipfs/go-ipfs-pq v0.0.3/go.mod h1:btNw5hsHBpRcSSgZtiNm/SLj5gYIZ18AKtv3kERkRb4= +github.com/ipfs/go-ipfs-redirects-file v0.1.1/go.mod h1:tAwRjCV0RjLTjH8DR/AU7VYvfQECg+lpUy2Mdzv7gyk= +github.com/ipfs/go-ipfs-routing v0.1.0/go.mod h1:hYoUkJLyAUKhF58tysKpids8RNDPO42BVMgK5dNsoqY= github.com/ipfs/go-ipfs-routing v0.2.1/go.mod h1:xiNNiwgjmLqPS1cimvAw6EyB9rkVDbiocA4yY+wRNLM= github.com/ipfs/go-ipfs-routing v0.3.0 h1:9W/W3N+g+y4ZDeffSgqhgo7BsBSJwPMcyssET9OWevc= github.com/ipfs/go-ipfs-routing v0.3.0/go.mod h1:dKqtTFIql7e1zYsEuWLyuOU+E0WJWW8JjbTPLParDWo= @@ -1073,16 +1144,30 @@ github.com/ipfs/go-ipfs-util v0.0.1/go.mod h1:spsl5z8KUnrve+73pOhSVZND1SIxPW5RyB github.com/ipfs/go-ipfs-util v0.0.2/go.mod h1:CbPtkWJzjLdEcezDns2XYaehFVNXG9zrdrtMecczcsQ= github.com/ipfs/go-ipfs-util v0.0.3 h1:2RFdGez6bu2ZlZdI+rWfIdbQb1KudQp3VGwPtdNCmE0= github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn17xAKWBvs= +github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= +github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= +github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= +github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-format v0.3.1/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= +github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= +github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= github.com/ipfs/go-ipld-legacy v0.2.1/go.mod h1:782MOUghNzMO2DER0FlBR94mllfdCJCkTtDtPM51otM= +github.com/ipfs/go-ipns v0.2.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-ipns v0.3.0/go.mod h1:3cLT2rbvgPZGkHJoPO1YMJeh6LtkxopCkKFcio/wE24= +github.com/ipfs/go-libipfs v0.1.0/go.mod h1:qX0d9h+wu53PFtCTXxdXVBakd6ZCvGDdkZUKmdLMLx0= +github.com/ipfs/go-libipfs v0.3.0/go.mod h1:pSUHZ5qPJTAidsxe9bAeHp3KIiw2ODEW2a2kM3v+iXI= +github.com/ipfs/go-libipfs v0.4.0/go.mod h1:XsU2cP9jBhDrXoJDe0WxikB8XcVmD3k2MEZvB3dbYu8= +github.com/ipfs/go-libipfs v0.6.0 h1:3FuckAJEm+zdHbHbf6lAyk0QUzc45LsFcGw102oBCZM= +github.com/ipfs/go-libipfs v0.6.0/go.mod h1:UjjDIuehp2GzlNP0HEr5I9GfFT7zWgst+YfpUEIThtw= github.com/ipfs/go-log v0.0.1/go.mod h1:kL1d2/hzSpI0thNYjiKfjanbVNU+IIGA/WnNESY9leM= github.com/ipfs/go-log v1.0.2/go.mod h1:1MNjMxe0u6xvJZgeqbJ8vdo2TKaGwZ1a0Bpza+sr2Sk= github.com/ipfs/go-log v1.0.3/go.mod h1:OsLySYkwIbiSUR/yBTdv1qPtcE4FW3WPWk/ewz9Ru+A= @@ -1098,29 +1183,65 @@ github.com/ipfs/go-log/v2 v2.3.0/go.mod h1:QqGoj30OTpnKaG/LKTGTxoP2mmQtjVMEnK72g github.com/ipfs/go-log/v2 v2.5.0/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= github.com/ipfs/go-log/v2 v2.5.1 h1:1XdUzF7048prq4aBjDQQ4SL5RxftpRGdXhNRwKSAlcY= github.com/ipfs/go-log/v2 v2.5.1/go.mod h1:prSpmC1Gpllc9UYWxDiZDreBYw7zp4Iqp1kOLU9U5UI= +github.com/ipfs/go-merkledag v0.2.3/go.mod h1:SQiXrtSts3KGNmgOzMICy5c0POOpUNQLvB3ClKnBAlk= +github.com/ipfs/go-merkledag v0.3.2/go.mod h1:fvkZNNZixVW6cKSZ/JfLlON5OlgTXNdRLz0p6QG/I2M= github.com/ipfs/go-merkledag v0.5.1/go.mod h1:cLMZXx8J08idkp5+id62iVftUQV+HlYJ3PIhDfZsjA4= +github.com/ipfs/go-merkledag v0.6.0/go.mod h1:9HSEwRd5sV+lbykiYP+2NC/3o6MZbKNaa4hfNcH5iH0= +github.com/ipfs/go-merkledag v0.9.0/go.mod h1:bPHqkHt5OZ0p1n3iqPeDiw2jIBkjAytRjS3WSBwjq90= +github.com/ipfs/go-merkledag v0.10.0/go.mod h1:zkVav8KiYlmbzUzNM6kENzkdP5+qR7+2mCwxkQ6GIj8= github.com/ipfs/go-merkledag v0.11.0 h1:DgzwK5hprESOzS4O1t/wi6JDpyVQdvm9Bs59N/jqfBY= github.com/ipfs/go-merkledag v0.11.0/go.mod h1:Q4f/1ezvBiJV0YCIXvt51W/9/kqJGH4I1LsA7+djsM4= github.com/ipfs/go-metrics-interface v0.0.1 h1:j+cpbjYvu4R8zbleSs36gvB7jR+wsL2fGD6n0jO4kdg= github.com/ipfs/go-metrics-interface v0.0.1/go.mod h1:6s6euYU4zowdslK0GKHmqaIZ3j/b/tL7HTWtJ4VPgWY= +github.com/ipfs/go-namesys v0.7.0/go.mod h1:KYSZBVZG3VJC34EfqqJPG7T48aWgxseoMPAPA5gLyyQ= +github.com/ipfs/go-path v0.1.1/go.mod h1:vC8q4AKOtrjJz2NnllIrmr2ZbGlF5fW2OKKyhV9ggb0= +github.com/ipfs/go-path v0.3.0/go.mod h1:NOScsVgxfC/eIw4nz6OiGwK42PjaSJ4Y/ZFPn1Xe07I= +github.com/ipfs/go-peertaskqueue v0.1.0/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= +github.com/ipfs/go-peertaskqueue v0.1.1/go.mod h1:Jmk3IyCcfl1W3jTW3YpghSwSEC6IJ3Vzz/jUmWw8Z0U= +github.com/ipfs/go-peertaskqueue v0.2.0/go.mod h1:5/eNrBEbtSKWCG+kQK8K8fGNixoYUnr+P7jivavs9lY= github.com/ipfs/go-peertaskqueue v0.7.0/go.mod h1:M/akTIE/z1jGNXMU7kFB4TeSEFvj68ow0Rrb04donIU= +github.com/ipfs/go-peertaskqueue v0.8.0/go.mod h1:cz8hEnnARq4Du5TGqiWKgMr/BOSQ5XOgMOh1K5YYKKM= github.com/ipfs/go-peertaskqueue v0.8.1 h1:YhxAs1+wxb5jk7RvS0LHdyiILpNmRIRnZVztekOF0pg= github.com/ipfs/go-peertaskqueue v0.8.1/go.mod h1:Oxxd3eaK279FxeydSPPVGHzbwVeHjatZ2GA8XD+KbPU= +github.com/ipfs/go-unixfs v0.2.4/go.mod h1:SUdisfUjNoSDzzhGVxvCL9QO/nKdwXdr+gbMUdqcbYw= +github.com/ipfs/go-unixfs v0.3.1/go.mod h1:h4qfQYzghiIc8ZNFKiLMFWOTzrWIAtzYQ59W/pCFf1o= +github.com/ipfs/go-unixfs v0.4.3/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= +github.com/ipfs/go-unixfs v0.4.4/go.mod h1:TSG7G1UuT+l4pNj91raXAPkX0BhJi3jST1FDTfQ5QyM= +github.com/ipfs/go-unixfs v0.4.5 h1:wj8JhxvV1G6CD7swACwSKYa+NgtdWC1RUit+gFnymDU= +github.com/ipfs/go-unixfs v0.4.5/go.mod h1:BIznJNvt/gEx/ooRMI4Us9K8+qeGO7vx1ohnbk8gjFg= +github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvPwZjIEkfV6s= +github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= +github.com/ipfs/go-unixfsnode v1.5.1/go.mod h1:ed79DaG9IEuZITJVQn4U6MZDftv6I3ygUBLPfhEbHvk= +github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/interface-go-ipfs-core v0.9.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= +github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= +github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= github.com/ipld/go-car v0.6.2 h1:Hlnl3Awgnq8icK+ze3iRghk805lu8YNq3wlREDTF2qc= github.com/ipld/go-car v0.6.2/go.mod h1:oEGXdwp6bmxJCZ+rARSkDliTeYnVzv3++eXajZ+Bmr8= github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZzeVNeFcI= +github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= +github.com/ipld/go-car/v2 v2.8.0/go.mod h1:a+BnAxUqgr7wcWxW/lI6ctyEQ2v9gjBChPytwFMp2f4= +github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s= github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= +github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= +github.com/ipld/go-codec-dagpb v1.4.1/go.mod h1:XdXTO/TUD/ra9RcK/NfmwBfr1JpFxM2uRKaB9oe4LxE= +github.com/ipld/go-codec-dagpb v1.5.0/go.mod h1:0yRIutEFD8o1DGVqw4RSHh+BUTlJA9XWldxaaWR/o4g= github.com/ipld/go-codec-dagpb v1.6.0 h1:9nYazfyu9B1p3NAgfVdpRco3Fs2nFC72DqVsMj6rOcc= github.com/ipld/go-codec-dagpb v1.6.0/go.mod h1:ANzFhfP2uMJxRBr8CE+WQWs5UsNa0pYtmKZ+agnUw9s= +github.com/ipld/go-ipld-prime v0.9.0/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.9.1-0.20210324083106-dc342a9917db/go.mod h1:KvBLMr4PX1gWptgkzRjVZCrLmSGcZCb/jioOQwCqZN8= github.com/ipld/go-ipld-prime v0.11.0/go.mod h1:+WIAkokurHmZ/KwzDOMUuoeJgaRQktHtEaLglS3ZeV8= github.com/ipld/go-ipld-prime v0.14.0/go.mod h1:9ASQLwUFLptCov6lIYc70GRB4V7UTyLD0IJtrDJe6ZM= -github.com/ipld/go-ipld-prime v0.20.0 h1:Ud3VwE9ClxpO2LkCYP7vWPc0Fo+dYdYzgxUJZ3uRG4g= +github.com/ipld/go-ipld-prime v0.16.0/go.mod h1:axSCuOCBPqrH+gvXr2w9uAOulJqBPhHPT2PjoiiU1qA= +github.com/ipld/go-ipld-prime v0.18.0/go.mod h1:735yXW548CKrLwVCYXzqx90p5deRJMVVxM9eJ4Qe+qE= +github.com/ipld/go-ipld-prime v0.19.0/go.mod h1:Q9j3BaVXwaA3o5JUDNvptDDr/x8+F7FG6XJ8WI3ILg4= github.com/ipld/go-ipld-prime v0.20.0/go.mod h1:PzqZ/ZR981eKbgdr3y2DJYeD/8bgMawdGVlJDE8kK+M= +github.com/ipld/go-ipld-prime v0.21.0 h1:n4JmcpOlPDIxBcY037SVfpd1G+Sj1nKZah0m6QH9C2E= +github.com/ipld/go-ipld-prime v0.21.0/go.mod h1:3RLqy//ERg/y5oShXXdx5YIp50cFGOanyMctpPjsvxQ= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20211210234204-ce2a1c70cd73/go.mod h1:2PJ0JgxyB08t0b2WKrcuqI3di0V+5n6RS/LTUJhkoxY= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd h1:gMlw/MhNr2Wtp5RwGdsW23cs+yCuj9k2ON7i9MiJlRo= github.com/ipld/go-ipld-prime/storage/bsadapter v0.0.0-20230102063945-1a409dc236dd/go.mod h1:wZ8hH8UxeryOs4kJEJaiui/s00hDSbE37OKsL47g+Sw= @@ -1139,6 +1260,7 @@ github.com/jbenet/goprocess v0.1.4 h1:DRGOFReOMqqDNXwW70QkacFW0YN9QnwLV0Vqk+3oU0 github.com/jbenet/goprocess v0.1.4/go.mod h1:5yspPrukOVuOLORacaBi858NqyClJPQxYZlqdZVfqY4= github.com/jedisct1/go-minisign v0.0.0-20190909160543-45766022959e/go.mod h1:G1CVv03EnqU1wYL2dFwXxW2An0az9JTl/ZsqXQeBlkU= github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU= +github.com/jellydator/ttlcache/v2 v2.11.1/go.mod h1:RtE5Snf0/57e+2cLWFYWCCsLas2Hy3c5Z4n14XmSvTI= github.com/jessevdk/go-flags v0.0.0-20141203071132-1679536dcc89/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jessevdk/go-flags v1.4.0/go.mod h1:4FA24M0QyGHXBuZZK/XkWh8h0e1EYbRYJSGM75WSRxI= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= @@ -1184,7 +1306,9 @@ github.com/klauspost/compress v1.11.7/go.mod h1:aoV0uJVorq1K+umq18yTdKaF57EivdYs github.com/klauspost/compress v1.12.3/go.mod h1:8dP1Hq4DHOhN9w426knH3Rhby4rFm6D8eO+e+Dq5Gzg= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= +github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= @@ -1192,6 +1316,11 @@ github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa02 github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.12/go.mod h1:g2LTdtYhdyuGPqyWyv7qRAmj1WBqxuObKfj5c0PQa7c= +github.com/klauspost/cpuid/v2 v2.1.0/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= +github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= @@ -1200,8 +1329,10 @@ github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7v github.com/klauspost/reedsolomon v1.11.8/go.mod h1:4bXRN+cVzMdml6ti7qLouuYi32KHJ5MGv0Qd8a47h6A= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= github.com/konsorten/go-windows-terminal-sequences v1.0.3/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/koron/go-ssdp v0.0.0-20180514024734-4a0ed625a78b/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.0-20191105050749-2e1c40ed0b5d/go.mod h1:5Ky9EC2xfoUKUor0Hjgi2BJhCSXJfMOFlmyYrVKGQMk= github.com/koron/go-ssdp v0.0.2/go.mod h1:XoLfkAiA2KeZsYh4DbHxD7h3nR2AZNqVQOa+LJuqPYs= +github.com/koron/go-ssdp v0.0.3/go.mod h1:b2MxI6yh02pKrsyNoQUsk4+YNikaGhe4894J+Q5lDvA= github.com/koron/go-ssdp v0.0.4 h1:1IDwrghSKYM7yLf7XCzbByg2sJ/JcNOZRXS2jczTwz0= github.com/koron/go-ssdp v0.0.4/go.mod h1:oDXq+E5IL5q0U8uSBcoAXzTzInwy5lEgC91HoKtbmZk= github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= @@ -1246,32 +1377,44 @@ github.com/libp2p/go-flow-metrics v0.0.1/go.mod h1:Iv1GH0sG8DtYN3SVJ2eG221wMiNpZ github.com/libp2p/go-flow-metrics v0.0.3/go.mod h1:HeoSNUrOJVK1jEpDqVEiUOIXqhbnS27omG0uWU5slZs= github.com/libp2p/go-flow-metrics v0.1.0 h1:0iPhMI8PskQwzh57jB9WxIuIOQ0r+15PChFGkx3Q3WM= github.com/libp2p/go-flow-metrics v0.1.0/go.mod h1:4Xi8MX8wj5aWNDAZttg6UPmc0ZrnFNsMtpsYUClFtro= +github.com/libp2p/go-libp2p v0.1.0/go.mod h1:6D/2OBauqLUoqcADOJpn9WbKqvaM07tDw68qHM0BxUM= +github.com/libp2p/go-libp2p v0.1.1/go.mod h1:I00BRo1UuUSdpuc8Q2mN7yDF/oTUTRAX6JWpTiK9Rp8= github.com/libp2p/go-libp2p v0.6.1/go.mod h1:CTFnWXogryAHjXAKEbOf1OWY+VeAP3lDMZkfEI5sT54= github.com/libp2p/go-libp2p v0.7.0/go.mod h1:hZJf8txWeCduQRDC/WSqBGMxaTHCOYHt2xSU1ivxn0k= github.com/libp2p/go-libp2p v0.7.4/go.mod h1:oXsBlTLF1q7pxr+9w6lqzS1ILpyHsaBPniVO7zIHGMw= github.com/libp2p/go-libp2p v0.8.1/go.mod h1:QRNH9pwdbEBpx5DTJYg+qxcVaDMAz3Ee/qDKwXujH5o= +github.com/libp2p/go-libp2p v0.13.0/go.mod h1:pM0beYdACRfHO1WcJlp65WXyG2A6NqYM+t2DTVAJxMo= github.com/libp2p/go-libp2p v0.14.3/go.mod h1:d12V4PdKbpL0T1/gsUNN8DfgMuRPDX8bS2QxCZlwRH0= github.com/libp2p/go-libp2p v0.19.0/go.mod h1:Ki9jJXLO2YqrTIFxofV7Twyd3INWPT97+r8hGt7XPjI= +github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBxSjRksxxU4= +github.com/libp2p/go-libp2p v0.23.4/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= +github.com/libp2p/go-libp2p v0.25.0/go.mod h1:vXHmFpcfl+xIGN4qW58Bw3a0/SKGAesr5/T4IuJHE3o= +github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= +github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= github.com/libp2p/go-libp2p-asn-util v0.3.0/go.mod h1:B1mcOrKUE35Xq/ASTmQ4tN3LNzVVaMNmq2NACuqyB9w= +github.com/libp2p/go-libp2p-autonat v0.1.0/go.mod h1:1tLf2yXxiE/oKGtDwPYWTSYG3PtvYlJmg7NeVtPRqH8= github.com/libp2p/go-libp2p-autonat v0.1.1/go.mod h1:OXqkeGOY2xJVWKAGV2inNF5aKN/djNA3fdpCWloIudE= github.com/libp2p/go-libp2p-autonat v0.2.0/go.mod h1:DX+9teU4pEEoZUqR1PiMlqliONQdNbfzE1C718tcViI= github.com/libp2p/go-libp2p-autonat v0.2.1/go.mod h1:MWtAhV5Ko1l6QBsHQNSuM6b1sRkXrpk0/LqCr+vCVxI= github.com/libp2p/go-libp2p-autonat v0.2.2/go.mod h1:HsM62HkqZmHR2k1xgX34WuWDzk/nBwNHoeyyT4IWV6A= +github.com/libp2p/go-libp2p-autonat v0.4.0/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= github.com/libp2p/go-libp2p-autonat v0.4.2/go.mod h1:YxaJlpr81FhdOv3W3BTconZPfhaYivRdf53g+S2wobk= github.com/libp2p/go-libp2p-blankhost v0.1.1/go.mod h1:pf2fvdLJPsC1FsVrNP3DUUvMzUts2dsLLBEpo1vW1ro= github.com/libp2p/go-libp2p-blankhost v0.1.4/go.mod h1:oJF0saYsAXQCSfDq254GMNmLNz6ZTHTOvtF4ZydUvwU= github.com/libp2p/go-libp2p-blankhost v0.2.0/go.mod h1:eduNKXGTioTuQAUcZ5epXi9vMl+t4d8ugUBRQ4SqaNQ= github.com/libp2p/go-libp2p-blankhost v0.3.0/go.mod h1:urPC+7U01nCGgJ3ZsV8jdwTp6Ji9ID0dMTvq+aJ+nZU= +github.com/libp2p/go-libp2p-circuit v0.1.0/go.mod h1:Ahq4cY3V9VJcHcn1SBXjr78AbFkZeIRmfunbA7pmFh8= github.com/libp2p/go-libp2p-circuit v0.1.4/go.mod h1:CY67BrEjKNDhdTk8UgBX1Y/H5c3xkAcs3gnksxY7osU= github.com/libp2p/go-libp2p-circuit v0.2.1/go.mod h1:BXPwYDN5A8z4OEY9sOfr2DUQMLQvKt/6oku45YUmjIo= github.com/libp2p/go-libp2p-circuit v0.4.0/go.mod h1:t/ktoFIUzM6uLQ+o1G6NuBl2ANhBKN9Bc8jRIk31MoA= github.com/libp2p/go-libp2p-circuit v0.6.0/go.mod h1:kB8hY+zCpMeScyvFrKrGicRdid6vNXbunKE4rXATZ0M= github.com/libp2p/go-libp2p-core v0.0.1/go.mod h1:g/VxnTZ/1ygHxH3dKok7Vno1VfpvGcGip57wjTU4fco= github.com/libp2p/go-libp2p-core v0.0.2/go.mod h1:9dAcntw/n46XycV4RnlBq3BpgrmyUi9LuoTNdPrbUco= +github.com/libp2p/go-libp2p-core v0.0.3/go.mod h1:j+YQMNz9WNSkNezXOsahp9kwZBKBvxLpKD316QWSJXE= github.com/libp2p/go-libp2p-core v0.0.4/go.mod h1:jyuCQP356gzfCFtRKyvAbNkyeuxb7OlyhWZ3nls5d2I= github.com/libp2p/go-libp2p-core v0.2.0/go.mod h1:X0eyB0Gy93v0DZtSYbEM7RnMChm9Uv3j7yRXjO77xSI= github.com/libp2p/go-libp2p-core v0.2.2/go.mod h1:8fcwTbsG2B+lTgRJ1ICZtiM5GWCWZVoVrLaDRvIRng0= @@ -1297,12 +1440,18 @@ github.com/libp2p/go-libp2p-core v0.11.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQR github.com/libp2p/go-libp2p-core v0.12.0/go.mod h1:ECdxehoYosLYHgDDFa2N4yE8Y7aQRAMf0sX9mf2sbGg= github.com/libp2p/go-libp2p-core v0.14.0/go.mod h1:tLasfcVdTXnixsLB0QYaT1syJOhsbrhG7q6pGrHtBg8= github.com/libp2p/go-libp2p-core v0.15.1/go.mod h1:agSaboYM4hzB1cWekgVReqV5M4g5M+2eNNejV+1EEhs= +github.com/libp2p/go-libp2p-core v0.19.0/go.mod h1:AkA+FUKQfYt1FLNef5fOPlo/naAWjKy/RCjkcPjqzYg= github.com/libp2p/go-libp2p-crypto v0.1.0/go.mod h1:sPUokVISZiy+nNuTTH/TY+leRSxnFj/2GLjtOTW90hI= +github.com/libp2p/go-libp2p-discovery v0.1.0/go.mod h1:4F/x+aldVHjHDHuX85x1zWoFTGElt8HnoDzwkFZm29g= github.com/libp2p/go-libp2p-discovery v0.2.0/go.mod h1:s4VGaxYMbw4+4+tsoQTqh7wfxg97AEdo4GYBt6BadWg= github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQOu38Fu7LJGEOK2gQltw= github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= +github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= +github.com/libp2p/go-libp2p-kad-dht v0.21.0/go.mod h1:Bhm9diAFmc6qcWAr084bHNL159srVZRKADdp96Qqd1I= github.com/libp2p/go-libp2p-kad-dht v0.25.0 h1:T2SXQ/VlXTQVLChWY/+OyOsmGMRJvB5kiR+eJt7jtvI= github.com/libp2p/go-libp2p-kad-dht v0.25.0/go.mod h1:P6fz+J+u4tPigvS5J0kxQ1isksqAhmXiS/pNaEw/nFI= +github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= +github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= github.com/libp2p/go-libp2p-kbucket v0.6.3/go.mod h1:RCseT7AH6eJWxxk2ol03xtP9pEHetYSPXOaJnOiD8i0= github.com/libp2p/go-libp2p-loggables v0.1.0/go.mod h1:EyumB2Y6PrYjr55Q3/tiJ/o3xoDasoRYM7nOzEpoa90= @@ -1313,15 +1462,18 @@ github.com/libp2p/go-libp2p-mplex v0.2.3/go.mod h1:CK3p2+9qH9x+7ER/gWWDYJ3QW5ZxW github.com/libp2p/go-libp2p-mplex v0.4.0/go.mod h1:yCyWJE2sc6TBTnFpjvLuEJgTSw/u+MamvzILKdX7asw= github.com/libp2p/go-libp2p-mplex v0.4.1/go.mod h1:cmy+3GfqfM1PceHTLL7zQzAAYaryDu6iPSC+CIb094g= github.com/libp2p/go-libp2p-mplex v0.5.0/go.mod h1:eLImPJLkj3iG5t5lq68w3Vm5NAQ5BcKwrrb2VmOYb3M= +github.com/libp2p/go-libp2p-nat v0.0.4/go.mod h1:N9Js/zVtAXqaeT99cXgTV9e75KpnWCvVOiGzlcHmBbY= github.com/libp2p/go-libp2p-nat v0.0.5/go.mod h1:1qubaE5bTZMJE+E/uu2URroMbzdubFz1ChgiN79yKPE= github.com/libp2p/go-libp2p-nat v0.0.6/go.mod h1:iV59LVhB3IkFvS6S6sauVTSOrNEANnINbI/fkaLimiw= github.com/libp2p/go-libp2p-nat v0.1.0/go.mod h1:DQzAG+QbDYjN1/C3B6vXucLtz3u9rEonLVPtZVzQqks= github.com/libp2p/go-libp2p-netutil v0.1.0/go.mod h1:3Qv/aDqtMLTUyQeundkKsA+YCThNdbQD54k3TqjpbFU= +github.com/libp2p/go-libp2p-noise v0.1.1/go.mod h1:QDFLdKX7nluB7DEnlVPbz7xlLHdwHFA9HiohJRr3vwM= github.com/libp2p/go-libp2p-noise v0.2.0/go.mod h1:IEbYhBBzGyvdLBoxxULL/SGbJARhUeqlO8lVSREYu2Q= github.com/libp2p/go-libp2p-noise v0.4.0/go.mod h1:BzzY5pyzCYSyJbQy9oD8z5oP2idsafjt4/X42h9DjZU= github.com/libp2p/go-libp2p-peer v0.2.0/go.mod h1:RCffaCvUyW2CJmG2gAWVqwePwW7JMgxjsHm7+J5kjWY= github.com/libp2p/go-libp2p-peerstore v0.1.0/go.mod h1:2CeHkQsr8svp4fZ+Oi9ykN1HBb6u0MOvdJ7YIsmcwtY= github.com/libp2p/go-libp2p-peerstore v0.1.3/go.mod h1:BJ9sHlm59/80oSkpWgr1MyY1ciXAXV397W6h1GH/uKI= +github.com/libp2p/go-libp2p-peerstore v0.1.4/go.mod h1:+4BDbDiiKf4PzpANZDAT+knVdLxvqh7hXOujessqdzs= github.com/libp2p/go-libp2p-peerstore v0.2.0/go.mod h1:N2l3eVIeAitSg3Pi2ipSrJYnqhVnMNQZo9nkSCuAbnQ= github.com/libp2p/go-libp2p-peerstore v0.2.1/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= github.com/libp2p/go-libp2p-peerstore v0.2.2/go.mod h1:NQxhNjWxf1d4w6PihR8btWIRjwRLBr4TYKfNgrUkOPA= @@ -1340,6 +1492,7 @@ github.com/libp2p/go-libp2p-record v0.1.0/go.mod h1:ujNc8iuE5dlKWVy6wuL6dd58t0n7 github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3queqa1NkQ0= github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= +github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= github.com/libp2p/go-libp2p-routing-helpers v0.7.1 h1:kc0kWCZecbBPAiFEHhxfGJZPqjg1g9zV+X+ovR4Tmnc= github.com/libp2p/go-libp2p-routing-helpers v0.7.1/go.mod h1:cHStPSRC/wgbfpb5jYdMP7zaSmc2wWcb1mkzNr6AR8o= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= @@ -1351,6 +1504,7 @@ github.com/libp2p/go-libp2p-swarm v0.2.2/go.mod h1:fvmtQ0T1nErXym1/aa1uJEyN7JzaT github.com/libp2p/go-libp2p-swarm v0.2.3/go.mod h1:P2VO/EpxRyDxtChXz/VPVXyTnszHvokHKRhfkEgFKNM= github.com/libp2p/go-libp2p-swarm v0.2.8/go.mod h1:JQKMGSth4SMqonruY0a8yjlPVIkb0mdNSwckW7OYziM= github.com/libp2p/go-libp2p-swarm v0.3.0/go.mod h1:hdv95GWCTmzkgeJpP+GK/9D9puJegb7H57B5hWQR5Kk= +github.com/libp2p/go-libp2p-swarm v0.4.0/go.mod h1:XVFcO52VoLoo0eitSxNQWYq4D6sydGOweTOAjJNraCw= github.com/libp2p/go-libp2p-swarm v0.5.0/go.mod h1:sU9i6BoHE0Ve5SKz3y9WfKrh8dUat6JknzUehFx8xW4= github.com/libp2p/go-libp2p-swarm v0.8.0/go.mod h1:sOMp6dPuqco0r0GHTzfVheVBh6UEL0L1lXUZ5ot2Fvc= github.com/libp2p/go-libp2p-swarm v0.10.0/go.mod h1:71ceMcV6Rg/0rIQ97rsZWMzto1l9LnNquef+efcRbmA= @@ -1367,6 +1521,7 @@ github.com/libp2p/go-libp2p-testing v0.5.0/go.mod h1:QBk8fqIL1XNcno/l3/hhaIEn4aL github.com/libp2p/go-libp2p-testing v0.7.0/go.mod h1:OLbdn9DbgdMwv00v+tlp1l3oe2Cl+FAjoWIA2pa0X6E= github.com/libp2p/go-libp2p-testing v0.9.0/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= github.com/libp2p/go-libp2p-testing v0.9.2/go.mod h1:Td7kbdkWqYTJYQGTwzlgXwaqldraIanyjuRiAbK/XQU= +github.com/libp2p/go-libp2p-testing v0.11.0/go.mod h1:qG4sF27dfKFoK9KlVzK2y52LQKhp0VEmLjV5aDqr1Hg= github.com/libp2p/go-libp2p-testing v0.12.0 h1:EPvBb4kKMWO29qP4mZGyhVzUyR25dvfUIK5WDu6iPUA= github.com/libp2p/go-libp2p-testing v0.12.0/go.mod h1:KcGDRXyN7sQCllucn1cOOS+Dmm7ujhfEyXQL5lvkcPg= github.com/libp2p/go-libp2p-tls v0.1.3/go.mod h1:wZfuewxOndz5RTnCAxFliGjvYSDA40sKitV4c50uI1M= @@ -1375,17 +1530,21 @@ github.com/libp2p/go-libp2p-tls v0.4.1/go.mod h1:EKCixHEysLNDlLUoKxv+3f/Lp90O2EX github.com/libp2p/go-libp2p-transport-upgrader v0.1.1/go.mod h1:IEtA6or8JUbsV07qPW4r01GnTenLW4oi3lOPbUMGJJA= github.com/libp2p/go-libp2p-transport-upgrader v0.2.0/go.mod h1:mQcrHj4asu6ArfSoMuyojOdjx73Q47cYD7s5+gZOlns= github.com/libp2p/go-libp2p-transport-upgrader v0.3.0/go.mod h1:i+SKzbRnvXdVbU3D1dwydnTmKRPXiAR/fyvi1dXuL4o= +github.com/libp2p/go-libp2p-transport-upgrader v0.4.0/go.mod h1:J4ko0ObtZSmgn5BX5AmegP+dK3CSnU2lMCKsSq/EY0s= github.com/libp2p/go-libp2p-transport-upgrader v0.4.2/go.mod h1:NR8ne1VwfreD5VIWIU62Agt/J18ekORFU/j1i2y8zvk= github.com/libp2p/go-libp2p-transport-upgrader v0.5.0/go.mod h1:Rc+XODlB3yce7dvFV4q/RmyJGsFcCZRkeZMu/Zdg0mo= github.com/libp2p/go-libp2p-transport-upgrader v0.7.0/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= github.com/libp2p/go-libp2p-transport-upgrader v0.7.1/go.mod h1:GIR2aTRp1J5yjVlkUoFqMkdobfob6RnAwYg/RZPhrzg= +github.com/libp2p/go-libp2p-xor v0.1.0/go.mod h1:LSTM5yRnjGZbWNTA/hRwq2gGFrvRIbQJscoIL/u6InY= github.com/libp2p/go-libp2p-yamux v0.2.0/go.mod h1:Db2gU+XfLpm6E4rG5uGCFX6uXA8MEXOxFcRoXUODaK8= +github.com/libp2p/go-libp2p-yamux v0.2.1/go.mod h1:1FBXiHDk1VyRM1C0aez2bCfHQ4vMZKkAQzZbkSQt5fI= github.com/libp2p/go-libp2p-yamux v0.2.2/go.mod h1:lIohaR0pT6mOt0AZ0L2dFze9hds9Req3OfS+B+dv4qw= github.com/libp2p/go-libp2p-yamux v0.2.5/go.mod h1:Zpgj6arbyQrmZ3wxSZxfBmbdnWtbZ48OpsfmQVTErwA= github.com/libp2p/go-libp2p-yamux v0.2.7/go.mod h1:X28ENrBMU/nm4I3Nx4sZ4dgjZ6VhLEn0XhIoZ5viCwU= github.com/libp2p/go-libp2p-yamux v0.2.8/go.mod h1:/t6tDqeuZf0INZMTgd0WxIRbtK2EzI2h7HbFm9eAKI4= github.com/libp2p/go-libp2p-yamux v0.4.0/go.mod h1:+DWDjtFMzoAwYLVkNZftoucn7PelNoy5nm3tZ3/Zw30= github.com/libp2p/go-libp2p-yamux v0.5.0/go.mod h1:AyR8k5EzyM2QN9Bbdg6X1SkVVuqLwTGf0L4DFq9g6po= +github.com/libp2p/go-libp2p-yamux v0.5.1/go.mod h1:dowuvDu8CRWmr0iqySMiSxK+W0iL5cMVO9S94Y6gkv4= github.com/libp2p/go-libp2p-yamux v0.5.4/go.mod h1:tfrXbyaTqqSU654GTvK3ocnSZL3BuHoeTSqhcel1wsE= github.com/libp2p/go-libp2p-yamux v0.8.0/go.mod h1:yTkPgN2ib8FHyU1ZcVD7aelzyAqXXwEPbyx+aSKm9h8= github.com/libp2p/go-libp2p-yamux v0.8.1/go.mod h1:rUozF8Jah2dL9LLGyBaBeTQeARdwhefMCTQVQt6QobE= @@ -1400,12 +1559,15 @@ github.com/libp2p/go-mplex v0.1.2/go.mod h1:Xgz2RDCi3co0LeZfgjm4OgUF15+sVR8SRcu3 github.com/libp2p/go-mplex v0.2.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= github.com/libp2p/go-mplex v0.3.0/go.mod h1:0Oy/A9PQlwBytDRp4wSkFnzHYDKcpLot35JQ6msjvYQ= github.com/libp2p/go-mplex v0.4.0/go.mod h1:y26Lx+wNVtMYMaPu300Cbot5LkEZ4tJaNYeHeT9dh6E= +github.com/libp2p/go-mplex v0.7.0/go.mod h1:rW8ThnRcYWft/Jb2jeORBmPd6xuG3dGxWN/W168L9EU= github.com/libp2p/go-msgio v0.0.2/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= +github.com/libp2p/go-msgio v0.0.3/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.4/go.mod h1:63lBBgOTDKQL6EWazRMCwXsEeEeK9O2Cd+0+6OOuipQ= github.com/libp2p/go-msgio v0.0.6/go.mod h1:4ecVB6d9f4BDSL5fqvPiC4A3KivjWn+Venn/1ALLMWA= github.com/libp2p/go-msgio v0.2.0/go.mod h1:dBVM1gW3Jk9XqHkU4eKdGvVHdLa51hoGfll6jMJMSlY= github.com/libp2p/go-msgio v0.3.0 h1:mf3Z8B1xcFN314sWX+2vOTShIE0Mmn2TXn3YCUQGNj0= github.com/libp2p/go-msgio v0.3.0/go.mod h1:nyRM819GmVaF9LX3l03RMh10QdOroF++NBbxAb0mmDM= +github.com/libp2p/go-nat v0.0.3/go.mod h1:88nUEt0k0JD45Bk93NIwDqjlhiOwOoV36GchpcVc1yI= github.com/libp2p/go-nat v0.0.4/go.mod h1:Nmw50VAvKuk38jUBcmNh6p9lUJLoODbJRvYAa/+KSDo= github.com/libp2p/go-nat v0.0.5/go.mod h1:B7NxsVNPZmRLvMOwiEO1scOSyjA56zxYAGv1yQgRkEU= github.com/libp2p/go-nat v0.1.0/go.mod h1:X7teVkwRHNInVNWQiO/tAiAVRwSr5zoRz4YSTC3uRBM= @@ -1423,9 +1585,11 @@ github.com/libp2p/go-openssl v0.0.3/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO github.com/libp2p/go-openssl v0.0.4/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.5/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= github.com/libp2p/go-openssl v0.0.7/go.mod h1:unDrJpgy3oFr+rqXsarWifmJuNnJR4chtO1HmaZjggc= +github.com/libp2p/go-openssl v0.1.0/go.mod h1:OiOxwPpL3n4xlenjx2h7AwSGaFSC/KZvf6gNdOBQMtc= github.com/libp2p/go-reuseport v0.0.1/go.mod h1:jn6RmB1ufnQwl0Q1f+YxAj8isJgDCQzaaxIFYDhcYEA= github.com/libp2p/go-reuseport v0.0.2/go.mod h1:SPD+5RwGC7rcnzngoYC86GjPzjSywuQyMVAheVBD9nQ= github.com/libp2p/go-reuseport v0.1.0/go.mod h1:bQVn9hmfcTaoo0c9v5pBhOarsU1eNOBZdaAd2hzXRKU= +github.com/libp2p/go-reuseport v0.2.0/go.mod h1:bvVho6eLMm6Bz5hmU0LYN3ixd3nPPvtIlaURZZgOY4k= github.com/libp2p/go-reuseport v0.4.0 h1:nR5KU7hD0WxXCJbmw7r2rhRYruNRl2koHw8fQscQm2s= github.com/libp2p/go-reuseport v0.4.0/go.mod h1:ZtI03j/wO5hZVDFo2jKywN6bYKWLOy8Se6DrI2E1cLU= github.com/libp2p/go-reuseport-transport v0.0.2/go.mod h1:YkbSDrvjUVDL6b8XqriyA20obEtsW9BLkuOUyQAOCbs= @@ -1442,34 +1606,44 @@ github.com/libp2p/go-stream-muxer-multistream v0.4.0/go.mod h1:nb+dGViZleRP4XcyH github.com/libp2p/go-tcp-transport v0.1.0/go.mod h1:oJ8I5VXryj493DEJ7OsBieu8fcg2nHGctwtInJVpipc= github.com/libp2p/go-tcp-transport v0.1.1/go.mod h1:3HzGvLbx6etZjnFlERyakbaYPdfjg2pWP97dFZworkY= github.com/libp2p/go-tcp-transport v0.2.0/go.mod h1:vX2U0CnWimU4h0SGSEsg++AzvBcroCGYw28kh94oLe0= +github.com/libp2p/go-tcp-transport v0.2.1/go.mod h1:zskiJ70MEfWz2MKxvFB/Pv+tPIB1PpPUrHIWQ8aFw7M= github.com/libp2p/go-tcp-transport v0.2.3/go.mod h1:9dvr03yqrPyYGIEN6Dy5UvdJZjyPFvl1S/igQ5QD1SU= github.com/libp2p/go-tcp-transport v0.4.0/go.mod h1:0y52Rwrn4076xdJYu/51/qJIdxz+EWDAOG2S45sV3VI= github.com/libp2p/go-tcp-transport v0.5.0/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y= github.com/libp2p/go-tcp-transport v0.5.1/go.mod h1:UPPL0DIjQqiWRwVAb+CEQlaAG0rp/mCqJfIhFcLHc4Y= +github.com/libp2p/go-testutil v0.1.0/go.mod h1:81b2n5HypcVyrCg/MJx4Wgfp/VHojytjVe/gLzZ2Ehc= +github.com/libp2p/go-ws-transport v0.1.0/go.mod h1:rjw1MG1LU9YDC6gzmwObkPd/Sqwhw7yT74kj3raBFuo= github.com/libp2p/go-ws-transport v0.2.0/go.mod h1:9BHJz/4Q5A9ludYWKoGCFC5gUElzlHoKzu0yY9p/klM= github.com/libp2p/go-ws-transport v0.3.0/go.mod h1:bpgTJmRZAvVHrgHybCVyqoBmyLQ1fiZuEaBYusP5zsk= github.com/libp2p/go-ws-transport v0.4.0/go.mod h1:EcIEKqf/7GDjth6ksuS/6p7R49V4CBY6/E7R/iyhYUA= github.com/libp2p/go-ws-transport v0.6.0/go.mod h1:dXqtI9e2JV9FtF1NOtWVZSKXh5zXvnuwPXfj8GPBbYU= github.com/libp2p/go-yamux v1.2.2/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= +github.com/libp2p/go-yamux v1.2.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.0/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.3/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.5/go.mod h1:FGTiPvoV/3DVdgWpX+tM0OW3tsM+W5bSE3gZwqQTcow= github.com/libp2p/go-yamux v1.3.7/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.0/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= github.com/libp2p/go-yamux v1.4.1/go.mod h1:fr7aVgmdNGJK+N1g+b6DW6VxzbRCjCOejR/hkmpooHE= +github.com/libp2p/go-yamux/v2 v2.0.0/go.mod h1:NVWira5+sVUIU6tu1JWvaRn1dRnG+cawOJiflsAM+7U= github.com/libp2p/go-yamux/v2 v2.2.0/go.mod h1:3So6P6TV6r75R9jiBpiIKgU/66lOarCZjqROGxzPpPQ= github.com/libp2p/go-yamux/v3 v3.0.1/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= github.com/libp2p/go-yamux/v3 v3.0.2/go.mod h1:s2LsDhHbh+RfCsQoICSYt58U2f8ijtPANFD8BmE74Bo= github.com/libp2p/go-yamux/v3 v3.1.1/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= +github.com/libp2p/go-yamux/v3 v3.1.2/go.mod h1:jeLEQgLXqE2YqX1ilAClIfCMDY+0uXQUKmmb/qp0gT4= +github.com/libp2p/go-yamux/v4 v4.0.0/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/libp2p/go-yamux/v4 v4.0.1 h1:FfDR4S1wj6Bw2Pqbc8Uz7pCxeRBPbwsBbEdfwiCypkQ= github.com/libp2p/go-yamux/v4 v4.0.1/go.mod h1:NWjl8ZTLOGlozrXSOZ/HlfG++39iKNnM5wwmtQP1YB4= github.com/libp2p/zeroconf/v2 v2.1.1/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= +github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= github.com/lucas-clemente/quic-go v0.27.0/go.mod h1:AzgQoPda7N+3IqMMMkywBKggIFo2KT6pfnlrQ2QieeI= +github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= +github.com/lucas-clemente/quic-go v0.29.1/go.mod h1:CTcNfLYJS2UuRNB+zcNlgvkjBhxX6Hm3WUxxAQx2mgE= github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG+X8f+meRWHuqUOewBOhvHkl7M76pdGEansxW4= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= @@ -1494,10 +1668,15 @@ github.com/marten-seemann/qtls-go1-16 v0.1.4/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZE github.com/marten-seemann/qtls-go1-16 v0.1.5/go.mod h1:gNpI2Ol+lRS3WwSOtIUUtRwZEQMXjYK+dQSBFbethAk= github.com/marten-seemann/qtls-go1-17 v0.1.0/go.mod h1:fz4HIxByo+LlWcreM4CZOYNuz3taBQ8rN2X6FqvaWo8= github.com/marten-seemann/qtls-go1-17 v0.1.1/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= +github.com/marten-seemann/qtls-go1-17 v0.1.2/go.mod h1:C2ekUKcDdz9SDWxec1N/MvcXBpaX9l3Nx67XaR84L5s= github.com/marten-seemann/qtls-go1-18 v0.1.0-beta.1/go.mod h1:PUhIQk19LoFt2174H4+an8TYvWOGjb/hHwphBeaDHwI= github.com/marten-seemann/qtls-go1-18 v0.1.1/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-18 v0.1.2/go.mod h1:mJttiymBAByA49mhlNZZGrH5u1uXYZJ+RW28Py7f4m4= +github.com/marten-seemann/qtls-go1-19 v0.1.0-beta.1/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= +github.com/marten-seemann/qtls-go1-19 v0.1.0/go.mod h1:5HTDWtVudo/WFsHKRNuOhWlbdjrfs5JHrYb0wIJqGpI= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd h1:br0buuQ854V8u83wA0rVZ8ttrq5CpaPZdvrK0LP2lOk= github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd/go.mod h1:QuCEs1Nt24+FYQEqAAncTDPJIuGs+LxK1MCiFL25pMU= +github.com/marten-seemann/webtransport-go v0.1.1/go.mod h1:kBEh5+RSvOA4troP1vyOVBWK4MIMzDICXVrvCPrYcrM= github.com/matryer/moq v0.0.0-20190312154309-6cfb0558e1bd/go.mod h1:9ELz6aaclSIGnZBoaSLZ3NAl1VTufbOrXBPvtcy6WiQ= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= @@ -1517,8 +1696,10 @@ github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= +github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.4/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -1535,9 +1716,11 @@ github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00v github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/miekg/dns v1.1.28/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= +github.com/miekg/dns v1.1.31/go.mod h1:KNUDUusw/aVsxyTYZM1oqvCicbwhgbNgztCETuNZ7xM= github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJysuI= github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= +github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= @@ -1615,6 +1798,9 @@ github.com/multiformats/go-multiaddr v0.3.3/go.mod h1:lCKNGP1EQ1eZ35Za2wlqnabm9x github.com/multiformats/go-multiaddr v0.4.0/go.mod h1:YcpyLH8ZPudLxQlemYBPhSm0/oCXAT8Z4mzFpyoPyRc= github.com/multiformats/go-multiaddr v0.4.1/go.mod h1:3afI9HfVW8csiF8UZqtpYRiDyew8pRX7qLIGHu9FLuM= github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar+bR+gh4zgbfr3SNug= +github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= +github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= +github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= @@ -1635,11 +1821,18 @@ github.com/multiformats/go-multiaddr-net v0.1.5/go.mod h1:ilNnaM9HbmVFqsb/qcNysj github.com/multiformats/go-multiaddr-net v0.2.0/go.mod h1:gGdH3UXny6U3cKKYCvpXI5rnK7YaOIEOPVDI9tsJbEA= github.com/multiformats/go-multibase v0.0.1/go.mod h1:bja2MqRZ3ggyXtZSEDKpl0uO/gviWFaSteVbWT51qgs= github.com/multiformats/go-multibase v0.0.3/go.mod h1:5+1R4eQrT3PkYZ24C3W2Ue2tPwIdYQD509ZjSb5y9Oc= +github.com/multiformats/go-multibase v0.1.1/go.mod h1:ZEjHE+IsUrgp5mhlEAYjMtZwK1k4haNkcaPg9aoe1a8= github.com/multiformats/go-multibase v0.2.0 h1:isdYCVLvksgWlMW9OZRYJEa9pZETFivncJHmHnnd87g= github.com/multiformats/go-multibase v0.2.0/go.mod h1:bFBZX4lKCA/2lyOFSAoKH5SS6oPyjtnzK/XTFDPkNuk= github.com/multiformats/go-multicodec v0.3.0/go.mod h1:qGGaQmioCDh+TeFOnxrbU0DaIPw8yFgAZgFG0V7p1qQ= github.com/multiformats/go-multicodec v0.3.1-0.20210902112759-1539a079fd61/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.3.1-0.20211210143421-a526f306ed2c/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= github.com/multiformats/go-multicodec v0.4.1/go.mod h1:1Hj/eHRaVWSXiSNNfcEPcwZleTmdNP81xlxDLnWU9GQ= +github.com/multiformats/go-multicodec v0.5.0/go.mod h1:DiY2HFaEp5EhEXb/iYzVAunmyX/aSFMxq2KMKfWEues= +github.com/multiformats/go-multicodec v0.6.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.7.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.8.0/go.mod h1:GUC8upxSBE4oG+q3kWZRw/+6yC1BqO550bjhWsJbZlw= +github.com/multiformats/go-multicodec v0.8.1/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multicodec v0.9.0 h1:pb/dlPnzee/Sxv/j4PmkDRxCOi3hXTz3IbPKOXWJkmg= github.com/multiformats/go-multicodec v0.9.0/go.mod h1:L3QTQvMIaVBkXOXXtVmYE+LI16i14xuaojr/H7Ai54k= github.com/multiformats/go-multihash v0.0.1/go.mod h1:w/5tugSrLEbWqlcgJabL3oHFKTwfvkofsjW2Qa1ct4U= @@ -1650,14 +1843,19 @@ github.com/multiformats/go-multihash v0.0.13/go.mod h1:VdAWLKTwram9oKAatUcLxBNUj github.com/multiformats/go-multihash v0.0.14/go.mod h1:VdAWLKTwram9oKAatUcLxBNUjdtcVwxObEQBtRfuyjc= github.com/multiformats/go-multihash v0.0.15/go.mod h1:D6aZrWNLFTV/ynMpKsNtB40mJzmCl4jb1alC0OvHiHg= github.com/multiformats/go-multihash v0.1.0/go.mod h1:RJlXsxt6vHGaia+S8We0ErjhojtKzPP2AH4+kYM7k84= +github.com/multiformats/go-multihash v0.2.0/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= +github.com/multiformats/go-multihash v0.2.1/go.mod h1:WxoMcYG85AZVQUyRyo9s4wULvW5qrI9vb2Lt6evduFc= github.com/multiformats/go-multihash v0.2.3 h1:7Lyc8XfX/IY2jWb/gI7JP+o7JEq9hOa7BFvVU9RSh+U= github.com/multiformats/go-multihash v0.2.3/go.mod h1:dXgKXCXjBzdscBLk9JkjINiEsCKRVch90MdaGiKsvSM= github.com/multiformats/go-multistream v0.1.0/go.mod h1:fJTiDfXJVmItycydCnNx4+wSzZ5NwG2FEVAI30fiovg= github.com/multiformats/go-multistream v0.1.1/go.mod h1:KmHZ40hzVxiaiwlj3MEbYgK9JFk2/9UktWZAF54Du38= +github.com/multiformats/go-multistream v0.2.0/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.1/go.mod h1:5GZPQZbkWOLOn3J2y4Y99vVW7vOfsAflxARk3x14o6k= github.com/multiformats/go-multistream v0.2.2/go.mod h1:UIcnm7Zuo8HKG+HkWgfQsGL+/MIEhyTqbODbIUwSXKs= github.com/multiformats/go-multistream v0.3.0/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-multistream v0.3.1/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= +github.com/multiformats/go-multistream v0.4.0/go.mod h1:BS6ZSYcA4NwYEaIMeCtpJydp2Dc+fNRA6uJMSu/m8+4= github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -1703,6 +1901,13 @@ github.com/onsi/ginkgo v1.16.2/go.mod h1:CObGmKUOKaSC0RjmoAK7tKyn4Azo5P2IWuoMnvw github.com/onsi/ginkgo v1.16.4/go.mod h1:dX+/inL/fNMqNlz0e9LfyB9TswhZpCVdJM/Z6Vvnwo0= github.com/onsi/ginkgo v1.16.5 h1:8xi0RTUf59SOSfEtZMvwTvXYMzG4gV23XVHOZiXNtnE= github.com/onsi/ginkgo v1.16.5/go.mod h1:+E8gABHa3K6zRBolWtd+ROzc/U5bkGt0FwiG042wbpU= +github.com/onsi/ginkgo/v2 v2.1.3/go.mod h1:vw5CSIxN1JObi/U8gcbwft7ZxR2dgaR70JSE3/PpL4c= +github.com/onsi/ginkgo/v2 v2.1.4/go.mod h1:um6tUpWM/cxCK3/FK8BXqEiUMUwRgSM4JXG47RKZmLU= +github.com/onsi/ginkgo/v2 v2.1.6/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7AG4VIk= +github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= +github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= +github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= @@ -1712,6 +1917,12 @@ github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7J github.com/onsi/gomega v1.9.0/go.mod h1:Ho0h+IUsWyvy1OpqCwxlQ/21gkhVunqlU8fDGcoTdcA= github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.17.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= +github.com/onsi/gomega v1.19.0/go.mod h1:LY+I3pBVzYsTBU1AnDwOSxaYi9WoWiqgwooUqq9yPro= +github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeREyVo= +github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= +github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= +github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= @@ -1765,6 +1976,7 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -1775,6 +1987,7 @@ github.com/pkg/term v0.0.0-20180730021639-bffc007b7fd5/go.mod h1:eCbImbZ95eXtAUI github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/polydawn/refmt v0.0.0-20190221155625-df39d6c2d992/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= +github.com/polydawn/refmt v0.0.0-20190408063855-01bf1e26dd14/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20190807091052-3d65705ee9f1/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.0.0-20201211092308-30ac6d18308e/go.mod h1:uIp+gprXxxrWSjjklXD+mN4wed/tMfjMMmN/9+JsA9o= github.com/polydawn/refmt v0.89.0 h1:ADJTApkvkeBZsN0tBTx8QjpD9JkmxbKp0cxfr9qszm4= @@ -1791,6 +2004,8 @@ github.com/prometheus/client_golang v1.9.0/go.mod h1:FqZLKOZnGdFAhOK4nqGHa7D66Id github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= +github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= +github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= @@ -1799,6 +2014,7 @@ github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1: github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= @@ -1814,6 +2030,7 @@ github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16 github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= +github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -1826,6 +2043,7 @@ github.com/prometheus/procfs v0.2.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= +github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= @@ -1837,10 +2055,15 @@ github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKA github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= +github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= +github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= +github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= github.com/quic-go/quic-go v0.37.6 h1:2IIUmQzT5YNxAiaPGjs++Z4hGOtIR0q79uS5qE9ccfY= github.com/quic-go/quic-go v0.37.6/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= +github.com/quic-go/webtransport-go v0.5.1/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= @@ -1880,6 +2103,7 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= +github.com/samber/lo v1.36.0/go.mod h1:HLeWcJRRyLKp3+/XBJvOrerCQn9mhdKMHyd7IRlgeQ8= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/sasha-s/go-deadlock v0.3.1 h1:sqv7fDNShgjcaxkO0JNcOAlr8B9+cV5Ey/OB71efZx0= github.com/sasha-s/go-deadlock v0.3.1/go.mod h1:F73l+cr82YSh10GxyRI6qZiCgK64VaZjwesgfQ1/iLM= @@ -1922,9 +2146,11 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v0.0.0-20190222223459-a17d461953aa/go.mod h1:2RVY1rIf+2J2o/IM9+vPq9RzmHDSseB7FoXiSNIUsoU= +github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/goconvey v1.7.2/go.mod h1:Vw0tHAZW6lzCRk3xgdin6fKYcG+G3Pg9vgXWeJpQFMM= @@ -1981,6 +2207,7 @@ github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/ github.com/stretchr/testify v1.7.1/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO+kdMU+MU= github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= +github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= @@ -1994,6 +2221,7 @@ github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2l github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= github.com/tendermint/tm-db v0.6.7/go.mod h1:byQDzFkZV1syXr/ReXS808NxA2xvyuuVgXOJ/088L6I= +github.com/thoas/go-funk v0.9.1/go.mod h1:+IWnUfUmFO1+WVYQWQtIJHeRRdaIyyYglZN7xzUPe4Q= github.com/tidwall/btree v1.5.0 h1:iV0yVY/frd7r6qGBXfEYs7DH0gTDgrKTrDjS7xt/IyQ= github.com/tidwall/btree v1.5.0/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= github.com/tidwall/gjson v1.6.0/go.mod h1:P256ACg0Mn+j1RXIDXoss50DeIABTYK1PULOJHhxOls= @@ -2011,6 +2239,7 @@ github.com/tidwall/pretty v1.2.1/go.mod h1:ITEVvHYasfjBbM0u2Pg8T2nJnzm8xPwvNhhso github.com/tidwall/sjson v1.2.4/go.mod h1:098SZ494YoMWPmMO6ct4dcFnqxwj9r/gF0Etp19pSNM= github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= +github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= @@ -2024,6 +2253,7 @@ github.com/tyler-smith/go-bip39 v1.0.1-0.20181017060643-dbb3b84ba2ef/go.mod h1:s github.com/tyler-smith/go-bip39 v1.0.2/go.mod h1:sJ5fKU0s6JVwZjjcUEX2zFOnvq0ASQ2K9Zr6cf67kNs= github.com/tyler-smith/go-bip39 v1.1.0 h1:5eUemwrMargf3BSLRRCalXT93Ns6pQJIjYQN2nyfOP8= github.com/tyler-smith/go-bip39 v1.1.0/go.mod h1:gUYDtqQw1JS3ZJ8UWVcGTGqqr6YIN3CWg+kkNaLt55U= +github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRWicNw3S7XpJ8sK/smGwU9WcSVU3dy9qahYBM= github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= @@ -2046,25 +2276,34 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/vmihailenco/msgpack/v5 v5.3.5/go.mod h1:7xyJ9e+0+9SaZT0Wt1RGleJXzli6Q/V5KbhBonMG9jc= github.com/vmihailenco/tagparser/v2 v2.0.0/go.mod h1:Wri+At7QHww0WTrCBeu4J6bNtoV6mEfg5OIWRZA9qds= +github.com/wangjia184/sortedset v0.0.0-20160527075905-f5d03557ba30/go.mod h1:YkocrP2K2tcw938x9gCOmT5G5eCD6jsTz0SZuyAqwIE= github.com/warpfork/go-testmark v0.3.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= -github.com/warpfork/go-testmark v0.11.0 h1:J6LnV8KpceDvo7spaNU4+DauH2n1x+6RaO2rJrmpQ9U= +github.com/warpfork/go-testmark v0.10.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= github.com/warpfork/go-testmark v0.11.0/go.mod h1:jhEf8FVxd+F17juRubpmut64NEG6I2rgkUhlcqqXwE0= +github.com/warpfork/go-testmark v0.12.1 h1:rMgCpJfwy1sJ50x0M0NgyphxYYPMOODIJHhsXyEHU0s= +github.com/warpfork/go-testmark v0.12.1/go.mod h1:kHwy7wfvGSPh1rQJYKayD4AbtNaeyZdcGi9tNJTaa5Y= github.com/warpfork/go-wish v0.0.0-20180510122957-5ad1f5abf436/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/warpfork/go-wish v0.0.0-20190328234359-8b3e70f8e830/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20200122115046-b9ea61034e4a/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0 h1:GDDkbFiaK8jsSDJfjId/PEGEShv6ugrt4kYsC5UIDaQ= github.com/warpfork/go-wish v0.0.0-20220906213052-39a1cc7a02d0/go.mod h1:x6AKhvSSexNrVSrViXSHUEbICjmGXhtgABaHIySUSGw= +github.com/whyrusleeping/base32 v0.0.0-20170828182744-c30ac30633cc/go.mod h1:r45hJU7yEoA81k6MWNhpMj/kms0n14dkzkxYHoB96UM= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 h1:5HZfQkwe0mIfyDmc1Em5GqlNRzcdtlv4HTNmdpt7XH0= github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11/go.mod h1:Wlo/SzPmxVp6vXpGt/zaXhHH0fn4IxgqZc82aKg6bpQ= github.com/whyrusleeping/cbor-gen v0.0.0-20200123233031-1cdf64d27158/go.mod h1:Xj/M2wWU+QdTdRbu/L/1dIZY8/Wb2K9pAhtroQuxJJI= -github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa h1:EyA027ZAkuaCLoxVX4r1TZMPy1d31fM6hbfQ4OU4I5o= +github.com/whyrusleeping/cbor-gen v0.0.0-20221220214510-0333c149dec0/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/cbor-gen v0.0.0-20230126041949-52956bd4c9aa/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= +github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 h1:yVYDLoN2gmB3OdBXFW8e1UwgVbmCvNlnAKhvHPaNARI= +github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25/go.mod h1:fgkXqYy7bV2cFeIEOkVTZS/WjXARfBqSH6Q2qHL33hQ= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f h1:jQa4QT2UP9WYv2nzyawpKMOCl+Z/jW7djv2/J50lj9E= github.com/whyrusleeping/chunker v0.0.0-20181014151217-fe64bd25879f/go.mod h1:p9UJB6dDgdPgMJZs7UjUOdulKyRr9fqkS+6JKAInPy8= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 h1:EKhdznlJHPMoKr0XTrX+IlJs1LH3lyx2nfr1dOlZ79k= github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1/go.mod h1:8UvriyWtv5Q5EOgjHaSseUEdkQfvwFv1I/In/O2M9gc= github.com/whyrusleeping/go-logging v0.0.0-20170515211332-0457bb6b88fc/go.mod h1:bopw91TMyo8J3tvftk8xmU2kPmlrt4nScJQZU2hE5EM= github.com/whyrusleeping/go-logging v0.0.1/go.mod h1:lDPYj54zutzG1XYfHAhcc7oNXEburHQBn+Iqd4yS4vE= +github.com/whyrusleeping/go-notifier v0.0.0-20170827234753-097c5d47330f/go.mod h1:cZNvX9cFybI01GriPRMXDtczuvUhgbcYr9iCGaNlRv8= github.com/whyrusleeping/mafmt v1.2.8/go.mod h1:faQJFPbLSxzD9xpA02ttW/tS9vZykNvXwGvqIpk20FA= +github.com/whyrusleeping/mdns v0.0.0-20180901202407-ef14215e6b30/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/mdns v0.0.0-20190826153040-b9b60ed33aa9/go.mod h1:j4l84WPFclQPj320J9gp0XwNKBb3U0zt5CBqjPp22G4= github.com/whyrusleeping/multiaddr-filter v0.0.0-20160516205228-e903e4adabd7/go.mod h1:X2c0RVCI1eSUFI8eLcY3c0423ykwiUdxLJtkDvruhjI= github.com/willf/bitset v1.1.3/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= @@ -2112,6 +2351,8 @@ go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 h1:EbmAUG9hEAMXyfWEa go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0/go.mod h1:rD9feqRYP24P14t5kmhNMqsqm1jvKmpx2H2rKVw52V8= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= +go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= +go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= @@ -2135,6 +2376,8 @@ go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh4 go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= +go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= +go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -2146,10 +2389,13 @@ go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.6.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= go.uber.org/atomic v1.9.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= +go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= +go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= +go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ= go.uber.org/fx v1.20.0/go.mod h1:qCUj0btiR3/JnanEr1TYEePfSw6o/4qYJscgvzQ5Ub0= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= @@ -2165,6 +2411,7 @@ go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKY go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= go.uber.org/multierr v1.7.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= go.uber.org/multierr v1.8.0/go.mod h1:7EAYxJLBy9rStEaz58O2t4Uvip6FSURkq8/ppBp95ak= +go.uber.org/multierr v1.9.0/go.mod h1:X2jQV1h+kxSjClGpnseKVIxpmcjrj7MNnI0bnlfKTVQ= go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/tools v0.0.0-20190618225709-2cfd321de3ee/go.mod h1:vJERXedbb3MVM5f9Ejo0C68/HhF8uaILCdgjnY+goOA= @@ -2176,6 +2423,9 @@ go.uber.org/zap v1.15.0/go.mod h1:Mb2vm2krFEG5DV0W9qcHBYFtp/Wku1cvYaqPsS/WYfc= go.uber.org/zap v1.16.0/go.mod h1:MA8QOfq0BHJwdXa996Y4dYkAqRKB8/1K1QMMZVaNZjQ= go.uber.org/zap v1.19.1/go.mod h1:j3DNczoxDZroyBnOT1L/Q79cfUMGZxlv/9dzN7SM1rI= go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= +go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= +go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= +go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= @@ -2192,6 +2442,7 @@ golang.org/x/crypto v0.0.0-20190313024323-a1f597ede03a/go.mod h1:djNgcEr1/C05ACk golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190510104115-cbcb75029529/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190513172903-22d7a77e9e5f/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190618222545-ea8f1a30c443/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= @@ -2202,6 +2453,7 @@ golang.org/x/crypto v0.0.0-20191206172530-e9b2fee46413/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20200109152110-61a87790db17/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200115085410-6d4e4cb37c7d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200221231518-2aa609cf4a9d/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= +golang.org/x/crypto v0.0.0-20200423211502-4bdfaf469ed5/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200510223506-06a226fb4e37/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200602180216-279210d13fed/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= @@ -2217,7 +2469,14 @@ golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5y golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.0.0-20220525230936-793ad666bf5e/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= +golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= +golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= +golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2236,7 +2495,12 @@ golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EH golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= golang.org/x/exp v0.0.0-20200331195152-e8c3332aa8e5/go.mod h1:4M0jN8W1tt0AVLNr8HDosyJCDCDuyL9N9+3m7wDWgKw= golang.org/x/exp v0.0.0-20210615023648-acb5c1269671/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= -golang.org/x/exp v0.0.0-20210714144626-1041f73d31d8/go.mod h1:DVyR6MI7P4kEQgvZJSj1fQGrWIi2RzIrfYWycwheUAc= +golang.org/x/exp v0.0.0-20220303212507-bbda1eaf7a17/go.mod h1:lgLbSvA5ygNOMpwM/9anMpWVlVJ7Z+cHWq/eFuinpGE= +golang.org/x/exp v0.0.0-20220722155223-a9213eeb770e/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= +golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE= +golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= +golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= @@ -2268,8 +2532,12 @@ golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.5.1/go.mod h1:5OXOZSfqPIIbmVBIIKWRFfZjPR0E5r58TLhUjH0a2Ro= +golang.org/x/mod v0.6.0-dev.0.20211013180041-c96bc1413d57/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2K1YbXdKPJLVgG5VJdoTa1poYQBtP1AY= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= +golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2292,7 +2560,9 @@ golang.org/x/net v0.0.0-20190313220215-9f648a60d977/go.mod h1:t9HGtf8HONx5eT2rtn golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190501004415-9ce7a6920f09/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190503192946-f4e77d36d62c/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190522155817-f3200d17e092/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190611141213-3f473d35a33a/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190628185345-da137c7871d7/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -2349,9 +2619,18 @@ golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.0.0-20220617184016-355a448f1bc9/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220624214902-1bab6f366d9e/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= +golang.org/x/net v0.0.0-20220812174116-3211cb980234/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20220909164309-bea034e7d591/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220920183852-bf014ff85ad5/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= +golang.org/x/net v0.0.0-20220923203811-8be639271d50/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.0.0-20221014081412-f15817d10f9b/go.mod h1:YDH+HFinaLZZlnHAfSS6ZXJJ9M9t4Dl22yv3iI2vPwk= golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= +golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= +golang.org/x/net v0.3.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= +golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= +golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2398,6 +2677,7 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -2416,6 +2696,7 @@ golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20190219092855-153ac476189d/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190302025703-b6889370fb10/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190312061237-fead79001313/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190316082340-a2f829d7f35f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190405154228-4b34438f7a67/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2423,8 +2704,10 @@ golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190502145724-3ef323f4f1fd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190507160741-ecd444e8653b/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190524122548-abf6ff778158/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190526052359-791d8a0f4d09/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190610200419-93c9922d18ae/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190624142023-c5567b49c5d0/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190626221950-04f50cda93cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190726091711-fc99dfbffb4e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2503,6 +2786,7 @@ golang.org/x/sys v0.0.0-20210809222454-d867a43fc93e/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210816183151-1e6c022a8912/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2514,20 +2798,29 @@ golang.org/x/sys v0.0.0-20220128215802-99c3d69c2c27/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220209214540-3681064d5158/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220227234510-4e6760a101f9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220310020820-b874c991c1a5/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220328115105-d36c6a25d886/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220405210540-1e041c57c461/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220412211240-33da011f77ad/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220422013727-9388b58f7150/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220429233432-b5fbb4746d32/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220502124256-b6088ccd6cba/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220615213510-4f61da869c0c/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220624220833-87e55d714810/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220704084225-05e143d24a9e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220728004956-3c1f35247d10/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220811171246-fbc7d0a398ab/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220908164124-27713097b956/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220919091848-fb04ddd9f9c8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20221010170243-090e33056c14/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= @@ -2536,6 +2829,10 @@ golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXR golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= +golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= +golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= +golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= +golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2549,6 +2846,9 @@ golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= @@ -2622,6 +2922,7 @@ golang.org/x/tools v0.0.0-20201224043029-2b0845dc783e/go.mod h1:emZCQorbCU4vsT4f golang.org/x/tools v0.0.0-20210105154028-b0ab187a4818/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210106214847-113979e3529a/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.0.0-20210108195828-e2f9c7f1fc8e/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= +golang.org/x/tools v0.0.0-20210112230658-8b4aab62c064/go.mod h1:emZCQorbCU4vsT4fOWvOPXz4eW1wZW4PmDk9uLelYpA= golang.org/x/tools v0.1.0/go.mod h1:xkSsbof2nBLbhDlRMhhhyNLN/zl3eTqcnHD5viDpcZ0= golang.org/x/tools v0.1.1/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= @@ -2629,8 +2930,11 @@ golang.org/x/tools v0.1.3/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.4/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.5/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= golang.org/x/tools v0.1.6-0.20210726203631-07bc1bf47fb2/go.mod h1:o0xws9oXOQQZyjljx8fwUC0k7L1pTE6eaCbjGeHmOkk= +golang.org/x/tools v0.1.8-0.20211029000441-d6a9af8af023/go.mod h1:nABZi5QlRsZVlzPpHl034qft6wpY4eDcsTt5AaioBiU= golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E= golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= +golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= +golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -2902,6 +3206,7 @@ gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLks gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20190902080502-41f04d3bba15/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/check.v1 v1.0.0-20200227125254-8fa46927fb4f/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20200902074654-038fdea0a05b/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c h1:Hei/4ADfdWqJk1ZMxUNpqntNwaWcugrBjAiHlqqRiVk= gopkg.in/check.v1 v1.0.0-20201130134442-10cb98267c6c/go.mod h1:JHkPIbrfpd72SG/EVd6muEfDQjcINNoR0C8j2r3qZ4Q= @@ -2936,6 +3241,7 @@ gopkg.in/yaml.v2 v2.3.0/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.4.0 h1:D8xgwECY7CYvx+Y2n4sBz93Jn9JRvxdiyyo8CTfuKaY= gopkg.in/yaml.v2 v2.4.0/go.mod h1:RDklbk79AGWmwhnvt/jBztapEOGDOx6ZbXqjP6csGnQ= gopkg.in/yaml.v3 v3.0.0-20200313102051-9f266ea9e77c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= +gopkg.in/yaml.v3 v3.0.0-20200605160147-a5ece683394c/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.0-20210107192922-496545a6307b/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gopkg.in/yaml.v3 v3.0.1 h1:fxVm/GzAzEWqLHuvctI91KS9hhNmmWOoWu0XTYJS7CA= gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= From b6efbc91eaaa16041a5a32c8d306c52803c13bca Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 23 Aug 2023 14:28:58 +0200 Subject: [PATCH 144/388] nodebuilder/core: Set default ports for RPC and GRPC (#2596) Now that we short-circuit on the IP address being empty, it's fine to set defaults for RPC and GRPC ports This fixes swamp tests as well --- nodebuilder/core/config.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder/core/config.go b/nodebuilder/core/config.go index d0455be86c..4affcd3087 100644 --- a/nodebuilder/core/config.go +++ b/nodebuilder/core/config.go @@ -19,8 +19,8 @@ type Config struct { func DefaultConfig() Config { return Config{ IP: "", - RPCPort: "", - GRPCPort: "", + RPCPort: "26657", + GRPCPort: "9090", } } From bbe970f8b2151ce1ad2a4bc3fefb5319fbf0ca58 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 23 Aug 2023 18:15:55 +0300 Subject: [PATCH 145/388] feat(cmd/node): add logs cmd to dynamically change the log level (#2601) ## Overview Added cmds that allow to change log level during runtime ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- cmd/celestia/logs.go | 48 ++++++++++++++++++++++++++++++++++++++++++++ cmd/celestia/rpc.go | 1 + cmd/flags_misc.go | 16 +++++++-------- 3 files changed, 57 insertions(+), 8 deletions(-) create mode 100644 cmd/celestia/logs.go diff --git a/cmd/celestia/logs.go b/cmd/celestia/logs.go new file mode 100644 index 0000000000..ac302ff6dd --- /dev/null +++ b/cmd/celestia/logs.go @@ -0,0 +1,48 @@ +package main + +import ( + "fmt" + "strings" + + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-node/cmd" +) + +var logCmd = &cobra.Command{ + Use: cmd.LogLevelFlag, + Args: cobra.ExactArgs(1), + Short: "Allows to set log level for all modules to " + + "`DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL and their lower-case forms`", + + RunE: func(c *cobra.Command, args []string) error { + client, err := rpcClient(c.Context()) + if err != nil { + return err + } + return client.Node.LogLevelSet(c.Context(), "*", args[0]) + }, +} + +var logModuleCmd = &cobra.Command{ + Use: cmd.LogLevelModuleFlag, + Args: cobra.MinimumNArgs(1), + Short: "Allows to set log level for a particular module in format :", + RunE: func(c *cobra.Command, args []string) error { + client, err := rpcClient(c.Context()) + if err != nil { + return err + } + for _, ll := range args { + params := strings.Split(ll, ":") + if len(params) != 2 { + return fmt.Errorf("cmd: %s arg must be in form :,"+ + "e.g. pubsub:debug", cmd.LogLevelModuleFlag) + } + if err = client.Node.LogLevelSet(c.Context(), params[0], params[1]); err != nil { + return err + } + } + return nil + }, +} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index dd50f07fc2..b8724fa789 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -60,6 +60,7 @@ func init() { false, "Print JSON-RPC request along with the response", ) + rpcCmd.AddCommand(logCmd, logModuleCmd) rpcCmd.AddCommand(blobCmd) rootCmd.AddCommand(rpcCmd) } diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index 4a11977b86..cd539bde4c 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -20,8 +20,8 @@ import ( ) var ( - logLevelFlag = "log.level" - logLevelModuleFlag = "log.level.module" + LogLevelFlag = "log.level" + LogLevelModuleFlag = "log.level.module" pprofFlag = "pprof" tracingFlag = "tracing" tracingEndpointFlag = "tracing.endpoint" @@ -40,14 +40,14 @@ func MiscFlags() *flag.FlagSet { flags := &flag.FlagSet{} flags.String( - logLevelFlag, + LogLevelFlag, "INFO", `DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL and their lower-case forms`, ) flags.StringSlice( - logLevelModuleFlag, + LogLevelModuleFlag, nil, ":, e.g. pubsub:debug", ) @@ -123,24 +123,24 @@ and their lower-case forms`, // ParseMiscFlags parses miscellaneous flags from the given cmd and applies values to Env. func ParseMiscFlags(ctx context.Context, cmd *cobra.Command) (context.Context, error) { - logLevel := cmd.Flag(logLevelFlag).Value.String() + logLevel := cmd.Flag(LogLevelFlag).Value.String() if logLevel != "" { level, err := logging.LevelFromString(logLevel) if err != nil { - return ctx, fmt.Errorf("cmd: while parsing '%s': %w", logLevelFlag, err) + return ctx, fmt.Errorf("cmd: while parsing '%s': %w", LogLevelFlag, err) } logs.SetAllLoggers(level) } - logModules, err := cmd.Flags().GetStringSlice(logLevelModuleFlag) + logModules, err := cmd.Flags().GetStringSlice(LogLevelModuleFlag) if err != nil { panic(err) } for _, ll := range logModules { params := strings.Split(ll, ":") if len(params) != 2 { - return ctx, fmt.Errorf("cmd: %s arg must be in form :, e.g. pubsub:debug", logLevelModuleFlag) + return ctx, fmt.Errorf("cmd: %s arg must be in form :, e.g. pubsub:debug", LogLevelModuleFlag) } err := logging.SetLogLevel(params[0], params[1]) From 440e232999331adf1d1dc034e5ceb8b14e9db5c7 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 24 Aug 2023 20:15:57 +0800 Subject: [PATCH 146/388] chore(deps): bump dagstore (#2602) Updating dagstore should fix the issue with panic of mmap on bitswap reads. --- go.mod | 4 ++-- go.sum | 11 ++++++----- 2 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index cf2329de01..0d26924835 100644 --- a/go.mod +++ b/go.mod @@ -212,7 +212,7 @@ require ( github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect - github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 // indirect + github.com/ipld/go-car/v2 v2.11.0 // indirect github.com/ipld/go-codec-dagpb v1.6.0 // indirect github.com/ipld/go-ipld-prime v0.21.0 // indirect github.com/jackpal/go-nat-pmp v1.0.2 // indirect @@ -337,7 +337,7 @@ require ( replace ( github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 - github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230823095320-1d66549bbeb3 + github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 diff --git a/go.sum b/go.sum index 90c6e89ec9..6adeef7f7c 100644 --- a/go.sum +++ b/go.sum @@ -362,8 +362,8 @@ github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHV github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 h1:PckXGxLJjXv97VO3xS8NPHN5oO83X5nvJLbc/4s8jUM= github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14/go.mod h1:70Go8qNy7YAb1PUcHCChRHNX2ke7c9jgUIEklUX+Mac= -github.com/celestiaorg/dagstore v0.0.0-20230823095320-1d66549bbeb3 h1:2/UcuBL0HQ8Y7QXhBjaNMI2LT8or1F67wArTJIs3Vos= -github.com/celestiaorg/dagstore v0.0.0-20230823095320-1d66549bbeb3/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= +github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= +github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= @@ -1213,8 +1213,9 @@ github.com/ipfs/go-unixfsnode v1.1.2/go.mod h1:5dcE2x03pyjHk4JjamXmunTMzz+VUtqvP github.com/ipfs/go-unixfsnode v1.4.0/go.mod h1:qc7YFFZ8tABc58p62HnIYbUMwj9chhUuFWmxSokfePo= github.com/ipfs/go-unixfsnode v1.5.1/go.mod h1:ed79DaG9IEuZITJVQn4U6MZDftv6I3ygUBLPfhEbHvk= github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE0LvBqHY+ks= -github.com/ipfs/go-unixfsnode v1.7.1 h1:RRxO2b6CSr5UQ/kxnGzaChTjp5LWTdf3Y4n8ANZgB/s= github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= +github.com/ipfs/go-unixfsnode v1.7.4/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/interface-go-ipfs-core v0.9.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= @@ -1224,8 +1225,8 @@ github.com/ipld/go-car/v2 v2.1.1/go.mod h1:+2Yvf0Z3wzkv7NeI69i8tuZ+ft7jyjPYIWZze github.com/ipld/go-car/v2 v2.5.1/go.mod h1:jKjGOqoCj5zn6KjnabD6JbnCsMntqU2hLiU6baZVO3E= github.com/ipld/go-car/v2 v2.8.0/go.mod h1:a+BnAxUqgr7wcWxW/lI6ctyEQ2v9gjBChPytwFMp2f4= github.com/ipld/go-car/v2 v2.10.1/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= -github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33 h1:0OZwzSYWIuiKEOXd/2vm5cMcEmmGLFn+1h6lHELCm3s= -github.com/ipld/go-car/v2 v2.10.2-0.20230622090957-499d0c909d33/go.mod h1:sQEkXVM3csejlb1kCCb+vQ/pWBKX9QtvsrysMQjOgOg= +github.com/ipld/go-car/v2 v2.11.0 h1:lkAPwbbTFqbdfawgm+bfmFc8PjGC7D12VcaLXPCLNfM= +github.com/ipld/go-car/v2 v2.11.0/go.mod h1:aDszqev0zjtU8l96g4lwXHaU9bzArj56Y7eEN0q/xqA= github.com/ipld/go-codec-dagpb v1.3.0/go.mod h1:ga4JTU3abYApDC3pZ00BC2RSvC3qfBb9MSJkMLSwnhA= github.com/ipld/go-codec-dagpb v1.3.1/go.mod h1:ErNNglIi5KMur/MfFE/svtgQthzVvf+43MrzLbpcIZY= github.com/ipld/go-codec-dagpb v1.4.1/go.mod h1:XdXTO/TUD/ra9RcK/NfmwBfr1JpFxM2uRKaB9oe4LxE= From 12054379426ebe8b91b73604373ec1665477ebe8 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Thu, 24 Aug 2023 10:23:55 -0400 Subject: [PATCH 147/388] refactor: address namespace follow-ups from celestia-app v1.0.0-rc12 bump (#2597) Addresses two follow-ups from https://github.com/celestiaorg/celestia-node/pull/2581 Closes https://github.com/celestiaorg/celestia-node/issues/2561 --- share/namespace.go | 14 +++++++------- share/namespace_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+), 7 deletions(-) diff --git a/share/namespace.go b/share/namespace.go index e7ed7dc0d6..df4ad74058 100644 --- a/share/namespace.go +++ b/share/namespace.go @@ -20,13 +20,13 @@ var ( // MinSecondaryReservedNamespace is the lowest secondary reserved namespace // reserved for protocol use. Namespaces higher than this are reserved for // protocol use. - MinSecondaryReservedNamespace = Namespace(appns.MinSecondaryReservedNamespace.Bytes()) - ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) - TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) - ReservedPaddingNamespace = Namespace(appns.PrimaryReservedPaddingNamespace.Bytes()) - TxNamespace = Namespace(appns.TxNamespace.Bytes()) - PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) - ISRNamespace = Namespace(appns.IntermediateStateRootsNamespace.Bytes()) + MinSecondaryReservedNamespace = Namespace(appns.MinSecondaryReservedNamespace.Bytes()) + ParitySharesNamespace = Namespace(appns.ParitySharesNamespace.Bytes()) + TailPaddingNamespace = Namespace(appns.TailPaddingNamespace.Bytes()) + PrimaryReservedPaddingNamespace = Namespace(appns.PrimaryReservedPaddingNamespace.Bytes()) + TxNamespace = Namespace(appns.TxNamespace.Bytes()) + PayForBlobNamespace = Namespace(appns.PayForBlobNamespace.Bytes()) + ISRNamespace = Namespace(appns.IntermediateStateRootsNamespace.Bytes()) ) // Namespace represents namespace of a Share. diff --git a/share/namespace_test.go b/share/namespace_test.go index 7c18d233c8..786441b043 100644 --- a/share/namespace_test.go +++ b/share/namespace_test.go @@ -173,6 +173,16 @@ func TestValidateForBlob(t *testing.T) { ns: append([]byte{appns.NamespaceVersionMax}, bytes.Repeat([]byte{0x0}, appns.NamespaceIDSize)...), wantErr: true, }, + { + name: "invalid blob namespace: primary reserved namespace", + ns: primaryReservedNamespace(0x10), + wantErr: true, + }, + { + name: "invalid blob namespace: secondary reserved namespace", + ns: secondaryReservedNamespace(0x10), + wantErr: true, + }, } for _, tc := range testCases { @@ -186,5 +196,21 @@ func TestValidateForBlob(t *testing.T) { assert.NoError(t, err) }) } +} + +func primaryReservedNamespace(lastByte byte) Namespace { + result := make([]byte, NamespaceSize) + result = append(result, appns.NamespaceVersionZero) + result = append(result, appns.NamespaceVersionZeroPrefix...) + result = append(result, bytes.Repeat([]byte{0x0}, appns.NamespaceVersionZeroIDSize-1)...) + result = append(result, lastByte) + return result +} +func secondaryReservedNamespace(lastByte byte) Namespace { + result := make([]byte, NamespaceSize) + result = append(result, appns.NamespaceVersionMax) + result = append(result, bytes.Repeat([]byte{0xFF}, appns.NamespaceIDSize-1)...) + result = append(result, lastByte) + return result } From 628559bc92fd7e8caaac5f264efff0fce5df2215 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Fri, 25 Aug 2023 14:21:16 +0200 Subject: [PATCH 148/388] feat(edssser): introduce EDS Store Stresser and cel-shed utility (#2482) --- cmd/cel-shed/eds_store_stress.go | 165 +++++++++++++++++++++++++++++ cmd/cel-shed/main.go | 9 +- libs/edssser/edssser.go | 172 +++++++++++++++++++++++++++++++ nodebuilder/init.go | 9 +- 4 files changed, 351 insertions(+), 4 deletions(-) create mode 100644 cmd/cel-shed/eds_store_stress.go create mode 100644 libs/edssser/edssser.go diff --git a/cmd/cel-shed/eds_store_stress.go b/cmd/cel-shed/eds_store_stress.go new file mode 100644 index 0000000000..62ea5cb772 --- /dev/null +++ b/cmd/cel-shed/eds_store_stress.go @@ -0,0 +1,165 @@ +package main + +import ( + "context" + "errors" + _ "expvar" + "fmt" + "math" + "net/http" + "os" + "time" + + logging "github.com/ipfs/go-log/v2" + "github.com/mitchellh/go-homedir" + "github.com/pyroscope-io/client/pyroscope" + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-node/libs/edssser" + "github.com/celestiaorg/celestia-node/nodebuilder" + "github.com/celestiaorg/celestia-node/nodebuilder/node" +) + +const ( + edsStorePathFlag = "path" + edsWritesFlag = "writes" + edsSizeFlag = "size" + edsDisableLogFlag = "disable-log" + edsLogStatFreqFlag = "log-stat-freq" + edsCleanupFlag = "cleanup" + edsFreshStartFlag = "fresh" + + pyroscopeEndpointFlag = "pyroscope" + putTimeoutFlag = "timeout" + badgerLogLevelFlag = "badger-log-level" +) + +func init() { + edsStoreCmd.AddCommand(edsStoreStress) + + defaultPath := "~/.edssser" + path, err := homedir.Expand(defaultPath) + if err != nil { + panic(err) + } + + pathFlagUsage := fmt.Sprintf("Directory path to use for stress test. Uses %s by default.", defaultPath) + edsStoreStress.Flags().String(edsStorePathFlag, path, pathFlagUsage) + edsStoreStress.Flags().String(pyroscopeEndpointFlag, "", + "Pyroscope address. If no address provided, pyroscope will be disabled") + edsStoreStress.Flags().Int(edsWritesFlag, math.MaxInt, "Total EDS writes to make. MaxInt by default.") + edsStoreStress.Flags().Int(edsSizeFlag, 128, "Chooses EDS size. 128 by default.") + edsStoreStress.Flags().Bool(edsDisableLogFlag, false, "Disables logging. Enabled by default.") + edsStoreStress.Flags().Int(edsLogStatFreqFlag, 10, "Write statistic logging frequency. 10 by default.") + edsStoreStress.Flags().Bool(edsCleanupFlag, false, "Cleans up the store on stop. Disabled by default.") + edsStoreStress.Flags().Bool(edsFreshStartFlag, false, "Cleanup previous state on start. Disabled by default.") + edsStoreStress.Flags().Int(putTimeoutFlag, 30, "Sets put timeout in seconds. 30 sec by default.") + edsStoreStress.Flags().String(badgerLogLevelFlag, "INFO", "Badger log level, Defaults to INFO") + + // kill redundant print + nodebuilder.PrintKeyringInfo = false +} + +var edsStoreCmd = &cobra.Command{ + Use: "eds-store [subcommand]", + Short: "Collection of eds-store related utilities", +} + +var edsStoreStress = &cobra.Command{ + Use: "stress", + Short: `Runs eds.Store stress test over default node.Store Datastore backend (e.g. Badger).`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, args []string) (err error) { + // expose expvar vars over http + go http.ListenAndServe(":9999", http.DefaultServeMux) //nolint:errcheck,gosec + + endpoint, _ := cmd.Flags().GetString(pyroscopeEndpointFlag) + if endpoint != "" { + _, err = pyroscope.Start(pyroscope.Config{ + ApplicationName: "cel-shred.stresser", + ServerAddress: endpoint, + ProfileTypes: []pyroscope.ProfileType{ + pyroscope.ProfileCPU, + pyroscope.ProfileAllocObjects, + pyroscope.ProfileAllocSpace, + pyroscope.ProfileInuseObjects, + pyroscope.ProfileInuseSpace, + }, + }) + if err != nil { + fmt.Printf("failed to launch pyroscope with addr: %s err: %s\n", endpoint, err.Error()) + } else { + fmt.Println("connected pyroscope to:", endpoint) + } + } + + path, _ := cmd.Flags().GetString(edsStorePathFlag) + fmt.Printf("using %s\n", path) + + freshStart, _ := cmd.Flags().GetBool(edsFreshStartFlag) + if freshStart { + err = os.RemoveAll(path) + if err != nil { + return err + } + } + + cleanup, _ := cmd.Flags().GetBool(edsCleanupFlag) + if cleanup { + defer func() { + err = errors.Join(err, os.RemoveAll(path)) + }() + } + + loglevel, _ := cmd.Flags().GetString(badgerLogLevelFlag) + if err = logging.SetLogLevel("badger", loglevel); err != nil { + return err + } + + disableLog, _ := cmd.Flags().GetBool(edsDisableLogFlag) + logFreq, _ := cmd.Flags().GetInt(edsLogStatFreqFlag) + edsWrites, _ := cmd.Flags().GetInt(edsWritesFlag) + edsSize, _ := cmd.Flags().GetInt(edsSizeFlag) + putTimeout, _ := cmd.Flags().GetInt(putTimeoutFlag) + + cfg := edssser.Config{ + EDSSize: edsSize, + EDSWrites: edsWrites, + EnableLog: !disableLog, + LogFilePath: path, + StatLogFreq: logFreq, + OpTimeout: time.Duration(putTimeout) * time.Second, + } + + err = nodebuilder.Init(*nodebuilder.DefaultConfig(node.Full), path, node.Full) + if err != nil { + return err + } + + nodestore, err := nodebuilder.OpenStore(path, nil) + if err != nil { + return err + } + defer func() { + err = errors.Join(err, nodestore.Close()) + }() + + datastore, err := nodestore.Datastore() + if err != nil { + return err + } + + stresser, err := edssser.NewEDSsser(path, datastore, cfg) + if err != nil { + return err + } + + stats, err := stresser.Run(cmd.Context()) + if !errors.Is(err, context.Canceled) { + return err + } + + fmt.Printf("%s", stats.Finalize()) + return nil + }, +} diff --git a/cmd/cel-shed/main.go b/cmd/cel-shed/main.go index 7982cfc1be..872bbb48a9 100644 --- a/cmd/cel-shed/main.go +++ b/cmd/cel-shed/main.go @@ -3,12 +3,14 @@ package main import ( "context" "os" + "os/signal" + "syscall" "github.com/spf13/cobra" ) func init() { - rootCmd.AddCommand(p2pCmd, headerCmd) + rootCmd.AddCommand(p2pCmd, headerCmd, edsStoreCmd) } var rootCmd = &cobra.Command{ @@ -26,5 +28,8 @@ func main() { } func run() error { - return rootCmd.ExecuteContext(context.Background()) + ctx, cancel := signal.NotifyContext(context.Background(), syscall.SIGINT, syscall.SIGTERM) + defer cancel() + + return rootCmd.ExecuteContext(ctx) } diff --git a/libs/edssser/edssser.go b/libs/edssser/edssser.go new file mode 100644 index 0000000000..fd11b47fcf --- /dev/null +++ b/libs/edssser/edssser.go @@ -0,0 +1,172 @@ +package edssser + +import ( + "context" + "errors" + "fmt" + "os" + "sync" + "testing" + "time" + + "github.com/ipfs/go-datastore" + + "github.com/celestiaorg/celestia-app/pkg/da" + + "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" +) + +type Config struct { + EDSSize int + EDSWrites int + EnableLog bool + LogFilePath string + StatLogFreq int + OpTimeout time.Duration +} + +// EDSsser stand for EDS Store Stresser. +type EDSsser struct { + config Config + datastore datastore.Batching + edsstoreMu sync.Mutex + edsstore *eds.Store + + statsFileMu sync.Mutex + statsFile *os.File +} + +func NewEDSsser(path string, datastore datastore.Batching, cfg Config) (*EDSsser, error) { + edsstore, err := eds.NewStore(path, datastore) + if err != nil { + return nil, err + } + + return &EDSsser{ + config: cfg, + datastore: datastore, + edsstore: edsstore, + }, nil +} + +func (ss *EDSsser) Run(ctx context.Context) (stats Stats, err error) { + ss.edsstoreMu.Lock() + defer ss.edsstoreMu.Unlock() + + err = ss.edsstore.Start(ctx) + if err != nil { + return stats, err + } + defer func() { + err = errors.Join(err, ss.edsstore.Stop(ctx)) + }() + + edsHashes, err := ss.edsstore.List() + if err != nil { + return stats, err + } + fmt.Printf("recovered %d EDSes\n\n", len(edsHashes)) + + t := &testing.T{} + for toWrite := ss.config.EDSWrites - len(edsHashes); ctx.Err() == nil && toWrite > 0; toWrite-- { + took, err := ss.put(ctx, t) + + stats.TotalWritten++ + stats.TotalTime += took + if took < stats.MinTime || stats.MinTime == 0 { + stats.MinTime = took + } else if took > stats.MaxTime { + stats.MaxTime = took + } + + if ss.config.EnableLog { + if stats.TotalWritten%ss.config.StatLogFreq == 0 { + stats := stats.Finalize() + fmt.Println(stats) + go func() { + err := ss.dumpStat(stats) + if err != nil { + fmt.Printf("error dumping stats: %s\n", err.Error()) + } + }() + } + if err != nil { + fmt.Printf("ERROR put: %s, took: %v, at: %v\n", err.Error(), took, time.Now()) + continue + } + if took > ss.config.OpTimeout/2 { + fmt.Println("long put", "size", ss.config.EDSSize, "took", took, "at", time.Now()) + continue + } + + fmt.Println("square written", "size", ss.config.EDSSize, "took", took, "at", time.Now()) + } + } + return stats, nil +} + +func (ss *EDSsser) dumpStat(stats Stats) (err error) { + ss.statsFileMu.Lock() + defer ss.statsFileMu.Unlock() + + ss.statsFile, err = os.Create(ss.config.LogFilePath + "/edssser_stats.txt") + if err != nil { + return err + } + + _, err = ss.statsFile.Write([]byte(stats.String())) + if err != nil { + return err + } + + return ss.statsFile.Close() +} + +type Stats struct { + TotalWritten int + TotalTime, MinTime, MaxTime, AvgTime time.Duration + // Deviation ? +} + +func (stats Stats) Finalize() Stats { + if stats.TotalTime != 0 { + stats.AvgTime = stats.TotalTime / time.Duration(stats.TotalWritten) + } + return stats +} + +func (stats Stats) String() string { + return fmt.Sprintf(` +TotalWritten %d +TotalWritingTime %v +MaxTime %s +MinTime %s +AvgTime %s +`, + stats.TotalWritten, + stats.TotalTime, + stats.MaxTime, + stats.MinTime, + stats.AvgTime, + ) +} + +func (ss *EDSsser) put(ctx context.Context, t *testing.T) (time.Duration, error) { + ctx, cancel := context.WithTimeout(ctx, ss.config.OpTimeout) + if ss.config.OpTimeout == 0 { + ctx, cancel = context.WithCancel(ctx) + } + defer cancel() + + // divide by 2 to get ODS size as expected by RandEDS + square := edstest.RandEDS(t, ss.config.EDSSize/2) + dah, err := da.NewDataAvailabilityHeader(square) + if err != nil { + return 0, err + } + + now := time.Now() + err = ss.edsstore.Put(ctx, dah.Hash(), square) + return time.Since(now), err +} diff --git a/nodebuilder/init.go b/nodebuilder/init.go index 2cabfc8abd..0593d88560 100644 --- a/nodebuilder/init.go +++ b/nodebuilder/init.go @@ -18,6 +18,9 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/state" ) +// PrintKeyringInfo whether to print keyring information during init. +var PrintKeyringInfo = true + // Init initializes the Node FileSystem Store for the given Node Type 'tp' in the directory under // 'path'. func Init(cfg Config, path string, tp node.Type) error { @@ -213,8 +216,10 @@ func generateKeys(cfg Config, ksPath string) error { if err != nil { return err } - fmt.Printf("\nNAME: %s\nADDRESS: %s\nMNEMONIC (save this somewhere safe!!!): \n%s\n\n", - keyInfo.Name, addr.String(), mn) + if PrintKeyringInfo { + fmt.Printf("\nNAME: %s\nADDRESS: %s\nMNEMONIC (save this somewhere safe!!!): \n%s\n\n", + keyInfo.Name, addr.String(), mn) + } return nil } From 4b960224e735c7a7e2036f730b6ccc06f8688a2b Mon Sep 17 00:00:00 2001 From: "[NODERS]TEAM" <94483941+nodersteam@users.noreply.github.com> Date: Fri, 25 Aug 2023 19:34:06 +0700 Subject: [PATCH 149/388] feat(share/p2p/discovery): Warning for discovery loop if haven't found wanted peers in 5 minutes (#2573) --- share/p2p/discovery/discovery.go | 33 +++++++++++++++++++++----------- 1 file changed, 22 insertions(+), 11 deletions(-) diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index af6fef1f89..c880b9b3c2 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -33,6 +33,10 @@ const ( // retryTimeout defines time interval between discovery and advertise attempts. retryTimeout = time.Second + + // logInterval defines the time interval at which a warning message will be logged + // if the desired number of nodes is not detected. + logInterval = 5 * time.Minute ) // discoveryRetryTimeout defines time interval between discovery attempts, needed for tests @@ -195,26 +199,33 @@ func (d *Discovery) Advertise(ctx context.Context) { } // discoveryLoop ensures we always have '~peerLimit' connected peers. -// It starts peer discovery per request and restarts the process until the soft limit reached. +// It initiates peer discovery upon request and restarts the process until the soft limit is reached. func (d *Discovery) discoveryLoop(ctx context.Context) { t := time.NewTicker(discoveryRetryTimeout) defer t.Stop() + + warnTicker := time.NewTicker(logInterval) + defer warnTicker.Stop() + for { - // drain all previous ticks from channel + // drain all previous ticks from the channel drainChannel(t.C) select { case <-t.C: - found := d.discover(ctx) - if !found { - // rerun discovery if amount of peers didn't reach the limit + if !d.discover(ctx) { + // rerun discovery if the number of peers hasn't reached the limit continue } - case <-ctx.Done(): - return - } - - select { - case <-d.triggerDisc: + case <-warnTicker.C: + if d.set.Size() < d.set.Limit() { + log.Warnf( + "Potentially degraded connectivity, unable to discover the desired amount of full node peers in %v. "+ + "Number of peers discovered: %d. Required: %d.", + logInterval, d.set.Size(), d.set.Limit(), + ) + } + // Do not break the loop; just continue + continue case <-ctx.Done(): return } From 196e849e08b95e363c0c33d2c9b00bca83206521 Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 28 Aug 2023 14:37:01 +0200 Subject: [PATCH 150/388] fix(eds/blockstore): Puts on EDSStore Blockstore (#2532) Closes #2424 Alternative Design Discussions: - Using in-memory instead of in on disk: During an unexpected shutdown or in the case of AsyncGetter, cleanup would not occur and blocks would be stuck in the store indefinitely. Using an in-memory blockstore would fix this issue upon restarts - Using a local Blockgetter to pass to NewErrByzantine: Instead of putting the blocks into the EDS blockstore, the retrieval session could make an in-memory blockstore on the fly to hand to NewErrByzantine when needed. This would look cleaner in the code/make sense architecturally, but it would mean full nodes cannot share these shares with each other during reconstruction. --- go.mod | 2 +- nodebuilder/tests/reconstruct_test.go | 151 ++++++++++++++++++++++++++ share/eds/blockstore.go | 77 +++++++++---- share/eds/inverted_index.go | 6 +- share/eds/retriever.go | 2 + share/eds/store.go | 2 +- share/getters/getter_test.go | 9 +- 7 files changed, 221 insertions(+), 28 deletions(-) diff --git a/go.mod b/go.mod index 0d26924835..63f6879041 100644 --- a/go.mod +++ b/go.mod @@ -202,7 +202,7 @@ require ( github.com/ipfs/bbloom v0.0.4 // indirect github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect - github.com/ipfs/go-ipfs-ds-help v1.1.0 // indirect + github.com/ipfs/go-ipfs-ds-help v1.1.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index 8b1490272c..d8640c5249 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -14,6 +14,7 @@ import ( "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" + ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" @@ -81,6 +82,156 @@ func TestFullReconstructFromBridge(t *testing.T) { require.NoError(t, errg.Wait()) } +/* +Test-Case: Full Node reconstructs blocks from each other, after unsuccessfully syncing the complete +block from LN subnetworks. Analog to TestShareAvailable_DisconnectedFullNodes. +*/ +func TestFullReconstructFromFulls(t *testing.T) { + light.DefaultSampleAmount = 10 // s + const ( + blocks = 10 + btime = time.Millisecond * 300 + bsize = 8 // k + lnodes = 12 // c - total number of nodes on two subnetworks + ) + + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) + + sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) + + const defaultTimeInterval = time.Second * 5 + bridge := sw.NewBridgeNode() + + sw.SetBootstrapper(t, bridge) + require.NoError(t, bridge.Start(ctx)) + + // TODO: This is required to avoid flakes coming from unfinished retry + // mechanism for the same peer in go-header + _, err := bridge.HeaderServ.WaitForHeight(ctx, uint64(blocks)) + require.NoError(t, err) + + lights1 := make([]*nodebuilder.Node, lnodes/2) + lights2 := make([]*nodebuilder.Node, lnodes/2) + subs := make([]event.Subscription, lnodes) + errg, errCtx := errgroup.WithContext(ctx) + for i := 0; i < lnodes/2; i++ { + i := i + errg.Go(func() error { + lnConfig := nodebuilder.DefaultConfig(node.Light) + setTimeInterval(lnConfig, defaultTimeInterval) + light := sw.NewNodeWithConfig(node.Light, lnConfig) + sub, err := light.Host.EventBus().Subscribe(&event.EvtPeerIdentificationCompleted{}) + if err != nil { + return err + } + subs[i] = sub + lights1[i] = light + return light.Start(errCtx) + }) + errg.Go(func() error { + lnConfig := nodebuilder.DefaultConfig(node.Light) + setTimeInterval(lnConfig, defaultTimeInterval) + light := sw.NewNodeWithConfig(node.Light, lnConfig) + sub, err := light.Host.EventBus().Subscribe(&event.EvtPeerIdentificationCompleted{}) + if err != nil { + return err + } + subs[(lnodes/2)+i] = sub + lights2[i] = light + return light.Start(errCtx) + }) + } + + require.NoError(t, errg.Wait()) + + for i := 0; i < lnodes; i++ { + select { + case <-ctx.Done(): + t.Fatal("peer was not found") + case <-subs[i].Out(): + require.NoError(t, subs[i].Close()) + continue + } + } + + // Remove bootstrappers to prevent FNs from connecting to bridge + sw.Bootstrappers = []ma.Multiaddr{} + // Use light nodes from respective subnetworks as bootstrappers to prevent connection to bridge + lnBootstrapper1, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(lights1[0].Host)) + require.NoError(t, err) + lnBootstrapper2, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(lights2[0].Host)) + require.NoError(t, err) + + cfg := nodebuilder.DefaultConfig(node.Full) + setTimeInterval(cfg, defaultTimeInterval) + cfg.Share.UseShareExchange = false + cfg.Share.Discovery.PeersLimit = 0 + cfg.Header.TrustedPeers = []string{lnBootstrapper1[0].String()} + full1 := sw.NewNodeWithConfig(node.Full, cfg) + cfg.Header.TrustedPeers = []string{lnBootstrapper2[0].String()} + full2 := sw.NewNodeWithConfig(node.Full, cfg) + require.NoError(t, full1.Start(ctx)) + require.NoError(t, full2.Start(ctx)) + + // Form topology + for i := 0; i < lnodes/2; i++ { + // Separate light nodes into two subnetworks + for j := 0; j < lnodes/2; j++ { + sw.Disconnect(t, lights1[i], lights2[j]) + if i != j { + sw.Connect(t, lights1[i], lights1[j]) + sw.Connect(t, lights2[i], lights2[j]) + } + } + + sw.Connect(t, full1, lights1[i]) + sw.Disconnect(t, full1, lights2[i]) + + sw.Connect(t, full2, lights2[i]) + sw.Disconnect(t, full2, lights1[i]) + } + + // Ensure the fulls are not connected to the bridge + sw.Disconnect(t, full1, full2) + sw.Disconnect(t, full1, bridge) + sw.Disconnect(t, full2, bridge) + + h, err := full1.HeaderServ.WaitForHeight(ctx, uint64(10+blocks-1)) + require.NoError(t, err) + + // Ensure that the full nodes cannot reconstruct before being connected to each other + ctxErr, cancelErr := context.WithTimeout(ctx, time.Second*30) + errg, errCtx = errgroup.WithContext(ctxErr) + errg.Go(func() error { + return full1.ShareServ.SharesAvailable(errCtx, h.DAH) + }) + errg.Go(func() error { + return full2.ShareServ.SharesAvailable(errCtx, h.DAH) + }) + require.Error(t, errg.Wait()) + cancelErr() + + // Reconnect FNs + sw.Connect(t, full1, full2) + + errg, bctx := errgroup.WithContext(ctx) + for i := 10; i < blocks+11; i++ { + h, err := full1.HeaderServ.WaitForHeight(bctx, uint64(i)) + require.NoError(t, err) + errg.Go(func() error { + return full1.ShareServ.SharesAvailable(bctx, h.DAH) + }) + errg.Go(func() error { + return full2.ShareServ.SharesAvailable(bctx, h.DAH) + }) + } + + require.NoError(t, <-fillDn) + require.NoError(t, errg.Wait()) +} + /* Test-Case: Full Node reconstructs blocks only from Light Nodes Pre-Reqs: diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go index 9fc9d7f2f8..22ef28f821 100644 --- a/share/eds/blockstore.go +++ b/share/eds/blockstore.go @@ -10,12 +10,15 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + dshelp "github.com/ipfs/go-ipfs-ds-help" ipld "github.com/ipfs/go-ipld-format" ) var _ bstore.Blockstore = (*blockstore)(nil) var ( + blockstoreCacheKey = datastore.NewKey("bs-cache") errUnsupportedOperation = errors.New("unsupported operation") ) @@ -30,29 +33,42 @@ var ( type blockstore struct { store *Store cache *blockstoreCache + ds datastore.Batching } -func newBlockstore(store *Store, cache *blockstoreCache) *blockstore { +func newBlockstore(store *Store, cache *blockstoreCache, ds datastore.Batching) *blockstore { return &blockstore{ store: store, cache: cache, + ds: namespace.Wrap(ds, blockstoreCacheKey), } } func (bs *blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) - if errors.Is(err, ErrNotFound) { - return false, nil + if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { + // key wasn't found in top level blockstore, but could be in datastore while being reconstructed + dsHas, dsErr := bs.ds.Has(ctx, dshelp.MultihashToDsKey(cid.Hash())) + if dsErr != nil { + return false, nil + } + return dsHas, nil } if err != nil { - return false, fmt.Errorf("failed to find shards containing multihash: %w", err) + return false, err } + return len(keys) > 0, nil } func (bs *blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) { blockstr, err := bs.getReadOnlyBlockstore(ctx, cid) - if errors.Is(err, ErrNotFound) { + if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { + k := dshelp.MultihashToDsKey(cid.Hash()) + blockData, err := bs.ds.Get(ctx, k) + if err == nil { + return blocks.NewBlockWithCid(blockData, cid) + } // nmt's GetNode expects an ipld.ErrNotFound when a cid is not found. return nil, ipld.ErrNotFound{Cid: cid} } @@ -65,7 +81,12 @@ func (bs *blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error func (bs *blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { blockstr, err := bs.getReadOnlyBlockstore(ctx, cid) - if errors.Is(err, ErrNotFound) { + if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { + k := dshelp.MultihashToDsKey(cid.Hash()) + size, err := bs.ds.GetSize(ctx, k) + if err == nil { + return size, nil + } // nmt's GetSize expects an ipld.ErrNotFound when a cid is not found. return 0, ipld.ErrNotFound{Cid: cid} } @@ -75,27 +96,35 @@ func (bs *blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { return blockstr.GetSize(ctx, cid) } -// DeleteBlock is a noop on the EDS blockstore that returns an errUnsupportedOperation when called. -func (bs *blockstore) DeleteBlock(context.Context, cid.Cid) error { - return errUnsupportedOperation +func (bs *blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error { + k := dshelp.MultihashToDsKey(cid.Hash()) + return bs.ds.Delete(ctx, k) } -// Put is a noop on the EDS blockstore, but it does not return an error because it is called by -// bitswap. For clarification, an implementation of Put does not make sense in this context because -// it is unclear which CAR file the block should be written to. -// -// TODO: throw errUnsupportedOperation after issue #1440 -func (bs *blockstore) Put(context.Context, blocks.Block) error { - return nil +func (bs *blockstore) Put(ctx context.Context, blk blocks.Block) error { + k := dshelp.MultihashToDsKey(blk.Cid().Hash()) + // note: we leave duplicate resolution to the underlying datastore + return bs.ds.Put(ctx, k, blk.RawData()) } -// PutMany is a noop on the EDS blockstore, but it does not return an error because it is called by -// bitswap. For clarification, an implementation of PutMany does not make sense in this context -// because it is unclear which CAR file the blocks should be written to. -// -// TODO: throw errUnsupportedOperation after issue #1440 -func (bs *blockstore) PutMany(context.Context, []blocks.Block) error { - return nil +func (bs *blockstore) PutMany(ctx context.Context, blocks []blocks.Block) error { + if len(blocks) == 1 { + // performance fast-path + return bs.Put(ctx, blocks[0]) + } + + t, err := bs.ds.Batch(ctx) + if err != nil { + return err + } + for _, b := range blocks { + k := dshelp.MultihashToDsKey(b.Cid().Hash()) + err = t.Put(ctx, k, b.RawData()) + if err != nil { + return err + } + } + return t.Commit(ctx) } // AllKeysChan is a noop on the EDS blockstore because the keys are not stored in a single CAR file. @@ -112,7 +141,7 @@ func (bs *blockstore) HashOnRead(bool) { // getReadOnlyBlockstore finds the underlying blockstore of the shard that contains the given CID. func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (dagstore.ReadBlockstore, error) { keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) - if errors.Is(err, datastore.ErrNotFound) { + if errors.Is(err, datastore.ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { return nil, ErrNotFound } if err != nil { diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index 81df5d92ef..8b9dcb5d95 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -2,6 +2,7 @@ package eds import ( "context" + "errors" "fmt" "github.com/filecoin-project/dagstore/index" @@ -14,6 +15,9 @@ import ( const invertedIndexPath = "/inverted_index/" +// ErrNotFoundInIndex is returned instead of ErrNotFound if the multihash doesn't exist in the index +var ErrNotFoundInIndex = fmt.Errorf("does not exist in index") + // simpleInvertedIndex is an inverted index that only stores a single shard key per multihash. Its // implementation is modified from the default upstream implementation in dagstore/index. type simpleInvertedIndex struct { @@ -76,7 +80,7 @@ func (s *simpleInvertedIndex) GetShardsForMultihash(ctx context.Context, mh mult key := ds.NewKey(string(mh)) sbz, err := s.ds.Get(ctx, key) if err != nil { - return nil, fmt.Errorf("failed to lookup index for mh %s, err: %w", mh, err) + return nil, errors.Join(ErrNotFoundInIndex, err) } return []shard.Key{shard.KeyFromString(string(sbz))}, nil diff --git a/share/eds/retriever.go b/share/eds/retriever.go index a870e07e22..e7837ae23a 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -122,6 +122,7 @@ type retrievalSession struct { // newSession creates a new retrieval session and kicks off requesting process. func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHeader) (*retrievalSession, error) { size := len(dah.RowRoots) + treeFn := func(_ rsmt2d.Axis, index uint) rsmt2d.Tree { // use proofs adder if provided, to cache collected proofs while recomputing the eds var opts []nmt.Option @@ -152,6 +153,7 @@ func (r *Retriever) newSession(ctx context.Context, dah *da.DataAvailabilityHead for i := range ses.squareCellsLks { ses.squareCellsLks[i] = make([]sync.Mutex, size) } + go ses.request(ctx) return ses, nil } diff --git a/share/eds/store.go b/share/eds/store.go index fd679e2591..24a96c9fe4 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -119,7 +119,7 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { mounts: r, cache: cache, } - store.bs = newBlockstore(store, cache) + store.bs = newBlockstore(store, cache, ds) return store, nil } diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 5cc73aa7cb..ae90074625 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -163,7 +163,8 @@ func TestIPLDGetter(t *testing.T) { err = edsStore.Start(ctx) require.NoError(t, err) - bserv := bsrv.New(edsStore.Blockstore(), offline.Exchange(edsStore.Blockstore())) + bStore := edsStore.Blockstore() + bserv := bsrv.New(bStore, offline.Exchange(bStore)) sg := NewIPLDGetter(bserv) t.Run("GetShare", func(t *testing.T) { @@ -200,6 +201,12 @@ func TestIPLDGetter(t *testing.T) { retrievedEDS, err := sg.GetEDS(ctx, &dah) require.NoError(t, err) assert.True(t, randEds.Equals(retrievedEDS)) + + // Ensure blocks still exist after cleanup + colRoots, _ := retrievedEDS.ColRoots() + has, err := bStore.Has(ctx, ipld.MustCidFromNamespacedSha256(colRoots[0])) + assert.NoError(t, err) + assert.True(t, has) }) t.Run("GetSharesByNamespace", func(t *testing.T) { From 8deffba77b0e68f6a2229bc8266359eda704c584 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 29 Aug 2023 12:43:52 +0200 Subject: [PATCH 151/388] fix: fixing panic on GetShare with out of bounds indexes (#2605) GetShare would panic on RPC calls if the passed indicies were out of bounds. --- share/getter.go | 2 ++ share/getters/cascade.go | 8 ++++---- share/getters/getter_test.go | 8 ++++++++ share/getters/ipld.go | 6 ++++++ share/getters/store.go | 6 ++++++ 5 files changed, 26 insertions(+), 4 deletions(-) diff --git a/share/getter.go b/share/getter.go index a33ea7aa87..19e4498458 100644 --- a/share/getter.go +++ b/share/getter.go @@ -13,6 +13,8 @@ import ( var ( // ErrNotFound is used to indicate that requested data could not be found. ErrNotFound = errors.New("share: data not found") + // ErrOutOfBounds is used to indicate that a passed row or column index is out of bounds of the square size. + ErrOutOfBounds = errors.New("share: row or column index is larger than square size") ) // Getter interface provides a set of accessors for shares by the Root. diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 98f29c87ca..63d7713d3d 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -3,7 +3,6 @@ package getters import ( "context" "errors" - "fmt" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" @@ -38,9 +37,10 @@ func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, co attribute.Int("col", col), )) defer span.End() - if row >= len(root.RowRoots) || col >= len(root.ColumnRoots) { - err := fmt.Errorf("cascade/get-share: invalid indexes were provided:rowIndex=%d, colIndex=%d."+ - "squarewidth=%d", row, col, len(root.RowRoots)) + + upperBound := len(root.RowRoots) + if row >= upperBound || col >= upperBound { + err := share.ErrOutOfBounds span.RecordError(err) return nil, err } diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index ae90074625..02e075459b 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -107,6 +107,10 @@ func TestStoreGetter(t *testing.T) { } } + // doesn't panic on indexes too high + _, err := sg.GetShare(ctx, &dah, squareSize, squareSize) + require.ErrorIs(t, err, share.ErrOutOfBounds) + // root not found _, dah = randomEDS(t) _, err = sg.GetShare(ctx, &dah, 0, 0) @@ -184,6 +188,10 @@ func TestIPLDGetter(t *testing.T) { } } + // doesn't panic on indexes too high + _, err := sg.GetShare(ctx, &dah, squareSize+1, squareSize+1) + require.ErrorIs(t, err, share.ErrOutOfBounds) + // root not found _, dah = randomEDS(t) _, err = sg.GetShare(ctx, &dah, 0, 0) diff --git a/share/getters/ipld.go b/share/getters/ipld.go index af6e7673ba..a892e0fc82 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -48,6 +48,12 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in utils.SetStatusAndEnd(span, err) }() + upperBound := len(dah.RowRoots) + if row >= upperBound || col >= upperBound { + err := share.ErrOutOfBounds + span.RecordError(err) + return nil, err + } root, leaf := ipld.Translate(dah, row, col) // wrap the blockservice in a session if it has been signaled in the context. diff --git a/share/getters/store.go b/share/getters/store.go index 28c6c6d021..989649f795 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -43,6 +43,12 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i utils.SetStatusAndEnd(span, err) }() + upperBound := len(dah.RowRoots) + if row >= upperBound || col >= upperBound { + err := share.ErrOutOfBounds + span.RecordError(err) + return nil, err + } root, leaf := ipld.Translate(dah, row, col) bs, err := sg.store.CARBlockstore(ctx, dah.Hash()) if errors.Is(err, eds.ErrNotFound) { From 9d663cf7d948fefa961d9b3c8d0afb8dca3ec8ab Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 30 Aug 2023 13:12:01 +0200 Subject: [PATCH 152/388] chore: bump go-header and go-fraud (#2619) This PR bumps go-header and go-fraud which are both breaking - lots of changes here so please review carefully. Note that this also contains re-introduction of VerifyCommitLightTrusting which caused us issues in `mocha` due to rapid voting power changes. At the moment, there are no test utilities to test that specific case, so we need to wait to roll it out to `mocha`. Related: https://github.com/celestiaorg/celestia-app/issues/2382 --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- api/docgen/examples.go | 2 +- core/exchange.go | 11 +-- core/header_test.go | 2 +- core/listener.go | 4 +- core/listener_test.go | 4 +- das/coordinator_test.go | 8 +-- das/daser.go | 8 +-- das/daser_test.go | 46 ++++++++---- das/state.go | 15 ++-- das/worker.go | 2 +- go.mod | 4 +- go.sum | 8 +-- header/header.go | 91 ++++++++++++++++-------- header/headertest/testing.go | 8 +-- header/headertest/verify_test.go | 49 +++---------- header/serde.go | 2 +- header/verify.go | 76 -------------------- nodebuilder/das/constructors.go | 6 +- nodebuilder/das/module.go | 5 +- nodebuilder/fraud/constructors.go | 72 +++++++++++-------- nodebuilder/fraud/fraud.go | 84 ++++++++++++++++++++++ nodebuilder/fraud/lifecycle.go | 24 ++++--- nodebuilder/fraud/module.go | 14 ++-- nodebuilder/fraud/service.go | 87 ---------------------- nodebuilder/fraud/unmarshaler.go | 32 +++++++++ nodebuilder/header/constructors.go | 33 +++++---- nodebuilder/header/module.go | 48 +++++++------ nodebuilder/header/module_test.go | 8 +-- nodebuilder/header/service.go | 8 +-- nodebuilder/header/service_test.go | 4 +- nodebuilder/module.go | 5 +- nodebuilder/p2p/pubsub.go | 13 ++-- nodebuilder/settings.go | 3 +- nodebuilder/state/core.go | 6 +- nodebuilder/state/module.go | 7 +- nodebuilder/testing.go | 5 +- nodebuilder/tests/api_test.go | 4 +- nodebuilder/tests/sync_test.go | 4 +- share/eds/byzantine/bad_encoding.go | 41 ++++------- share/eds/byzantine/bad_encoding_test.go | 2 +- share/eds/retriever_test.go | 4 +- share/p2p/discovery/discovery.go | 3 +- share/p2p/peers/manager.go | 2 +- share/p2p/peers/manager_test.go | 8 +-- state/core_access.go | 13 ++-- state/integration_test.go | 6 +- 46 files changed, 449 insertions(+), 442 deletions(-) delete mode 100644 header/verify.go delete mode 100644 nodebuilder/fraud/service.go create mode 100644 nodebuilder/fraud/unmarshaler.go diff --git a/api/docgen/examples.go b/api/docgen/examples.go index 3456880c4f..b873e7e050 100644 --- a/api/docgen/examples.go +++ b/api/docgen/examples.go @@ -56,7 +56,7 @@ var ExampleValues = map[reflect.Type]interface{}{ reflect.TypeOf(node.Full): node.Full, reflect.TypeOf(auth.Permission("admin")): auth.Permission("admin"), reflect.TypeOf(byzantine.BadEncoding): byzantine.BadEncoding, - reflect.TypeOf((*fraud.Proof)(nil)).Elem(): byzantine.CreateBadEncodingProof( + reflect.TypeOf((*fraud.Proof[*header.ExtendedHeader])(nil)).Elem(): byzantine.CreateBadEncodingProof( []byte("bad encoding proof"), 42, &byzantine.ErrByzantine{ diff --git a/core/exchange.go b/core/exchange.go index bed2404195..f8e1606a3e 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -78,7 +78,7 @@ func (ce *Exchange) GetVerifiedRange( from *header.ExtendedHeader, amount uint64, ) ([]*header.ExtendedHeader, error) { - headers, err := ce.GetRangeByHeight(ctx, uint64(from.Height())+1, amount) + headers, err := ce.GetRangeByHeight(ctx, from.Height()+1, amount) if err != nil { return nil, err } @@ -115,7 +115,7 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende return nil, fmt.Errorf("extending block data for height %d: %w", &block.Height, err) } // construct extended header - eh, err := ce.construct(ctx, &block.Header, comm, vals, eds) + eh, err := ce.construct(&block.Header, comm, vals, eds) if err != nil { panic(fmt.Errorf("constructing extended header for height %d: %w", &block.Height, err)) } @@ -133,7 +133,10 @@ func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.Extende return eh, nil } -func (ce *Exchange) Head(ctx context.Context) (*header.ExtendedHeader, error) { +func (ce *Exchange) Head( + ctx context.Context, + _ ...libhead.HeadOption[*header.ExtendedHeader], +) (*header.ExtendedHeader, error) { log.Debug("requesting head") return ce.getExtendedHeaderByHeight(ctx, nil) } @@ -157,7 +160,7 @@ func (ce *Exchange) getExtendedHeaderByHeight(ctx context.Context, height *int64 return nil, fmt.Errorf("extending block data for height %d: %w", b.Header.Height, err) } // create extended header - eh, err := ce.construct(ctx, &b.Header, &b.Commit, &b.ValidatorSet, eds) + eh, err := ce.construct(&b.Header, &b.Commit, &b.ValidatorSet, eds) if err != nil { panic(fmt.Errorf("constructing extended header for height %d: %w", b.Header.Height, err)) } diff --git a/core/header_test.go b/core/header_test.go index 1c89db9d6b..c942ea7875 100644 --- a/core/header_test.go +++ b/core/header_test.go @@ -33,7 +33,7 @@ func TestMakeExtendedHeaderForEmptyBlock(t *testing.T) { eds, err := extendBlock(b.Data, b.Header.Version.App) require.NoError(t, err) - headerExt, err := header.MakeExtendedHeader(ctx, &b.Header, comm, val, eds) + headerExt, err := header.MakeExtendedHeader(&b.Header, comm, val, eds) require.NoError(t, err) assert.Equal(t, header.EmptyDAH(), *headerExt.DAH) diff --git a/core/listener.go b/core/listener.go index 565fc62032..1c79fbbe71 100644 --- a/core/listener.go +++ b/core/listener.go @@ -160,7 +160,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS return fmt.Errorf("extending block data: %w", err) } // generate extended header - eh, err := cl.construct(ctx, &b.Header, &b.Commit, &b.ValidatorSet, eds) + eh, err := cl.construct(&b.Header, &b.Commit, &b.ValidatorSet, eds) if err != nil { panic(fmt.Errorf("making extended header: %w", err)) } @@ -181,7 +181,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS if !syncing { err = cl.hashBroadcaster(ctx, shrexsub.Notification{ DataHash: eh.DataHash.Bytes(), - Height: uint64(eh.Height()), + Height: eh.Height(), }) if err != nil && !errors.Is(err, context.Canceled) { log.Errorw("listener: broadcasting data hash", diff --git a/core/listener_test.go b/core/listener_test.go index 7d4b12310a..8b3d05bea9 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -31,8 +31,8 @@ func TestListener(t *testing.T) { // create mocknet with two pubsub endpoints ps0, ps1 := createMocknetWithTwoPubsubEndpoints(ctx, t) subscriber := p2p.NewSubscriber[*header.ExtendedHeader](ps1, header.MsgID, networkID) - err := subscriber.AddValidator(func(context.Context, *header.ExtendedHeader) pubsub.ValidationResult { - return pubsub.ValidationAccept + err := subscriber.SetVerifier(func(context.Context, *header.ExtendedHeader) error { + return nil }) require.NoError(t, err) require.NoError(t, subscriber.Start(ctx)) diff --git a/das/coordinator_test.go b/das/coordinator_test.go index 188cb0d222..55ed01dd4e 100644 --- a/das/coordinator_test.go +++ b/das/coordinator_test.go @@ -366,7 +366,7 @@ func (m *mockSampler) sample(ctx context.Context, h *header.ExtendedHeader) erro m.lock.Lock() defer m.lock.Unlock() - height := uint64(h.Height()) + height := h.Height() m.done[height]++ if len(m.done) > int(m.NetworkHead-m.SampleFrom) && !m.isFinished { @@ -503,7 +503,7 @@ func (o *checkOrder) middleWare(out sampleFn) sampleFn { if len(o.queue) > 0 { // check last item in queue to be same as input - if o.queue[0] != uint64(h.Height()) { + if o.queue[0] != h.Height() { defer o.lock.Unlock() return fmt.Errorf("expected height: %v,got: %v", o.queue[0], h.Height()) } @@ -573,7 +573,7 @@ func (l *lock) releaseAll(except ...uint64) { func (l *lock) middleWare(out sampleFn) sampleFn { return func(ctx context.Context, h *header.ExtendedHeader) error { l.m.Lock() - ch, blocked := l.blockList[uint64(h.Height())] + ch, blocked := l.blockList[h.Height()] l.m.Unlock() if !blocked { return out(ctx, h) @@ -589,7 +589,7 @@ func (l *lock) middleWare(out sampleFn) sampleFn { } func onceMiddleWare(out sampleFn) sampleFn { - db := make(map[int64]int) + db := make(map[uint64]int) m := sync.Mutex{} return func(ctx context.Context, h *header.ExtendedHeader) error { m.Lock() diff --git a/das/daser.go b/das/daser.go index d4ad0ee641..9d3e43a91b 100644 --- a/das/daser.go +++ b/das/daser.go @@ -25,7 +25,7 @@ type DASer struct { params Parameters da share.Availability - bcast fraud.Broadcaster + bcast fraud.Broadcaster[*header.ExtendedHeader] hsub libhead.Subscriber[*header.ExtendedHeader] // listens for new headers in the network getter libhead.Getter[*header.ExtendedHeader] // retrieves past headers @@ -47,7 +47,7 @@ func NewDASer( hsub libhead.Subscriber[*header.ExtendedHeader], getter libhead.Getter[*header.ExtendedHeader], dstore datastore.Datastore, - bcast fraud.Broadcaster, + bcast fraud.Broadcaster[*header.ExtendedHeader], shrexBroadcast shrexsub.BroadcastFn, options ...Option, ) (*DASer, error) { @@ -99,7 +99,7 @@ func (d *DASer) Start(ctx context.Context) error { // attempt to get head info. No need to handle error, later DASer // will be able to find new head from subscriber after it is started if h, err := d.getter.Head(ctx); err == nil { - cp.NetworkHead = uint64(h.Height()) + cp.NetworkHead = h.Height() } } log.Info("starting DASer from checkpoint: ", cp.String()) @@ -152,7 +152,7 @@ func (d *DASer) sample(ctx context.Context, h *header.ExtendedHeader) error { var byzantineErr *byzantine.ErrByzantine if errors.As(err, &byzantineErr) { log.Warn("Propagating proof...") - sendErr := d.bcast.Broadcast(ctx, byzantine.CreateBadEncodingProof(h.Hash(), uint64(h.Height()), byzantineErr)) + sendErr := d.bcast.Broadcast(ctx, byzantine.CreateBadEncodingProof(h.Hash(), h.Height(), byzantineErr)) if sendErr != nil { log.Errorw("fraud proof propagating failed", "err", sendErr) } diff --git a/das/daser_test.go b/das/daser_test.go index 7398310a6b..68f6e01ef2 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -159,21 +159,37 @@ func TestDASer_stopsAfter_BEFP(t *testing.T) { mockGet, sub, _ := createDASerSubcomponents(t, bServ, 15, 15) // create fraud service and break one header - getter := func(ctx context.Context, height uint64) (libhead.Header, error) { + getter := func(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { return mockGet.GetByHeight(ctx, height) } - f := fraudserv.NewProofService(ps, net.Hosts()[0], getter, ds, false, "private") - require.NoError(t, f.Start(ctx)) + unmarshaler := fraud.MultiUnmarshaler[*header.ExtendedHeader]{ + Unmarshalers: map[fraud.ProofType]func([]byte) (fraud.Proof[*header.ExtendedHeader], error){ + byzantine.BadEncoding: func(data []byte) (fraud.Proof[*header.ExtendedHeader], error) { + befp := &byzantine.BadEncodingProof{} + return befp, befp.UnmarshalBinary(data) + }, + }, + } + + fserv := fraudserv.NewProofService[*header.ExtendedHeader](ps, + net.Hosts()[0], + getter, + unmarshaler, + ds, + false, + "private", + ) + require.NoError(t, fserv.Start(ctx)) mockGet.headers[1], _ = headertest.CreateFraudExtHeader(t, mockGet.headers[1], bServ) newCtx := context.Background() // create and start DASer - daser, err := NewDASer(avail, sub, mockGet, ds, f, newBroadcastMock(1)) + daser, err := NewDASer(avail, sub, mockGet, ds, fserv, newBroadcastMock(1)) require.NoError(t, err) resultCh := make(chan error) - go fraud.OnProof(newCtx, f, byzantine.BadEncoding, - func(fraud.Proof) { + go fraud.OnProof[*header.ExtendedHeader](newCtx, fserv, byzantine.BadEncoding, + func(fraud.Proof[*header.ExtendedHeader]) { resultCh <- daser.Stop(newCtx) }) @@ -210,10 +226,10 @@ func TestDASerSampleTimeout(t *testing.T) { ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) sub := new(headertest.Subscriber) - f := new(fraudtest.DummyService) + fserv := &fraudtest.DummyService[*header.ExtendedHeader]{} // create and start DASer - daser, err := NewDASer(avail, sub, getter, ds, f, newBroadcastMock(1), WithSampleTimeout(1)) + daser, err := NewDASer(avail, sub, getter, ds, fserv, newBroadcastMock(1), WithSampleTimeout(1)) require.NoError(t, err) require.NoError(t, daser.Start(ctx)) @@ -235,9 +251,9 @@ func createDASerSubcomponents( bServ blockservice.BlockService, numGetter, numSub int, -) (*mockGetter, *headertest.Subscriber, *fraudtest.DummyService) { +) (*mockGetter, *headertest.Subscriber, *fraudtest.DummyService[*header.ExtendedHeader]) { mockGet, sub := createMockGetterAndSub(t, bServ, numGetter, numSub) - fraud := new(fraudtest.DummyService) + fraud := &fraudtest.DummyService[*header.ExtendedHeader]{} return mockGet, sub, fraud } @@ -313,7 +329,10 @@ func (m *mockGetter) generateHeaders(t *testing.T, bServ blockservice.BlockServi m.head = int64(startHeight + endHeight) } -func (m *mockGetter) Head(context.Context) (*header.ExtendedHeader, error) { +func (m *mockGetter) Head( + context.Context, + ...libhead.HeadOption[*header.ExtendedHeader], +) (*header.ExtendedHeader, error) { return m.headers[m.head], nil } @@ -354,7 +373,10 @@ func (m benchGetterStub) GetByHeight(context.Context, uint64) (*header.ExtendedH type getterStub struct{} -func (m getterStub) Head(context.Context) (*header.ExtendedHeader, error) { +func (m getterStub) Head( + context.Context, + ...libhead.HeadOption[*header.ExtendedHeader], +) (*header.ExtendedHeader, error) { return &header.ExtendedHeader{RawHeader: header.RawHeader{Height: 1}}, nil } diff --git a/das/state.go b/das/state.go index 6af0b7d8d8..bd3a018a40 100644 --- a/das/state.go +++ b/das/state.go @@ -132,30 +132,29 @@ func (s *coordinatorState) handleRetryResult(res result) { } } -func (s *coordinatorState) isNewHead(newHead int64) bool { +func (s *coordinatorState) isNewHead(newHead uint64) bool { // seen this header before - if uint64(newHead) <= s.networkHead { + if newHead <= s.networkHead { log.Warnf("received head height: %v, which is lower or the same as previously known: %v", newHead, s.networkHead) return false } return true } -func (s *coordinatorState) updateHead(newHead int64) { +func (s *coordinatorState) updateHead(newHead uint64) { if s.networkHead == s.sampleFrom { log.Infow("found first header, starting sampling") } - s.networkHead = uint64(newHead) + s.networkHead = newHead log.Debugw("updated head", "from_height", s.networkHead, "to_height", newHead) s.checkDone() } // recentJob creates a job to process a recent header. func (s *coordinatorState) recentJob(header *header.ExtendedHeader) job { - height := uint64(header.Height()) // move next, to prevent catchup job from processing same height - if s.next == height { + if s.next == header.Height() { s.next++ } s.nextJobID++ @@ -163,8 +162,8 @@ func (s *coordinatorState) recentJob(header *header.ExtendedHeader) job { id: s.nextJobID, jobType: recentJob, header: header, - from: height, - to: height, + from: header.Height(), + to: header.Height(), } } diff --git a/das/worker.go b/das/worker.go index 746324ec48..f2e8c4d821 100644 --- a/das/worker.go +++ b/das/worker.go @@ -135,7 +135,7 @@ func (w *worker) sample(ctx context.Context, timeout time.Duration, height uint6 if w.state.job.jobType == recentJob { err = w.broadcast(ctx, shrexsub.Notification{ DataHash: h.DataHash.Bytes(), - Height: uint64(h.Height()), + Height: h.Height(), }) if err != nil { log.Warn("failed to broadcast availability message", diff --git a/go.mod b/go.mod index 63f6879041..846f4316e6 100644 --- a/go.mod +++ b/go.mod @@ -14,8 +14,8 @@ require ( github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.0.0-rc12 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 - github.com/celestiaorg/go-fraud v0.1.2 - github.com/celestiaorg/go-header v0.2.13 + github.com/celestiaorg/go-fraud v0.2.0 + github.com/celestiaorg/go-header v0.3.0 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.18.1 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 6adeef7f7c..09bd8ab26e 100644 --- a/go.sum +++ b/go.sum @@ -366,10 +366,10 @@ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= -github.com/celestiaorg/go-fraud v0.1.2 h1:Bf7yIN3lZ4IR/Vlu5OtmcVCVNESBKEJ/xwu28rRKGA8= -github.com/celestiaorg/go-fraud v0.1.2/go.mod h1:kHZXQY+6gd1kYkoWRFFKgWyrLPWRgDN3vd1Ll9gE/oo= -github.com/celestiaorg/go-header v0.2.13 h1:sUJLXYs8ViPpxLXyIIaW3h4tPFgtVYMhzsLC4GHfS8I= -github.com/celestiaorg/go-header v0.2.13/go.mod h1:NhiWq97NtAYyRBu8quzYOUghQULjgOzO2Ql0iVEFOf0= +github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= +github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= +github.com/celestiaorg/go-header v0.3.0 h1:9fhxSgldPiWWq3yd9u7oSk5vYqaLV1JkeTnJdGcisFo= +github.com/celestiaorg/go-header v0.3.0/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= diff --git a/header/header.go b/header/header.go index d69b11d998..92f8538696 100644 --- a/header/header.go +++ b/header/header.go @@ -2,12 +2,12 @@ package header import ( "bytes" - "context" "encoding/json" "fmt" "time" tmjson "github.com/tendermint/tendermint/libs/json" + "github.com/tendermint/tendermint/light" core "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" @@ -18,7 +18,6 @@ import ( // ConstructFn aliases a function that creates an ExtendedHeader. type ConstructFn = func( - context.Context, *core.Header, *core.Commit, *core.ValidatorSet, @@ -45,31 +44,8 @@ type ExtendedHeader struct { DAH *DataAvailabilityHeader `json:"dah"` } -func (eh *ExtendedHeader) New() libhead.Header { - return new(ExtendedHeader) -} - -func (eh *ExtendedHeader) IsZero() bool { - return eh == nil -} - -func (eh *ExtendedHeader) ChainID() string { - return eh.RawHeader.ChainID -} - -func (eh *ExtendedHeader) Height() int64 { - return eh.RawHeader.Height -} - -func (eh *ExtendedHeader) Time() time.Time { - return eh.RawHeader.Time -} - -var _ libhead.Header = &ExtendedHeader{} - // MakeExtendedHeader assembles new ExtendedHeader. func MakeExtendedHeader( - _ context.Context, h *core.Header, comm *core.Commit, vals *core.ValidatorSet, @@ -95,14 +71,34 @@ func MakeExtendedHeader( Commit: comm, ValidatorSet: vals, } - return eh, eh.Validate() + return eh, nil +} + +func (eh *ExtendedHeader) New() *ExtendedHeader { + return new(ExtendedHeader) +} + +func (eh *ExtendedHeader) IsZero() bool { + return eh == nil +} + +func (eh *ExtendedHeader) ChainID() string { + return eh.RawHeader.ChainID +} + +func (eh *ExtendedHeader) Height() uint64 { + return uint64(eh.RawHeader.Height) +} + +func (eh *ExtendedHeader) Time() time.Time { + return eh.RawHeader.Time } // Hash returns Hash of the wrapped RawHeader. // NOTE: It purposely overrides Hash method of RawHeader to get it directly from Commit without // recomputing. func (eh *ExtendedHeader) Hash() libhead.Hash { - return libhead.Hash(eh.Commit.BlockID.Hash) + return eh.Commit.BlockID.Hash.Bytes() } // LastHeader returns the Hash of the last wrapped RawHeader. @@ -158,7 +154,8 @@ func (eh *ExtendedHeader) Validate() error { return fmt.Errorf("commit signs block %X, header is block %X", chash, hhash) } - if err := eh.ValidatorSet.VerifyCommitLight(eh.ChainID(), eh.Commit.BlockID, eh.Height(), eh.Commit); err != nil { + err = eh.ValidatorSet.VerifyCommitLight(eh.ChainID(), eh.Commit.BlockID, int64(eh.Height()), eh.Commit) + if err != nil { return fmt.Errorf("VerifyCommitLight error at height %d: %w", eh.Height(), err) } @@ -169,6 +166,42 @@ func (eh *ExtendedHeader) Validate() error { return nil } +// Verify validates given untrusted Header against trusted ExtendedHeader. +func (eh *ExtendedHeader) Verify(untrst *ExtendedHeader) error { + isAdjacent := eh.Height()+1 == untrst.Height() + if isAdjacent { + // Optimized verification for adjacent headers + // Check the validator hashes are the same + if !bytes.Equal(untrst.ValidatorsHash, eh.NextValidatorsHash) { + return &libhead.VerifyError{ + Reason: fmt.Errorf("expected old header next validators (%X) to match those from new header (%X)", + eh.NextValidatorsHash, + untrst.ValidatorsHash, + ), + } + } + + if !bytes.Equal(untrst.LastHeader(), eh.Hash()) { + return &libhead.VerifyError{ + Reason: fmt.Errorf("expected new header to point to last header hash (%X), but got %X)", + eh.Hash(), + untrst.LastHeader(), + ), + } + } + + return nil + } + + if err := eh.ValidatorSet.VerifyCommitLightTrusting(eh.ChainID(), untrst.Commit, light.DefaultTrustLevel); err != nil { + return &libhead.VerifyError{ + Reason: err, + SoftFailure: true, + } + } + return nil +} + // MarshalBinary marshals ExtendedHeader to binary. func (eh *ExtendedHeader) MarshalBinary() ([]byte, error) { return MarshalExtendedHeader(eh) @@ -240,3 +273,5 @@ func (eh *ExtendedHeader) UnmarshalJSON(data []byte) error { eh.RawHeader = *rawHeader return nil } + +var _ libhead.Header[*ExtendedHeader] = &ExtendedHeader{} diff --git a/header/headertest/testing.go b/header/headertest/testing.go index b20d389452..65ae8c950f 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -158,9 +158,9 @@ func (s *TestSuite) NextHeader() *header.ExtendedHeader { } func (s *TestSuite) GenRawHeader( - height int64, lastHeader, lastCommit, dataHash libhead.Hash) *header.RawHeader { + height uint64, lastHeader, lastCommit, dataHash libhead.Hash) *header.RawHeader { rh := RandRawHeader(s.t) - rh.Height = height + rh.Height = int64(height) rh.Time = time.Now() rh.LastBlockID = types.BlockID{Hash: bytes.HexBytes(lastHeader)} rh.LastCommitHash = bytes.HexBytes(lastCommit) @@ -299,7 +299,7 @@ func RandBlockID(*testing.T) types.BlockID { // FraudMaker creates a custom ConstructFn that breaks the block at the given height. func FraudMaker(t *testing.T, faultHeight int64, bServ blockservice.BlockService) header.ConstructFn { log.Warn("Corrupting block...", "height", faultHeight) - return func(ctx context.Context, + return func( h *types.Header, comm *types.Commit, vals *types.ValidatorSet, @@ -318,7 +318,7 @@ func FraudMaker(t *testing.T, faultHeight int64, bServ blockservice.BlockService } return eh, nil } - return header.MakeExtendedHeader(ctx, h, comm, vals, eds) + return header.MakeExtendedHeader(h, comm, vals, eds) } } diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 33bcf72642..7ef16afc8d 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -3,34 +3,32 @@ package headertest import ( "strconv" "testing" - "time" "github.com/stretchr/testify/assert" tmrand "github.com/tendermint/tendermint/libs/rand" - "github.com/celestiaorg/celestia-app/pkg/appconsts" - libhead "github.com/celestiaorg/go-header" + "github.com/celestiaorg/celestia-node/header" ) func TestVerify(t *testing.T) { h := NewTestSuite(t, 2).GenExtendedHeaders(3) trusted, untrustedAdj, untrustedNonAdj := h[0], h[1], h[2] tests := []struct { - prepare func() libhead.Header + prepare func() *header.ExtendedHeader err bool }{ { - prepare: func() libhead.Header { return untrustedAdj }, + prepare: func() *header.ExtendedHeader { return untrustedAdj }, err: false, }, { - prepare: func() libhead.Header { + prepare: func() *header.ExtendedHeader { return untrustedNonAdj }, err: false, }, { - prepare: func() libhead.Header { + prepare: func() *header.ExtendedHeader { untrusted := *untrustedAdj untrusted.ValidatorsHash = tmrand.Bytes(32) return &untrusted @@ -38,7 +36,7 @@ func TestVerify(t *testing.T) { err: true, }, { - prepare: func() libhead.Header { + prepare: func() *header.ExtendedHeader { untrusted := *untrustedAdj untrusted.RawHeader.LastBlockID.Hash = tmrand.Bytes(32) return &untrusted @@ -46,37 +44,10 @@ func TestVerify(t *testing.T) { err: true, }, { - prepare: func() libhead.Header { - untrustedAdj.RawHeader.Time = untrustedAdj.RawHeader.Time.Add(time.Minute) - return untrustedAdj - }, - err: true, - }, - { - prepare: func() libhead.Header { - untrustedAdj.RawHeader.Time = untrustedAdj.RawHeader.Time.Truncate(time.Hour) - return untrustedAdj - }, - err: true, - }, - { - prepare: func() libhead.Header { - untrustedAdj.RawHeader.ChainID = "toaster" - return untrustedAdj - }, - err: true, - }, - { - prepare: func() libhead.Header { - untrustedAdj.RawHeader.Height++ - return untrustedAdj - }, - err: true, - }, - { - prepare: func() libhead.Header { - untrustedAdj.RawHeader.Version.App = appconsts.LatestVersion + 1 - return untrustedAdj + prepare: func() *header.ExtendedHeader { + untrusted := *untrustedNonAdj + untrusted.Commit = NewTestSuite(t, 2).Commit(RandRawHeader(t)) + return &untrusted }, err: true, }, diff --git a/header/serde.go b/header/serde.go index f4763e3b3b..a511a1352b 100644 --- a/header/serde.go +++ b/header/serde.go @@ -61,7 +61,7 @@ func UnmarshalExtendedHeader(data []byte) (*ExtendedHeader, error) { return nil, err } - return out, out.Validate() + return out, nil } func ExtendedHeaderToProto(eh *ExtendedHeader) (*header_pb.ExtendedHeader, error) { diff --git a/header/verify.go b/header/verify.go deleted file mode 100644 index 827f6c1d1b..0000000000 --- a/header/verify.go +++ /dev/null @@ -1,76 +0,0 @@ -package header - -import ( - "bytes" - "fmt" - "time" - - libhead "github.com/celestiaorg/go-header" -) - -// Verify validates given untrusted Header against trusted ExtendedHeader. -func (eh *ExtendedHeader) Verify(untrusted libhead.Header) error { - untrst, ok := untrusted.(*ExtendedHeader) - if !ok { - // if the header of the type was given, something very wrong happens - panic(fmt.Sprintf("invalid header type: expected %T, got %T", eh, untrusted)) - } - - if err := eh.verify(untrst); err != nil { - return &libhead.VerifyError{Reason: err} - } - - isAdjacent := eh.Height()+1 == untrst.Height() - if isAdjacent { - // Optimized verification for adjacent headers - // Check the validator hashes are the same - if !bytes.Equal(untrst.ValidatorsHash, eh.NextValidatorsHash) { - return &libhead.VerifyError{ - Reason: fmt.Errorf("expected old header next validators (%X) to match those from new header (%X)", - eh.NextValidatorsHash, - untrst.ValidatorsHash, - ), - } - } - - if !bytes.Equal(untrst.LastHeader(), eh.Hash()) { - return &libhead.VerifyError{ - Reason: fmt.Errorf("expected new header to point to last header hash (%X), but got %X)", - eh.Hash(), - untrst.LastHeader(), - ), - } - } - - return nil - } - - return nil -} - -// clockDrift defines how much new header's time can drift into -// the future relative to the now time during verification. -var clockDrift = 10 * time.Second - -// verify performs basic verification of untrusted header. -func (eh *ExtendedHeader) verify(untrst libhead.Header) error { - if untrst.Height() <= eh.Height() { - return fmt.Errorf("untrusted header height(%d) <= current trusted header(%d)", untrst.Height(), eh.Height()) - } - - if untrst.ChainID() != eh.ChainID() { - return fmt.Errorf("untrusted header has different chain %s, not %s", untrst.ChainID(), eh.ChainID()) - } - - if !untrst.Time().After(eh.Time()) { - return fmt.Errorf("untrusted header time(%v) must be after current trusted header(%v)", untrst.Time(), eh.Time()) - } - - now := time.Now() - if !untrst.Time().Before(now.Add(clockDrift)) { - return fmt.Errorf( - "new untrusted header has a time from the future %v (now: %v, clockDrift: %v)", untrst.Time(), now, clockDrift) - } - - return nil -} diff --git a/nodebuilder/das/constructors.go b/nodebuilder/das/constructors.go index 18f6962f40..7c6b5bed4f 100644 --- a/nodebuilder/das/constructors.go +++ b/nodebuilder/das/constructors.go @@ -42,16 +42,16 @@ func newDASer( hsub libhead.Subscriber[*header.ExtendedHeader], store libhead.Store[*header.ExtendedHeader], batching datastore.Batching, - fraudServ fraud.Service, + fraudServ fraud.Service[*header.ExtendedHeader], bFn shrexsub.BroadcastFn, options ...das.Option, -) (*das.DASer, *modfraud.ServiceBreaker[*das.DASer], error) { +) (*das.DASer, *modfraud.ServiceBreaker[*das.DASer, *header.ExtendedHeader], error) { ds, err := das.NewDASer(da, hsub, store, batching, fraudServ, bFn, options...) if err != nil { return nil, nil, err } - return ds, &modfraud.ServiceBreaker[*das.DASer]{ + return ds, &modfraud.ServiceBreaker[*das.DASer, *header.ExtendedHeader]{ Service: ds, FraudServ: fraudServ, FraudType: byzantine.BadEncoding, diff --git a/nodebuilder/das/module.go b/nodebuilder/das/module.go index 61c935fd40..d9f7e700e2 100644 --- a/nodebuilder/das/module.go +++ b/nodebuilder/das/module.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "github.com/celestiaorg/celestia-node/das" + "github.com/celestiaorg/celestia-node/header" modfraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/node" ) @@ -41,10 +42,10 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { baseComponents, fx.Provide(fx.Annotate( newDASer, - fx.OnStart(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*das.DASer]) error { + fx.OnStart(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*das.DASer, *header.ExtendedHeader]) error { return breaker.Start(ctx) }), - fx.OnStop(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*das.DASer]) error { + fx.OnStop(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*das.DASer, *header.ExtendedHeader]) error { return breaker.Stop(ctx) }), )), diff --git a/nodebuilder/fraud/constructors.go b/nodebuilder/fraud/constructors.go index a70ee3e3d4..eee85d4139 100644 --- a/nodebuilder/fraud/constructors.go +++ b/nodebuilder/fraud/constructors.go @@ -1,8 +1,6 @@ package fraud import ( - "context" - "github.com/ipfs/go-datastore" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" @@ -16,32 +14,46 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/p2p" ) -func newFraudService(syncerEnabled bool) func( - fx.Lifecycle, - *pubsub.PubSub, - host.Host, - libhead.Store[*header.ExtendedHeader], - datastore.Batching, - p2p.Network, -) (Module, fraud.Service, error) { - return func( - lc fx.Lifecycle, - sub *pubsub.PubSub, - host host.Host, - hstore libhead.Store[*header.ExtendedHeader], - ds datastore.Batching, - network p2p.Network, - ) (Module, fraud.Service, error) { - getter := func(ctx context.Context, height uint64) (libhead.Header, error) { - return hstore.GetByHeight(ctx, height) - } - pservice := fraudserv.NewProofService(sub, host, getter, ds, syncerEnabled, network.String()) - lc.Append(fx.Hook{ - OnStart: pservice.Start, - OnStop: pservice.Stop, - }) - return &Service{ - Service: pservice, - }, pservice, nil - } +func fraudUnmarshaler() fraud.ProofUnmarshaler[*header.ExtendedHeader] { + return defaultProofUnmarshaler +} + +func newFraudServiceWithSync( + lc fx.Lifecycle, + sub *pubsub.PubSub, + host host.Host, + hstore libhead.Store[*header.ExtendedHeader], + registry fraud.ProofUnmarshaler[*header.ExtendedHeader], + ds datastore.Batching, + network p2p.Network, +) (Module, fraud.Service[*header.ExtendedHeader], error) { + syncerEnabled := true + pservice := fraudserv.NewProofService(sub, host, hstore.GetByHeight, registry, ds, syncerEnabled, network.String()) + lc.Append(fx.Hook{ + OnStart: pservice.Start, + OnStop: pservice.Stop, + }) + return &module{ + Service: pservice, + }, pservice, nil +} + +func newFraudServiceWithoutSync( + lc fx.Lifecycle, + sub *pubsub.PubSub, + host host.Host, + hstore libhead.Store[*header.ExtendedHeader], + registry fraud.ProofUnmarshaler[*header.ExtendedHeader], + ds datastore.Batching, + network p2p.Network, +) (Module, fraud.Service[*header.ExtendedHeader], error) { + syncerEnabled := false + pservice := fraudserv.NewProofService(sub, host, hstore.GetByHeight, registry, ds, syncerEnabled, network.String()) + lc.Append(fx.Hook{ + OnStart: pservice.Start, + OnStop: pservice.Stop, + }) + return &module{ + Service: pservice, + }, pservice, nil } diff --git a/nodebuilder/fraud/fraud.go b/nodebuilder/fraud/fraud.go index 8d10d34e88..45c3863d6f 100644 --- a/nodebuilder/fraud/fraud.go +++ b/nodebuilder/fraud/fraud.go @@ -2,8 +2,12 @@ package fraud import ( "context" + "encoding/json" + "errors" "github.com/celestiaorg/go-fraud" + + "github.com/celestiaorg/celestia-node/header" ) var _ Module = (*API)(nil) @@ -35,3 +39,83 @@ func (api *API) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-cha func (api *API) Get(ctx context.Context, proofType fraud.ProofType) ([]Proof, error) { return api.Internal.Get(ctx, proofType) } + +var _ Module = (*module)(nil) + +// module is an implementation of Module that uses fraud.module as a backend. It is used to +// provide fraud proofs as a non-interface type to the API, and wrap fraud.Subscriber with a +// channel of Proofs. +type module struct { + fraud.Service[*header.ExtendedHeader] +} + +func (s *module) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-chan Proof, error) { + subscription, err := s.Service.Subscribe(proofType) + if err != nil { + return nil, err + } + proofs := make(chan Proof) + go func() { + defer close(proofs) + defer subscription.Cancel() + for { + proof, err := subscription.Proof(ctx) + if err != nil { + if !errors.Is(err, context.DeadlineExceeded) && !errors.Is(err, context.Canceled) { + log.Errorw("fetching proof from subscription", "err", err) + } + return + } + select { + case <-ctx.Done(): + return + case proofs <- Proof{Proof: proof}: + } + } + }() + return proofs, nil +} + +func (s *module) Get(ctx context.Context, proofType fraud.ProofType) ([]Proof, error) { + originalProofs, err := s.Service.Get(ctx, proofType) + if err != nil { + return nil, err + } + proofs := make([]Proof, len(originalProofs)) + for i, originalProof := range originalProofs { + proofs[i].Proof = originalProof + } + return proofs, nil +} + +// Proof embeds the fraud.Proof interface type to provide a concrete type for JSON serialization. +type Proof struct { + fraud.Proof[*header.ExtendedHeader] +} + +type fraudProofJSON struct { + ProofType fraud.ProofType `json:"proof_type"` + Data []byte `json:"data"` +} + +func (f *Proof) UnmarshalJSON(data []byte) error { + var fp fraudProofJSON + err := json.Unmarshal(data, &fp) + if err != nil { + return err + } + f.Proof, err = defaultProofUnmarshaler.Unmarshal(fp.ProofType, fp.Data) + return err +} + +func (f *Proof) MarshalJSON() ([]byte, error) { + marshaledProof, err := f.MarshalBinary() + if err != nil { + return nil, err + } + fraudProof := &fraudProofJSON{ + ProofType: f.Type(), + Data: marshaledProof, + } + return json.Marshal(fraudProof) +} diff --git a/nodebuilder/fraud/lifecycle.go b/nodebuilder/fraud/lifecycle.go index cffa4d0f56..1a6702aafa 100644 --- a/nodebuilder/fraud/lifecycle.go +++ b/nodebuilder/fraud/lifecycle.go @@ -2,11 +2,13 @@ package fraud import ( "context" + "errors" "fmt" "github.com/ipfs/go-datastore" "github.com/celestiaorg/go-fraud" + libhead "github.com/celestiaorg/go-header" ) // service defines minimal interface with service lifecycle methods @@ -18,30 +20,30 @@ type service interface { // ServiceBreaker wraps any service with fraud proof subscription of a specific type. // If proof happens the service is Stopped automatically. // TODO(@Wondertan): Support multiple fraud types. -type ServiceBreaker[S service] struct { +type ServiceBreaker[S service, H libhead.Header[H]] struct { Service S FraudType fraud.ProofType - FraudServ fraud.Service + FraudServ fraud.Service[H] ctx context.Context cancel context.CancelFunc - sub fraud.Subscription + sub fraud.Subscription[H] } // Start starts the inner service if there are no fraud proofs stored. // Subscribes for fraud and stops the service whenever necessary. -func (breaker *ServiceBreaker[S]) Start(ctx context.Context) error { +func (breaker *ServiceBreaker[S, H]) Start(ctx context.Context) error { if breaker == nil { return nil } proofs, err := breaker.FraudServ.Get(ctx, breaker.FraudType) - switch err { + switch { default: return fmt.Errorf("getting proof(%s): %w", breaker.FraudType, err) - case nil: - return &fraud.ErrFraudExists{Proof: proofs} - case datastore.ErrNotFound: + case err == nil: + return &fraud.ErrFraudExists[H]{Proof: proofs} + case errors.Is(err, datastore.ErrNotFound): } err = breaker.Service.Start(ctx) @@ -60,7 +62,7 @@ func (breaker *ServiceBreaker[S]) Start(ctx context.Context) error { } // Stop stops the service and cancels subscription. -func (breaker *ServiceBreaker[S]) Stop(ctx context.Context) error { +func (breaker *ServiceBreaker[S, H]) Stop(ctx context.Context) error { if breaker == nil { return nil } @@ -75,13 +77,13 @@ func (breaker *ServiceBreaker[S]) Stop(ctx context.Context) error { return breaker.Service.Stop(ctx) } -func (breaker *ServiceBreaker[S]) awaitProof() { +func (breaker *ServiceBreaker[S, H]) awaitProof() { _, err := breaker.sub.Proof(breaker.ctx) if err != nil { return } - if err := breaker.Stop(breaker.ctx); err != nil && err != context.Canceled { + if err := breaker.Stop(breaker.ctx); err != nil && !errors.Is(err, context.Canceled) { log.Errorw("stopping service: %s", err.Error()) } } diff --git a/nodebuilder/fraud/module.go b/nodebuilder/fraud/module.go index 718b702f84..bf353f63c6 100644 --- a/nodebuilder/fraud/module.go +++ b/nodebuilder/fraud/module.go @@ -6,27 +6,31 @@ import ( "github.com/celestiaorg/go-fraud" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" ) var log = logging.Logger("module/fraud") func ConstructModule(tp node.Type) fx.Option { - baseComponent := fx.Provide(func(serv fraud.Service) fraud.Getter { - return serv - }) + baseComponent := fx.Options( + fx.Provide(fraudUnmarshaler), + fx.Provide(func(serv fraud.Service[*header.ExtendedHeader]) fraud.Getter[*header.ExtendedHeader] { + return serv + }), + ) switch tp { case node.Light: return fx.Module( "fraud", baseComponent, - fx.Provide(newFraudService(true)), + fx.Provide(newFraudServiceWithSync), ) case node.Full, node.Bridge: return fx.Module( "fraud", baseComponent, - fx.Provide(newFraudService(false)), + fx.Provide(newFraudServiceWithoutSync), ) default: panic("invalid node type") diff --git a/nodebuilder/fraud/service.go b/nodebuilder/fraud/service.go deleted file mode 100644 index 0337c375ef..0000000000 --- a/nodebuilder/fraud/service.go +++ /dev/null @@ -1,87 +0,0 @@ -package fraud - -import ( - "context" - "encoding/json" - - "github.com/celestiaorg/go-fraud" -) - -var _ Module = (*Service)(nil) - -// Service is an implementation of Module that uses fraud.Service as a backend. It is used to -// provide fraud proofs as a non-interface type to the API, and wrap fraud.Subscriber with a -// channel of Proofs. -type Service struct { - fraud.Service -} - -func (s *Service) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-chan Proof, error) { - subscription, err := s.Service.Subscribe(proofType) - if err != nil { - return nil, err - } - proofs := make(chan Proof) - go func() { - defer close(proofs) - for { - proof, err := subscription.Proof(ctx) - if err != nil { - if err != context.DeadlineExceeded && err != context.Canceled { - log.Errorw("fetching proof from subscription", "err", err) - } - return - } - select { - case <-ctx.Done(): - return - case proofs <- Proof{Proof: proof}: - } - } - }() - return proofs, nil -} - -func (s *Service) Get(ctx context.Context, proofType fraud.ProofType) ([]Proof, error) { - originalProofs, err := s.Service.Get(ctx, proofType) - if err != nil { - return nil, err - } - proofs := make([]Proof, len(originalProofs)) - for i, originalProof := range originalProofs { - proofs[i].Proof = originalProof - } - return proofs, nil -} - -// Proof embeds the fraud.Proof interface type to provide a concrete type for JSON serialization. -type Proof struct { - fraud.Proof -} - -type fraudProofJSON struct { - ProofType fraud.ProofType `json:"proof_type"` - Data []byte `json:"data"` -} - -func (f *Proof) UnmarshalJSON(data []byte) error { - var fp fraudProofJSON - err := json.Unmarshal(data, &fp) - if err != nil { - return err - } - f.Proof, err = fraud.Unmarshal(fp.ProofType, fp.Data) - return err -} - -func (f *Proof) MarshalJSON() ([]byte, error) { - marshaledProof, err := f.MarshalBinary() - if err != nil { - return nil, err - } - fraudProof := &fraudProofJSON{ - ProofType: f.Type(), - Data: marshaledProof, - } - return json.Marshal(fraudProof) -} diff --git a/nodebuilder/fraud/unmarshaler.go b/nodebuilder/fraud/unmarshaler.go new file mode 100644 index 0000000000..d5e0461f01 --- /dev/null +++ b/nodebuilder/fraud/unmarshaler.go @@ -0,0 +1,32 @@ +package fraud + +import ( + "github.com/celestiaorg/go-fraud" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share/eds/byzantine" +) + +var defaultProofUnmarshaler proofRegistry + +type proofRegistry struct{} + +func (pr proofRegistry) List() []fraud.ProofType { + return []fraud.ProofType{ + byzantine.BadEncoding, + } +} + +func (pr proofRegistry) Unmarshal(proofType fraud.ProofType, data []byte) (fraud.Proof[*header.ExtendedHeader], error) { + switch proofType { + case byzantine.BadEncoding: + befp := &byzantine.BadEncodingProof{} + err := befp.UnmarshalBinary(data) + if err != nil { + return nil, err + } + return befp, nil + default: + return nil, &fraud.ErrNoUnmarshaler{ProofType: proofType} + } +} diff --git a/nodebuilder/header/constructors.go b/nodebuilder/header/constructors.go index 7d70f0f5a8..267f0c30f7 100644 --- a/nodebuilder/header/constructors.go +++ b/nodebuilder/header/constructors.go @@ -15,21 +15,20 @@ import ( "github.com/celestiaorg/go-header/store" "github.com/celestiaorg/go-header/sync" - "github.com/celestiaorg/celestia-node/header" modfraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" modp2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/share/eds/byzantine" ) // newP2PExchange constructs a new Exchange for headers. -func newP2PExchange( +func newP2PExchange[H libhead.Header[H]]( lc fx.Lifecycle, bpeers modp2p.Bootstrappers, network modp2p.Network, host host.Host, conngater *conngater.BasicConnectionGater, cfg Config, -) (libhead.Exchange[*header.ExtendedHeader], error) { +) (libhead.Exchange[H], error) { peers, err := cfg.trustedPeers(bpeers) if err != nil { return nil, err @@ -39,7 +38,7 @@ func newP2PExchange( ids[index] = peer.ID host.Peerstore().AddAddrs(peer.ID, peer.Addrs, peerstore.PermanentAddrTTL) } - exchange, err := p2p.NewExchange[*header.ExtendedHeader](host, ids, conngater, + exchange, err := p2p.NewExchange[H](host, ids, conngater, p2p.WithParams(cfg.Client), p2p.WithNetworkID[p2p.ClientParameters](network.String()), p2p.WithChainID(network.String()), @@ -60,14 +59,14 @@ func newP2PExchange( } // newSyncer constructs new Syncer for headers. -func newSyncer( - ex libhead.Exchange[*header.ExtendedHeader], - fservice libfraud.Service, - store InitStore, - sub libhead.Subscriber[*header.ExtendedHeader], +func newSyncer[H libhead.Header[H]]( + ex libhead.Exchange[H], + fservice libfraud.Service[H], + store InitStore[H], + sub libhead.Subscriber[H], cfg Config, -) (*sync.Syncer[*header.ExtendedHeader], *modfraud.ServiceBreaker[*sync.Syncer[*header.ExtendedHeader]], error) { - syncer, err := sync.NewSyncer[*header.ExtendedHeader](ex, store, sub, +) (*sync.Syncer[H], *modfraud.ServiceBreaker[*sync.Syncer[H], H], error) { + syncer, err := sync.NewSyncer[H](ex, store, sub, sync.WithParams(cfg.Syncer), sync.WithBlockTime(modp2p.BlockTime), ) @@ -75,7 +74,7 @@ func newSyncer( return nil, nil, err } - return syncer, &modfraud.ServiceBreaker[*sync.Syncer[*header.ExtendedHeader]]{ + return syncer, &modfraud.ServiceBreaker[*sync.Syncer[H], H]{ Service: syncer, FraudType: byzantine.BadEncoding, FraudServ: fservice, @@ -84,16 +83,16 @@ func newSyncer( // InitStore is a type representing initialized header store. // NOTE: It is needed to ensure that Store is always initialized before Syncer is started. -type InitStore libhead.Store[*header.ExtendedHeader] +type InitStore[H libhead.Header[H]] libhead.Store[H] // newInitStore constructs an initialized store -func newInitStore( +func newInitStore[H libhead.Header[H]]( lc fx.Lifecycle, cfg Config, net modp2p.Network, - s libhead.Store[*header.ExtendedHeader], - ex libhead.Exchange[*header.ExtendedHeader], -) (InitStore, error) { + s libhead.Store[H], + ex libhead.Exchange[H], +) (InitStore[H], error) { trustedHash, err := cfg.trustedHash(net) if err != nil { return nil, err diff --git a/nodebuilder/header/module.go b/nodebuilder/header/module.go index 77e7c5eb99..5e02e94fe1 100644 --- a/nodebuilder/header/module.go +++ b/nodebuilder/header/module.go @@ -22,7 +22,7 @@ import ( var log = logging.Logger("module/header") -func ConstructModule(tp node.Type, cfg *Config) fx.Option { +func ConstructModule[H libhead.Header[H]](tp node.Type, cfg *Config) fx.Option { // sanitize config values before constructing module cfgErr := cfg.Validate(tp) @@ -31,61 +31,63 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { fx.Error(cfgErr), fx.Provide(newHeaderService), fx.Provide(fx.Annotate( - func(ds datastore.Batching) (libhead.Store[*header.ExtendedHeader], error) { - return store.NewStore[*header.ExtendedHeader](ds, store.WithParams(cfg.Store)) + func(ds datastore.Batching) (libhead.Store[H], error) { + return store.NewStore[H](ds, store.WithParams(cfg.Store)) }, - fx.OnStart(func(ctx context.Context, store libhead.Store[*header.ExtendedHeader]) error { - return store.Start(ctx) + fx.OnStart(func(ctx context.Context, str libhead.Store[H]) error { + s := str.(*store.Store[H]) + return s.Start(ctx) }), - fx.OnStop(func(ctx context.Context, store libhead.Store[*header.ExtendedHeader]) error { - return store.Stop(ctx) + fx.OnStop(func(ctx context.Context, str libhead.Store[H]) error { + s := str.(*store.Store[H]) + return s.Stop(ctx) }), )), - fx.Provide(newInitStore), - fx.Provide(func(subscriber *p2p.Subscriber[*header.ExtendedHeader]) libhead.Subscriber[*header.ExtendedHeader] { + fx.Provide(newInitStore[H]), + fx.Provide(func(subscriber *p2p.Subscriber[H]) libhead.Subscriber[H] { return subscriber }), fx.Provide(fx.Annotate( - newSyncer, + newSyncer[H], fx.OnStart(func( ctx context.Context, - breaker *modfraud.ServiceBreaker[*sync.Syncer[*header.ExtendedHeader]], + breaker *modfraud.ServiceBreaker[*sync.Syncer[H], H], ) error { return breaker.Start(ctx) }), fx.OnStop(func( ctx context.Context, - breaker *modfraud.ServiceBreaker[*sync.Syncer[*header.ExtendedHeader]], + breaker *modfraud.ServiceBreaker[*sync.Syncer[H], H], ) error { return breaker.Stop(ctx) }), )), fx.Provide(fx.Annotate( - func(ps *pubsub.PubSub, network modp2p.Network) *p2p.Subscriber[*header.ExtendedHeader] { - return p2p.NewSubscriber[*header.ExtendedHeader](ps, header.MsgID, network.String()) + func(ps *pubsub.PubSub, network modp2p.Network) *p2p.Subscriber[H] { + return p2p.NewSubscriber[H](ps, header.MsgID, network.String()) }, - fx.OnStart(func(ctx context.Context, sub *p2p.Subscriber[*header.ExtendedHeader]) error { + fx.OnStart(func(ctx context.Context, sub *p2p.Subscriber[H]) error { return sub.Start(ctx) }), - fx.OnStop(func(ctx context.Context, sub *p2p.Subscriber[*header.ExtendedHeader]) error { + fx.OnStop(func(ctx context.Context, sub *p2p.Subscriber[H]) error { return sub.Stop(ctx) }), )), fx.Provide(fx.Annotate( func( host host.Host, - store libhead.Store[*header.ExtendedHeader], + store libhead.Store[H], network modp2p.Network, - ) (*p2p.ExchangeServer[*header.ExtendedHeader], error) { - return p2p.NewExchangeServer[*header.ExtendedHeader](host, store, + ) (*p2p.ExchangeServer[H], error) { + return p2p.NewExchangeServer[H](host, store, p2p.WithParams(cfg.Server), p2p.WithNetworkID[p2p.ServerParameters](network.String()), ) }, - fx.OnStart(func(ctx context.Context, server *p2p.ExchangeServer[*header.ExtendedHeader]) error { + fx.OnStart(func(ctx context.Context, server *p2p.ExchangeServer[H]) error { return server.Start(ctx) }), - fx.OnStop(func(ctx context.Context, server *p2p.ExchangeServer[*header.ExtendedHeader]) error { + fx.OnStop(func(ctx context.Context, server *p2p.ExchangeServer[H]) error { return server.Stop(ctx) }), )), @@ -96,13 +98,13 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { return fx.Module( "header", baseComponents, - fx.Provide(newP2PExchange), + fx.Provide(newP2PExchange[H]), ) case node.Bridge: return fx.Module( "header", baseComponents, - fx.Provide(func(subscriber *p2p.Subscriber[*header.ExtendedHeader]) libhead.Broadcaster[*header.ExtendedHeader] { + fx.Provide(func(subscriber *p2p.Subscriber[H]) libhead.Broadcaster[H] { return subscriber }), fx.Supply(header.MakeExtendedHeader), diff --git a/nodebuilder/header/module_test.go b/nodebuilder/header/module_test.go index 89293e4ab4..6a35e35284 100644 --- a/nodebuilder/header/module_test.go +++ b/nodebuilder/header/module_test.go @@ -38,7 +38,7 @@ func TestConstructModule_StoreParams(t *testing.T) { fx.Provide(func() datastore.Batching { return datastore.NewMapDatastore() }), - ConstructModule(node.Light, &cfg), + ConstructModule[*header.ExtendedHeader](node.Light, &cfg), fx.Invoke( func(s libhead.Store[*header.ExtendedHeader]) { ss := s.(*store.Store[*header.ExtendedHeader]) @@ -72,10 +72,10 @@ func TestConstructModule_SyncerParams(t *testing.T) { fx.Provide(func() datastore.Batching { return datastore.NewMapDatastore() }), - fx.Provide(func() fraud.Service { + fx.Provide(func() fraud.Service[*header.ExtendedHeader] { return nil }), - ConstructModule(node.Light, &cfg), + ConstructModule[*header.ExtendedHeader](node.Light, &cfg), fx.Invoke(func(s *sync.Syncer[*header.ExtendedHeader]) { syncer = s }), @@ -100,7 +100,7 @@ func TestConstructModule_ExchangeParams(t *testing.T) { fx.Provide(func() datastore.Batching { return datastore.NewMapDatastore() }), - ConstructModule(node.Light, &cfg), + ConstructModule[*header.ExtendedHeader](node.Light, &cfg), fx.Provide(func(b datastore.Batching) (*conngater.BasicConnectionGater, error) { return conngater.NewBasicConnectionGater(b) }), diff --git a/nodebuilder/header/service.go b/nodebuilder/header/service.go index f410c04f04..2b208cb88d 100644 --- a/nodebuilder/header/service.go +++ b/nodebuilder/header/service.go @@ -65,9 +65,9 @@ func (s *Service) GetByHeight(ctx context.Context, height uint64) (*header.Exten switch { case err != nil: return nil, err - case uint64(head.Height()) == height: + case head.Height() == height: return head, nil - case uint64(head.Height())+1 < height: + case head.Height()+1 < height: return nil, fmt.Errorf("header: given height is from the future: "+ "networkHeight: %d, requestedHeight: %d", head.Height(), height) } @@ -78,10 +78,10 @@ func (s *Service) GetByHeight(ctx context.Context, height uint64) (*header.Exten switch { case err != nil: return nil, err - case uint64(head.Height()) == height: + case head.Height() == height: return head, nil // `+1` allows for one header network lag, e.g. user request header that is milliseconds away - case uint64(head.Height())+1 < height: + case head.Height()+1 < height: return nil, fmt.Errorf("header: syncing in progress: "+ "localHeadHeight: %d, requestedHeight: %d", head.Height(), height) default: diff --git a/nodebuilder/header/service_test.go b/nodebuilder/header/service_test.go index 6493d3d51d..14d5ada87d 100644 --- a/nodebuilder/header/service_test.go +++ b/nodebuilder/header/service_test.go @@ -25,9 +25,9 @@ func TestGetByHeightHandlesError(t *testing.T) { }) } -type errorSyncer[H libhead.Header] struct{} +type errorSyncer[H libhead.Header[H]] struct{} -func (d *errorSyncer[H]) Head(context.Context) (H, error) { +func (d *errorSyncer[H]) Head(context.Context, ...libhead.HeadOption[H]) (H, error) { var zero H return zero, fmt.Errorf("dummy error") } diff --git a/nodebuilder/module.go b/nodebuilder/module.go index 719705e35c..3068113102 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -5,13 +5,14 @@ import ( "go.uber.org/fx" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/fxutil" "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/gateway" - "github.com/celestiaorg/celestia-node/nodebuilder/header" + modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/rpc" @@ -46,7 +47,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store // modules provided by the node p2p.ConstructModule(tp, &cfg.P2P), state.ConstructModule(tp, &cfg.State, &cfg.Core), - header.ConstructModule(tp, &cfg.Header), + modhead.ConstructModule[*header.ExtendedHeader](tp, &cfg.Header), share.ConstructModule(tp, &cfg.Share), rpc.ConstructModule(tp, &cfg.RPC), gateway.ConstructModule(tp, &cfg.Gateway), diff --git a/nodebuilder/p2p/pubsub.go b/nodebuilder/p2p/pubsub.go index 0061ab9eea..13d812e3ce 100644 --- a/nodebuilder/p2p/pubsub.go +++ b/nodebuilder/p2p/pubsub.go @@ -18,6 +18,8 @@ import ( "github.com/celestiaorg/go-fraud" "github.com/celestiaorg/go-fraud/fraudserv" headp2p "github.com/celestiaorg/go-header/p2p" + + "github.com/celestiaorg/celestia-node/header" ) func init() { @@ -66,7 +68,7 @@ func pubSub(cfg Config, params pubSubParams) (*pubsub.PubSub, error) { // * https://github.com/libp2p/specs/blob/master/pubsub/gossipsub/gossipsub-v1.1.md#peer-scoring // * lotus // * prysm - topicScores := topicScoreParams(params.Network) + topicScores := topicScoreParams(params) peerScores, err := peerScoreParams(params.Bootstrappers, cfg) if err != nil { return nil, err @@ -105,15 +107,16 @@ type pubSubParams struct { Host hst.Host Bootstrappers Bootstrappers Network Network + Unmarshaler fraud.ProofUnmarshaler[*header.ExtendedHeader] } -func topicScoreParams(network Network) map[string]*pubsub.TopicScoreParams { +func topicScoreParams(params pubSubParams) map[string]*pubsub.TopicScoreParams { mp := map[string]*pubsub.TopicScoreParams{ - headp2p.PubsubTopicID(network.String()): &headp2p.GossibSubScore, + headp2p.PubsubTopicID(params.Network.String()): &headp2p.GossibSubScore, } - for _, pt := range fraud.Registered() { - mp[fraudserv.PubsubTopicID(pt.String(), network.String())] = &fraudserv.GossibSubScore + for _, pt := range params.Unmarshaler.List() { + mp[fraudserv.PubsubTopicID(pt.String(), params.Network.String())] = &fraudserv.GossibSubScore } return mp diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 97440fa7dc..d56125209c 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -22,6 +22,7 @@ import ( "github.com/celestiaorg/go-fraud" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder/das" modheader "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" @@ -80,7 +81,7 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti } state.WithMetrics(ca) }), - fx.Invoke(fraud.WithMetrics), + fx.Invoke(fraud.WithMetrics[*header.ExtendedHeader]), fx.Invoke(node.WithMetrics), fx.Invoke(modheader.WithMetrics), fx.Invoke(share.WithDiscoveryMetrics), diff --git a/nodebuilder/state/core.go b/nodebuilder/state/core.go index 4636e0f099..f8f8508540 100644 --- a/nodebuilder/state/core.go +++ b/nodebuilder/state/core.go @@ -18,11 +18,11 @@ func coreAccessor( corecfg core.Config, signer *apptypes.KeyringSigner, sync *sync.Syncer[*header.ExtendedHeader], - fraudServ libfraud.Service, -) (*state.CoreAccessor, Module, *modfraud.ServiceBreaker[*state.CoreAccessor]) { + fraudServ libfraud.Service[*header.ExtendedHeader], +) (*state.CoreAccessor, Module, *modfraud.ServiceBreaker[*state.CoreAccessor, *header.ExtendedHeader]) { ca := state.NewCoreAccessor(signer, sync, corecfg.IP, corecfg.RPCPort, corecfg.GRPCPort) - return ca, ca, &modfraud.ServiceBreaker[*state.CoreAccessor]{ + return ca, ca, &modfraud.ServiceBreaker[*state.CoreAccessor, *header.ExtendedHeader]{ Service: ca, FraudType: byzantine.BadEncoding, FraudServ: fraudServ, diff --git a/nodebuilder/state/module.go b/nodebuilder/state/module.go index fe90d023eb..733419a918 100644 --- a/nodebuilder/state/module.go +++ b/nodebuilder/state/module.go @@ -6,6 +6,7 @@ import ( logging "github.com/ipfs/go-log/v2" "go.uber.org/fx" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/fxutil" "github.com/celestiaorg/celestia-node/nodebuilder/core" modfraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" @@ -26,10 +27,12 @@ func ConstructModule(tp node.Type, cfg *Config, coreCfg *core.Config) fx.Option fx.Error(cfgErr), fxutil.ProvideIf(coreCfg.IsEndpointConfigured(), fx.Annotate( coreAccessor, - fx.OnStart(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*state.CoreAccessor]) error { + fx.OnStart(func(ctx context.Context, + breaker *modfraud.ServiceBreaker[*state.CoreAccessor, *header.ExtendedHeader]) error { return breaker.Start(ctx) }), - fx.OnStop(func(ctx context.Context, breaker *modfraud.ServiceBreaker[*state.CoreAccessor]) error { + fx.OnStop(func(ctx context.Context, + breaker *modfraud.ServiceBreaker[*state.CoreAccessor, *header.ExtendedHeader]) error { return breaker.Stop(ctx) }), )), diff --git a/nodebuilder/testing.go b/nodebuilder/testing.go index 6cb40a2b6c..36f2c2f47f 100644 --- a/nodebuilder/testing.go +++ b/nodebuilder/testing.go @@ -11,9 +11,10 @@ import ( apptypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/core" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/libs/fxutil" - "github.com/celestiaorg/celestia-node/nodebuilder/header" + modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -47,7 +48,7 @@ func TestNodeWithConfig(t *testing.T, tp node.Type, cfg *Config, opts ...fx.Opti // temp dir for the eds store FIXME: Should be in mem fx.Replace(node.StorePath(t.TempDir())), // avoid requesting trustedPeer during initialization - fxutil.ReplaceAs(headertest.NewStore(t), new(header.InitStore)), + fxutil.ReplaceAs(headertest.NewStore(t), new(modhead.InitStore[*header.ExtendedHeader])), ) // in fact, we don't need core.Client in tests, but Bridge requires is a valid one diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 1bc1c261de..3a66c4e58c 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -80,12 +80,12 @@ func TestGetByHeight(t *testing.T) { networkHead, err := client.Header.NetworkHead(ctx) require.NoError(t, err) - _, err = client.Header.GetByHeight(ctx, uint64(networkHead.Height()+1)) + _, err = client.Header.GetByHeight(ctx, networkHead.Height()+1) require.Nil(t, err, "Requesting syncer.Head()+1 shouldn't return an error") networkHead, err = client.Header.NetworkHead(ctx) require.NoError(t, err) - _, err = client.Header.GetByHeight(ctx, uint64(networkHead.Height()+2)) + _, err = client.Header.GetByHeight(ctx, networkHead.Height()+2) require.ErrorContains(t, err, "given height is from the future") } diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index dfa3577599..234556a3aa 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -295,7 +295,7 @@ func TestSyncLightAgainstFull(t *testing.T) { require.NoError(t, err) bridgeHead, err := bridge.HeaderServ.LocalHead(ctx) require.NoError(t, err) - _, err = full.HeaderServ.WaitForHeight(ctx, uint64(bridgeHead.Height())) + _, err = full.HeaderServ.WaitForHeight(ctx, bridgeHead.Height()) require.NoError(t, err) // reset suite bootstrapper list and set full node as a bootstrapper for @@ -316,7 +316,7 @@ func TestSyncLightAgainstFull(t *testing.T) { require.NoError(t, err) fullHead, err := full.HeaderServ.LocalHead(ctx) require.NoError(t, err) - _, err = light.HeaderServ.WaitForHeight(ctx, uint64(fullHead.Height())) + _, err = light.HeaderServ.WaitForHeight(ctx, fullHead.Height()) require.NoError(t, err) // wait for the core block filling process to exit diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index 3c5bc6951b..e3a862e38a 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -7,7 +7,6 @@ import ( "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/go-fraud" - libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" @@ -22,10 +21,6 @@ const ( BadEncoding fraud.ProofType = "badencoding" + version ) -func init() { - fraud.Register(&BadEncodingProof{}) -} - type BadEncodingProof struct { headerHash []byte BlockHeight uint64 @@ -46,8 +41,7 @@ func CreateBadEncodingProof( hash []byte, height uint64, errByzantine *ErrByzantine, -) fraud.Proof { - +) fraud.Proof[*header.ExtendedHeader] { return &BadEncodingProof{ headerHash: hash, BlockHeight: height, @@ -112,34 +106,29 @@ func (p *BadEncodingProof) UnmarshalBinary(data []byte) error { // Validate checks that provided Merkle Proofs correspond to the shares, // rebuilds bad row or col from received shares, computes Merkle Root // and compares it with block's Merkle Root. -func (p *BadEncodingProof) Validate(hdr libhead.Header) error { - header, ok := hdr.(*header.ExtendedHeader) - if !ok { - panic(fmt.Sprintf("invalid header type received during BEFP validation: expected %T, got %T", header, hdr)) - } - - if header.Height() != int64(p.BlockHeight) { +func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { + if hdr.Height() != p.BlockHeight { return fmt.Errorf("incorrect block height during BEFP validation: expected %d, got %d", - p.BlockHeight, header.Height(), + p.BlockHeight, hdr.Height(), ) } - if len(header.DAH.RowRoots) != len(header.DAH.ColumnRoots) { + if len(hdr.DAH.RowRoots) != len(hdr.DAH.ColumnRoots) { // NOTE: This should never happen as callers of this method should not feed it with a // malformed extended header. panic(fmt.Sprintf( "invalid extended header: length of row and column roots do not match. (rowRoots=%d) (colRoots=%d)", - len(header.DAH.RowRoots), - len(header.DAH.ColumnRoots)), + len(hdr.DAH.RowRoots), + len(hdr.DAH.ColumnRoots)), ) } // merkleRoots are the roots against which we are going to check the inclusion of the received // shares. Changing the order of the roots to prove the shares relative to the orthogonal axis, // because inside the rsmt2d library rsmt2d.Row = 0 and rsmt2d.Col = 1 - merkleRoots := header.DAH.RowRoots + merkleRoots := hdr.DAH.RowRoots if p.Axis == rsmt2d.Row { - merkleRoots = header.DAH.ColumnRoots + merkleRoots = hdr.DAH.ColumnRoots } if int(p.Index) >= len(merkleRoots) { @@ -196,7 +185,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { rebuiltShares, err := codec.Decode(shares) if err != nil { log.Infow("failed to decode shares at height", - "height", header.Height(), "err", err, + "height", hdr.Height(), "err", err, ) return nil } @@ -204,7 +193,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { rebuiltExtendedShares, err := codec.Encode(rebuiltShares[0:odsWidth]) if err != nil { log.Infow("failed to encode shares at height", - "height", header.Height(), "err", err, + "height", hdr.Height(), "err", err, ) return nil } @@ -215,7 +204,7 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { err = tree.Push(share) if err != nil { log.Infow("failed to build a tree from the reconstructed shares at height", - "height", header.Height(), "err", err, + "height", hdr.Height(), "err", err, ) return nil } @@ -224,15 +213,15 @@ func (p *BadEncodingProof) Validate(hdr libhead.Header) error { expectedRoot, err := tree.Root() if err != nil { log.Infow("failed to build a tree root at height", - "height", header.Height(), "err", err, + "height", hdr.Height(), "err", err, ) return nil } // root is a merkle root of the row/col where ErrByzantine occurred - root := header.DAH.RowRoots[p.Index] + root := hdr.DAH.RowRoots[p.Index] if p.Axis == rsmt2d.Col { - root = header.DAH.ColumnRoots[p.Index] + root = hdr.DAH.ColumnRoots[p.Index] } // comparing rebuilt Merkle Root of bad row/col with respective Merkle Root of row/col from block. diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 49cf64c2c2..b5dcea3452 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -86,7 +86,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { }, } - proof := CreateBadEncodingProof(h.Hash(), uint64(h.Height()), &fakeError) + proof := CreateBadEncodingProof(h.Hash(), h.Height(), &fakeError) err = proof.Validate(h) require.Error(t, err) } diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index ebccf0e384..12b1c11083 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -145,7 +145,7 @@ func TestFraudProofValidation(t *testing.T) { faultHeader, err := generateByzantineError(ctx, t, size, bServ) require.True(t, errors.As(err, &errByz)) - p := byzantine.CreateBadEncodingProof([]byte("hash"), uint64(faultHeader.Height()), errByz) + p := byzantine.CreateBadEncodingProof([]byte("hash"), faultHeader.Height(), errByz) err = p.Validate(faultHeader) require.NoError(t, err) }) @@ -197,7 +197,7 @@ func BenchmarkBEFPValidation(b *testing.B) { for i := 0; i < b.N; i++ { b.ReportAllocs() - p := byzantine.CreateBadEncodingProof([]byte("hash"), uint64(h.Height()), errByz) + p := byzantine.CreateBadEncodingProof([]byte("hash"), h.Height(), errByz) err = p.Validate(h) require.NoError(b, err) } diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index c880b9b3c2..f24df2c88b 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -199,7 +199,8 @@ func (d *Discovery) Advertise(ctx context.Context) { } // discoveryLoop ensures we always have '~peerLimit' connected peers. -// It initiates peer discovery upon request and restarts the process until the soft limit is reached. +// It initiates peer discovery upon request and restarts the process until the soft limit is +// reached. func (d *Discovery) discoveryLoop(ctx context.Context) { t := time.NewTicker(discoveryRetryTimeout) defer t.Stop() diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index 2a7c1fee18..87f9361ee2 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -293,7 +293,7 @@ func (m *Manager) subscribeHeader(ctx context.Context, headerSub libhead.Subscri m.validatedPool(h.DataHash.String()) // store first header for validation purposes - if m.initialHeight.CompareAndSwap(0, uint64(h.Height())) { + if m.initialHeight.CompareAndSwap(0, h.Height()) { log.Debugw("stored initial height", "height", h.Height()) } } diff --git a/share/p2p/peers/manager_test.go b/share/p2p/peers/manager_test.go index e10e820e84..ad04d2c7bd 100644 --- a/share/p2p/peers/manager_test.go +++ b/share/p2p/peers/manager_test.go @@ -274,7 +274,7 @@ func TestManager(t *testing.T) { // create shrexSub msg with height lower than first header from headerSub msg := shrexsub.Notification{ DataHash: share.DataHash("datahash"), - Height: uint64(h.Height() - 1), + Height: h.Height() - 1, } result := manager.Validate(ctx, "peer", msg) require.Equal(t, pubsub.ValidationIgnore, result) @@ -298,7 +298,7 @@ func TestManager(t *testing.T) { // create shrexSub msg with height lower than first header from headerSub msg := shrexsub.Notification{ DataHash: share.DataHash("datahash"), - Height: uint64(h.Height() - 1), + Height: h.Height() - 1, } result := manager.Validate(ctx, "peer", msg) require.Equal(t, pubsub.ValidationIgnore, result) @@ -537,7 +537,7 @@ func (s *subLock) Subscribe() (libhead.Subscription[*header.ExtendedHeader], err return s, nil } -func (s *subLock) AddValidator(func(context.Context, *header.ExtendedHeader) pubsub.ValidationResult) error { +func (s *subLock) SetVerifier(func(context.Context, *header.ExtendedHeader) error) error { panic("implement me") } @@ -561,6 +561,6 @@ func (s *subLock) Cancel() { func newShrexSubMsg(h *header.ExtendedHeader) shrexsub.Notification { return shrexsub.Notification{ DataHash: h.DataHash.Bytes(), - Height: uint64(h.Height()), + Height: h.Height(), } } diff --git a/state/core_access.go b/state/core_access.go index 43f5313c5f..d8f6894e24 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -180,9 +180,10 @@ func (ca *CoreAccessor) constructSignedTx( return ca.signer.EncodeTx(tx) } -// SubmitPayForBlob builds, signs, and synchronously submits a MsgPayForBlob. It blocks until the transaction -// is committed and returns the TxReponse. If gasLim is set to 0, the method will automatically estimate the -// gas limit. If the fee is negative, the method will use the nodes min gas price multiplied by the gas limit. +// SubmitPayForBlob builds, signs, and synchronously submits a MsgPayForBlob. It blocks until the +// transaction is committed and returns the TxReponse. If gasLim is set to 0, the method will +// automatically estimate the gas limit. If the fee is negative, the method will use the nodes min +// gas price multiplied by the gas limit. func (ca *CoreAccessor) SubmitPayForBlob( ctx context.Context, fee Int, @@ -201,8 +202,8 @@ func (ca *CoreAccessor) SubmitPayForBlob( appblobs[i] = &b.Blob } - // we only estimate gas if the user wants us to (by setting the gasLim to 0). In the future we may want - // to make these arguments optional. + // we only estimate gas if the user wants us to (by setting the gasLim to 0). In the future we may + // want to make these arguments optional. if gasLim == 0 { blobSizes := make([]uint32, len(blobs)) for i, blob := range blobs { @@ -294,7 +295,7 @@ func (ca *CoreAccessor) BalanceForAddress(ctx context.Context, addr Address) (*B abciReq := abci.RequestQuery{ // TODO @renayay: once https://github.com/cosmos/cosmos-sdk/pull/12674 is merged, use const instead Path: fmt.Sprintf("store/%s/key", banktypes.StoreKey), - Height: head.Height() - 1, + Height: int64(head.Height() - 1), Data: prefixedAccountKey, Prove: true, } diff --git a/state/integration_test.go b/state/integration_test.go index 8862de1bf8..193e7bddc7 100644 --- a/state/integration_test.go +++ b/state/integration_test.go @@ -20,6 +20,7 @@ import ( "github.com/celestiaorg/celestia-app/test/util/testfactory" "github.com/celestiaorg/celestia-app/test/util/testnode" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" + libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/celestia-node/core" "github.com/celestiaorg/celestia-node/header" @@ -95,7 +96,10 @@ type localHeader struct { client rpcclient.Client } -func (l localHeader) Head(ctx context.Context) (*header.ExtendedHeader, error) { +func (l localHeader) Head( + ctx context.Context, + _ ...libhead.HeadOption[*header.ExtendedHeader], +) (*header.ExtendedHeader, error) { latest, err := l.client.Block(ctx, nil) if err != nil { return nil, err From 47047f39dbd7bd344ce47102e7da3aa34a06bba1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jose=20Ramon=20Ma=C3=B1es?= <32740567+jrmanes@users.noreply.github.com> Date: Thu, 31 Aug 2023 10:56:28 +0200 Subject: [PATCH 153/388] chore(nodebuilder/p2p)!: Bump arabica-10 (#2639) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit hello team! This PR contains the updates for `arabica-10`: - the genesis hash - chain-id - bootstrappers thanks in advance! 🚀 Jose Ramon Mañes --------- Signed-off-by: Jose Ramon Mañes --- nodebuilder/p2p/bootstrap.go | 8 ++++---- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index 376d95fb14..0718d8cf59 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -38,10 +38,10 @@ func bootstrappersFor(net Network) ([]string, error) { // NOTE: Every time we add a new long-running network, its bootstrap peers have to be added here. var bootstrapList = map[Network][]string{ Arabica: { - "/dns4/da-bridge-arabica-9.celestia-arabica.com/tcp/2121/p2p/12D3KooWBLvsfkbovAH74DbGGxHPpVW7DkvKdbQxhorrkv9tfGZU", - "/dns4/da-bridge-arabica-9-2.celestia-arabica.com/tcp/2121/p2p/12D3KooWNjJSk8JcY7VoLEjGGUz8CXp9Bxt495zXmdmccjaMPgHf", - "/dns4/da-full-1-arabica-9.celestia-arabica.com/tcp/2121/p2p/12D3KooWFUK2Z4WPsQN3p5n8tgBigxP32gbmABUet2UMby2Ha9ZK", - "/dns4/da-full-2-arabica-9.celestia-arabica.com/tcp/2121/p2p/12D3KooWKnmwsimoghxUT1DXr7f8yXbWCfmXDk4UGbQDsAks9XsN", + "/dns4/da-bridge.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWM3e9MWtyc8GkP8QRt74Riu17QuhGfZMytB2vq5NwkWAu", + "/dns4/da-bridge-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWKj8mcdiBGxQRe1jqhaMnh2tGoC3rPDmr5UH2q8H4WA9M", + "/dns4/da-full-1.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWBWkgmN7kmJSFovVrCjkeG47FkLGq7yEwJ2kEqNKCsBYk", + "/dns4/da-full-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWRByRF67a2kVM2j4MP5Po3jgTw7H2iL2Spu8aUwPkrRfP", }, Mocha: { "/dns4/bootstr-mocha-1.celestia-mocha.com/tcp/2121/p2p/12D3KooWDRSJMbH3PS4dRDa11H7Tk615aqTUgkeEKz4pwd4sS6fN", diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index 1fb6b10a55..31ea0b0ae9 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -23,7 +23,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ - Arabica: "7A5FABB19713D732D967B1DA84FA0DF5E87A7B62302D783F78743E216C1A3550", + Arabica: "5904E55478BA4B3002EE885621E007A2A6A2399662841912219AECD5D5CBE393", Mocha: "79A97034D569C4199A867439B1B7B77D4E1E1D9697212755E1CE6D920CDBB541", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index b1c3a5fbb7..29e0218bb2 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -12,7 +12,7 @@ const ( // DefaultNetwork is the default network of the current build. DefaultNetwork = Mocha // Arabica testnet. See: celestiaorg/networks. - Arabica Network = "arabica-9" + Arabica Network = "arabica-10" // Mocha testnet. See: celestiaorg/networks. Mocha Network = "mocha-3" // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. From 9ff98c4b611bac7b1d08135a4c249e74b1e7afa2 Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 4 Sep 2023 11:58:43 +0200 Subject: [PATCH 154/388] chore(deps): bump golanci-lint workflow (#2641) Self explanatory. The reason for the fix in core_accessor was an implicit memory aliasing. It was taking the address of the loop variable, which doesn't change in between iterations ofc, so the every blob pointed to the last blob --- .github/workflows/go-ci.yml | 2 +- .golangci.yml | 2 +- cmd/celestia/rpc.go | 2 +- nodebuilder/config.go | 10 +++++----- state/core_access.go | 6 +++--- 5 files changed, 11 insertions(+), 11 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 7435b7912f..41b3a82994 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -25,7 +25,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3.6.0 with: - version: v1.52.2 + version: v1.54.2 go_mod_tidy_check: name: Go Mod Tidy Check diff --git a/.golangci.yml b/.golangci.yml index 5f7b13e6a5..a0f2754a9b 100644 --- a/.golangci.yml +++ b/.golangci.yml @@ -4,7 +4,7 @@ run: linters: enable: - bodyclose - - depguard + # - depguard as of v1.54.2, the default config throws errors on our repo - dogsled - dupl - errcheck diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index b8724fa789..c263496b26 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -23,7 +23,7 @@ import ( "github.com/celestiaorg/celestia-node/state" ) -const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" +const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" //nolint:gosec var requestURL string var authTokenFlag string diff --git a/nodebuilder/config.go b/nodebuilder/config.go index 670bbf9bbd..41f24d6d3d 100644 --- a/nodebuilder/config.go +++ b/nodebuilder/config.go @@ -114,7 +114,7 @@ func removeConfig(path string) error { func UpdateConfig(tp node.Type, path string) (err error) { path, err = storePath(path) if err != nil { - return + return err } flock, err := fslock.Lock(lockPath(path)) @@ -122,7 +122,7 @@ func UpdateConfig(tp node.Type, path string) (err error) { if err == fslock.ErrLocked { err = ErrOpened } - return + return err } defer flock.Unlock() //nolint: errcheck @@ -131,18 +131,18 @@ func UpdateConfig(tp node.Type, path string) (err error) { cfgPath := configPath(path) cfg, err := LoadConfig(cfgPath) if err != nil { - return + return err } cfg, err = updateConfig(cfg, newCfg) if err != nil { - return + return err } // save the updated config err = removeConfig(cfgPath) if err != nil { - return + return err } return SaveConfig(cfgPath, cfg) } diff --git a/state/core_access.go b/state/core_access.go index d8f6894e24..2a49e70a03 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -195,11 +195,11 @@ func (ca *CoreAccessor) SubmitPayForBlob( } appblobs := make([]*apptypes.Blob, len(blobs)) - for i, b := range blobs { - if err := b.Namespace().ValidateForBlob(); err != nil { + for i := range blobs { + if err := blobs[i].Namespace().ValidateForBlob(); err != nil { return nil, err } - appblobs[i] = &b.Blob + appblobs[i] = &blobs[i].Blob } // we only estimate gas if the user wants us to (by setting the gasLim to 0). In the future we may From da841121a8c9eb1834f2aac5079124a2917ecec4 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 4 Sep 2023 18:33:07 +0800 Subject: [PATCH 155/388] feat(share/eds) add dagstore shards status metric (#2642) Adds metrics to expose shards statistics. --- share/eds/metrics.go | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 6547e239cd..1f430bf688 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -24,6 +24,8 @@ const ( longOpUnresolved longOpResult = "unresolved" longOpOK longOpResult = "ok" longOpFailed longOpResult = "failed" + + dagstoreShardStatusKey = "shard_status" ) var ( @@ -116,9 +118,31 @@ func (s *Store) WithMetrics() error { return err } + dagStoreShards, err := meter.Int64ObservableGauge("eds_store_dagstore_shards", + metric.WithDescription("dagstore amount of shards by status")) + if err != nil { + return err + } + if err = s.cache.withMetrics(); err != nil { return err } + + callback := func(ctx context.Context, observer metric.Observer) error { + stats := s.dgstr.Stats() + for status, amount := range stats { + observer.ObserveInt64(dagStoreShards, int64(amount), + metric.WithAttributes( + attribute.String(dagstoreShardStatusKey, status.String()), + )) + } + return nil + } + + if _, err := meter.RegisterCallback(callback, dagStoreShards); err != nil { + return err + } + s.metrics = &metrics{ putTime: putTime, getCARTime: getCARTime, From 613d229a70cd73c0b700c2383b7a92af703e6861 Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 4 Sep 2023 13:18:01 +0200 Subject: [PATCH 156/388] refactor(share)!: have NamespacedShares fields use snake_case in JSON (#2633) Related #2631 , does not fix without nmt PR and release --- share/getter.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/share/getter.go b/share/getter.go index 19e4498458..18d3873de1 100644 --- a/share/getter.go +++ b/share/getter.go @@ -13,7 +13,8 @@ import ( var ( // ErrNotFound is used to indicate that requested data could not be found. ErrNotFound = errors.New("share: data not found") - // ErrOutOfBounds is used to indicate that a passed row or column index is out of bounds of the square size. + // ErrOutOfBounds is used to indicate that a passed row or column index is out of bounds of the + // square size. ErrOutOfBounds = errors.New("share: row or column index is larger than square size") ) @@ -50,8 +51,8 @@ func (ns NamespacedShares) Flatten() []Share { // NamespacedRow represents all shares with proofs within a specific namespace of a single EDS row. type NamespacedRow struct { - Shares []Share - Proof *nmt.Proof + Shares []Share `json:"shares"` + Proof *nmt.Proof `json:"proof"` } // Verify validates NamespacedShares by checking every row with nmt inclusion proof. From ab56434799834415ba18915af02e9315c6dda15e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:39:27 +0000 Subject: [PATCH 157/388] chore(deps): bump golangci/golangci-lint-action from 3.6.0 to 3.7.0 (#2586) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [golangci/golangci-lint-action](https://github.com/golangci/golangci-lint-action) from 3.6.0 to 3.7.0.
Release notes

Sourced from golangci/golangci-lint-action's releases.

v3.7.0

What's Changed

Changes

Documentation

Dependencies

... (truncated)

Commits
  • 3a91952 feat: working-directory with only-new-issues (#795)
  • 5e67631 build(deps): bump @​actions/cache from 3.2.1 to 3.2.2 (#825)
  • 18dad33 build(deps-dev): bump @​typescript-eslint/eslint-plugin from 6.2.1 to 6.3.0 (#...
  • 945dc98 build(deps): bump @​types/node from 20.4.8 to 20.5.0 (#824)
  • cac24f5 build(deps-dev): bump eslint from 8.45.0 to 8.47.0 (#823)
  • ab66454 build(deps-dev): bump @​typescript-eslint/parser from 6.2.0 to 6.3.0 (#821)
  • a252750 build(deps): bump @​actions/http-client from 2.1.0 to 2.1.1 (#819)
  • 44d9998 build(deps-dev): bump eslint-config-prettier from 8.9.0 to 9.0.0 (#818)
  • b6bdfb3 build(deps-dev): bump prettier from 3.0.0 to 3.0.1 (#817)
  • c0cd965 build(deps-dev): bump @​typescript-eslint/eslint-plugin from 6.2.0 to 6.2.1 (#...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golangci/golangci-lint-action&package-manager=github_actions&previous-version=3.6.0&new-version=3.7.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/go-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 41b3a82994..f68825bec1 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -23,7 +23,7 @@ jobs: go-version: ${{ env.GO_VERSION }} - name: golangci-lint - uses: golangci/golangci-lint-action@v3.6.0 + uses: golangci/golangci-lint-action@v3.7.0 with: version: v1.54.2 From afc0504de67bf0ea20e528c9fbdad335df0f6f2e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:42:34 +0000 Subject: [PATCH 158/388] chore(deps): bump alpine from 3.18.2 to 3.18.3 (#2564) Bumps alpine from 3.18.2 to 3.18.3. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=alpine&package-manager=docker&previous-version=3.18.2&new-version=3.18.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index c8e9bb2e1c..1f1705c2f3 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . RUN make build && make cel-key -FROM docker.io/alpine:3.18.2 +FROM docker.io/alpine:3.18.3 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From 80d6dbaa7278ffc590a75d556daffbb6ac37d657 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 13:43:14 +0200 Subject: [PATCH 159/388] chore(deps): bump cosmossdk.io/math from 1.1.1 to 1.1.2 (#2594) Bumps [cosmossdk.io/math](https://github.com/cosmos/cosmos-sdk) from 1.1.1 to 1.1.2.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=cosmossdk.io/math&package-manager=go_modules&previous-version=1.1.1&new-version=1.1.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 846f4316e6..38b2068178 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1 require ( cosmossdk.io/errors v1.0.0 - cosmossdk.io/math v1.1.1 + cosmossdk.io/math v1.1.2 github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 diff --git a/go.sum b/go.sum index 09bd8ab26e..feaba34043 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= -cosmossdk.io/math v1.1.1 h1:Eqx44E6fSvG055Z6VNiCLWA9fra0JSyP0kQX7VvNNfk= -cosmossdk.io/math v1.1.1/go.mod h1:uFRkSZDz38KjWjm6jN+/sI8tJWQxbGwxcjOTzapWSpE= +cosmossdk.io/math v1.1.2 h1:ORZetZCTyWkI5GlZ6CZS28fMHi83ZYf+A2vVnHNzZBM= +cosmossdk.io/math v1.1.2/go.mod h1:l2Gnda87F0su8a/7FEKJfFdJrM0JZRXQaohlgJeyQh0= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= From d75a743d7c31b4edfdb8e3dbe8dbca0008bea9cb Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 4 Sep 2023 15:09:42 +0300 Subject: [PATCH 160/388] tests(swamp): befp testing (#2584) --- das/daser_test.go | 3 +- header/headertest/fraud/testing.go | 100 +++++++++++++++++ header/headertest/testing.go | 48 --------- nodebuilder/fraud/lifecycle.go | 2 +- nodebuilder/p2p/opts.go | 6 ++ nodebuilder/tests/fraud_test.go | 168 +++++++++++------------------ nodebuilder/tests/swamp/swamp.go | 14 +++ share/eds/edstest/testing.go | 8 +- share/getters/cascade.go | 10 +- share/getters/ipld.go | 5 + 10 files changed, 208 insertions(+), 156 deletions(-) create mode 100644 header/headertest/fraud/testing.go diff --git a/das/daser_test.go b/das/daser_test.go index 68f6e01ef2..e4e74dc7ff 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -23,6 +23,7 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" + headerfraud "github.com/celestiaorg/celestia-node/header/headertest/fraud" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/availability/full" "github.com/celestiaorg/celestia-node/share/availability/light" @@ -180,7 +181,7 @@ func TestDASer_stopsAfter_BEFP(t *testing.T) { "private", ) require.NoError(t, fserv.Start(ctx)) - mockGet.headers[1], _ = headertest.CreateFraudExtHeader(t, mockGet.headers[1], bServ) + mockGet.headers[1] = headerfraud.CreateFraudExtHeader(t, mockGet.headers[1], bServ) newCtx := context.Background() // create and start DASer diff --git a/header/headertest/fraud/testing.go b/header/headertest/fraud/testing.go new file mode 100644 index 0000000000..6a5cda733d --- /dev/null +++ b/header/headertest/fraud/testing.go @@ -0,0 +1,100 @@ +package headerfraud + +import ( + "context" + "testing" + "time" + + "github.com/ipfs/go-blockservice" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/bytes" + tmproto "github.com/tendermint/tendermint/proto/tendermint/types" + "github.com/tendermint/tendermint/types" + + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" + "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" +) + +// FraudMaker allows to produce an invalid header at the specified height in order to produce the +// BEFP. +type FraudMaker struct { + t *testing.T + + vals []types.PrivValidator + valSet *types.ValidatorSet + + // height of the invalid header + height int64 + + prevHash bytes.HexBytes +} + +func NewFraudMaker(t *testing.T, height int64, vals []types.PrivValidator, valSet *types.ValidatorSet) *FraudMaker { + return &FraudMaker{ + t: t, + vals: vals, + valSet: valSet, + height: height, + } +} + +func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *eds.Store) header.ConstructFn { + return func(h *types.Header, + comm *types.Commit, + vals *types.ValidatorSet, + eds *rsmt2d.ExtendedDataSquare, + ) (*header.ExtendedHeader, error) { + if h.Height < f.height { + return header.MakeExtendedHeader(h, comm, vals, eds) + } + + hdr := *h + if h.Height == f.height { + adder := ipld.NewProofsAdder(odsSize) + square := edstest.RandByzantineEDS(f.t, odsSize, nmt.NodeVisitor(adder.VisitFn())) + dah, err := da.NewDataAvailabilityHeader(square) + require.NoError(f.t, err) + hdr.DataHash = dah.Hash() + + ctx := ipld.CtxWithProofsAdder(context.Background(), adder) + require.NoError(f.t, edsStore.Put(ctx, h.DataHash.Bytes(), square)) + + *eds = *square + } + if h.Height > f.height { + hdr.LastBlockID.Hash = f.prevHash + } + + blockID := comm.BlockID + blockID.Hash = hdr.Hash() + voteSet := types.NewVoteSet(hdr.ChainID, hdr.Height, 0, tmproto.PrecommitType, f.valSet) + commit, err := headertest.MakeCommit(blockID, hdr.Height, 0, voteSet, f.vals, time.Now()) + require.NoError(f.t, err) + + *h = hdr + *comm = *commit + f.prevHash = h.Hash() + return header.MakeExtendedHeader(h, comm, vals, eds) + } +} +func CreateFraudExtHeader( + t *testing.T, + eh *header.ExtendedHeader, + serv blockservice.BlockService, +) *header.ExtendedHeader { + square := edstest.RandByzantineEDS(t, len(eh.DAH.RowRoots)) + err := ipld.ImportEDS(context.Background(), square, serv) + require.NoError(t, err) + dah, err := da.NewDataAvailabilityHeader(square) + require.NoError(t, err) + eh.DAH = &dah + eh.RawHeader.DataHash = dah.Hash() + return eh +} diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 65ae8c950f..f288556bd9 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -1,7 +1,6 @@ package headertest import ( - "context" "crypto/rand" "fmt" mrand "math/rand" @@ -9,8 +8,6 @@ import ( "testing" "time" - "github.com/ipfs/go-blockservice" - logging "github.com/ipfs/go-log/v2" "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/crypto/tmhash" "github.com/tendermint/tendermint/libs/bytes" @@ -26,12 +23,8 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" - "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/ipld" ) -var log = logging.Logger("headertest") - // TestSuite provides everything you need to test chain of Headers. // If not, please don't hesitate to extend it for your case. type TestSuite struct { @@ -296,32 +289,6 @@ func RandBlockID(*testing.T) types.BlockID { return bid } -// FraudMaker creates a custom ConstructFn that breaks the block at the given height. -func FraudMaker(t *testing.T, faultHeight int64, bServ blockservice.BlockService) header.ConstructFn { - log.Warn("Corrupting block...", "height", faultHeight) - return func( - h *types.Header, - comm *types.Commit, - vals *types.ValidatorSet, - eds *rsmt2d.ExtendedDataSquare, - ) (*header.ExtendedHeader, error) { - if h.Height == faultHeight { - eh := &header.ExtendedHeader{ - RawHeader: *h, - Commit: comm, - ValidatorSet: vals, - } - - eh, dataSq := CreateFraudExtHeader(t, eh, bServ) - if eds != nil { - *eds = *dataSq - } - return eh, nil - } - return header.MakeExtendedHeader(h, comm, vals, eds) - } -} - func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedDataSquare) *header.ExtendedHeader { valSet, vals := RandValidatorSet(10, 10) gen := RandRawHeader(t) @@ -348,21 +315,6 @@ func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedData return eh } -func CreateFraudExtHeader( - t *testing.T, - eh *header.ExtendedHeader, - serv blockservice.BlockService, -) (*header.ExtendedHeader, *rsmt2d.ExtendedDataSquare) { - square := edstest.RandByzantineEDS(t, len(eh.DAH.RowRoots)) - err := ipld.ImportEDS(context.Background(), square, serv) - require.NoError(t, err) - dah, err := da.NewDataAvailabilityHeader(square) - require.NoError(t, err) - eh.DAH = &dah - eh.RawHeader.DataHash = dah.Hash() - return eh, square -} - type Subscriber struct { headertest.Subscriber[*header.ExtendedHeader] } diff --git a/nodebuilder/fraud/lifecycle.go b/nodebuilder/fraud/lifecycle.go index 1a6702aafa..50f4e1035b 100644 --- a/nodebuilder/fraud/lifecycle.go +++ b/nodebuilder/fraud/lifecycle.go @@ -73,7 +73,7 @@ func (breaker *ServiceBreaker[S, H]) Stop(ctx context.Context) error { } breaker.sub.Cancel() - breaker.cancel() + defer breaker.cancel() return breaker.Service.Stop(ctx) } diff --git a/nodebuilder/p2p/opts.go b/nodebuilder/p2p/opts.go index 9501dfe8e1..8e5d714a64 100644 --- a/nodebuilder/p2p/opts.go +++ b/nodebuilder/p2p/opts.go @@ -3,6 +3,7 @@ package p2p import ( "encoding/hex" + "github.com/ipfs/go-blockservice" "github.com/libp2p/go-libp2p/core/crypto" hst "github.com/libp2p/go-libp2p/core/host" "go.uber.org/fx" @@ -34,3 +35,8 @@ func WithP2PKeyStr(key string) fx.Option { func WithHost(hst hst.Host) fx.Option { return fxutil.ReplaceAs(hst, new(HostBase)) } + +// WithBlockService allows to replace the default BlockService. +func WithBlockService(bServ blockservice.BlockService) fx.Option { + return fxutil.ReplaceAs(bServ, new(blockservice.BlockService)) +} diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index f652724d55..95c702c0c0 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -5,16 +5,20 @@ import ( "testing" "time" - mdutils "github.com/ipfs/go-merkledag/test" + "github.com/ipfs/go-datastore" + ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/types" + "go.uber.org/fx" - "github.com/celestiaorg/celestia-node/header/headertest" + headerfraud "github.com/celestiaorg/celestia-node/header/headertest/fraud" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" + "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/byzantine" ) @@ -29,156 +33,114 @@ Steps: 4. Start a FN. 5. Subscribe to a fraud proof and wait when it will be received. 6. Check FN is not synced to 15. -Note: 15 is not available because DASer will be stopped before reaching this height due to receiving a fraud proof. +Note: 15 is not available because DASer/Syncer will be stopped +before reaching this height due to receiving a fraud proof. Another note: this test disables share exchange to speed up test results. +7. Spawn a Light Node(LN) in order to sync a BEFP. +8. Ensure that the BEFP was received. +9. Try to start a Full Node(FN) that contains a BEFP in its store. */ -func TestFraudProofBroadcasting(t *testing.T) { - t.Skip("requires BEFP generation on app side to work") +func TestFraudProofHandling(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) const ( blocks = 15 - blockSize = 2 - blockTime = time.Millisecond * 300 + blockSize = 4 + blockTime = time.Second ) sw := swamp.NewSwamp(t, swamp.WithBlockTime(blockTime)) fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, blockSize, blocks) + set, val := sw.Validators(t) + fMaker := headerfraud.NewFraudMaker(t, 10, []types.PrivValidator{val}, set) + + tmpDir := t.TempDir() + ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) + edsStore, err := eds.NewStore(tmpDir, ds) + require.NoError(t, err) + require.NoError(t, edsStore.Start(ctx)) + t.Cleanup(func() { + _ = edsStore.Stop(ctx) + }) cfg := nodebuilder.DefaultConfig(node.Bridge) - cfg.Share.UseShareExchange = false + // 1. bridge := sw.NewNodeWithConfig( node.Bridge, cfg, - core.WithHeaderConstructFn(headertest.FraudMaker(t, 10, mdutils.Bserv())), + core.WithHeaderConstructFn(fMaker.MakeExtendedHeader(16, edsStore)), + fx.Replace(edsStore), ) - - err := bridge.Start(ctx) + // 2. + err = bridge.Start(ctx) require.NoError(t, err) + // 3. cfg = nodebuilder.DefaultConfig(node.Full) - cfg.Share.UseShareExchange = false addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) require.NoError(t, err) cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) - + cfg.Share.UseShareExchange = false store := nodebuilder.MockStore(t, cfg) full := sw.NewNodeWithStore(node.Full, store) + // 4. err = full.Start(ctx) require.NoError(t, err) - // subscribe to fraud proof before node starts helps - // to prevent flakiness when fraud proof is propagating before subscribing on it - subscr, err := full.FraudServ.Subscribe(ctx, byzantine.BadEncoding) + // 5. + subCtx, subCancel := context.WithCancel(ctx) + subscr, err := full.FraudServ.Subscribe(subCtx, byzantine.BadEncoding) require.NoError(t, err) - select { case p := <-subscr: require.Equal(t, 10, int(p.Height())) + subCancel() case <-ctx.Done(): - t.Fatal("fraud proof was not received in time") + subCancel() + t.Fatal("full node did not receive a fraud proof in time") } + // This is an obscure way to check if the Syncer was stopped. // If we cannot get a height header within a timeframe it means the syncer was stopped // FIXME: Eventually, this should be a check on service registry managing and keeping // lifecycles of each Module. - syncCtx, syncCancel := context.WithTimeout(context.Background(), btime) - _, err = full.HeaderServ.WaitForHeight(syncCtx, 100) + // 6. + syncCtx, syncCancel := context.WithTimeout(context.Background(), blockTime*5) + _, err = full.HeaderServ.WaitForHeight(syncCtx, 15) require.ErrorIs(t, err, context.DeadlineExceeded) syncCancel() - sw.StopNode(ctx, full) - - full = sw.NewNodeWithStore(node.Full, store) - - require.Error(t, full.Start(ctx)) - proofs, err := full.FraudServ.Get(ctx, byzantine.BadEncoding) - require.NoError(t, err) - require.NotNil(t, proofs) - require.NoError(t, <-fillDn) -} - -/* -Test-Case: Light node receives a fraud proof using Fraud Sync -Pre-Requisites: -- CoreClient is started by swamp. -Steps: -1. Create a Bridge Node(BN) with broken extended header at height 10. -2. Start a BN. -3. Create a Full Node(FN) with a connection to BN as a trusted peer. -4. Start a FN. -5. Subscribe to a fraud proof and wait when it will be received. -6. Start LN once a fraud proof is received and verified by FN. -7. Wait until LN will be connected to FN and fetch a fraud proof. -Note: this test disables share exchange to speed up test results. -*/ -func TestFraudProofSyncing(t *testing.T) { - t.Skip("requires BEFP generation on app side to work") - - const ( - blocks = 15 - bsize = 2 - btime = time.Millisecond * 300 - ) - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) - t.Cleanup(cancel) - - fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, blocks) - cfg := nodebuilder.DefaultConfig(node.Bridge) - cfg.Share.UseShareExchange = false - store := nodebuilder.MockStore(t, cfg) - bridge := sw.NewNodeWithStore( - node.Bridge, - store, - core.WithHeaderConstructFn(headertest.FraudMaker(t, 10, mdutils.Bserv())), - ) - - err := bridge.Start(ctx) - require.NoError(t, err) - addr := host.InfoFromHost(bridge.Host) - addrs, err := peer.AddrInfoToP2pAddrs(addr) - require.NoError(t, err) - - fullCfg := nodebuilder.DefaultConfig(node.Full) - fullCfg.Share.UseShareExchange = false - fullCfg.Header.TrustedPeers = append(fullCfg.Header.TrustedPeers, addrs[0].String()) - full := sw.NewNodeWithStore(node.Full, nodebuilder.MockStore(t, fullCfg)) - - lightCfg := nodebuilder.DefaultConfig(node.Light) - lightCfg.Header.TrustedPeers = append(lightCfg.Header.TrustedPeers, addrs[0].String()) - ln := sw.NewNodeWithStore(node.Light, nodebuilder.MockStore(t, lightCfg)) - require.NoError(t, full.Start(ctx)) + // 7. + cfg = nodebuilder.DefaultConfig(node.Light) + cfg.Header.TrustedPeers = append(cfg.Header.TrustedPeers, addrs[0].String()) + lnStore := nodebuilder.MockStore(t, cfg) + light := sw.NewNodeWithStore(node.Light, lnStore) + require.NoError(t, light.Start(ctx)) - subsFN, err := full.FraudServ.Subscribe(ctx, byzantine.BadEncoding) + // 8. + subCtx, subCancel = context.WithCancel(ctx) + subscr, err = light.FraudServ.Subscribe(subCtx, byzantine.BadEncoding) require.NoError(t, err) - select { - case <-subsFN: + case p := <-subscr: + require.Equal(t, 10, int(p.Height())) + subCancel() case <-ctx.Done(): - t.Fatal("full node didn't get FP in time") + subCancel() + t.Fatal("light node did not receive a fraud proof in time") } - // start LN to enforce syncing logic, not the PubSub's broadcasting - err = ln.Start(ctx) - require.NoError(t, err) - - // internal subscription for the fraud proof is done in order to ensure that light node - // receives the BEFP. - subsLN, err := ln.FraudServ.Subscribe(ctx, byzantine.BadEncoding) - require.NoError(t, err) - - // ensure that the full and light node are connected to speed up test - // alternatively, they would discover each other - err = ln.Host.Connect(ctx, *host.InfoFromHost(full.Host)) + // 9. + fN := sw.NewNodeWithStore(node.Full, store) + require.Error(t, fN.Start(ctx)) + proofs, err := fN.FraudServ.Get(ctx, byzantine.BadEncoding) require.NoError(t, err) + require.NotNil(t, proofs) - select { - case <-subsLN: - case <-ctx.Done(): - t.Fatal("light node didn't get FP in time") - } + sw.StopNode(ctx, bridge) + sw.StopNode(ctx, full) + sw.StopNode(ctx, light) require.NoError(t, <-fillDn) } diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 58584912be..4f07d96b51 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -16,6 +16,8 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" ma "github.com/multiformats/go-multiaddr" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/privval" + "github.com/tendermint/tendermint/types" "go.uber.org/fx" "golang.org/x/exp/maps" @@ -335,3 +337,15 @@ func (s *Swamp) SetBootstrapper(t *testing.T, bootstrappers ...*nodebuilder.Node s.Bootstrappers = append(s.Bootstrappers, addrs[0]) } } + +// Validators retrieves keys from the app node in order to build the validators. +func (s *Swamp) Validators(t *testing.T) (*types.ValidatorSet, types.PrivValidator) { + privPath := s.cfg.TmConfig.PrivValidatorKeyFile() + statePath := s.cfg.TmConfig.PrivValidatorStateFile() + priv := privval.LoadFilePV(privPath, statePath) + key, err := priv.GetPubKey() + require.NoError(t, err) + validator := types.NewValidator(key, 100) + set := types.NewValidatorSet([]*types.Validator{validator}) + return set, priv +} diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go index ddca285f0c..f75e8b619b 100644 --- a/share/eds/edstest/testing.go +++ b/share/eds/edstest/testing.go @@ -7,17 +7,21 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/sharetest" ) -func RandByzantineEDS(t *testing.T, size int) *rsmt2d.ExtendedDataSquare { +func RandByzantineEDS(t *testing.T, size int, options ...nmt.Option) *rsmt2d.ExtendedDataSquare { eds := RandEDS(t, size) shares := eds.Flattened() copy(share.GetData(shares[0]), share.GetData(shares[1])) // corrupting eds - eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) + eds, err := rsmt2d.ImportExtendedDataSquare(shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(size), + options...)) require.NoError(t, err, "failure to recompute the extended data square") return eds } diff --git a/share/getters/cascade.go b/share/getters/cascade.go index 63d7713d3d..eb3e969c1c 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -11,6 +11,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/byzantine" ) var _ share.Getter = (*CascadeGetter)(nil) @@ -130,8 +131,15 @@ func cascadeGetters[V any]( continue } - err = errors.Join(err, getErr) span.RecordError(getErr, trace.WithAttributes(attribute.Int("getter_idx", i))) + var byzantineErr *byzantine.ErrByzantine + if errors.As(getErr, &byzantineErr) { + // short circuit if byzantine error was detected (to be able to handle it correctly + // and create the BEFP) + return zero, byzantineErr + } + + err = errors.Join(err, getErr) if ctx.Err() != nil { return zero, err } diff --git a/share/getters/ipld.go b/share/getters/ipld.go index a892e0fc82..8e11a389bd 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -16,6 +16,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" + "github.com/celestiaorg/celestia-node/share/eds/byzantine" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -82,6 +83,10 @@ func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d // convert error to satisfy getter interface contract err = share.ErrNotFound } + var errByz *byzantine.ErrByzantine + if errors.As(err, &errByz) { + return nil, err + } if err != nil { return nil, fmt.Errorf("getter/ipld: failed to retrieve eds: %w", err) } From 42caaee5df1ebe31336e2534b644dcda5e030a14 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:20:36 +0200 Subject: [PATCH 161/388] chore(deps): bump github.com/ipfs/go-ipld-cbor from 0.0.6 to 0.1.0 (#2622) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/go-ipld-cbor](https://github.com/ipfs/go-ipld-cbor) from 0.0.6 to 0.1.0.
Release notes

Sourced from github.com/ipfs/go-ipld-cbor's releases.

v0.1.0

What's Changed

  • allow configuration of ipldStores default hash function by @​whyrusleeping in ipfs/go-ipld-cbor#86
  • adopt PL unified-ci
  • dependency upgrades to latest
  • Minimum go version bump to v1.20.

Full Changelog: https://github.com/ipfs/go-ipld-cbor/compare/v0.0.6...v0.1.0

Commits
  • 2d0b3fb v0.1.0 bump
  • c10cddb chore: update deps
  • f314046 chore: bump go.mod to Go 1.20 and run go fix
  • 16a0369 chore: add or force update version.json
  • da23a18 chore: add or force update .github/workflows/tagpush.yml
  • 8da0e33 chore: add or force update .github/workflows/release-check.yml
  • 5698d0f chore: add or force update .github/workflows/releaser.yml
  • 5f282bb chore: add or force update .github/workflows/go-check.yml
  • 2312e76 chore: add or force update .github/workflows/go-test.yml
  • 1dc9bde chore: delete templates [skip ci] (#91)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/go-ipld-cbor&package-manager=go_modules&previous-version=0.0.6&new-version=0.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 38b2068178..9206bddaea 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/ipfs/go-blockservice v0.5.1 // down 1 version, 0.5.2 is marked as deprecated and raises alerts github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 - github.com/ipfs/go-ipld-cbor v0.0.6 + github.com/ipfs/go-ipld-cbor v0.1.0 github.com/ipfs/go-ipld-format v0.5.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.11.0 diff --git a/go.sum b/go.sum index feaba34043..b2afe6be69 100644 --- a/go.sum +++ b/go.sum @@ -1147,8 +1147,9 @@ github.com/ipfs/go-ipfs-util v0.0.3/go.mod h1:LHzG1a0Ig4G+iZ26UUOMjHd+lfM84LZCrn github.com/ipfs/go-ipld-cbor v0.0.2/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.3/go.mod h1:wTBtrQZA3SoFKMVkp6cn6HMRteIB1VsmHA0AQFOn7Nc= github.com/ipfs/go-ipld-cbor v0.0.5/go.mod h1:BkCduEx3XBCO6t2Sfo5BaHzuok7hbhdMm9Oh8B2Ftq4= -github.com/ipfs/go-ipld-cbor v0.0.6 h1:pYuWHyvSpIsOOLw4Jy7NbBkCyzLDcl64Bf/LZW7eBQ0= github.com/ipfs/go-ipld-cbor v0.0.6/go.mod h1:ssdxxaLJPXH7OjF5V4NSjBbcfh+evoR4ukuru0oPXMA= +github.com/ipfs/go-ipld-cbor v0.1.0 h1:dx0nS0kILVivGhfWuB6dUpMa/LAwElHPw1yOGYopoYs= +github.com/ipfs/go-ipld-cbor v0.1.0/go.mod h1:U2aYlmVrJr2wsUBU67K4KgepApSZddGRDWBYR0H4sCk= github.com/ipfs/go-ipld-format v0.0.1/go.mod h1:kyJtbkDALmFHv3QR6et67i35QzO3S0dCDnkOJhcZkms= github.com/ipfs/go-ipld-format v0.0.2/go.mod h1:4B6+FM2u9OJ9zCV+kSbgFAZlOrv1Hqbf0INGQgiKf9k= github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg1BuydPSdzQs= From 9415cf0b21c490aa23374b2d503bc3b62148f3f4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:40:09 +0200 Subject: [PATCH 162/388] chore(deps): bump github.com/ipfs/go-ipld-format from 0.5.0 to 0.6.0 (#2621) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/go-ipld-format](https://github.com/ipfs/go-ipld-format) from 0.5.0 to 0.6.0.
Release notes

Sourced from github.com/ipfs/go-ipld-format's releases.

v0.6.0

What's Changed

New Contributors

Full Changelog: https://github.com/ipfs/go-ipld-format/compare/v0.5.0...v0.6.0

Commits
  • 608bf9d v0.6.0 bump
  • 93da70a chore: update deps
  • 6ce9a5e fix: stop using the deprecated io/ioutil package
  • 1c2f1ce chore: bump go.mod to Go 1.20 and run go fix
  • ad47740 ci: uci/copy-templates (#83)
  • 706e93c chore: delete templates [skip ci] (#82)
  • c3da866 chore: Update .github/workflows/stale.yml [skip ci]
  • ef09f69 chore: Update .github/workflows/stale.yml [skip ci]
  • 530b600 Bump golang.org/x/crypto from 0.0.0-20190211182817-74369b46fc67 to 0.1.0
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/go-ipld-format&package-manager=go_modules&previous-version=0.5.0&new-version=0.6.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 9206bddaea..7fb3056a9e 100644 --- a/go.mod +++ b/go.mod @@ -38,7 +38,7 @@ require ( github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-cbor v0.1.0 - github.com/ipfs/go-ipld-format v0.5.0 + github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipfs/go-merkledag v0.11.0 github.com/ipld/go-car v0.6.2 diff --git a/go.sum b/go.sum index b2afe6be69..91e56dd9ca 100644 --- a/go.sum +++ b/go.sum @@ -1156,8 +1156,9 @@ github.com/ipfs/go-ipld-format v0.2.0/go.mod h1:3l3C1uKoadTPbeNfrDi+xMInYKlx2Cvg github.com/ipfs/go-ipld-format v0.3.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.3.1/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= github.com/ipfs/go-ipld-format v0.4.0/go.mod h1:co/SdBE8h99968X0hViiw1MNlh6fvxxnHpvVLnH7jSM= -github.com/ipfs/go-ipld-format v0.5.0 h1:WyEle9K96MSrvr47zZHKKcDxJ/vlpET6PSiQsAFO+Ds= github.com/ipfs/go-ipld-format v0.5.0/go.mod h1:ImdZqJQaEouMjCvqCe0ORUS+uoBmf7Hf+EO/jh+nk3M= +github.com/ipfs/go-ipld-format v0.6.0 h1:VEJlA2kQ3LqFSIm5Vu6eIlSxD/Ze90xtc4Meten1F5U= +github.com/ipfs/go-ipld-format v0.6.0/go.mod h1:g4QVMTn3marU3qXchwjpKPKgJv+zF+OlaKMyhJ4LHPg= github.com/ipfs/go-ipld-legacy v0.1.0/go.mod h1:86f5P/srAmh9GcIcWQR9lfFLZPrIyyXQeVlOWeeWEuI= github.com/ipfs/go-ipld-legacy v0.1.1/go.mod h1:8AyKFCjgRPsQFf15ZQgDB8Din4DML/fOmKZkkFkrIEg= github.com/ipfs/go-ipld-legacy v0.2.1 h1:mDFtrBpmU7b//LzLSypVrXsD8QxkEWxu5qVxN99/+tk= From a31a8081a1e4471d01320aa61e42abc4ebc338de Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 4 Sep 2023 14:51:30 +0200 Subject: [PATCH 163/388] feat: local docker telemetry infra (#2296) Supersedes #1990 Adds local telemetry infra to the Makefile. This includes pyroscope, grafana, jaeger, an otel-collector, prometheus, and loki. --------- Co-authored-by: derrandz Co-authored-by: ramin --- Makefile | 12 +++ README.md | 10 ++- docker/telemetry/docker-compose.yml | 89 +++++++++++++++++++ .../telemetry/grafana/datasources/config.yml | 14 +++ docker/telemetry/loki/config.yml | 11 +++ docker/telemetry/otel-collector/config.yml | 32 +++++++ docker/telemetry/prometheus/prometheus.yml | 25 ++++++ docker/telemetry/promtail/config.yml | 29 ++++++ 8 files changed, 218 insertions(+), 4 deletions(-) create mode 100644 docker/telemetry/docker-compose.yml create mode 100644 docker/telemetry/grafana/datasources/config.yml create mode 100644 docker/telemetry/loki/config.yml create mode 100644 docker/telemetry/otel-collector/config.yml create mode 100644 docker/telemetry/prometheus/prometheus.yml create mode 100644 docker/telemetry/promtail/config.yml diff --git a/Makefile b/Makefile index feef6172aa..d4b3d2572e 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,6 @@ SHELL=/usr/bin/env bash PROJECTNAME=$(shell basename "$(PWD)") +DIR_FULLPATH=$(shell pwd) versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node" LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" ifeq (${PREFIX},) @@ -176,3 +177,14 @@ adr-gen: @echo "--> Generating ADR" @curl -sSL https://raw.githubusercontent.com/celestiaorg/.github/main/adr-template.md > docs/architecture/adr-$(NUM)-$(TITLE).md .PHONY: adr-gen + +## telemetry-infra-up: launches local telemetry infrastructure. This includes grafana, jaeger, loki, pyroscope, and an otel-collector. +## you can access the grafana instance at localhost:3000 and login with admin:admin. +telemetry-infra-up: + PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml up +.PHONY: telemetry-infra-up + +## telemetry-infra-down: tears the telemetry infrastructure down. The stores for grafana, prometheus, and loki will persist. +telemetry-infra-down: + PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml down +.PHONY: telemetry-infra-down diff --git a/README.md b/README.md index 0711c2d223..89cb67dc15 100644 --- a/README.md +++ b/README.md @@ -24,7 +24,7 @@ Continue reading [here](https://blog.celestia.org/celestia-mvp-release-data-avai - [API docs](#api-docs) - [Node types](#node-types) - [Run a node](#run-a-node) - - [Environment variables](#environment-variables) + - [Environment variables](#environment-variables) - [Package-specific documentation](#package-specific-documentation) - [Code of Conduct](#code-of-conduct) @@ -55,7 +55,7 @@ For more information on setting up a node and the hardware requirements needed, ## API docs -Celestia-node public API is documented [here](https://docs.celestia.org/category/node-api/). +The celestia-node public API is documented [here](https://docs.celestia.org/category/node-api/). ## Node types @@ -67,7 +67,7 @@ More information can be found [here](https://github.com/celestiaorg/celestia-nod ## Run a node -`` can be `bridge`, `full` or `light`. +`` can be: `bridge`, `full` or `light`. ```sh celestia init @@ -77,7 +77,9 @@ celestia init celestia start ``` -### Environment variables +Please refer to [this guide](https://docs.celestia.org/nodes/celestia-node/) for more information on running a node. + +## Environment variables | Variable | Explanation | Default value | Required | | ----------------------- | ----------------------------------- | ------------- | -------- | diff --git a/docker/telemetry/docker-compose.yml b/docker/telemetry/docker-compose.yml new file mode 100644 index 0000000000..9a95551b0c --- /dev/null +++ b/docker/telemetry/docker-compose.yml @@ -0,0 +1,89 @@ +--- +version: '3.8' + +services: + loki: + container_name: loki + image: grafana/loki:2.6.1 + expose: + - 3100 + ports: + - "3100:3100" + restart: unless-stopped + volumes: + - loki-data:/loki + + promtail: + container_name: promtail + image: grafana/promtail:latest + volumes: + # custom config will read logs from the containers of + # this project + - ${PWD}/promtail:/etc/promtail + # to read container labels and logs + - /var/run/docker.sock:/var/run/docker.sock + - /var/lib/docker/containers:/var/lib/docker/containers:ro + depends_on: + - loki + + prometheus: + container_name: prometheus + image: prom/prometheus + ports: + - 9000:9090 + volumes: + - ${PWD}/prometheus:/etc/prometheus + - prometheus-data:/prometheus + # yamllint disable-line rule:line-length + command: --web.enable-lifecycle --config.file=/etc/prometheus/prometheus.yml + extra_hosts: + - "host.docker.internal:host-gateway" + + otel-collector: + container_name: otel-collector + image: otel/opentelemetry-collector + command: ["--config=/root/otel-collector/config.yml"] + volumes: + - ${PWD}/otel-collector:/root/otel-collector/ + ports: + - "8888:8888" # Prometheus metrics exposed by the collector + - "8889:8889" # Prometheus exporter metrics + - "55681:55681" + - "13133:13133" # health_check extension + - "4317:4317" # OTLP gRPC receiver + - "4318:4318" # OTLP http receiver + + jaeger: + container_name: jaeger + image: jaegertracing/all-in-one:latest + ports: + - "16686:16686" + - "14268:14268" + - "14250:14250" + environment: + - COLLECTOR_OTLP_ENABLED=true + - LOG_LEVEL=debug + + grafana: + container_name: grafana + image: grafana/grafana:latest + user: "0" + ports: + - 3001:3000 + restart: unless-stopped + volumes: + - ${PWD}/grafana/:/etc/grafana/provisioning/ + - ${PWD}/grafana/:/var/lib/grafana/dashboards/ + - grafana-data:/var/lib/grafana + + pyroscope: + image: "pyroscope/pyroscope:latest" + ports: + - "4040:4040" + command: + - "server" + +volumes: + prometheus-data: + loki-data: + grafana-data: diff --git a/docker/telemetry/grafana/datasources/config.yml b/docker/telemetry/grafana/datasources/config.yml new file mode 100644 index 0000000000..a9ad009548 --- /dev/null +++ b/docker/telemetry/grafana/datasources/config.yml @@ -0,0 +1,14 @@ +--- +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + - name: Loki + type: loki + access: proxy + url: http://loki:3100 + jsonData: + maxLines: 1000 diff --git a/docker/telemetry/loki/config.yml b/docker/telemetry/loki/config.yml new file mode 100644 index 0000000000..467188f09b --- /dev/null +++ b/docker/telemetry/loki/config.yml @@ -0,0 +1,11 @@ +--- +auth_enabled: true + +http_prefix: + +server: + http_listen_address: 0.0.0.0 + grpc_listen_address: 0.0.0.0 + http_listen_port: 3100 + grpc_listen_port: 9095 + log_level: info diff --git a/docker/telemetry/otel-collector/config.yml b/docker/telemetry/otel-collector/config.yml new file mode 100644 index 0000000000..c74c8ca93c --- /dev/null +++ b/docker/telemetry/otel-collector/config.yml @@ -0,0 +1,32 @@ +--- +extensions: + health_check: + +receivers: + otlp: + protocols: + grpc: + # endpoint: "0.0.0.0:4317" + http: + # endpoint: "0.0.0.0:4318" + +exporters: + prometheus: + endpoint: "otel-collector:8889" + send_timestamps: true + metric_expiration: 1800m + jaeger: + endpoint: "jaeger:14250" + tls: + insecure: true + +service: + extensions: [health_check] + pipelines: + metrics: + receivers: [otlp] + exporters: [prometheus] + traces: + receivers: [otlp] + processors: [] + exporters: [jaeger] diff --git a/docker/telemetry/prometheus/prometheus.yml b/docker/telemetry/prometheus/prometheus.yml new file mode 100644 index 0000000000..7cc7395305 --- /dev/null +++ b/docker/telemetry/prometheus/prometheus.yml @@ -0,0 +1,25 @@ +--- +global: + scrape_interval: 15s + scrape_timeout: 10s + evaluation_interval: 15s + +scrape_configs: + - job_name: 'collector' + metrics_path: /metrics + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + scheme: http + static_configs: + - targets: + - 'otel-collector:8889' + - job_name: 'p2p-metrics' + metrics_path: /metrics + honor_timestamps: true + scrape_interval: 15s + scrape_timeout: 10s + scheme: http + static_configs: + - targets: + - 'host.docker.internal:8890' diff --git a/docker/telemetry/promtail/config.yml b/docker/telemetry/promtail/config.yml new file mode 100644 index 0000000000..95267000a9 --- /dev/null +++ b/docker/telemetry/promtail/config.yml @@ -0,0 +1,29 @@ +# https://grafana.com/docs/loki/latest/clients/promtail/configuration/ +# https://docs.docker.com/engine/api/v1.41/#operation/ContainerList +--- +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: flog_scrape + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + filters: + - name: label + values: ["logging=promtail"] + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container' + - source_labels: ['__meta_docker_container_log_stream'] + target_label: 'logstream' + - source_labels: ['__meta_docker_container_label_logging_jobname'] + target_label: 'job' From a01e51d74147f2573e7d7791ab07f05f6a46861c Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Sep 2023 15:15:44 +0200 Subject: [PATCH 164/388] chore(deps): bump github.com/ipfs/go-block-format from 0.1.2 to 0.2.0 (#2623) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/go-block-format](https://github.com/ipfs/go-block-format) from 0.1.2 to 0.2.0.
Release notes

Sourced from github.com/ipfs/go-block-format's releases.

v0.2.0

What's Changed

New Contributors

Full Changelog: https://github.com/ipfs/go-block-format/compare/v0.1.2...v0.2.0

Commits
  • f2d9400 v0.2.0 bump
  • ee6647c chore!: go@1.20, update deps, uci, remove gx
  • ffa6dd1 build(deps): bump golang.org/x/sys
  • 441d112 chore: Update .github/workflows/stale.yml [skip ci]
  • 1f4e90a chore: Update .github/workflows/stale.yml [skip ci]
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/go-block-format&package-manager=go_modules&previous-version=0.1.2&new-version=0.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 7fb3056a9e..c44e35e920 100644 --- a/go.mod +++ b/go.mod @@ -33,7 +33,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.11.0 - github.com/ipfs/go-block-format v0.1.2 + github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-blockservice v0.5.1 // down 1 version, 0.5.2 is marked as deprecated and raises alerts github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 91e56dd9ca..31b11a82e0 100644 --- a/go.sum +++ b/go.sum @@ -1040,8 +1040,9 @@ github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= -github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= +github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= +github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= From 81d6d3be82ea0be2e94fc0a6aa42e271ae0ec143 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Mon, 4 Sep 2023 11:36:11 -0500 Subject: [PATCH 165/388] chore!: bump celestia-app to v1.0.0-rc13 (#2646) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit bumps app to v1.0.0-rc13, core to v1.26.2-tm-v0.34.28, the sdk to v1.18.0-sdk-v0.46.14, and nmt to v0.19.0 per release notes: this is a consensus breaking change that is targetting mocha-4 and arabica-10 besides the already mentioned changes, this bumps app to go 1.21 🎉 and has the simplified programmatic transaction signing repo (we got rid of the KeyringSigner since it was originally designed for tx malleation). This package can be found here, and there are a few (hopefully straightforward) examples here, here, and here. When you all are ready, we'd like to remove the KeyringSigner, and would be delighted to help migrate. --- go.mod | 9 +++++---- go.sum | 29 ++++++++++++++++------------- 2 files changed, 21 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index c44e35e920..8c1fbb1349 100644 --- a/go.mod +++ b/go.mod @@ -12,12 +12,12 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc12 + github.com/celestiaorg/celestia-app v1.0.0-rc13 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.3.0 github.com/celestiaorg/go-libp2p-messenger v0.2.0 - github.com/celestiaorg/nmt v0.18.1 + github.com/celestiaorg/nmt v0.19.0 github.com/celestiaorg/rsmt2d v0.11.0 github.com/cosmos/cosmos-sdk v0.46.14 github.com/cosmos/cosmos-sdk/api v0.1.0 @@ -310,6 +310,7 @@ require ( go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect + go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect @@ -336,10 +337,10 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28 ) diff --git a/go.sum b/go.sum index 31b11a82e0..d738f67e10 100644 --- a/go.sum +++ b/go.sum @@ -356,12 +356,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc12 h1:ko9hPD4oz1UTS4ZqzikGVQ0wXi5+4kEhDb7decx5Ehs= -github.com/celestiaorg/celestia-app v1.0.0-rc12/go.mod h1:vXvKEudUpdJCvUr79qVKKJ0Xo7ofsuU80+Hs9aKGjvU= -github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28 h1:eXS3v26nob8Xs2+flKHVxcTzhzQW44KgTcooR3OxnK4= -github.com/celestiaorg/celestia-core v1.24.0-tm-v0.34.28/go.mod h1:J/GsBjoTZaFz71VeyrLZbG8rV+Rzi6oFEUZUipQ97hQ= -github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14 h1:PckXGxLJjXv97VO3xS8NPHN5oO83X5nvJLbc/4s8jUM= -github.com/celestiaorg/cosmos-sdk v1.17.0-sdk-v0.46.14/go.mod h1:70Go8qNy7YAb1PUcHCChRHNX2ke7c9jgUIEklUX+Mac= +github.com/celestiaorg/celestia-app v1.0.0-rc13 h1:BM9lBJw+RcAiFIUmVDd3XBYyXV9rKJBi8mDqc2wok1o= +github.com/celestiaorg/celestia-app v1.0.0-rc13/go.mod h1:JYu6i1NxJw26TVZ+XSllUdnw0Fw3nGNk5f3wm6RIcys= +github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28 h1:2efXQaggLFknz0wQufr4nUEz5G7pSVHS1j7NuJDsvII= +github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28/go.mod h1:++dNzzzjP9jYg+NopN9G8sg1HEZ58lv1TPtg71evZ0E= +github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 h1:dDfoQJOlVNj4HufJ1lBLTo2k3/L/255MIiKmEQziDmw= +github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14/go.mod h1:kkdiHo/zG6ar80730+bG1owdMAQXrGp4utFu7mbfADo= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= @@ -376,8 +376,8 @@ github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.18.1 h1:zU3apzW4y0fs0ilQA74XnEYW8FvRv0CUK2LXK66L3rA= -github.com/celestiaorg/nmt v0.18.1/go.mod h1:0l8q6UYRju1xNrxtvV6NwPdW3lfsN6KuZ0htRnModdc= +github.com/celestiaorg/nmt v0.19.0 h1:9VXFeI/gt+q8h5HeCE0RjXJhOxsFzxJUjHrkvF9CMYE= +github.com/celestiaorg/nmt v0.19.0/go.mod h1:Oz15Ub6YPez9uJV0heoU4WpFctxazuIhKyUtaYNio7E= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.11.0 h1:lcto/637WyTEZR3dLRoNvyuExfnUbxvdvKi3qz/2V4k= @@ -2091,8 +2091,9 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= -github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= +github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -2147,8 +2148,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= @@ -2369,6 +2370,8 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= +go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= @@ -2861,8 +2864,8 @@ golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20201208040808-7e3f01d25324/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20210220033141-f8bda1e9f3ba/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af h1:Yx9k8YCG3dvF87UAn2tu2HQLf2dt/eR1bXxpLMWeH+Y= -golang.org/x/time v0.0.0-20220922220347-f3bd1da661af/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.3.0 h1:rg5rLMjNzMS1RkNLzCG38eapWhnYLFYXDXj2gOlr8j4= +golang.org/x/time v0.3.0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180525024113-a5b4c53f6e8b/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= From a3220f4503f7b0666b1860148e910658fe130d85 Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 5 Sep 2023 08:17:45 +0100 Subject: [PATCH 166/388] fix(blob): typo, sumitter -> submitter (#2648) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Overview - simple change in `blob/service.go` where a typo of sumitter existed in the `Service` struct, renamed to submitter ## Checklist - [√] New and updated code has appropriate documentation - [√] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [n/a] Visual proof for any user facing features like CLI or documentation updates - [n/a] Linked issues closed with keywords --------- Co-authored-by: Ryan --- blob/service.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/blob/service.go b/blob/service.go index da14fc06c7..31cd259efc 100644 --- a/blob/service.go +++ b/blob/service.go @@ -32,7 +32,7 @@ type Submitter interface { type Service struct { // accessor dials the given celestia-core endpoint to submit blobs. - blobSumitter Submitter + blobSubmitter Submitter // shareGetter retrieves the EDS to fetch all shares from the requested header. shareGetter share.Getter // headerGetter fetches header by the provided height @@ -45,9 +45,9 @@ func NewService( headerGetter func(context.Context, uint64) (*header.ExtendedHeader, error), ) *Service { return &Service{ - blobSumitter: submitter, - shareGetter: getter, - headerGetter: headerGetter, + blobSubmitter: submitter, + shareGetter: getter, + headerGetter: headerGetter, } } @@ -58,7 +58,7 @@ func NewService( func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { log.Debugw("submitting blobs", "amount", len(blobs)) - resp, err := s.blobSumitter.SubmitPayForBlob(ctx, types.OneInt().Neg(), 0, blobs) + resp, err := s.blobSubmitter.SubmitPayForBlob(ctx, types.OneInt().Neg(), 0, blobs) if err != nil { return 0, err } From f9cf180ab4eb8cac0ffa6ac3b9723d4c77799e3a Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Sep 2023 10:22:44 +0200 Subject: [PATCH 167/388] chore(nodebuilder/tests): Clean up integration tests in `p2p_test` (#2166) Cleans up swamp tests in `p2p_test.go`. Based on #2162 --- nodebuilder/tests/p2p_test.go | 190 ++++++++++++++------------------- nodebuilder/tests/sync_test.go | 1 + 2 files changed, 79 insertions(+), 112 deletions(-) diff --git a/nodebuilder/tests/p2p_test.go b/nodebuilder/tests/p2p_test.go index 613dface94..083712dfdd 100644 --- a/nodebuilder/tests/p2p_test.go +++ b/nodebuilder/tests/p2p_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/libp2p/go-libp2p/core/event" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/network" "github.com/libp2p/go-libp2p/core/peer" @@ -18,22 +17,22 @@ import ( ) /* -Test-Case: Full/Light Nodes connection to Bridge as a Bootstapper +Test-Case: Full/Light Nodes connection to Bridge as a Bootstrapper Steps: 1. Create a Bridge Node(BN) 2. Start a BN -3. Create full/light nodes with bridge node as bootsrapped peer +3. Create full/light nodes with bridge node as bootstrap peer 4. Start full/light nodes 5. Check that nodes are connected to bridge */ -func TestUseBridgeNodeAsBootstraper(t *testing.T) { - sw := swamp.NewSwamp(t) - - bridge := sw.NewBridgeNode() - +func TestBridgeNodeAsBootstrapper(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) + sw := swamp.NewSwamp(t) + + // create and start BN + bridge := sw.NewBridgeNode() err := bridge.Start(ctx) require.NoError(t, err) @@ -41,39 +40,14 @@ func TestUseBridgeNodeAsBootstraper(t *testing.T) { full := sw.NewFullNode(nodebuilder.WithBootstrappers([]peer.AddrInfo{*addr})) light := sw.NewLightNode(nodebuilder.WithBootstrappers([]peer.AddrInfo{*addr})) - nodes := []*nodebuilder.Node{full, light} - for index := range nodes { - require.NoError(t, nodes[index].Start(ctx)) - assert.Equal(t, *addr, nodes[index].Bootstrappers[0]) - assert.True(t, nodes[index].Host.Network().Connectedness(addr.ID) == network.Connected) - } -} - -/* -Test-Case: Add peer to blacklist -Steps: -1. Create a Full Node(BN) -2. Start a FN -3. Create a Light Node(LN) -5. Start a LN -6. Explicitly block FN id by LN -7. Check FN is allowed to dial with LN -8. Check LN is not allowed to dial with FN -*/ -func TestAddPeerToBlackList(t *testing.T) { - sw := swamp.NewSwamp(t) - full := sw.NewFullNode() - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) - t.Cleanup(cancel) - require.NoError(t, full.Start(ctx)) - addr := host.InfoFromHost(full.Host) - light := sw.NewLightNode() - require.NoError(t, light.Start(ctx)) - require.NoError(t, light.ConnGater.BlockPeer(addr.ID)) - - require.True(t, full.ConnGater.InterceptPeerDial(host.InfoFromHost(light.Host).ID)) - require.False(t, light.ConnGater.InterceptPeerDial(addr.ID)) + for _, nd := range []*nodebuilder.Node{full, light} { + // start node and ensure that BN is correctly set as bootstrapper + require.NoError(t, nd.Start(ctx)) + assert.Equal(t, *addr, nd.Bootstrappers[0]) + // ensure that node is actually connected to BN + assert.True(t, nd.Host.Network().Connectedness(addr.ID) == network.Connected) + } } /* @@ -86,131 +60,123 @@ Steps: 5. Ensure that nodes are connected to bridge 6. Wait until light will find full node 7. Check that full and light nodes are connected to each other - 8. Stop FN and ensure that it's not connected to LN */ -func TestBootstrapNodesFromBridgeNode(t *testing.T) { +func TestFullDiscoveryViaBootstrapper(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) + t.Cleanup(cancel) + + const defaultTimeInterval = time.Second * 2 + sw := swamp.NewSwamp(t) + + // create and start a BN cfg := nodebuilder.DefaultConfig(node.Bridge) - const defaultTimeInterval = time.Second * 10 setTimeInterval(cfg, defaultTimeInterval) - bridge := sw.NewNodeWithConfig(node.Bridge, cfg) - - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) - t.Cleanup(cancel) - err := bridge.Start(ctx) require.NoError(t, err) - bridgeAddr := host.InfoFromHost(bridge.Host) + // use BN as the bootstrapper + bootstrapper := host.InfoFromHost(bridge.Host) + + // create FN with BN as bootstrapper cfg = nodebuilder.DefaultConfig(node.Full) setTimeInterval(cfg, defaultTimeInterval) full := sw.NewNodeWithConfig( node.Full, cfg, - nodebuilder.WithBootstrappers([]peer.AddrInfo{*bridgeAddr}), + nodebuilder.WithBootstrappers([]peer.AddrInfo{*bootstrapper}), ) + // create LN with BN as bootstrapper cfg = nodebuilder.DefaultConfig(node.Light) setTimeInterval(cfg, defaultTimeInterval) - cfg.P2P.PeerExchange = true light := sw.NewNodeWithConfig( node.Light, cfg, - nodebuilder.WithBootstrappers([]peer.AddrInfo{*bridgeAddr}), + nodebuilder.WithBootstrappers([]peer.AddrInfo{*bootstrapper}), ) + + // start FN and LN and ensure they are both connected to BN as a bootstrapper nodes := []*nodebuilder.Node{full, light} - ch := make(chan struct{}) - sub, err := light.Host.EventBus().Subscribe(&event.EvtPeerConnectednessChanged{}) - require.NoError(t, err) - defer sub.Close() for index := range nodes { require.NoError(t, nodes[index].Start(ctx)) - assert.Equal(t, *bridgeAddr, nodes[index].Bootstrappers[0]) - assert.True(t, nodes[index].Host.Network().Connectedness(bridgeAddr.ID) == network.Connected) + assert.Equal(t, *bootstrapper, nodes[index].Bootstrappers[0]) + assert.True(t, nodes[index].Host.Network().Connectedness(bootstrapper.ID) == network.Connected) } - addrFull := host.InfoFromHost(full.Host) - go func() { - for e := range sub.Out() { - connStatus := e.(event.EvtPeerConnectednessChanged) - if connStatus.Peer == full.Host.ID() && connStatus.Connectedness == network.NotConnected { - ch <- struct{}{} - } + + for { + if ctx.Err() != nil { + t.Fatal(ctx.Err()) + } + if light.Host.Network().Connectedness(host.InfoFromHost(full.Host).ID) == network.Connected { + // LN discovered FN successfully and is now connected + break } - }() - - // ensure that the light node is connected to the full node - assert.True(t, light.Host.Network().Connectedness(addrFull.ID) == network.Connected) - - sw.Disconnect(t, light, full) - require.NoError(t, full.Stop(ctx)) - select { - case <-ctx.Done(): - t.Fatal("peer was not disconnected") - case <-ch: - assert.True(t, light.Host.Network().Connectedness(addrFull.ID) == network.NotConnected) } } /* -Test-Case: Restart full node discovery after one node is disconnected +Test-Case: Full node discovery of disconnected full nodes Steps: 1. Create a Bridge Node(BN) 2. Start a BN -3. Create 2 full nodes with bridge node as bootstrapper peer and start them -4. Check that nodes are connected to each other -5. Create one more node with disabled discovery -6. Disconnect FNs from each other -7. Check that the last FN is connected to one of the nodes +3. Create 2 FNs with bridge node as bootstrapper peer and start them +4. Check that the FNs discover each other +5. Disconnect the FNs +6. Create one more node with discovery process disabled (however advertisement is still enabled) +7. Check that the FN with discovery disabled is still found by the other two FNs *NOTE*: this test will take some time because it relies on several cycles of peer discovery */ func TestRestartNodeDiscovery(t *testing.T) { - sw := swamp.NewSwamp(t) - cfg := nodebuilder.DefaultConfig(node.Bridge) - const defaultTimeInterval = time.Second * 2 - const fullNodes = 2 - - setTimeInterval(cfg, defaultTimeInterval) - cfg.Share.Discovery.PeersLimit = fullNodes - bridge := sw.NewNodeWithConfig(node.Bridge, cfg) - ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) + const ( + defaultTimeInterval = time.Second * 2 + numFulls = 2 + ) + + sw := swamp.NewSwamp(t) + + // create and start a BN as a bootstrapper + fullCfg := nodebuilder.DefaultConfig(node.Bridge) + setTimeInterval(fullCfg, defaultTimeInterval) + bridge := sw.NewNodeWithConfig(node.Bridge, fullCfg) err := bridge.Start(ctx) require.NoError(t, err) + bridgeAddr := host.InfoFromHost(bridge.Host) - nodes := make([]*nodebuilder.Node, fullNodes) - cfg = nodebuilder.DefaultConfig(node.Full) - setTimeInterval(cfg, defaultTimeInterval) - cfg.Share.Discovery.PeersLimit = fullNodes + fullCfg = nodebuilder.DefaultConfig(node.Full) + setTimeInterval(fullCfg, defaultTimeInterval) nodesConfig := nodebuilder.WithBootstrappers([]peer.AddrInfo{*bridgeAddr}) - for index := 0; index < fullNodes; index++ { - nodes[index] = sw.NewNodeWithConfig(node.Full, cfg, nodesConfig) - } - for index := 0; index < fullNodes; index++ { + // create two FNs and start them, ensuring they are connected to BN as + // bootstrapper + nodes := make([]*nodebuilder.Node, numFulls) + for index := 0; index < numFulls; index++ { + nodes[index] = sw.NewNodeWithConfig(node.Full, fullCfg, nodesConfig) require.NoError(t, nodes[index].Start(ctx)) assert.True(t, nodes[index].Host.Network().Connectedness(bridgeAddr.ID) == network.Connected) } - // ensure full nodes are connected to each other + // ensure FNs are connected to each other require.True(t, nodes[0].Host.Network().Connectedness(nodes[1].Host.ID()) == network.Connected) - // create one more node with disabled discovery - cfg = nodebuilder.DefaultConfig(node.Full) - setTimeInterval(cfg, defaultTimeInterval) - cfg.Share.Discovery.PeersLimit = 0 - node := sw.NewNodeWithConfig(node.Full, cfg, nodesConfig) - connectSub, err := nodes[0].Host.EventBus().Subscribe(&event.EvtPeerConnectednessChanged{}) - require.NoError(t, err) - defer connectSub.Close() + // disconnect the FNs sw.Disconnect(t, nodes[0], nodes[1]) - require.NoError(t, node.Start(ctx)) - // ensure that the last node is connected to one of the nodes - require.True(t, nodes[0].Host.Network().Connectedness(node.Host.ID()) == network.Connected) + // create and start one more FN with disabled discovery + fullCfg.Share.Discovery.PeersLimit = 0 + disabledDiscoveryFN := sw.NewNodeWithConfig(node.Full, fullCfg, nodesConfig) + err = disabledDiscoveryFN.Start(ctx) + require.NoError(t, err) + + // ensure that the FN with disabled discovery is discovered by both of the + // running FNs that have discovery enabled + require.True(t, nodes[0].Host.Network().Connectedness(disabledDiscoveryFN.Host.ID()) == network.Connected) + require.True(t, nodes[1].Host.Network().Connectedness(disabledDiscoveryFN.Host.ID()) == network.Connected) } func setTimeInterval(cfg *nodebuilder.Config, interval time.Duration) { diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index 234556a3aa..65db1332ff 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -297,6 +297,7 @@ func TestSyncLightAgainstFull(t *testing.T) { require.NoError(t, err) _, err = full.HeaderServ.WaitForHeight(ctx, bridgeHead.Height()) require.NoError(t, err) + assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // reset suite bootstrapper list and set full node as a bootstrapper for // LN to connect to From da8244e00fc434139fb1fd21414683660553c239 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 5 Sep 2023 11:06:33 +0200 Subject: [PATCH 168/388] refactor(blob): removing jsonProof (#2652) Closes #2634 . nmt.Proof has json serialization built in now so jsonProof is unnecessary and results in missing fields from the original struct. Only adds additional fields to the JSON response so not breaking --- blob/blob.go | 37 ------------------------------------- blob/service_test.go | 5 +++-- 2 files changed, 3 insertions(+), 39 deletions(-) diff --git a/blob/blob.go b/blob/blob.go index e9ad2b6255..c5fd04c782 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -12,7 +12,6 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/ipld" ) // Commitment is a Merkle Root of the subtree built from shares of the Blob. @@ -62,42 +61,6 @@ func (p Proof) equal(input Proof) error { return nil } -type jsonProof struct { - Start int `json:"start"` - End int `json:"end"` - Nodes [][]byte `json:"nodes"` -} - -func (p *Proof) MarshalJSON() ([]byte, error) { - proofs := make([]jsonProof, 0, p.Len()) - for _, pp := range *p { - proofs = append(proofs, jsonProof{ - Start: pp.Start(), - End: pp.End(), - Nodes: pp.Nodes(), - }) - } - - return json.Marshal(proofs) -} - -func (p *Proof) UnmarshalJSON(data []byte) error { - var proofs []jsonProof - err := json.Unmarshal(data, &proofs) - if err != nil { - return err - } - - nmtProofs := make([]*nmt.Proof, len(proofs)) - for i, jProof := range proofs { - nmtProof := nmt.NewInclusionProof(jProof.Start, jProof.End, jProof.Nodes, ipld.NMTIgnoreMaxNamespace) - nmtProofs[i] = &nmtProof - } - - *p = nmtProofs - return nil -} - // Blob represents any application-specific binary data that anyone can submit to Celestia. type Blob struct { types.Blob `json:"blob"` diff --git a/blob/service_test.go b/blob/service_test.go index 1dcabe7129..a45fe631af 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -4,6 +4,7 @@ import ( "bytes" "context" "crypto/sha256" + "encoding/json" "testing" "time" @@ -272,14 +273,14 @@ func TestBlobService_Get(t *testing.T) { doFn: func() (interface{}, error) { proof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) require.NoError(t, err) - return proof.MarshalJSON() + return json.Marshal(proof) }, expectedResult: func(i interface{}, err error) { require.NoError(t, err) jsonData, ok := i.([]byte) require.True(t, ok) var proof Proof - require.NoError(t, proof.UnmarshalJSON(jsonData)) + require.NoError(t, json.Unmarshal(jsonData, &proof)) newProof, err := service.GetProof(ctx, 1, blobs0[1].Namespace(), blobs0[1].Commitment) require.NoError(t, err) From 456d169582dc02aafb7d5df891da5973d54a20b7 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 5 Sep 2023 11:43:52 +0200 Subject: [PATCH 169/388] feat(eds/store): remove corrupted blocks from store (#2625) Related to #2335 If an OpShardFail is found or corruption is detected from GetSharesByNamespace, the shard is removed --- nodebuilder/share/constructors.go | 18 +++++++ nodebuilder/share/module.go | 79 ++++++++++++++++++++----------- share/eds/accessor_cache.go | 8 ++++ share/eds/metrics.go | 20 ++++++++ share/eds/store.go | 50 ++++++++++++++----- share/eds/store_test.go | 38 +++++++++++++++ share/getters/getter_test.go | 29 ++++++++++++ share/getters/shrex_test.go | 3 +- share/getters/store.go | 12 +++++ share/p2p/peers/manager.go | 28 ++++++++--- share/p2p/peers/manager_test.go | 5 +- share/p2p/peers/options.go | 17 +++++++ share/share.go | 14 ++++++ 13 files changed, 269 insertions(+), 52 deletions(-) diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index a1b7e39713..b962038d17 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -87,6 +87,24 @@ func lightGetter( return getters.NewCascadeGetter(cascade) } +// ShrexGetter is added to bridge nodes for the case that a shard is removed +// after detected shard corruption. This ensures the block is fetched and stored +// by shrex the next time the data is retrieved (meaning shard recovery is +// manual after corruption is detected). +func bridgeGetter( + store *eds.Store, + storeGetter *getters.StoreGetter, + shrexGetter *getters.ShrexGetter, + cfg Config, +) share.Getter { + var cascade []share.Getter + cascade = append(cascade, storeGetter) + if cfg.UseShareExchange { + cascade = append(cascade, getters.NewTeeGetter(shrexGetter, store)) + } + return getters.NewCascadeGetter(cascade) +} + func fullGetter( store *eds.Store, storeGetter *getters.StoreGetter, diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index b924cf8167..f7a84bf526 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -5,9 +5,12 @@ import ( "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/p2p/net/conngater" "go.uber.org/fx" - "github.com/celestiaorg/celestia-node/libs/fxutil" + libhead "github.com/celestiaorg/go-header" + + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" modp2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/share" @@ -48,6 +51,33 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option ), ) + shrexGetterComponents := fx.Options( + fx.Provide(func() peers.Parameters { + return cfg.PeerManagerParams + }), + fx.Provide( + func(host host.Host, network modp2p.Network) (*shrexnd.Client, error) { + cfg.ShrExNDParams.WithNetworkID(network.String()) + return shrexnd.NewClient(cfg.ShrExNDParams, host) + }, + ), + fx.Provide( + func(host host.Host, network modp2p.Network) (*shrexeds.Client, error) { + cfg.ShrExEDSParams.WithNetworkID(network.String()) + return shrexeds.NewClient(cfg.ShrExEDSParams, host) + }, + ), + fx.Provide(fx.Annotate( + getters.NewShrexGetter, + fx.OnStart(func(ctx context.Context, getter *getters.ShrexGetter) error { + return getter.Start(ctx) + }), + fx.OnStop(func(ctx context.Context, getter *getters.ShrexGetter) error { + return getter.Stop(ctx) + }), + )), + ) + bridgeAndFullComponents := fx.Options( fx.Provide(getters.NewStoreGetter), fx.Invoke(func(edsSrv *shrexeds.Server, ndSrc *shrexnd.Server) {}), @@ -112,32 +142,25 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option }), ) - shrexGetterComponents := fx.Options( - fx.Provide(func() peers.Parameters { - return cfg.PeerManagerParams - }), - fx.Provide(peers.NewManager), - fx.Provide( - func(host host.Host, network modp2p.Network) (*shrexnd.Client, error) { - cfg.ShrExNDParams.WithNetworkID(network.String()) - return shrexnd.NewClient(cfg.ShrExNDParams, host) - }, - ), + peerManagerWithShrexPools := fx.Options( fx.Provide( - func(host host.Host, network modp2p.Network) (*shrexeds.Client, error) { - cfg.ShrExEDSParams.WithNetworkID(network.String()) - return shrexeds.NewClient(cfg.ShrExEDSParams, host) + func( + params peers.Parameters, + discovery *disc.Discovery, + host host.Host, + connGater *conngater.BasicConnectionGater, + shrexSub *shrexsub.PubSub, + headerSub libhead.Subscriber[*header.ExtendedHeader], + ) (*peers.Manager, error) { + return peers.NewManager( + params, + discovery, + host, + connGater, + peers.WithShrexSubPools(shrexSub, headerSub), + ) }, ), - fx.Provide(fx.Annotate( - getters.NewShrexGetter, - fx.OnStart(func(ctx context.Context, getter *getters.ShrexGetter) error { - return getter.Start(ctx) - }), - fx.OnStop(func(ctx context.Context, getter *getters.ShrexGetter) error { - return getter.Stop(ctx) - }), - )), ) switch tp { @@ -145,10 +168,10 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option return fx.Module( "share", baseComponents, + fx.Provide(peers.NewManager), bridgeAndFullComponents, - fxutil.ProvideAs(func(getter *getters.StoreGetter) share.Getter { - return getter - }), + shrexGetterComponents, + fx.Provide(bridgeGetter), fx.Invoke(func(lc fx.Lifecycle, sub *shrexsub.PubSub) error { lc.Append(fx.Hook{ OnStart: sub.Start, @@ -160,6 +183,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option case node.Full: return fx.Module( "share", + peerManagerWithShrexPools, baseComponents, bridgeAndFullComponents, shrexGetterComponents, @@ -175,6 +199,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option light.WithSampleAmount(cfg.LightAvailability.SampleAmount), } }), + peerManagerWithShrexPools, shrexGetterComponents, fx.Invoke(ensureEmptyEDSInBS), fx.Provide(getters.NewIPLDGetter), diff --git a/share/eds/accessor_cache.go b/share/eds/accessor_cache.go index cd0f0537fa..9f70178be6 100644 --- a/share/eds/accessor_cache.go +++ b/share/eds/accessor_cache.go @@ -69,6 +69,14 @@ func (bc *blockstoreCache) evictFn() func(_ interface{}, val interface{}) { } } +func (bc *blockstoreCache) Remove(key shard.Key) bool { + lk := &bc.stripedLocks[shardKeyToStriped(key)] + lk.Lock() + defer lk.Unlock() + + return bc.cache.Remove(key) +} + // Get retrieves the blockstore for a given shard key from the cache. If the blockstore is not in // the cache, it returns an errCacheMiss func (bc *blockstoreCache) Get(shardContainingCid shard.Key) (*accessorWithBlockstore, error) { diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 1f430bf688..9d4b2a53ef 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -47,6 +47,8 @@ type metrics struct { listTime metric.Float64Histogram getAccessorTime metric.Float64Histogram + shardFailureCount metric.Int64Counter + longOpTime metric.Float64Histogram gcTime metric.Float64Histogram } @@ -106,6 +108,12 @@ func (s *Store) WithMetrics() error { return err } + shardFailureCount, err := meter.Int64Counter("eds_store_shard_failure_counter", + metric.WithDescription("eds store OpShardFail counter")) + if err != nil { + return err + } + longOpTime, err := meter.Float64Histogram("eds_store_long_operation_time_histogram", metric.WithDescription("eds store long operation time histogram(s)")) if err != nil { @@ -153,6 +161,7 @@ func (s *Store) WithMetrics() error { hasTime: hasTime, listTime: listTime, getAccessorTime: getAccessorTime, + shardFailureCount: shardFailureCount, longOpTime: longOpTime, gcTime: gcTime, } @@ -170,6 +179,17 @@ func (m *metrics) observeGCtime(ctx context.Context, dur time.Duration, failed b attribute.Bool(failedKey, failed))) } +func (m *metrics) observeShardFailure(ctx context.Context, shardKey string) { + if m == nil { + return + } + if ctx.Err() != nil { + ctx = context.Background() + } + + m.shardFailureCount.Add(ctx, 1, metric.WithAttributes(attribute.String("shard_key", shardKey))) +} + func (m *metrics) observePut(ctx context.Context, dur time.Duration, result putResult, size uint) { if m == nil { return diff --git a/share/eds/store.go b/share/eds/store.go index 24a96c9fe4..fa9a7f7c7e 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -4,7 +4,6 @@ import ( "bufio" "bytes" "context" - "encoding/hex" "errors" "fmt" "io" @@ -64,6 +63,8 @@ type Store struct { // lastGCResult is only stored on the store for testing purposes. lastGCResult atomic.Pointer[dagstore.GCResult] + shardFailures chan dagstore.ShardResult + metrics *metrics } @@ -92,6 +93,8 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { if err != nil { return nil, fmt.Errorf("failed to create index: %w", err) } + + failureChan := make(chan dagstore.ShardResult) dagStore, err := dagstore.NewDAGStore( dagstore.Config{ TransientsDir: basepath + transientsPath, @@ -99,6 +102,7 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { Datastore: ds, MountRegistry: r, TopLevelIndex: invertedIdx, + FailureCh: failureChan, }, ) if err != nil { @@ -111,13 +115,14 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { } store := &Store{ - basepath: basepath, - dgstr: dagStore, - carIdx: fsRepo, - invertedIdx: invertedIdx, - gcInterval: defaultGCInterval, - mounts: r, - cache: cache, + basepath: basepath, + dgstr: dagStore, + carIdx: fsRepo, + invertedIdx: invertedIdx, + gcInterval: defaultGCInterval, + mounts: r, + shardFailures: failureChan, + cache: cache, } store.bs = newBlockstore(store, cache, ds) return store, nil @@ -139,6 +144,8 @@ func (s *Store) Start(ctx context.Context) error { if s.gcInterval != 0 { go s.gc(runCtx) } + + go s.watchForFailures(runCtx) return nil } @@ -172,6 +179,23 @@ func (s *Store) gc(ctx context.Context) { } } +func (s *Store) watchForFailures(ctx context.Context) { + for { + select { + case <-ctx.Done(): + return + case res := <-s.shardFailures: + log.Errorw("removing shard after failure", "key", res.Key, "err", res.Error) + s.metrics.observeShardFailure(ctx, res.Key.String()) + k := share.MustDataHashFromString(res.Key.String()) + err := s.Remove(ctx, k) + if err != nil { + log.Errorw("failed to remove shard after failure", "key", res.Key, "err", err) + } + } + } +} + // Put stores the given data square with DataRoot's hash as a key. // // The square is verified on the Exchange level, and Put only stores the square, trusting it. @@ -437,6 +461,11 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) error { func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { key := root.String() + + // Remove from accessor cache, so that existing readers are closed and + // DestroyShard can be executed. + s.cache.Remove(shard.KeyFromString(key)) + ch := make(chan dagstore.ShardResult, 1) err = s.dgstr.DestroyShard(ctx, shard.KeyFromString(key), ch, dagstore.DestroyOpts{}) if err != nil { @@ -535,10 +564,7 @@ func (s *Store) list() ([]share.DataHash, error) { shards := s.dgstr.AllShardsInfo() hashes := make([]share.DataHash, 0, len(shards)) for shrd := range shards { - hash, err := hex.DecodeString(shrd.String()) - if err != nil { - return nil, err - } + hash := share.MustDataHashFromString(shrd.String()) hashes = append(hashes, hash) } return hashes, nil diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 4b263e7062..4f1d7f4c8b 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -122,6 +122,44 @@ func TestEDSStore(t *testing.T) { assert.ErrorContains(t, err, "no such file or directory") }) + t.Run("Remove after OpShardFail", func(t *testing.T) { + eds, dah := randomEDS(t) + + err = edsStore.Put(ctx, dah.Hash(), eds) + require.NoError(t, err) + + // assert that shard now exists + ok, err := edsStore.Has(ctx, dah.Hash()) + assert.NoError(t, err) + assert.True(t, ok) + + // assert that file now exists + path := edsStore.basepath + blocksPath + dah.String() + _, err = os.Stat(path) + assert.NoError(t, err) + + err = os.Remove(path) + assert.NoError(t, err) + + _, err = edsStore.GetCAR(ctx, dah.Hash()) + assert.Error(t, err) + + ticker := time.NewTicker(time.Millisecond * 100) + defer ticker.Stop() + for { + select { + case <-ticker.C: + has, err := edsStore.Has(ctx, dah.Hash()) + if err == nil && !has { + // shard no longer exists after OpShardFail was detected from GetCAR call + return + } + case <-ctx.Done(): + t.Fatal("timeout waiting for shard to be removed") + } + } + }) + t.Run("Has", func(t *testing.T) { eds, dah := randomEDS(t) diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 02e075459b..d19d0cf174 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -2,6 +2,7 @@ package getters import ( "context" + "os" "testing" "time" @@ -153,6 +154,34 @@ func TestStoreGetter(t *testing.T) { _, err = sg.GetSharesByNamespace(ctx, &root, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) + + t.Run("GetSharesFromNamespace removes corrupted shard", func(t *testing.T) { + randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, dah.Hash(), randEds) + require.NoError(t, err) + + // available + shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) + require.NoError(t, err) + require.NoError(t, shares.Verify(&dah, namespace)) + assert.Len(t, shares.Flatten(), 2) + + // 'corrupt' existing CAR by overwriting with a random EDS + f, err := os.OpenFile(tmpDir+"/blocks/"+dah.String(), os.O_WRONLY, 0644) + require.NoError(t, err) + edsToOverwriteWith, dah := randomEDS(t) + err = eds.WriteEDS(ctx, edsToOverwriteWith, f) + require.NoError(t, err) + + shares, err = sg.GetSharesByNamespace(ctx, &dah, namespace) + require.ErrorIs(t, err, share.ErrNotFound) + require.Nil(t, shares) + + // corruption detected, shard is removed + has, err := edsStore.Has(ctx, dah.Hash()) + require.False(t, has) + require.NoError(t, err) + }) } func TestIPLDGetter(t *testing.T) { diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 0ca807d0d4..236fae36e1 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -227,11 +227,10 @@ func testManager( } manager, err := peers.NewManager( peers.DefaultParameters(), - headerSub, - shrexSub, disc, host, connGater, + peers.WithShrexSubPools(shrexSub, headerSub), ) return manager, err } diff --git a/share/getters/store.go b/share/getters/store.go index 989649f795..5eca956faa 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -121,6 +121,18 @@ func (sg *StoreGetter) GetSharesByNamespace( // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) + if errors.Is(err, ipld.ErrNodeNotFound) { + // IPLD node not found after the index pointed to this shard and the CAR blockstore has been + // opened successfully is a strong indicator of corruption. We remove the block on bridges + // and fulls and return share.ErrNotFound to ensure the data is retrieved by the next + // getter. Note that this recovery is manual and will only be restored by an RPC call to + // fetch the same datahash that was removed. + err = sg.store.Remove(ctx, root.Hash()) + if err != nil { + log.Errorf("getter/store: failed to remove CAR after detected corruption: %w", err) + } + err = share.ErrNotFound + } if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve shares by namespace: %w", err) } diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index 87f9361ee2..caef242eec 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -99,11 +99,10 @@ type syncPool struct { func NewManager( params Parameters, - headerSub libhead.Subscriber[*header.ExtendedHeader], - shrexSub *shrexsub.PubSub, discovery *discovery.Discovery, host host.Host, connGater *conngater.BasicConnectionGater, + options ...Option, ) (*Manager, error) { if err := params.Validate(); err != nil { return nil, err @@ -111,8 +110,6 @@ func NewManager( s := &Manager{ params: params, - headerSub: headerSub, - shrexSub: shrexSub, connGater: connGater, disc: discovery, host: host, @@ -122,6 +119,13 @@ func NewManager( disconnectedPeersDone: make(chan struct{}), } + for _, opt := range options { + err := opt(s) + if err != nil { + return nil, err + } + } + s.fullNodes = newPool(s.params.PeerCooldown) discovery.WithOnPeersUpdate( @@ -147,12 +151,17 @@ func (m *Manager) Start(startCtx context.Context) error { ctx, cancel := context.WithCancel(context.Background()) m.cancel = cancel + // pools will only be populated with senders of shrexsub notifications if the WithShrexSubPools + // option is used. + if m.shrexSub == nil && m.headerSub == nil { + return nil + } + validatorFn := m.metrics.validationObserver(m.Validate) err := m.shrexSub.AddValidator(validatorFn) if err != nil { return fmt.Errorf("registering validator: %w", err) } - err = m.shrexSub.Start(startCtx) if err != nil { return fmt.Errorf("starting shrexsub: %w", err) @@ -168,16 +177,21 @@ func (m *Manager) Start(startCtx context.Context) error { return fmt.Errorf("subscribing to libp2p events: %w", err) } - go m.subscribeDisconnectedPeers(ctx, sub) go m.subscribeHeader(ctx, headerSub) + go m.subscribeDisconnectedPeers(ctx, sub) go m.GC(ctx) - return nil } func (m *Manager) Stop(ctx context.Context) error { m.cancel() + // we do not need to wait for headersub and disconnected peers to finish + // here, since they were never started + if m.headerSub == nil && m.shrexSub == nil { + return nil + } + select { case <-m.headerSubDone: case <-ctx.Done(): diff --git a/share/p2p/peers/manager_test.go b/share/p2p/peers/manager_test.go index ad04d2c7bd..c60a737baa 100644 --- a/share/p2p/peers/manager_test.go +++ b/share/p2p/peers/manager_test.go @@ -419,8 +419,6 @@ func TestIntegration(t *testing.T) { require.NoError(t, err) fnPeerManager, err := NewManager( DefaultParameters(), - nil, - nil, fnDisc, nil, connGater, @@ -469,11 +467,10 @@ func testManager(ctx context.Context, headerSub libhead.Subscriber[*header.Exten } manager, err := NewManager( DefaultParameters(), - headerSub, - shrexSub, disc, host, connGater, + WithShrexSubPools(shrexSub, headerSub), ) if err != nil { return nil, err diff --git a/share/p2p/peers/options.go b/share/p2p/peers/options.go index cfda906071..97ec30df4a 100644 --- a/share/p2p/peers/options.go +++ b/share/p2p/peers/options.go @@ -3,6 +3,11 @@ package peers import ( "fmt" "time" + + libhead "github.com/celestiaorg/go-header" + + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) type Parameters struct { @@ -21,6 +26,8 @@ type Parameters struct { EnableBlackListing bool } +type Option func(*Manager) error + // Validate validates the values in Parameters func (p *Parameters) Validate() error { if p.PoolValidationTimeout <= 0 { @@ -56,6 +63,16 @@ func DefaultParameters() Parameters { } } +// WithShrexSubPools passes a shrexsub and headersub instance to be used to populate and validate +// pools from shrexsub notifications. +func WithShrexSubPools(shrexSub *shrexsub.PubSub, headerSub libhead.Subscriber[*header.ExtendedHeader]) Option { + return func(m *Manager) error { + m.shrexSub = shrexSub + m.headerSub = headerSub + return nil + } +} + // WithMetrics turns on metric collection in peer manager. func (m *Manager) WithMetrics() error { metrics, err := initMetrics(m) diff --git a/share/share.go b/share/share.go index 02ccd73909..4079028d82 100644 --- a/share/share.go +++ b/share/share.go @@ -2,6 +2,7 @@ package share import ( "bytes" + "encoding/hex" "fmt" "github.com/celestiaorg/celestia-app/pkg/appconsts" @@ -57,3 +58,16 @@ func (dh DataHash) String() string { func (dh DataHash) IsEmptyRoot() bool { return bytes.Equal(EmptyRoot().Hash(), dh) } + +// MustDataHashFromString converts a hex string to a valid datahash. +func MustDataHashFromString(datahash string) DataHash { + dh, err := hex.DecodeString(datahash) + if err != nil { + panic(fmt.Sprintf("datahash conversion: passed string was not valid hex: %s", datahash)) + } + err = DataHash(dh).Validate() + if err != nil { + panic(fmt.Sprintf("datahash validation: passed hex string failed: %s", err)) + } + return dh +} From c1139e4faf196d5fed6cb51a626c7102e427f340 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 11:04:05 +0100 Subject: [PATCH 170/388] chore(deps): bump go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp from 0.39.0 to 0.40.0 (#2635) Bumps [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp](https://github.com/open-telemetry/opentelemetry-go) from 0.39.0 to 0.40.0.
Release notes

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's releases.

Release v1.17.0/v0.40.0/v0.5.0

Added

  • Export the ManualReader struct in go.opentelemetry.io/otel/sdk/metric. (#4244)
  • Export the PeriodicReader struct in go.opentelemetry.io/otel/sdk/metric. (#4244)
  • Add support for exponential histogram aggregations. A histogram can be configured as an exponential histogram using a view with "go.opentelemetry.io/otel/sdk/metric".ExponentialHistogram as the aggregation. (#4245)
  • Export the Exporter struct in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc. (#4272)
  • Export the Exporter struct in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp. (#4272)
  • The exporters in go.opentelemetry.io/otel/exporters/otlp/otlpmetric now support the OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE environment variable. (#4287)
  • Add WithoutCounterSuffixes option in go.opentelemetry.io/otel/exporters/prometheus to disable addition of _total suffixes. (#4306)
  • Add info and debug logging to the metric SDK in go.opentelemetry.io/otel/sdk/metric. (#4315)
  • The go.opentelemetry.io/otel/semconv/v1.21.0 package. The package contains semantic conventions from the v1.21.0 version of the OpenTelemetry Semantic Conventions. (#4362)
  • Accept 201 to 299 HTTP status as success in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp and go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp. (#4365)
  • Document the Temporality and Aggregation methods of the "go.opentelemetry.io/otel/sdk/metric".Exporter" need to be concurrent safe. (#4381)
  • Expand the set of units supported by the Prometheus exporter, and don't add unit suffixes if they are already present in go.opentelemetry.op/otel/exporters/prometheus (#4374)
  • Move the Aggregation interface and its implementations from go.opentelemetry.io/otel/sdk/metric/aggregation to go.opentelemetry.io/otel/sdk/metric. (#4435)
  • The exporters in go.opentelemetry.io/otel/exporters/otlp/otlpmetric now support the OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION environment variable. (#4437)
  • Add the NewAllowKeysFilter and NewDenyKeysFilter functions to go.opentelemetry.io/otel/attribute to allow convenient creation of allow-keys and deny-keys filters. (#4444)

Changed

  • Starting from v1.21.0 of semantic conventions, go.opentelemetry.io/otel/semconv/{version}/httpconv and go.opentelemetry.io/otel/semconv/{version}/netconv packages will no longer be published. (#4145)
  • Log duplicate instrument conflict at a warning level instead of info in go.opentelemetry.io/otel/sdk/metric. (#4202)
  • Return an error on the creation of new instruments in go.opentelemetry.io/otel/sdk/metric if their name doesn't pass regexp validation. (#4210)
  • NewManualReader in go.opentelemetry.io/otel/sdk/metric returns *ManualReader instead of Reader. (#4244)
  • NewPeriodicReader in go.opentelemetry.io/otel/sdk/metric returns *PeriodicReader instead of Reader. (#4244)
  • Count the Collect time in the PeriodicReader timeout in go.opentelemetry.io/otel/sdk/metric. (#4221)
  • The function New in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc returns *Exporter instead of "go.opentelemetry.io/otel/sdk/metric".Exporter. (#4272)
  • The function New in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp returns *Exporter instead of "go.opentelemetry.io/otel/sdk/metric".Exporter. (#4272)
  • If an attribute set is omitted from an async callback, the previous value will no longer be exported in go.opentelemetry.io/otel/sdk/metric. (#4290)
  • If an attribute set is observed multiple times in an async callback in go.opentelemetry.io/otel/sdk/metric, the values will be summed instead of the last observation winning. (#4289)
  • Allow the explicit bucket histogram aggregation to be used for the up-down counter, observable counter, observable up-down counter, and observable gauge in the go.opentelemetry.io/otel/sdk/metric package. (#4332)
  • Restrict Meters in go.opentelemetry.io/otel/sdk/metric to only register and collect instruments it created. (#4333)
  • PeriodicReader.Shutdown and PeriodicReader.ForceFlush in go.opentelemetry.io/otel/sdk/metric now apply the periodic reader's timeout to the operation if the user provided context does not contain a deadline. (#4356, #4377)
  • Upgrade all use of go.opentelemetry.io/otel/semconv to use v1.21.0. (#4408)
  • Increase instrument name maximum length from 63 to 255 characters in go.opentelemetry.io/otel/sdk/metric. (#4434)
  • Add go.opentelemetry.op/otel/sdk/metric.WithProducer as an Option for "go.opentelemetry.io/otel/sdk/metric".NewManualReader and "go.opentelemetry.io/otel/sdk/metric".NewPeriodicReader. (#4346)

Removed

  • Remove Reader.RegisterProducer in go.opentelemetry.io/otel/metric. Use the added WithProducer option instead. (#4346)
  • Remove Reader.ForceFlush in go.opentelemetry.io/otel/metric. Notice that PeriodicReader.ForceFlush is still available. (#4375)

Fixed

  • Correctly format log messages from the go.opentelemetry.io/otel/exporters/zipkin exporter. (#4143)

... (truncated)

Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's changelog.

[1.17.0/0.40.0/0.0.5] 2023-08-28

Added

  • Export the ManualReader struct in go.opentelemetry.io/otel/sdk/metric. (#4244)
  • Export the PeriodicReader struct in go.opentelemetry.io/otel/sdk/metric. (#4244)
  • Add support for exponential histogram aggregations. A histogram can be configured as an exponential histogram using a view with "go.opentelemetry.io/otel/sdk/metric".ExponentialHistogram as the aggregation. (#4245)
  • Export the Exporter struct in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc. (#4272)
  • Export the Exporter struct in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp. (#4272)
  • The exporters in go.opentelemetry.io/otel/exporters/otlp/otlpmetric now support the OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE environment variable. (#4287)
  • Add WithoutCounterSuffixes option in go.opentelemetry.io/otel/exporters/prometheus to disable addition of _total suffixes. (#4306)
  • Add info and debug logging to the metric SDK in go.opentelemetry.io/otel/sdk/metric. (#4315)
  • The go.opentelemetry.io/otel/semconv/v1.21.0 package. The package contains semantic conventions from the v1.21.0 version of the OpenTelemetry Semantic Conventions. (#4362)
  • Accept 201 to 299 HTTP status as success in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp and go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp. (#4365)
  • Document the Temporality and Aggregation methods of the "go.opentelemetry.io/otel/sdk/metric".Exporter" need to be concurrent safe. (#4381)
  • Expand the set of units supported by the Prometheus exporter, and don't add unit suffixes if they are already present in go.opentelemetry.op/otel/exporters/prometheus (#4374)
  • Move the Aggregation interface and its implementations from go.opentelemetry.io/otel/sdk/metric/aggregation to go.opentelemetry.io/otel/sdk/metric. (#4435)
  • The exporters in go.opentelemetry.io/otel/exporters/otlp/otlpmetric now support the OTEL_EXPORTER_OTLP_METRICS_DEFAULT_HISTOGRAM_AGGREGATION environment variable. (#4437)
  • Add the NewAllowKeysFilter and NewDenyKeysFilter functions to go.opentelemetry.io/otel/attribute to allow convenient creation of allow-keys and deny-keys filters. (#4444)
  • Support Go 1.21. (#4463)

Changed

  • Starting from v1.21.0 of semantic conventions, go.opentelemetry.io/otel/semconv/{version}/httpconv and go.opentelemetry.io/otel/semconv/{version}/netconv packages will no longer be published. (#4145)
  • Log duplicate instrument conflict at a warning level instead of info in go.opentelemetry.io/otel/sdk/metric. (#4202)
  • Return an error on the creation of new instruments in go.opentelemetry.io/otel/sdk/metric if their name doesn't pass regexp validation. (#4210)
  • NewManualReader in go.opentelemetry.io/otel/sdk/metric returns *ManualReader instead of Reader. (#4244)
  • NewPeriodicReader in go.opentelemetry.io/otel/sdk/metric returns *PeriodicReader instead of Reader. (#4244)
  • Count the Collect time in the PeriodicReader timeout in go.opentelemetry.io/otel/sdk/metric. (#4221)
  • The function New in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetricgrpc returns *Exporter instead of "go.opentelemetry.io/otel/sdk/metric".Exporter. (#4272)
  • The function New in go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp returns *Exporter instead of "go.opentelemetry.io/otel/sdk/metric".Exporter. (#4272)
  • If an attribute set is omitted from an async callback, the previous value will no longer be exported in go.opentelemetry.io/otel/sdk/metric. (#4290)
  • If an attribute set is observed multiple times in an async callback in go.opentelemetry.io/otel/sdk/metric, the values will be summed instead of the last observation winning. (#4289)
  • Allow the explicit bucket histogram aggregation to be used for the up-down counter, observable counter, observable up-down counter, and observable gauge in the go.opentelemetry.io/otel/sdk/metric package. (#4332)
  • Restrict Meters in go.opentelemetry.io/otel/sdk/metric to only register and collect instruments it created. (#4333)
  • PeriodicReader.Shutdown and PeriodicReader.ForceFlush in go.opentelemetry.io/otel/sdk/metric now apply the periodic reader's timeout to the operation if the user provided context does not contain a deadline. (#4356, #4377)
  • Upgrade all use of go.opentelemetry.io/otel/semconv to use v1.21.0. (#4408)
  • Increase instrument name maximum length from 63 to 255 characters in go.opentelemetry.io/otel/sdk/metric. (#4434)
  • Add go.opentelemetry.op/otel/sdk/metric.WithProducer as an Option for "go.opentelemetry.io/otel/sdk/metric".NewManualReader and "go.opentelemetry.io/otel/sdk/metric".NewPeriodicReader. (#4346)

Removed

  • Remove Reader.RegisterProducer in go.opentelemetry.io/otel/metric. Use the added WithProducer option instead. (#4346)
  • Remove Reader.ForceFlush in go.opentelemetry.io/otel/metric. Notice that PeriodicReader.ForceFlush is still available. (#4375)

Fixed

... (truncated)

Commits
  • 3c476ce Release v1.17.0/v0.40.0/v0.0.5 (#4464)
  • 183e081 Bump github.com/golangci/golangci-lint in /internal/tools (#4465)
  • 6be116e Add testing support for Go 1.21 (#4463)
  • 69611bd Switch Stream back to having an AttributeFilter field and add `New*Filter...
  • f15ae16 Fix the broken sentence (#4456)
  • 16ce491 Fix guard of measured value to not record empty (#4452)
  • 9b47674 Make getBin and scaleChange methods of expoHistogramDataPoint (#4451)
  • 9d9b71f Remove the expoHistogramValues type (#4450)
  • a5ff7af Ignore +/- Inf and NaN for exponential histogram measurement (#4446)
  • d78820e Deprecate exporters/jaeger (#4423)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp&package-manager=go_modules&previous-version=0.39.0&new-version=0.40.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Co-authored-by: ramin --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 8c1fbb1349..470dde9bed 100644 --- a/go.mod +++ b/go.mod @@ -61,14 +61,14 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 - go.opentelemetry.io/otel v1.16.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0 + go.opentelemetry.io/otel v1.17.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 - go.opentelemetry.io/otel/metric v1.16.0 - go.opentelemetry.io/otel/sdk v1.16.0 - go.opentelemetry.io/otel/sdk/metric v0.39.0 - go.opentelemetry.io/otel/trace v1.16.0 + go.opentelemetry.io/otel/metric v1.17.0 + go.opentelemetry.io/otel/sdk v1.17.0 + go.opentelemetry.io/otel/sdk/metric v0.40.0 + go.opentelemetry.io/otel/trace v1.17.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 go.uber.org/zap v1.25.0 @@ -308,8 +308,8 @@ require ( github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect diff --git a/go.sum b/go.sum index d738f67e10..3835cf9304 100644 --- a/go.sum +++ b/go.sum @@ -2358,14 +2358,14 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.16.0 h1:Z7GVAX/UkAXPKsy94IU+i6thsQS4nb7LviLpnaNeW8s= -go.opentelemetry.io/otel v1.16.0/go.mod h1:vl0h9NUa1D5s1nv3A5vZOYWn8av4K8Ml6JDeHrT/bx4= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0 h1:t4ZwRPU+emrcvM2e9DHd0Fsf0JTPVcbfa/BhTDF03d0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.16.0/go.mod h1:vLarbg68dH2Wa77g71zmKQqlQ8+8Rq3GRG31uc0WcWI= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0 h1:f6BwB2OACc3FCbYVznctQ9V6KK7Vq6CjmYXJ7DeSs4E= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.39.0/go.mod h1:UqL5mZ3qs6XYhDnZaW1Ps4upD+PX6LipH40AoeuIlwU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0 h1:IZXpCEtI7BbX01DRQEWTGDkvjMB6hEhiEZXS+eg2YqY= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.39.0/go.mod h1:xY111jIZtWb+pUUgT4UiiSonAaY2cD2Ts5zvuKLki3o= +go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= +go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0 h1:eU0ffpYuEY7eQ75K+nKr9CI5KcY8h+GPk/9DDlEO1NI= +go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0/go.mod h1:9P5RK5JS2sjKepuCkqFwPp3etwV/57E0eigLw18Mn1k= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 h1:MZbjiZeMmn5wFMORhozpouGKDxj9POHTuU5UA8msBQk= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0/go.mod h1:C7tOYVCJmrDTCwxNny0MuUtnDIR3032vFHYke0F2ZrU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 h1:SZaSbubADNhH2Gxm+1GaZ/cFsGiYefZoodMMX79AOd4= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0/go.mod h1:N65FzQDfQH7NY7umgb0U+7ypGKVYKwwE24L6KXT4OA8= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= @@ -2373,20 +2373,20 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.16.0 h1:RbrpwVG1Hfv85LgnZ7+txXioPDoh6EdbZHo26Q3hqOo= -go.opentelemetry.io/otel/metric v1.16.0/go.mod h1:QE47cpOmkwipPiefDwo2wDzwJrlfxxNYodqc4xnGCo4= +go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc= +go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.16.0 h1:Z1Ok1YsijYL0CSJpHt4cS3wDDh7p572grzNrBMiMWgE= -go.opentelemetry.io/otel/sdk v1.16.0/go.mod h1:tMsIuKXuuIWPBAOrH+eHtvhTL+SntFtXF9QD68aP6p4= -go.opentelemetry.io/otel/sdk/metric v0.39.0 h1:Kun8i1eYf48kHH83RucG93ffz0zGV1sh46FAScOTuDI= -go.opentelemetry.io/otel/sdk/metric v0.39.0/go.mod h1:piDIRgjcK7u0HCL5pCA4e74qpK/jk3NiUoAHATVAmiI= +go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE= +go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= +go.opentelemetry.io/otel/sdk/metric v0.40.0 h1:qOM29YaGcxipWjL5FzpyZDpCYrDREvX0mVlmXdOjCHU= +go.opentelemetry.io/otel/sdk/metric v0.40.0/go.mod h1:dWxHtdzdJvg+ciJUKLTKwrMe5P6Dv3FyDbh8UkfgkVs= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.16.0 h1:8JRpaObFoW0pxuVPapkgH8UhHQj+bJW8jJsCZEu5MQs= -go.opentelemetry.io/otel/trace v1.16.0/go.mod h1:Yt9vYq1SdNz3xdjZZK7wcXv1qv2pwLkqr2QVwea0ef0= +go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= +go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= From 1751d62deeb750c13f7085e76374b5af1796318f Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Tue, 5 Sep 2023 12:19:09 +0200 Subject: [PATCH 171/388] chore(deps): migrate to boxo (#2595) Thanks to https://github.com/ipfs/boxo/pull/407, we can now fully migrate to boxo and get rid of the oldest `replace` directive in our go.mod for `verifcid`. TODO: Issue about go-car/v2 migration --------- Co-authored-by: Ryan --- Makefile | 5 +++- blob/service_test.go | 7 ++--- das/daser_test.go | 10 +++---- go.mod | 11 +++---- go.sum | 15 +++++----- header/headertest/fraud/testing.go | 5 ++-- nodebuilder/node.go | 2 +- nodebuilder/p2p/ipld.go | 12 -------- nodebuilder/p2p/module.go | 3 +- nodebuilder/share/constructors.go | 2 +- share/availability/cache/availability_test.go | 4 +-- share/availability/cache/testing.go | 6 ++-- share/availability/full/testing.go | 4 +-- share/availability/light/testing.go | 8 ++--- share/availability/test/testing.go | 4 +-- share/eds/adapters.go | 2 +- share/eds/byzantine/bad_encoding_test.go | 5 ++-- share/eds/byzantine/byzantine.go | 2 +- share/eds/byzantine/share_proof.go | 2 +- share/eds/byzantine/share_proof_test.go | 5 ++-- share/eds/retriever.go | 2 +- share/eds/retriever_test.go | 15 +++++----- share/getters/getter_test.go | 6 ++-- share/getters/ipld.go | 2 +- share/getters/utils.go | 2 +- share/ipld/add.go | 5 ++-- share/ipld/blockserv.go | 30 +++++++++++++++++++ share/ipld/get.go | 2 +- share/ipld/get_shares.go | 2 +- share/ipld/get_shares_test.go | 25 +++++++--------- share/ipld/namespace_data.go | 2 +- share/ipld/nmt.go | 2 +- share/ipld/nmt_adder.go | 4 +-- 33 files changed, 111 insertions(+), 102 deletions(-) delete mode 100644 nodebuilder/p2p/ipld.go create mode 100644 share/ipld/blockserv.go diff --git a/Makefile b/Makefile index d4b3d2572e..8813b65d23 100644 --- a/Makefile +++ b/Makefile @@ -169,7 +169,10 @@ lint-imports: ## sort-imports: Sort Go imports. sort-imports: - @goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" -output stdout . + @for file in `find . -type f -name '*.go'`; \ + do goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" $$file \ + || exit 1; \ + done; .PHONY: sort-imports ## adr-gen: Generate ADR from template. Must set NUM and TITLE parameters. diff --git a/blob/service_test.go b/blob/service_test.go index a45fe631af..64296b969c 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -10,7 +10,6 @@ import ( ds "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" tmrand "github.com/tendermint/tendermint/libs/rand" @@ -324,7 +323,7 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) rawShares = append(rawShares, append(rawShares1, padding1.ToBytes())...) - bs := mdutils.Bserv() + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) @@ -375,7 +374,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { rawShares = append(rawShares, append(rawShares0, padding0.ToBytes())...) } - bs := mdutils.Bserv() + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) @@ -397,7 +396,7 @@ func TestService_GetAllWithoutPadding(t *testing.T) { } func createService(ctx context.Context, t *testing.T, blobs []*Blob) *Service { - bs := mdutils.Bserv() + bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) require.NoError(t, err) diff --git a/das/daser_test.go b/das/daser_test.go index e4e74dc7ff..3e5bce0b97 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -6,10 +6,9 @@ import ( "time" "github.com/golang/mock/gomock" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" pubsub "github.com/libp2p/go-libp2p-pubsub" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/assert" @@ -31,6 +30,7 @@ import ( availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/eds/byzantine" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) var timeout = time.Second * 15 @@ -39,7 +39,7 @@ var timeout = time.Second * 15 // the DASer checkpoint is updated to network head. func TestDASerLifecycle(t *testing.T) { ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() avail := light.TestAvailability(getters.NewIPLDGetter(bServ)) // 15 headers from the past and 15 future headers mockGet, sub, mockService := createDASerSubcomponents(t, bServ, 15, 15) @@ -79,7 +79,7 @@ func TestDASerLifecycle(t *testing.T) { func TestDASer_Restart(t *testing.T) { ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() avail := light.TestAvailability(getters.NewIPLDGetter(bServ)) // 15 headers from the past and 15 future headers mockGet, sub, mockService := createDASerSubcomponents(t, bServ, 15, 15) @@ -147,7 +147,7 @@ func TestDASer_stopsAfter_BEFP(t *testing.T) { t.Cleanup(cancel) ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() // create mock network net, err := mocknet.FullMeshLinked(1) require.NoError(t, err) diff --git a/go.mod b/go.mod index 470dde9bed..ab1c199e97 100644 --- a/go.mod +++ b/go.mod @@ -4,8 +4,6 @@ go 1.21 toolchain go1.21.0 -replace github.com/ipfs/go-verifcid => github.com/celestiaorg/go-verifcid v0.0.1-lazypatch - require ( cosmossdk.io/errors v1.0.0 cosmossdk.io/math v1.1.2 @@ -32,15 +30,13 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru v1.0.2 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.11.0 - github.com/ipfs/go-block-format v0.2.0 - github.com/ipfs/go-blockservice v0.5.1 // down 1 version, 0.5.2 is marked as deprecated and raises alerts + github.com/ipfs/boxo v0.12.0 + github.com/ipfs/go-block-format v0.1.2 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-cbor v0.1.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 - github.com/ipfs/go-merkledag v0.11.0 github.com/ipld/go-car v0.6.2 github.com/libp2p/go-libp2p v0.30.0 github.com/libp2p/go-libp2p-kad-dht v0.25.0 @@ -200,15 +196,16 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/ipfs/bbloom v0.0.4 // indirect + github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.0 github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect - github.com/ipfs/go-ipfs-exchange-offline v0.3.0 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect github.com/ipfs/go-ipld-legacy v0.2.1 // indirect github.com/ipfs/go-log v1.0.5 // indirect + github.com/ipfs/go-merkledag v0.11.0 // indirect github.com/ipfs/go-metrics-interface v0.0.1 // indirect github.com/ipfs/go-peertaskqueue v0.8.1 // indirect github.com/ipfs/go-verifcid v0.0.2 // indirect diff --git a/go.sum b/go.sum index 3835cf9304..267e05154a 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,6 @@ github.com/celestiaorg/go-header v0.3.0 h1:9fhxSgldPiWWq3yd9u7oSk5vYqaLV1JkeTnJd github.com/celestiaorg/go-header v0.3.0/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= -github.com/celestiaorg/go-verifcid v0.0.1-lazypatch h1:9TSe3w1cmJmbWlweCwCTIZkan7jV8M+KwglXpdD+UG8= -github.com/celestiaorg/go-verifcid v0.0.1-lazypatch/go.mod h1:kXPYu0XqTNUKWA1h3M95UHjUqBzDwXVVt/RXZDjKJmQ= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.19.0 h1:9VXFeI/gt+q8h5HeCE0RjXJhOxsFzxJUjHrkvF9CMYE= @@ -1023,8 +1021,8 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.11.0 h1:urMxhZ3xoF4HssJVD3+0ssGT9pptEfHfbL8DYdoWFlg= -github.com/ipfs/boxo v0.11.0/go.mod h1:8IfDmp+FzFGcF4zjAgHMVPpwYw4AjN9ePEzDfkaYJ1w= +github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= +github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= @@ -1040,16 +1038,14 @@ github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= +github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= -github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= -github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= github.com/ipfs/go-blockservice v0.3.0/go.mod h1:P5ppi8IHDC7O+pA0AlGTF09jruB2h+oP3wVVaZl8sfk= +github.com/ipfs/go-blockservice v0.5.0 h1:B2mwhhhVQl2ntW2EIpaWPwSCxSuqr5fFA93Ms4bYLEY= github.com/ipfs/go-blockservice v0.5.0/go.mod h1:W6brZ5k20AehbmERplmERn8o2Ni3ZZubvAxaIUeaT6w= -github.com/ipfs/go-blockservice v0.5.1 h1:9pAtkyKAz/skdHTh0kH8VulzWp+qmSDD0aI17TYP/s0= -github.com/ipfs/go-blockservice v0.5.1/go.mod h1:VpMblFEqG67A/H2sHKAemeH9vlURVavlysbdUI632yk= github.com/ipfs/go-cid v0.0.1/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.2/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= github.com/ipfs/go-cid v0.0.3/go.mod h1:GHWU/WuQdMPmIosc4Yn1bcCT7dSeX4lBafM7iqUPQvM= @@ -1219,6 +1215,9 @@ github.com/ipfs/go-unixfsnode v1.5.2/go.mod h1:NlOebRwYx8lMCNMdhAhEspYPBD3obp7TE github.com/ipfs/go-unixfsnode v1.7.1/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= github.com/ipfs/go-unixfsnode v1.7.4 h1:iLvKyAVKUYOIAW2t4kDYqsT7VLGj31eXJE2aeqGfbwA= github.com/ipfs/go-unixfsnode v1.7.4/go.mod h1:PVfoyZkX1B34qzT3vJO4nsLUpRCyhnMuHBznRcXirlk= +github.com/ipfs/go-verifcid v0.0.1/go.mod h1:5Hrva5KBeIog4A+UpqlaIU+DEstipcJYQQZc0g37pY0= +github.com/ipfs/go-verifcid v0.0.2 h1:XPnUv0XmdH+ZIhLGKg6U2vaPaRDXb9urMyNVCE7uvTs= +github.com/ipfs/go-verifcid v0.0.2/go.mod h1:40cD9x1y4OWnFXbLNJYRe7MpNvWlMn3LZAG5Wb4xnPU= github.com/ipfs/interface-go-ipfs-core v0.9.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipfs/interface-go-ipfs-core v0.10.0/go.mod h1:F3EcmDy53GFkF0H3iEJpfJC320fZ/4G60eftnItrrJ0= github.com/ipld/go-car v0.5.0/go.mod h1:ppiN5GWpjOZU9PgpAZ9HbZd9ZgSpwPMr48fGRJOWmvE= diff --git a/header/headertest/fraud/testing.go b/header/headertest/fraud/testing.go index 6a5cda733d..e2ff13a4e0 100644 --- a/header/headertest/fraud/testing.go +++ b/header/headertest/fraud/testing.go @@ -5,7 +5,7 @@ import ( "testing" "time" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/stretchr/testify/require" "github.com/tendermint/tendermint/libs/bytes" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" @@ -46,7 +46,8 @@ func NewFraudMaker(t *testing.T, height int64, vals []types.PrivValidator, valSe } func (f *FraudMaker) MakeExtendedHeader(odsSize int, edsStore *eds.Store) header.ConstructFn { - return func(h *types.Header, + return func( + h *types.Header, comm *types.Commit, vals *types.ValidatorSet, eds *rsmt2d.ExtendedDataSquare, diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 6d83e6c4c3..58a182276c 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -6,8 +6,8 @@ import ( "fmt" "strings" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/exchange" - "github.com/ipfs/go-blockservice" logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" diff --git a/nodebuilder/p2p/ipld.go b/nodebuilder/p2p/ipld.go deleted file mode 100644 index ad32cc39f5..0000000000 --- a/nodebuilder/p2p/ipld.go +++ /dev/null @@ -1,12 +0,0 @@ -package p2p - -import ( - "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange" - "github.com/ipfs/go-blockservice" -) - -// blockService constructs IPFS's BlockService for fetching arbitrary Merkle structures. -func blockService(bs blockstore.Blockstore, ex exchange.Interface) blockservice.BlockService { - return blockservice.New(bs, ex) -} diff --git a/nodebuilder/p2p/module.go b/nodebuilder/p2p/module.go index e7f60956c7..1ddff02173 100644 --- a/nodebuilder/p2p/module.go +++ b/nodebuilder/p2p/module.go @@ -6,6 +6,7 @@ import ( "go.uber.org/fx" "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/share/ipld" ) var log = logging.Logger("module/p2p") @@ -27,7 +28,7 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { fx.Provide(routedHost), fx.Provide(pubSub), fx.Provide(dataExchange), - fx.Provide(blockService), + fx.Provide(ipld.NewBlockservice), fx.Provide(peerRouting), fx.Provide(contentRouting), fx.Provide(addrsFactory(cfg.AnnounceAddresses, cfg.NoAnnounceAddresses)), diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index b962038d17..0e8d17f208 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -5,7 +5,7 @@ import ( "errors" "github.com/filecoin-project/dagstore" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" diff --git a/share/availability/cache/availability_test.go b/share/availability/cache/availability_test.go index 03df47f848..ccc11a1d28 100644 --- a/share/availability/cache/availability_test.go +++ b/share/availability/cache/availability_test.go @@ -9,7 +9,6 @@ import ( "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -17,6 +16,7 @@ import ( "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/ipld" ) // TestCacheAvailability tests to ensure that the successful result of a @@ -86,7 +86,7 @@ func TestCacheAvailability_NoDuplicateSampling(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() // create root to cache - root := availability_test.RandFillBS(t, 16, mdutils.Bserv()) + root := availability_test.RandFillBS(t, 16, ipld.NewMemBlockservice()) // wrap dummyAvailability with a datastore ds := sync.MutexWrap(datastore.NewMapDatastore()) ca := NewShareAvailability(&dummyAvailability{counter: 0}, ds) diff --git a/share/availability/cache/testing.go b/share/availability/cache/testing.go index 978d51b6b5..00a1520114 100644 --- a/share/availability/cache/testing.go +++ b/share/availability/cache/testing.go @@ -5,18 +5,18 @@ import ( ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/availability/full" "github.com/celestiaorg/celestia-node/share/availability/light" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) // LightAvailabilityWithLocalRandSquare wraps light.GetterWithRandSquare with cache availability func LightAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availability, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() store := dssync.MutexWrap(ds.NewMapDatastore()) getter := getters.NewIPLDGetter(bServ) avail := NewShareAvailability( @@ -28,7 +28,7 @@ func LightAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availabili // FullAvailabilityWithLocalRandSquare wraps full.GetterWithRandSquare with cache availability func FullAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availability, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() store := dssync.MutexWrap(ds.NewMapDatastore()) getter := getters.NewIPLDGetter(bServ) avail := NewShareAvailability( diff --git a/share/availability/full/testing.go b/share/availability/full/testing.go index 680eb7abaf..4638fde1bc 100644 --- a/share/availability/full/testing.go +++ b/share/availability/full/testing.go @@ -4,20 +4,20 @@ import ( "testing" "time" - mdutils "github.com/ipfs/go-merkledag/test" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/p2p/discovery/routing" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/discovery" ) // GetterWithRandSquare provides a share.Getter filled with 'n' NMT // trees of 'n' random shares, essentially storing a whole square. func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) return getter, availability_test.RandFillBS(t, n, bServ) } diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index 59163d6356..991d143a62 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -3,18 +3,18 @@ package light import ( "testing" - "github.com/ipfs/go-blockservice" - mdutils "github.com/ipfs/go-merkledag/test" + "github.com/ipfs/boxo/blockservice" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" + "github.com/celestiaorg/celestia-node/share/ipld" ) // GetterWithRandSquare provides a share.Getter filled with 'n' NMT trees of 'n' random shares, // essentially storing a whole square. func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) return getter, availability_test.RandFillBS(t, n, bServ) } @@ -22,7 +22,7 @@ func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { // EmptyGetter provides an unfilled share.Getter with corresponding blockservice.BlockService than // can be filled by the test. func EmptyGetter() (share.Getter, blockservice.BlockService) { - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) return getter, bServ } diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 8977678198..8973ee8cb1 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -6,9 +6,9 @@ import ( "github.com/ipfs/boxo/bitswap" "github.com/ipfs/boxo/bitswap/network" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/routing/offline" - "github.com/ipfs/go-blockservice" ds "github.com/ipfs/go-datastore" dssync "github.com/ipfs/go-datastore/sync" record "github.com/libp2p/go-libp2p-record" @@ -94,7 +94,7 @@ func (dn *TestDagNet) NewTestNodeWithBlockstore(dstore ds.Datastore, bstore bloc ) nd := &TestNode{ net: dn, - BlockService: blockservice.New(bstore, bs), + BlockService: ipld.NewBlockservice(bstore, bs), Host: hst, } dn.nodes = append(dn.nodes, nd) diff --git a/share/eds/adapters.go b/share/eds/adapters.go index fe498400b3..8bf2340d91 100644 --- a/share/eds/adapters.go +++ b/share/eds/adapters.go @@ -5,8 +5,8 @@ import ( "sync" "github.com/filecoin-project/dagstore" + "github.com/ipfs/boxo/blockservice" blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ) diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index b5dcea3452..9d16955ee3 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" core "github.com/tendermint/tendermint/types" @@ -22,7 +21,7 @@ import ( func TestBadEncodingFraudProof(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() square := edstest.RandByzantineEDS(t, 16) dah, err := da.NewDataAvailabilityHeader(square) @@ -48,7 +47,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() squareSize := 8 shares := sharetest.RandShares(t, squareSize*squareSize) diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index 0fcd78273e..dfdf681f04 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "golang.org/x/sync/errgroup" "github.com/celestiaorg/celestia-app/pkg/da" diff --git a/share/eds/byzantine/share_proof.go b/share/eds/byzantine/share_proof.go index 411fd85818..98b58ebbec 100644 --- a/share/eds/byzantine/share_proof.go +++ b/share/eds/byzantine/share_proof.go @@ -4,7 +4,7 @@ import ( "context" "crypto/sha256" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" diff --git a/share/eds/byzantine/share_proof_test.go b/share/eds/byzantine/share_proof_test.go index db1db64f80..a9021d806d 100644 --- a/share/eds/byzantine/share_proof_test.go +++ b/share/eds/byzantine/share_proof_test.go @@ -7,7 +7,6 @@ import ( "time" "github.com/ipfs/go-cid" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" @@ -21,7 +20,7 @@ func TestGetProof(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() shares := sharetest.RandShares(t, width*width) in, err := ipld.AddShares(ctx, shares, bServ) @@ -58,7 +57,7 @@ func TestGetProofs(t *testing.T) { const width = 4 ctx, cancel := context.WithTimeout(context.Background(), time.Second*2) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() shares := sharetest.RandShares(t, width*width) in, err := ipld.AddShares(ctx, shares, bServ) diff --git a/share/eds/retriever.go b/share/eds/retriever.go index e7837ae23a..c2966c3953 100644 --- a/share/eds/retriever.go +++ b/share/eds/retriever.go @@ -7,7 +7,7 @@ import ( "sync/atomic" "time" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel" diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index 12b1c11083..f3f7ccca64 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -7,8 +7,7 @@ import ( "testing" "time" - "github.com/ipfs/go-blockservice" - mdutils "github.com/ipfs/go-merkledag/test" + "github.com/ipfs/boxo/blockservice" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -30,7 +29,7 @@ func TestRetriever_Retrieve(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) type test struct { @@ -73,7 +72,7 @@ func TestRetriever_ByzantineError(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) defer cancel() - bserv := mdutils.Bserv() + bserv := ipld.NewMemBlockservice() shares := edstest.RandEDS(t, width).Flattened() _, err := ipld.ImportShares(ctx, shares, bserv) require.NoError(t, err) @@ -110,7 +109,7 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) // generate EDS @@ -136,7 +135,7 @@ func TestRetriever_MultipleRandQuadrants(t *testing.T) { func TestFraudProofValidation(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() odsSize := []int{2, 4, 16, 32, 64, 128} for _, size := range odsSize { @@ -178,7 +177,7 @@ BenchmarkBEFPValidation/ods_size:128 259 4934375 ns/op 5418406 B func BenchmarkBEFPValidation(b *testing.B) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) defer b.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) t := &testing.T{} odsSize := []int{2, 4, 16, 32, 64, 128} @@ -217,7 +216,7 @@ func BenchmarkNewErrByzantineData(b *testing.B) { odsSize := []int{2, 4, 16, 32, 64, 128} ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() r := NewRetriever(bServ) t := &testing.T{} for _, size := range odsSize { diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index d19d0cf174..ddab46f290 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -7,10 +7,8 @@ import ( "time" "github.com/ipfs/boxo/exchange/offline" - bsrv "github.com/ipfs/go-blockservice" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -37,7 +35,7 @@ func TestTeeGetter(t *testing.T) { err = edsStore.Start(ctx) require.NoError(t, err) - bServ := mdutils.Bserv() + bServ := ipld.NewMemBlockservice() ig := NewIPLDGetter(bServ) tg := NewTeeGetter(ig, edsStore) @@ -197,7 +195,7 @@ func TestIPLDGetter(t *testing.T) { require.NoError(t, err) bStore := edsStore.Blockstore() - bserv := bsrv.New(bStore, offline.Exchange(bStore)) + bserv := ipld.NewBlockservice(bStore, offline.Exchange(edsStore.Blockstore())) sg := NewIPLDGetter(bserv) t.Run("GetShare", func(t *testing.T) { diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 8e11a389bd..4024a67f5b 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -7,7 +7,7 @@ import ( "sync" "sync/atomic" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/trace" diff --git a/share/getters/utils.go b/share/getters/utils.go index 5305c5c737..e3f24b3857 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -6,7 +6,7 @@ import ( "fmt" "time" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel" diff --git a/share/ipld/add.go b/share/ipld/add.go index 7e5909669d..fbb743148c 100644 --- a/share/ipld/add.go +++ b/share/ipld/add.go @@ -4,7 +4,7 @@ import ( "context" "fmt" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" @@ -52,7 +52,8 @@ func AddShares( func ImportShares( ctx context.Context, shares [][]byte, - adder blockservice.BlockService) (*rsmt2d.ExtendedDataSquare, error) { + adder blockservice.BlockService, +) (*rsmt2d.ExtendedDataSquare, error) { if len(shares) == 0 { return nil, fmt.Errorf("ipld: importing empty data") } diff --git a/share/ipld/blockserv.go b/share/ipld/blockserv.go new file mode 100644 index 0000000000..2ed2a21c77 --- /dev/null +++ b/share/ipld/blockserv.go @@ -0,0 +1,30 @@ +package ipld + +import ( + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/blockstore" + "github.com/ipfs/boxo/exchange" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/sync" +) + +// NewBlockservice constructs Blockservice for fetching NMTrees. +func NewBlockservice(bs blockstore.Blockstore, exchange exchange.Interface) blockservice.BlockService { + return blockservice.New(bs, exchange, blockservice.WithAllowlist(defaultAllowlist)) +} + +// NewMemBlockservice constructs Blockservice for fetching NMTrees with in-memory blockstore. +func NewMemBlockservice() blockservice.BlockService { + bstore := blockstore.NewBlockstore(sync.MutexWrap(datastore.NewMapDatastore())) + return NewBlockservice(bstore, nil) +} + +// defaultAllowlist keeps default list of hashes allowed in the network. +var defaultAllowlist allowlist + +type allowlist struct{} + +func (a allowlist) IsAllowed(code uint64) bool { + // we allow all codes except home-baked sha256NamespaceFlagged + return code == sha256NamespaceFlagged +} diff --git a/share/ipld/get.go b/share/ipld/get.go index 35f601853d..f263877dc0 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -7,7 +7,7 @@ import ( "sync/atomic" "github.com/gammazero/workerpool" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" diff --git a/share/ipld/get_shares.go b/share/ipld/get_shares.go index 0bed240fdc..3fda86941f 100644 --- a/share/ipld/get_shares.go +++ b/share/ipld/get_shares.go @@ -3,7 +3,7 @@ package ipld import ( "context" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" format "github.com/ipfs/go-ipld-format" diff --git a/share/ipld/get_shares_test.go b/share/ipld/get_shares_test.go index dd6c3a6636..580efcb69b 100644 --- a/share/ipld/get_shares_test.go +++ b/share/ipld/get_shares_test.go @@ -11,13 +11,8 @@ import ( "testing" "time" - "github.com/ipfs/boxo/blockstore" - "github.com/ipfs/boxo/exchange/offline" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - mdutils "github.com/ipfs/go-merkledag/test" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -35,7 +30,7 @@ func TestGetShare(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() // generate random shares for the nmt shares := sharetest.RandShares(t, size*size) @@ -155,7 +150,7 @@ func removeRandShares(data [][]byte, d int) [][]byte { func TestGetSharesByNamespace(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() var tests = []struct { rawData []share.Share @@ -201,7 +196,7 @@ func TestGetSharesByNamespace(t *testing.T) { func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() shares := sharetest.RandShares(t, 16) @@ -245,7 +240,7 @@ func TestCollectLeavesByNamespace_IncompleteData(t *testing.T) { func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() shares := sharetest.RandShares(t, 1024) @@ -290,7 +285,7 @@ func TestCollectLeavesByNamespace_AbsentNamespaceId(t *testing.T) { func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() shares := sharetest.RandShares(t, 16) @@ -332,7 +327,7 @@ func TestCollectLeavesByNamespace_MultipleRowsContainingSameNamespaceId(t *testi func TestGetSharesWithProofsByNamespace(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 5*time.Second) t.Cleanup(cancel) - bServ := mdutils.Bserv() + bServ := NewMemBlockservice() var tests = []struct { rawData []share.Share @@ -431,13 +426,13 @@ func TestBatchSize(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*time.Duration(tt.origWidth)) defer cancel() - bs := blockstore.NewBlockstore(dssync.MutexWrap(ds.NewMapDatastore())) + bs := NewMemBlockservice() randEds := edstest.RandEDS(t, tt.origWidth) - _, err := AddShares(ctx, randEds.FlattenedODS(), blockservice.New(bs, offline.Exchange(bs))) + _, err := AddShares(ctx, randEds.FlattenedODS(), bs) require.NoError(t, err) - out, err := bs.AllKeysChan(ctx) + out, err := bs.Blockstore().AllKeysChan(ctx) require.NoError(t, err) var count int diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index 38dfdb2169..a3f468ee47 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -7,7 +7,7 @@ import ( "sync" "sync/atomic" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" "go.opentelemetry.io/otel/attribute" diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index e7d6b4b513..b43d786a5f 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -8,8 +8,8 @@ import ( "hash" "math/rand" + "github.com/ipfs/boxo/blockservice" blocks "github.com/ipfs/go-block-format" - "github.com/ipfs/go-blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" diff --git a/share/ipld/nmt_adder.go b/share/ipld/nmt_adder.go index d090c679d9..7ce52859b2 100644 --- a/share/ipld/nmt_adder.go +++ b/share/ipld/nmt_adder.go @@ -5,10 +5,10 @@ import ( "fmt" "sync" - "github.com/ipfs/go-blockservice" + "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/boxo/ipld/merkledag" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "github.com/ipfs/go-merkledag" "github.com/celestiaorg/nmt" ) From 34bc95f4bc7c8c7e7265a05a74c1a5878cd72297 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 5 Sep 2023 10:52:59 +0000 Subject: [PATCH 172/388] chore(deps): bump github.com/ipfs/go-ipfs-ds-help from 1.1.0 to 1.1.1 (#2624) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/go-ipfs-ds-help](https://github.com/ipfs/go-ipfs-ds-help) from 1.1.0 to 1.1.1.
Release notes

Sourced from github.com/ipfs/go-ipfs-ds-help's releases.

v1.1.1

What's Changed

Full Changelog: https://github.com/ipfs/go-ipfs-ds-help/compare/v1.1.0...v1.1.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/go-ipfs-ds-help&package-manager=go_modules&previous-version=1.1.0&new-version=1.1.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 3 ++- share/eds/blockstore.go | 2 +- 3 files changed, 4 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ab1c199e97..e8b8fb5795 100644 --- a/go.mod +++ b/go.mod @@ -199,7 +199,7 @@ require ( github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect - github.com/ipfs/go-ipfs-ds-help v1.1.0 + github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect github.com/ipfs/go-ipfs-exchange-interface v0.2.1 // indirect github.com/ipfs/go-ipfs-pq v0.0.3 // indirect github.com/ipfs/go-ipfs-util v0.0.3 // indirect diff --git a/go.sum b/go.sum index 267e05154a..8360dee057 100644 --- a/go.sum +++ b/go.sum @@ -1109,8 +1109,9 @@ github.com/ipfs/go-ipfs-delay v0.0.1 h1:r/UXYyRcddO6thwOnhiznIAiSvxMECGgtv35Xs1I github.com/ipfs/go-ipfs-delay v0.0.1/go.mod h1:8SP1YXK1M1kXuc4KJZINY3TQQ03J2rwBG9QfXmbRPrw= github.com/ipfs/go-ipfs-ds-help v0.0.1/go.mod h1:gtP9xRaZXqIQRh1HRpp595KbBEdgqWFxefeVKOV8sxo= github.com/ipfs/go-ipfs-ds-help v0.1.1/go.mod h1:SbBafGJuGsPI/QL3j9Fc5YPLeAu+SzOkI0gFwAg+mOs= -github.com/ipfs/go-ipfs-ds-help v1.1.0 h1:yLE2w9RAsl31LtfMt91tRZcrx+e61O5mDxFRR994w4Q= github.com/ipfs/go-ipfs-ds-help v1.1.0/go.mod h1:YR5+6EaebOhfcqVCyqemItCLthrpVNot+rsOU/5IatU= +github.com/ipfs/go-ipfs-ds-help v1.1.1 h1:B5UJOH52IbcfS56+Ul+sv8jnIV10lbjLF5eOO0C66Nw= +github.com/ipfs/go-ipfs-ds-help v1.1.1/go.mod h1:75vrVCkSdSFidJscs8n4W+77AtTpCIAdDGAwjitJMIo= github.com/ipfs/go-ipfs-exchange-interface v0.0.1/go.mod h1:c8MwfHjtQjPoDyiy9cFquVtVHkO9b9Ob3FG91qJnWCM= github.com/ipfs/go-ipfs-exchange-interface v0.1.0/go.mod h1:ych7WPlyHqFvCi/uQI48zLZuAWVP5iTQPXEfVaw5WEI= github.com/ipfs/go-ipfs-exchange-interface v0.2.0/go.mod h1:z6+RhJuDQbqKguVyslSOuVDhqF9JtTrO3eptSAiW2/Y= diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go index 22ef28f821..2abe4219f1 100644 --- a/share/eds/blockstore.go +++ b/share/eds/blockstore.go @@ -7,11 +7,11 @@ import ( "github.com/filecoin-project/dagstore" bstore "github.com/ipfs/boxo/blockstore" + dshelp "github.com/ipfs/boxo/datastore/dshelp" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" "github.com/ipfs/go-datastore/namespace" - dshelp "github.com/ipfs/go-ipfs-ds-help" ipld "github.com/ipfs/go-ipld-format" ) From 96388ea79a5f738a1fe633e40c7a412aadcfd07e Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Sep 2023 14:38:01 +0200 Subject: [PATCH 173/388] chore(go.mod): bump go-header (#2657) Bumps go-header. Resolves https://github.com/celestiaorg/go-header/issues/102 which was blocking sync for arabica-10 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e8b8fb5795..c978655477 100644 --- a/go.mod +++ b/go.mod @@ -13,7 +13,7 @@ require ( github.com/celestiaorg/celestia-app v1.0.0-rc13 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.3.0 + github.com/celestiaorg/go-header v0.3.1 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.19.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 8360dee057..5c4e483b6f 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.3.0 h1:9fhxSgldPiWWq3yd9u7oSk5vYqaLV1JkeTnJdGcisFo= -github.com/celestiaorg/go-header v0.3.0/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.3.1 h1:rJuFPVMoI20Du4KHWLz0IMllEIp0XgIWi9lHQLPWmq8= +github.com/celestiaorg/go-header v0.3.1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= From c146a0a7a47b7abe02746ef583bc31ad52d130f5 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Sep 2023 16:07:32 +0200 Subject: [PATCH 174/388] feat(nodebuilder/header): Bootstrap from previously seen peers (#2507) Provides `PIDStore` to header module so that it can be used in `peerTracker` and replaces mem `peerstore.Peerstore` with on-disk `peerstore.Peerstore` so that `peerTracker` can quickly bootstrap itself with previously-seen peers and allow syncer to initialise its sync target from tracked peers rather than trusted so long as it has a subjective head within the trusting period. Overrides #2133 Closes #1851, mitigates issues resulting from #1623 Swamp integration tests to follow (tracked in #2506) ### Future note: This PR introduces a soon-to-be deprecated feature from libp2p (on-disk peerstore). Once libp2p deprecates and removes this feature, the PIDStore will have to become a PeerAddrStore such that it can save addr info of good peers to disk instead of just their IDs. --- go.mod | 1 + go.sum | 5 +++++ libs/pidstore/pidstore.go | 14 ++++++++++++-- libs/pidstore/pidstore_test.go | 14 +++++++++++++- nodebuilder/header/constructors.go | 5 +++-- nodebuilder/header/module.go | 4 ++++ nodebuilder/header/module_test.go | 3 +++ nodebuilder/p2p/misc.go | 9 +++++---- 8 files changed, 46 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index c978655477..0890bf64f1 100644 --- a/go.mod +++ b/go.mod @@ -185,6 +185,7 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect diff --git a/go.sum b/go.sum index 5c4e483b6f..7d9c2fba70 100644 --- a/go.sum +++ b/go.sum @@ -214,6 +214,7 @@ github.com/99designs/keyring v1.2.1 h1:tYLp1ULvO7i3fI5vE21ReQuj99QFSs7lGm0xWyJo8 github.com/99designs/keyring v1.2.1/go.mod h1:fc+wB5KTk9wQ9sDx0kFXB3A0MaeGHM9AwRStKOQ5vOA= github.com/AndreasBriese/bbloom v0.0.0-20180913140656-343706a395b7/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/AndreasBriese/bbloom v0.0.0-20190306092124-e2d15f34fcf9/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= +github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96 h1:cTp8I5+VIoKjsnZuH8vjyaysT/ses3EvZeaV/1UkF2M= github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOvUY6CB00SOBii9/FifXqc0awNKxLFCL/+pkDPuyl8= github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= @@ -535,6 +536,7 @@ github.com/dgraph-io/badger v1.5.5-0.20190226225317-8115aed38f8f/go.mod h1:VZxzA github.com/dgraph-io/badger v1.6.0-rc1/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.0/go.mod h1:zwt7syl517jmP8s94KqSxTlM6IMsdhYy6psNgSztDR4= github.com/dgraph-io/badger v1.6.1/go.mod h1:FRmFw3uxvcpa8zG3Rxs0th+hCLIuaQg8HlNV5bjgnuU= +github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x8= github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= @@ -965,6 +967,8 @@ github.com/hashicorp/golang-lru v0.5.4/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru v0.5.5-0.20210104140557-80c98217689d/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= github.com/hashicorp/golang-lru v1.0.2 h1:dV3g9Z/unq5DpblPpw+Oqcv4dU/1omnb4Ok8iPY6p1c= github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uGBxy+E8yxSoD4= +github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= +github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= @@ -1081,6 +1085,7 @@ github.com/ipfs/go-ds-badger v0.0.5/go.mod h1:g5AuuCGmr7efyzQhLL8MzwqcauPojGPUaH github.com/ipfs/go-ds-badger v0.0.7/go.mod h1:qt0/fWzZDoPW6jpQeqUjR5kBfhDNB65jd9YlmAvpQBk= github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9DrGVwx/NOwE= github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= +github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= diff --git a/libs/pidstore/pidstore.go b/libs/pidstore/pidstore.go index 2d4eb870a8..7e416d98a0 100644 --- a/libs/pidstore/pidstore.go +++ b/libs/pidstore/pidstore.go @@ -24,10 +24,20 @@ type PeerIDStore struct { } // NewPeerIDStore creates a new peer ID store backed by the given datastore. -func NewPeerIDStore(ds datastore.Datastore) *PeerIDStore { - return &PeerIDStore{ +func NewPeerIDStore(ctx context.Context, ds datastore.Datastore) (*PeerIDStore, error) { + pidstore := &PeerIDStore{ ds: namespace.Wrap(ds, storePrefix), } + // check if pidstore is already initialized, and if not, + // initialize the pidstore + exists, err := pidstore.ds.Has(ctx, peersKey) + if err != nil { + return nil, err + } + if !exists { + return pidstore, pidstore.Put(ctx, []peer.ID{}) + } + return pidstore, nil } // Load loads the peers from datastore and returns them. diff --git a/libs/pidstore/pidstore_test.go b/libs/pidstore/pidstore_test.go index eafceff3fe..d8d214c83e 100644 --- a/libs/pidstore/pidstore_test.go +++ b/libs/pidstore/pidstore_test.go @@ -19,7 +19,19 @@ func TestPutLoad(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer t.Cleanup(cancel) - peerstore := NewPeerIDStore(sync.MutexWrap(datastore.NewMapDatastore())) + ds := sync.MutexWrap(datastore.NewMapDatastore()) + + t.Run("unitialized-pidstore", func(t *testing.T) { + testPutLoad(ctx, ds, t) + }) + t.Run("initialized-pidstore", func(t *testing.T) { + testPutLoad(ctx, ds, t) + }) +} + +func testPutLoad(ctx context.Context, ds datastore.Datastore, t *testing.T) { + peerstore, err := NewPeerIDStore(ctx, ds) + require.NoError(t, err) ids, err := generateRandomPeerList(10) require.NoError(t, err) diff --git a/nodebuilder/header/constructors.go b/nodebuilder/header/constructors.go index 267f0c30f7..984d551434 100644 --- a/nodebuilder/header/constructors.go +++ b/nodebuilder/header/constructors.go @@ -23,11 +23,12 @@ import ( // newP2PExchange constructs a new Exchange for headers. func newP2PExchange[H libhead.Header[H]]( lc fx.Lifecycle, + cfg Config, bpeers modp2p.Bootstrappers, network modp2p.Network, host host.Host, conngater *conngater.BasicConnectionGater, - cfg Config, + pidstore p2p.PeerIDStore, ) (libhead.Exchange[H], error) { peers, err := cfg.trustedPeers(bpeers) if err != nil { @@ -42,6 +43,7 @@ func newP2PExchange[H libhead.Header[H]]( p2p.WithParams(cfg.Client), p2p.WithNetworkID[p2p.ClientParameters](network.String()), p2p.WithChainID(network.String()), + p2p.WithPeerIDStore[p2p.ClientParameters](pidstore), ) if err != nil { return nil, err @@ -55,7 +57,6 @@ func newP2PExchange[H libhead.Header[H]]( }, }) return exchange, nil - } // newSyncer constructs new Syncer for headers. diff --git a/nodebuilder/header/module.go b/nodebuilder/header/module.go index 5e02e94fe1..eaee6b047f 100644 --- a/nodebuilder/header/module.go +++ b/nodebuilder/header/module.go @@ -15,6 +15,7 @@ import ( "github.com/celestiaorg/go-header/sync" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/libs/pidstore" modfraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/node" modp2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" @@ -99,6 +100,9 @@ func ConstructModule[H libhead.Header[H]](tp node.Type, cfg *Config) fx.Option { "header", baseComponents, fx.Provide(newP2PExchange[H]), + fx.Provide(func(ctx context.Context, ds datastore.Batching) (p2p.PeerIDStore, error) { + return pidstore.NewPeerIDStore(ctx, ds) + }), ) case node.Bridge: return fx.Module( diff --git a/nodebuilder/header/module_test.go b/nodebuilder/header/module_test.go index 6a35e35284..c31cf546d8 100644 --- a/nodebuilder/header/module_test.go +++ b/nodebuilder/header/module_test.go @@ -20,6 +20,7 @@ import ( "github.com/celestiaorg/go-header/sync" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/libs/pidstore" "github.com/celestiaorg/celestia-node/nodebuilder/node" modp2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" ) @@ -94,6 +95,8 @@ func TestConstructModule_ExchangeParams(t *testing.T) { var exchangeServer *p2p.ExchangeServer[*header.ExtendedHeader] app := fxtest.New(t, + fx.Provide(pidstore.NewPeerIDStore), + fx.Provide(context.Background), fx.Supply(modp2p.Private), fx.Supply(modp2p.Bootstrappers{}), fx.Provide(libp2p.New), diff --git a/nodebuilder/p2p/misc.go b/nodebuilder/p2p/misc.go index 9b7d8d8108..0d842e0601 100644 --- a/nodebuilder/p2p/misc.go +++ b/nodebuilder/p2p/misc.go @@ -1,12 +1,13 @@ package p2p import ( + "context" "time" "github.com/ipfs/go-datastore" connmgri "github.com/libp2p/go-libp2p/core/connmgr" "github.com/libp2p/go-libp2p/core/peerstore" - "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoremem" + "github.com/libp2p/go-libp2p/p2p/host/peerstore/pstoreds" //nolint:staticcheck "github.com/libp2p/go-libp2p/p2p/net/conngater" "github.com/libp2p/go-libp2p/p2p/net/connmgr" @@ -71,7 +72,7 @@ func connectionGater(ds datastore.Batching) (*conngater.BasicConnectionGater, er return conngater.NewBasicConnectionGater(ds) } -// peerStore constructs a PeerStore. -func peerStore() (peerstore.Peerstore, error) { - return pstoremem.NewPeerstore() +// peerStore constructs an on-disk PeerStore. +func peerStore(ctx context.Context, ds datastore.Batching) (peerstore.Peerstore, error) { + return pstoreds.NewPeerstore(ctx, ds, pstoreds.DefaultOpts()) } From a5e01b9ca1c7384b5b6807ac2455bd8f89593167 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 6 Sep 2023 12:32:07 +0200 Subject: [PATCH 175/388] feat(modp2p): listen on WebTransport by default (#2660) --- nodebuilder/p2p/config.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/nodebuilder/p2p/config.go b/nodebuilder/p2p/config.go index 589945ae9f..99f3fe8879 100644 --- a/nodebuilder/p2p/config.go +++ b/nodebuilder/p2p/config.go @@ -40,6 +40,8 @@ type Config struct { func DefaultConfig(tp node.Type) Config { return Config{ ListenAddresses: []string{ + "/ip4/0.0.0.0/udp/2121/quic-v1/webtransport", + "/ip6/::/udp/2121/quic-v1/webtransport", "/ip4/0.0.0.0/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1", "/ip4/0.0.0.0/tcp/2121", @@ -47,6 +49,9 @@ func DefaultConfig(tp node.Type) Config { }, AnnounceAddresses: []string{}, NoAnnounceAddresses: []string{ + "/ip4/127.0.0.1/udp/2121/quic-v1/webtransport", + "/ip4/0.0.0.0/udp/2121/quic-v1/webtransport", + "/ip6/::/udp/2121/quic-v1/webtransport", "/ip4/0.0.0.0/udp/2121/quic-v1", "/ip4/127.0.0.1/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1", From 1c84b0409b6b57d909869cd07c2bb4224d039833 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 6 Sep 2023 19:44:41 +0800 Subject: [PATCH 176/388] fix(share/eds): skip concurrent writes on edsstore Put (#2613) Short circuit parallel operations Closes #2608 --- share/eds/store.go | 7 +++++++ share/eds/store_test.go | 26 ++++++++++++++++++++++++++ 2 files changed, 33 insertions(+) diff --git a/share/eds/store.go b/share/eds/store.go index fa9a7f7c7e..19cbd83057 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -8,6 +8,7 @@ import ( "fmt" "io" "os" + "sync" "sync/atomic" "time" @@ -63,6 +64,8 @@ type Store struct { // lastGCResult is only stored on the store for testing purposes. lastGCResult atomic.Pointer[dagstore.GCResult] + // stripedLocks is used to synchronize parallel operations + stripedLocks [256]sync.Mutex shardFailures chan dagstore.ShardResult metrics *metrics @@ -221,6 +224,10 @@ func (s *Store) Put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext } func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.ExtendedDataSquare) (err error) { + lk := &s.stripedLocks[root[len(root)-1]] + lk.Lock() + defer lk.Unlock() + // if root already exists, short-circuit if has, _ := s.Has(ctx, root); has { return dagstore.ErrShardExists diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 4f1d7f4c8b..616e5c2874 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -3,9 +3,11 @@ package eds import ( "context" "os" + "sync" "testing" "time" + "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" @@ -209,6 +211,30 @@ func TestEDSStore(t *testing.T) { assert.Contains(t, hashesOut, hash) } }) + + t.Run("Parallel put", func(t *testing.T) { + const amount = 20 + eds, dah := randomEDS(t) + + wg := sync.WaitGroup{} + for i := 1; i < amount; i++ { + wg.Add(1) + go func() { + defer wg.Done() + err := edsStore.Put(ctx, dah.Hash(), eds) + if err != nil { + require.ErrorIs(t, err, dagstore.ErrShardExists) + } + }() + } + wg.Wait() + + eds, err := edsStore.Get(ctx, dah.Hash()) + require.NoError(t, err) + newDah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + require.Equal(t, dah.Hash(), newDah.Hash()) + }) } // TestEDSStore_GC verifies that unused transient shards are collected by the GC periodically. From 7601c08a59097de8e5d3a5771d70d5359fd1dc1c Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 6 Sep 2023 19:45:17 +0800 Subject: [PATCH 177/388] chore(lint/sort-imports): fix absolute path for sort-imports (#2640) If repo folder name is different from "celestia-node" lint and sort-imprort commands from Makefile will not work. --- Makefile | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/Makefile b/Makefile index 8813b65d23..30b0c18be3 100644 --- a/Makefile +++ b/Makefile @@ -159,20 +159,18 @@ openrpc-gen: .PHONY: openrpc-gen ## lint-imports: Lint only Go imports. +## flag -set-exit-status doesn't exit with code 1 as it should, so we use find until it is fixed by goimports-reviser lint-imports: @echo "--> Running imports linter" @for file in `find . -type f -name '*.go'`; \ - do goimports-reviser -list-diff -set-exit-status -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" -output stdout $$file \ + do goimports-reviser -list-diff -set-exit-status -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/celestia-node" -output stdout $$file \ || exit 1; \ done; .PHONY: lint-imports ## sort-imports: Sort Go imports. sort-imports: - @for file in `find . -type f -name '*.go'`; \ - do goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/"$(PROJECTNAME)"" $$file \ - || exit 1; \ - done; + @goimports-reviser -company-prefixes "github.com/celestiaorg" -project-name "github.com/celestiaorg/celestia-node" -output stdout ./... .PHONY: sort-imports ## adr-gen: Generate ADR from template. Must set NUM and TITLE parameters. From cb46dc1f23575cbdafbecb287cca3aaa6a34d451 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 7 Sep 2023 11:45:35 +0200 Subject: [PATCH 178/388] chore: cleanup usage of `share.Root`(DAH) (#2481) Somewhere we were using Root and somewhere DAH. Also, we mostly use DAH generator from the app. These PRs clean up all the usages and localize the Root/DAH type in share pkg as was originally intended. Up to discussion is rename of share.Root to share.DAH to minimize discrepancies between app and node. Alternatively, if we wanna keep Root naming as it is, we should also rename DAH name on the ExtendedHeader(and son tags) and in other places, which is breaking. Now, I think that calling it Root was not right and we should be calling the DataHash type Root, which is in fact the Merkle root of all the shares in the EDS. --- core/eds_test.go | 3 +- core/header_test.go | 3 +- core/listener_test.go | 3 +- das/coordinator_test.go | 3 +- das/daser_test.go | 4 +- header/header.go | 28 ++++----- header/headertest/testing.go | 18 +++--- nodebuilder/share/share_test.go | 4 +- share/availability.go | 11 ++++ share/availability/cache/availability.go | 7 +-- share/availability/cache/availability_test.go | 4 +- share/availability/light/availability_test.go | 11 ++-- share/availability/test/testing.go | 6 +- share/eds/byzantine/bad_encoding_test.go | 5 +- share/eds/eds.go | 3 +- share/eds/eds_test.go | 6 +- share/eds/edstest/testing.go | 5 +- share/eds/store.go | 5 +- share/eds/store_test.go | 8 +-- share/empty.go | 7 +-- share/getters/getter_test.go | 49 ++++++++------- share/getters/shrex_test.go | 59 +++++-------------- share/getters/testing.go | 4 +- share/p2p/shrexeds/exchange_test.go | 9 ++- share/p2p/shrexnd/exchange_test.go | 5 +- 25 files changed, 117 insertions(+), 153 deletions(-) diff --git a/core/eds_test.go b/core/eds_test.go index 6a2026ee58..723cb77ad1 100644 --- a/core/eds_test.go +++ b/core/eds_test.go @@ -9,7 +9,6 @@ import ( "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-node/share" ) @@ -47,7 +46,7 @@ func TestEmptySquareWithZeroTxs(t *testing.T) { eds, err = app.ExtendBlock(data, appconsts.LatestVersion) require.NoError(t, err) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) assert.Equal(t, share.EmptyRoot().Hash(), dah.Hash()) } diff --git a/core/header_test.go b/core/header_test.go index c942ea7875..ee5d10170e 100644 --- a/core/header_test.go +++ b/core/header_test.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" + "github.com/celestiaorg/celestia-node/share" ) func TestMakeExtendedHeaderForEmptyBlock(t *testing.T) { @@ -36,7 +37,7 @@ func TestMakeExtendedHeaderForEmptyBlock(t *testing.T) { headerExt, err := header.MakeExtendedHeader(&b.Header, comm, val, eds) require.NoError(t, err) - assert.Equal(t, header.EmptyDAH(), *headerExt.DAH) + assert.Equal(t, share.EmptyRoot(), headerExt.DAH) } func TestMismatchedDataHash_ComputedRoot(t *testing.T) { diff --git a/core/listener_test.go b/core/listener_test.go index 8b3d05bea9..90bdefb72f 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -17,6 +17,7 @@ import ( "github.com/celestiaorg/celestia-node/header" nodep2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -96,7 +97,7 @@ func TestListenerWithNonEmptyBlocks(t *testing.T) { require.NoError(t, err) t.Cleanup(sub.Cancel) - empty := header.EmptyDAH() + empty := share.EmptyRoot() // TODO extract 16 for i := 0; i < 16; i++ { _, err := cctx.FillBlock(16, cfg.Accounts, flags.BroadcastBlock) diff --git a/das/coordinator_test.go b/das/coordinator_test.go index 55ed01dd4e..1ae54c470f 100644 --- a/das/coordinator_test.go +++ b/das/coordinator_test.go @@ -13,6 +13,7 @@ import ( "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -431,7 +432,7 @@ func (m *mockSampler) discover(ctx context.Context, newHeight uint64, emit liste emit(ctx, &header.ExtendedHeader{ Commit: &types.Commit{}, RawHeader: header.RawHeader{Height: int64(newHeight)}, - DAH: &header.DataAvailabilityHeader{RowRoots: make([][]byte, 0)}, + DAH: &share.Root{RowRoots: make([][]byte, 0)}, }) } diff --git a/das/daser_test.go b/das/daser_test.go index 3e5bce0b97..a14da325f2 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -365,7 +365,7 @@ type benchGetterStub struct { func newBenchGetter() benchGetterStub { return benchGetterStub{header: &header.ExtendedHeader{ - DAH: &header.DataAvailabilityHeader{RowRoots: make([][]byte, 0)}}} + DAH: &share.Root{RowRoots: make([][]byte, 0)}}} } func (m benchGetterStub) GetByHeight(context.Context, uint64) (*header.ExtendedHeader, error) { @@ -385,7 +385,7 @@ func (m getterStub) GetByHeight(_ context.Context, height uint64) (*header.Exten return &header.ExtendedHeader{ Commit: &types.Commit{}, RawHeader: header.RawHeader{Height: int64(height)}, - DAH: &header.DataAvailabilityHeader{RowRoots: make([][]byte, 0)}}, nil + DAH: &share.Root{RowRoots: make([][]byte, 0)}}, nil } func (m getterStub) GetRangeByHeight(context.Context, uint64, uint64) ([]*header.ExtendedHeader, error) { diff --git a/header/header.go b/header/header.go index 92f8538696..1129284817 100644 --- a/header/header.go +++ b/header/header.go @@ -11,9 +11,10 @@ import ( core "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" - "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" ) // ConstructFn aliases a function that creates an ExtendedHeader. @@ -24,11 +25,6 @@ type ConstructFn = func( *rsmt2d.ExtendedDataSquare, ) (*ExtendedHeader, error) -type DataAvailabilityHeader = da.DataAvailabilityHeader - -// EmptyDAH provides DAH of the empty block. -var EmptyDAH = da.MinDataAvailabilityHeader - // RawHeader is an alias to core.Header. It is // "raw" because it is not yet wrapped to include // the DataAvailabilityHeader. @@ -39,9 +35,9 @@ type RawHeader = core.Header // block headers and perform Data Availability Sampling. type ExtendedHeader struct { RawHeader `json:"header"` - Commit *core.Commit `json:"commit"` - ValidatorSet *core.ValidatorSet `json:"validator_set"` - DAH *DataAvailabilityHeader `json:"dah"` + Commit *core.Commit `json:"commit"` + ValidatorSet *core.ValidatorSet `json:"validator_set"` + DAH *share.Root `json:"dah"` } // MakeExtendedHeader assembles new ExtendedHeader. @@ -52,22 +48,22 @@ func MakeExtendedHeader( eds *rsmt2d.ExtendedDataSquare, ) (*ExtendedHeader, error) { var ( - dah DataAvailabilityHeader + dah *share.Root err error ) switch eds { case nil: - dah = EmptyDAH() + dah = share.EmptyRoot() default: - dah, err = da.NewDataAvailabilityHeader(eds) - } - if err != nil { - return nil, err + dah, err = share.NewRoot(eds) + if err != nil { + return nil, err + } } eh := &ExtendedHeader{ RawHeader: *h, - DAH: &dah, + DAH: dah, Commit: comm, ValidatorSet: vals, } diff --git a/header/headertest/testing.go b/header/headertest/testing.go index f288556bd9..0b3c6a34dc 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -17,12 +17,12 @@ import ( "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" - "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/go-header/headertest" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" ) // TestSuite provides everything you need to test chain of Headers. @@ -52,7 +52,7 @@ func NewTestSuite(t *testing.T, num int) *TestSuite { } func (s *TestSuite) genesis() *header.ExtendedHeader { - dah := header.EmptyDAH() + dah := share.EmptyRoot() gen := RandRawHeader(s.t) @@ -70,7 +70,7 @@ func (s *TestSuite) genesis() *header.ExtendedHeader { RawHeader: *gen, Commit: commit, ValidatorSet: s.valSet, - DAH: &dah, + DAH: dah, } require.NoError(s.t, eh.Validate()) return eh @@ -137,14 +137,14 @@ func (s *TestSuite) NextHeader() *header.ExtendedHeader { return s.head } - dah := da.MinDataAvailabilityHeader() + dah := share.EmptyRoot() height := s.Head().Height() + 1 rh := s.GenRawHeader(height, s.Head().Hash(), libhead.Hash(s.Head().Commit.Hash()), dah.Hash()) s.head = &header.ExtendedHeader{ RawHeader: *rh, Commit: s.Commit(rh), ValidatorSet: s.valSet, - DAH: &dah, + DAH: dah, } require.NoError(s.t, s.head.Validate()) return s.head @@ -203,7 +203,7 @@ func (s *TestSuite) nextProposer() *types.Validator { // RandExtendedHeader provides an ExtendedHeader fixture. func RandExtendedHeader(t *testing.T) *header.ExtendedHeader { - dah := header.EmptyDAH() + dah := share.EmptyRoot() rh := RandRawHeader(t) rh.DataHash = dah.Hash() @@ -220,7 +220,7 @@ func RandExtendedHeader(t *testing.T) *header.ExtendedHeader { RawHeader: *rh, Commit: commit, ValidatorSet: valSet, - DAH: &dah, + DAH: dah, } } @@ -292,7 +292,7 @@ func RandBlockID(*testing.T) types.BlockID { func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedDataSquare) *header.ExtendedHeader { valSet, vals := RandValidatorSet(10, 10) gen := RandRawHeader(t) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) gen.DataHash = dah.Hash() @@ -309,7 +309,7 @@ func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedData RawHeader: *gen, Commit: commit, ValidatorSet: valSet, - DAH: &dah, + DAH: dah, } require.NoError(t, eh.Validate()) return eh diff --git a/nodebuilder/share/share_test.go b/nodebuilder/share/share_test.go index 7c440a6dbf..57119b5003 100644 --- a/nodebuilder/share/share_test.go +++ b/nodebuilder/share/share_test.go @@ -9,8 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" ) @@ -27,7 +25,7 @@ func Test_EmptyCARExists(t *testing.T) { require.NoError(t, err) eds := share.EmptyExtendedDataSquare() - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) // add empty EDS to store diff --git a/share/availability.go b/share/availability.go index b6b44271c8..cd4b3a777f 100644 --- a/share/availability.go +++ b/share/availability.go @@ -5,6 +5,7 @@ import ( "errors" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/rsmt2d" ) // ErrNotAvailable is returned whenever DA sampling fails. @@ -14,6 +15,16 @@ var ErrNotAvailable = errors.New("share: data not available") // In practice, it is a commitment to all the Data in a square. type Root = da.DataAvailabilityHeader +// NewRoot generates Root(DataAvailabilityHeader) using the +// provided extended data square. +func NewRoot(eds *rsmt2d.ExtendedDataSquare) (*Root, error) { + dah, err := da.NewDataAvailabilityHeader(eds) + if err != nil { + return nil, err + } + return &dah, nil +} + // Availability defines interface for validation of Shares' availability. type Availability interface { // SharesAvailable subjectively validates if Shares committed to the given Root are available on diff --git a/share/availability/cache/availability.go b/share/availability/cache/availability.go index d6496f7ff8..a530d94d1d 100644 --- a/share/availability/cache/availability.go +++ b/share/availability/cache/availability.go @@ -10,14 +10,11 @@ import ( "github.com/ipfs/go-datastore/namespace" logging "github.com/ipfs/go-log/v2" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" ) var ( - log = logging.Logger("share/cache") - minRoot = da.MinDataAvailabilityHeader() + log = logging.Logger("share/cache") cacheAvailabilityPrefix = datastore.NewKey("sampling_result") writeBatchSize = 2048 @@ -96,5 +93,5 @@ func rootKey(root *share.Root) datastore.Key { // isMinRoot returns whether the given root is a minimum (empty) // DataAvailabilityHeader (DAH). func isMinRoot(root *share.Root) bool { - return bytes.Equal(minRoot.Hash(), root.Hash()) + return bytes.Equal(share.EmptyRoot().Hash(), root.Hash()) } diff --git a/share/availability/cache/availability_test.go b/share/availability/cache/availability_test.go index ccc11a1d28..458ce5cc95 100644 --- a/share/availability/cache/availability_test.go +++ b/share/availability/cache/availability_test.go @@ -110,9 +110,7 @@ func TestCacheAvailability_MinRoot(t *testing.T) { defer cancel() fullLocalAvail, _ := FullAvailabilityWithLocalRandSquare(t, 16) - minDAH := da.MinDataAvailabilityHeader() - - err := fullLocalAvail.SharesAvailable(ctx, &minDAH) + err := fullLocalAvail.SharesAvailable(ctx, share.EmptyRoot()) assert.NoError(t, err) } diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 48813a33f9..549ea14ce5 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -9,9 +9,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" - - "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/sharetest" @@ -33,8 +30,8 @@ func TestSharesAvailableFailed(t *testing.T) { getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) - empty := header.EmptyDAH() - err := avail.SharesAvailable(ctx, &empty) + empty := share.EmptyRoot() + err := avail.SharesAvailable(ctx, empty) assert.Error(t, err) } @@ -132,10 +129,10 @@ func TestGetShares(t *testing.T) { eds, err := getter.GetEDS(ctx, dah) require.NoError(t, err) - gotDAH, err := da.NewDataAvailabilityHeader(eds) + gotDAH, err := share.NewRoot(eds) require.NoError(t, err) - require.True(t, dah.Equals(&gotDAH)) + require.True(t, dah.Equals(gotDAH)) } func TestService_GetSharesByNamespaceNotFound(t *testing.T) { diff --git a/share/availability/test/testing.go b/share/availability/test/testing.go index 8973ee8cb1..64e8d23bb7 100644 --- a/share/availability/test/testing.go +++ b/share/availability/test/testing.go @@ -17,8 +17,6 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/sharetest" @@ -34,9 +32,9 @@ func RandFillBS(t *testing.T, n int, bServ blockservice.BlockService) *share.Roo func FillBS(t *testing.T, bServ blockservice.BlockService, shares []share.Share) *share.Root { eds, err := ipld.AddShares(context.TODO(), shares, bServ) require.NoError(t, err) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) - return &dah + return dah } type TestNode struct { diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 9d16955ee3..3e245c1ab3 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -13,6 +13,7 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/sharetest" @@ -55,7 +56,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { eds, err := ipld.AddShares(ctx, shares, bServ) require.NoError(t, err) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) // get an arbitrary row @@ -77,7 +78,7 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { RawHeader: core.Header{ Height: 420, }, - DAH: &dah, + DAH: dah, Commit: &core.Commit{ BlockID: core.BlockID{ Hash: []byte("made up hash"), diff --git a/share/eds/eds.go b/share/eds/eds.go index e689aec31c..e0433a1b6b 100644 --- a/share/eds/eds.go +++ b/share/eds/eds.go @@ -13,7 +13,6 @@ import ( "github.com/ipld/go-car" "github.com/ipld/go-car/util" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -260,7 +259,7 @@ func ReadEDS(ctx context.Context, r io.Reader, root share.DataHash) (eds *rsmt2d return nil, fmt.Errorf("share: computing eds: %w", err) } - newDah, err := da.NewDataAvailabilityHeader(eds) + newDah, err := share.NewRoot(eds) if err != nil { return nil, err } diff --git a/share/eds/eds_test.go b/share/eds/eds_test.go index 6ebca8b779..ffb05343b9 100644 --- a/share/eds/eds_test.go +++ b/share/eds/eds_test.go @@ -138,7 +138,7 @@ func TestWriteEDSInQuadrantOrder(t *testing.T) { func TestReadWriteRoundtrip(t *testing.T) { eds := writeRandomEDS(t) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) f := openWrittenEDS(t) defer f.Close() @@ -196,7 +196,7 @@ func BenchmarkReadWriteEDS(b *testing.B) { b.Cleanup(cancel) for originalDataWidth := 4; originalDataWidth <= 64; originalDataWidth *= 2 { eds := edstest.RandEDS(b, originalDataWidth) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(b, err) b.Run(fmt.Sprintf("Writing %dx%d", originalDataWidth, originalDataWidth), func(b *testing.B) { b.ReportAllocs() @@ -268,7 +268,7 @@ func createTestData(t *testing.T, testDir string) { //nolint:unused err = WriteEDS(ctx, eds, f) require.NoError(t, err, "writing EDS to file") f.Close() - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) header, err := json.MarshalIndent(dah, "", "") diff --git a/share/eds/edstest/testing.go b/share/eds/edstest/testing.go index f75e8b619b..bf5e664f90 100644 --- a/share/eds/edstest/testing.go +++ b/share/eds/edstest/testing.go @@ -5,7 +5,6 @@ import ( "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -39,11 +38,11 @@ func RandEDSWithNamespace( t require.TestingT, namespace share.Namespace, size int, -) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader) { +) (*rsmt2d.ExtendedDataSquare, *share.Root) { shares := sharetest.RandSharesWithNamespace(t, namespace, size*size) eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) require.NoError(t, err, "failure to recompute the extended data square") - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) return eds, dah } diff --git a/share/eds/store.go b/share/eds/store.go index 19cbd83057..974f147292 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -24,7 +24,6 @@ import ( "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/ipld" @@ -394,13 +393,13 @@ func (s *Store) getDAH(ctx context.Context, root share.DataHash) (*share.Root, e } // dahFromCARHeader returns the DataAvailabilityHeader stored in the CIDs of a CARv1 header. -func dahFromCARHeader(carHeader *carv1.CarHeader) *header.DataAvailabilityHeader { +func dahFromCARHeader(carHeader *carv1.CarHeader) *share.Root { rootCount := len(carHeader.Roots) rootBytes := make([][]byte, 0, rootCount) for _, root := range carHeader.Roots { rootBytes = append(rootBytes, ipld.NamespacedSha256FromCID(root)) } - return &header.DataAvailabilityHeader{ + return &share.Root{ RowRoots: rootBytes[:rootCount/2], ColumnRoots: rootBytes[rootCount/2:], } diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 616e5c2874..21239e320a 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -352,7 +352,7 @@ func BenchmarkStore(b *testing.B) { // pause the timer for initializing test data b.StopTimer() eds := edstest.RandEDS(b, 128) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(b, err) b.StartTimer() @@ -368,7 +368,7 @@ func BenchmarkStore(b *testing.B) { // pause the timer for initializing test data b.StopTimer() eds := edstest.RandEDS(b, 128) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(b, err) _ = edsStore.Put(ctx, dah.Hash(), eds) b.StartTimer() @@ -387,9 +387,9 @@ func newStore(t *testing.T) (*Store, error) { return NewStore(tmpDir, ds) } -func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { +func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) return eds, dah diff --git a/share/empty.go b/share/empty.go index 07d48f2f07..ef3d088e1d 100644 --- a/share/empty.go +++ b/share/empty.go @@ -54,16 +54,15 @@ func initEmpty() { } emptyBlockEDS = eds - dah, err := da.NewDataAvailabilityHeader(eds) + emptyBlockRoot, err = NewRoot(eds) if err != nil { panic(fmt.Errorf("failed to create empty DAH: %w", err)) } minDAH := da.MinDataAvailabilityHeader() - if !bytes.Equal(minDAH.Hash(), dah.Hash()) { + if !bytes.Equal(minDAH.Hash(), emptyBlockRoot.Hash()) { panic(fmt.Sprintf("mismatch in calculated minimum DAH and minimum DAH from celestia-app, "+ - "expected %s, got %s", minDAH.String(), dah.String())) + "expected %s, got %s", minDAH.String(), emptyBlockRoot.String())) } - emptyBlockRoot = &dah // precompute Hash, so it's cached internally to avoid potential races emptyBlockRoot.Hash() diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index ddab46f290..bee7b3a963 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -12,7 +12,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" "github.com/celestiaorg/rsmt2d" @@ -49,7 +48,7 @@ func TestTeeGetter(t *testing.T) { assert.False(t, ok) assert.NoError(t, err) - retrievedEDS, err := tg.GetEDS(ctx, &dah) + retrievedEDS, err := tg.GetEDS(ctx, dah) require.NoError(t, err) require.True(t, randEds.Equals(retrievedEDS)) @@ -67,12 +66,12 @@ func TestTeeGetter(t *testing.T) { _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) require.NoError(t, err) - retrievedEDS, err := tg.GetEDS(ctx, &dah) + retrievedEDS, err := tg.GetEDS(ctx, dah) require.NoError(t, err) require.True(t, randEds.Equals(retrievedEDS)) // no error should be returned, even though the EDS identified by the DAH already exists - retrievedEDS, err = tg.GetEDS(ctx, &dah) + retrievedEDS, err = tg.GetEDS(ctx, dah) require.NoError(t, err) require.True(t, randEds.Equals(retrievedEDS)) }) @@ -100,19 +99,19 @@ func TestStoreGetter(t *testing.T) { squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { - share, err := sg.GetShare(ctx, &dah, i, j) + share, err := sg.GetShare(ctx, dah, i, j) require.NoError(t, err) assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } // doesn't panic on indexes too high - _, err := sg.GetShare(ctx, &dah, squareSize, squareSize) + _, err := sg.GetShare(ctx, dah, squareSize, squareSize) require.ErrorIs(t, err, share.ErrOutOfBounds) // root not found _, dah = randomEDS(t) - _, err = sg.GetShare(ctx, &dah, 0, 0) + _, err = sg.GetShare(ctx, dah, 0, 0) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -121,7 +120,7 @@ func TestStoreGetter(t *testing.T) { err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - retrievedEDS, err := sg.GetEDS(ctx, &dah) + retrievedEDS, err := sg.GetEDS(ctx, dah) require.NoError(t, err) assert.True(t, randEds.Equals(retrievedEDS)) @@ -136,14 +135,14 @@ func TestStoreGetter(t *testing.T) { err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) + shares, err := sg.GetSharesByNamespace(ctx, dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, namespace)) + require.NoError(t, shares.Verify(dah, namespace)) assert.Len(t, shares.Flatten(), 2) // namespace not found randNamespace := sharetest.RandV0Namespace() - emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, randNamespace) + emptyShares, err := sg.GetSharesByNamespace(ctx, dah, randNamespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) @@ -159,9 +158,9 @@ func TestStoreGetter(t *testing.T) { require.NoError(t, err) // available - shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) + shares, err := sg.GetSharesByNamespace(ctx, dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, namespace)) + require.NoError(t, shares.Verify(dah, namespace)) assert.Len(t, shares.Flatten(), 2) // 'corrupt' existing CAR by overwriting with a random EDS @@ -171,7 +170,7 @@ func TestStoreGetter(t *testing.T) { err = eds.WriteEDS(ctx, edsToOverwriteWith, f) require.NoError(t, err) - shares, err = sg.GetSharesByNamespace(ctx, &dah, namespace) + shares, err = sg.GetSharesByNamespace(ctx, dah, namespace) require.ErrorIs(t, err, share.ErrNotFound) require.Nil(t, shares) @@ -209,19 +208,19 @@ func TestIPLDGetter(t *testing.T) { squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { - share, err := sg.GetShare(ctx, &dah, i, j) + share, err := sg.GetShare(ctx, dah, i, j) require.NoError(t, err) assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } // doesn't panic on indexes too high - _, err := sg.GetShare(ctx, &dah, squareSize+1, squareSize+1) + _, err := sg.GetShare(ctx, dah, squareSize+1, squareSize+1) require.ErrorIs(t, err, share.ErrOutOfBounds) // root not found _, dah = randomEDS(t) - _, err = sg.GetShare(ctx, &dah, 0, 0) + _, err = sg.GetShare(ctx, dah, 0, 0) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -233,7 +232,7 @@ func TestIPLDGetter(t *testing.T) { err = edsStore.Put(ctx, dah.Hash(), randEds) require.NoError(t, err) - retrievedEDS, err := sg.GetEDS(ctx, &dah) + retrievedEDS, err := sg.GetEDS(ctx, dah) require.NoError(t, err) assert.True(t, randEds.Equals(retrievedEDS)) @@ -253,14 +252,14 @@ func TestIPLDGetter(t *testing.T) { require.NoError(t, err) // first check that shares are returned correctly if they exist - shares, err := sg.GetSharesByNamespace(ctx, &dah, namespace) + shares, err := sg.GetSharesByNamespace(ctx, dah, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(&dah, namespace)) + require.NoError(t, shares.Verify(dah, namespace)) assert.Len(t, shares.Flatten(), 2) // namespace not found randNamespace := sharetest.RandV0Namespace() - emptyShares, err := sg.GetSharesByNamespace(ctx, &dah, randNamespace) + emptyShares, err := sg.GetSharesByNamespace(ctx, dah, randNamespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) @@ -272,16 +271,16 @@ func TestIPLDGetter(t *testing.T) { }) } -func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, share.Root) { +func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) return eds, dah } // randomEDSWithDoubledNamespace generates a random EDS and ensures that there are two shares in the // middle that share a namespace. -func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedDataSquare, []byte, share.Root) { +func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedDataSquare, []byte, *share.Root) { n := size * size randShares := sharetest.RandShares(t, n) idx1 := (n - 1) / 2 @@ -303,7 +302,7 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData wrapper.NewConstructor(uint64(size)), ) require.NoError(t, err, "failure to recompute the extended data square") - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) return eds, share.GetNamespace(randShares[idx1]), dah diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 236fae36e1..38611ed0ef 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -1,12 +1,9 @@ package getters import ( - "bytes" "context" "encoding/binary" "errors" - "math/rand" - "sort" "testing" "time" @@ -19,8 +16,6 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/wrapper" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" @@ -69,16 +64,16 @@ func TestShrexGetter(t *testing.T) { // generate test data namespace := sharetest.RandV0Namespace() - randEDS, dah := singleNamespaceEds(t, namespace, 64) + randEDS, dah := edstest.RandEDSWithNamespace(t, namespace, 64) require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - got, err := getter.GetSharesByNamespace(ctx, &dah, namespace) + got, err := getter.GetSharesByNamespace(ctx, dah, namespace) require.NoError(t, err) - require.NoError(t, got.Verify(&dah, namespace)) + require.NoError(t, got.Verify(dah, namespace)) }) t.Run("ND_err_not_found", func(t *testing.T) { @@ -92,7 +87,7 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - _, err := getter.GetSharesByNamespace(ctx, &dah, namespace) + _, err := getter.GetSharesByNamespace(ctx, dah, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -108,16 +103,16 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - namespace, err := addToNamespace(maxNamespace, -1) + nID, err := addToNamespace(maxNamespace, -1) require.NoError(t, err) // check for namespace to be between max and min namespace in root - require.Len(t, filterRootsByNamespace(&dah, namespace), 1) + require.Len(t, filterRootsByNamespace(dah, nID), 1) - emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, namespace) + emptyShares, err := getter.GetSharesByNamespace(ctx, dah, nID) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(&dah, namespace)) + require.Nil(t, emptyShares.Verify(dah, nID)) }) t.Run("ND_namespace_not_in_dah", func(t *testing.T) { @@ -135,13 +130,13 @@ func TestShrexGetter(t *testing.T) { namespace, err := addToNamespace(maxNamesapce, 1) require.NoError(t, err) // check for namespace to be not in root - require.Len(t, filterRootsByNamespace(&dah, namespace), 0) + require.Len(t, filterRootsByNamespace(dah, namespace), 0) - emptyShares, err := getter.GetSharesByNamespace(ctx, &dah, namespace) + emptyShares, err := getter.GetSharesByNamespace(ctx, dah, namespace) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) - require.Nil(t, emptyShares.Verify(&dah, namespace)) + require.Nil(t, emptyShares.Verify(dah, namespace)) }) t.Run("EDS_Available", func(t *testing.T) { @@ -156,7 +151,7 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - got, err := getter.GetEDS(ctx, &dah) + got, err := getter.GetEDS(ctx, dah) require.NoError(t, err) require.Equal(t, randEDS.Flattened(), got.Flattened()) }) @@ -172,7 +167,7 @@ func TestShrexGetter(t *testing.T) { }) cancel() - _, err := getter.GetEDS(ctx, &dah) + _, err := getter.GetEDS(ctx, dah) require.ErrorIs(t, err, context.Canceled) }) @@ -187,7 +182,7 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - _, err := getter.GetEDS(ctx, &dah) + _, err := getter.GetEDS(ctx, dah) require.ErrorIs(t, err, share.ErrNotFound) }) } @@ -200,9 +195,9 @@ func newStore(t *testing.T) (*eds.Store, error) { return eds.NewStore(tmpDir, ds) } -func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader, share.Namespace) { +func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root, share.Namespace) { eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) max := nmt.MaxNamespace(dah.RowRoots[(len(dah.RowRoots))/2-1], share.NamespaceSize) return eds, dah, max @@ -374,25 +369,3 @@ func TestAddToNamespace(t *testing.T) { }) } } - -func singleNamespaceEds( - t require.TestingT, - namespace share.Namespace, - size int, -) (*rsmt2d.ExtendedDataSquare, da.DataAvailabilityHeader) { - shares := make([]share.Share, size*size) - rnd := rand.New(rand.NewSource(time.Now().Unix())) - for i := range shares { - shr := make([]byte, share.Size) - copy(share.GetNamespace(shr), namespace) - _, err := rnd.Read(share.GetData(shr)) - require.NoError(t, err) - shares[i] = shr - } - sort.Slice(shares, func(i, j int) bool { return bytes.Compare(shares[i], shares[j]) < 0 }) - eds, err := rsmt2d.ComputeExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), wrapper.NewConstructor(uint64(size))) - require.NoError(t, err, "failure to recompute the extended data square") - dah, err := da.NewDataAvailabilityHeader(eds) - require.NoError(t, err) - return eds, dah -} diff --git a/share/getters/testing.go b/share/getters/testing.go index 71c6231f3c..aa0cc0e479 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -17,11 +17,11 @@ import ( // TestGetter provides a testing SingleEDSGetter and the root of the EDS it holds. func TestGetter(t *testing.T) (share.Getter, *share.Root) { eds := edstest.RandEDS(t, 8) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) return &SingleEDSGetter{ EDS: eds, - }, &dah + }, dah } // SingleEDSGetter contains a single EDS where data is retrieved from. diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index 14e220a8f9..e9305a96b9 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -14,8 +14,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" - + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" "github.com/celestiaorg/celestia-node/share/p2p" @@ -35,7 +34,7 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is immediately available t.Run("EDS_Available", func(t *testing.T) { eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) err = store.Put(ctx, dah.Hash(), eds) require.NoError(t, err) @@ -48,7 +47,7 @@ func TestExchange_RequestEDS(t *testing.T) { // Testcase: EDS is unavailable initially, but is found after multiple requests t.Run("EDS_AvailableAfterDelay", func(t *testing.T) { eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) lock := make(chan struct{}) @@ -85,7 +84,7 @@ func TestExchange_RequestEDS(t *testing.T) { timeoutCtx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) _, err = client.RequestEDS(timeoutCtx, dah.Hash(), server.host.ID()) require.ErrorIs(t, err, p2p.ErrNotFound) diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 1ca0736cd6..50324844b1 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -13,7 +13,6 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -45,12 +44,12 @@ func TestExchange_RequestND_NotFound(t *testing.T) { t.Cleanup(cancel) eds := edstest.RandEDS(t, 4) - dah, err := da.NewDataAvailabilityHeader(eds) + dah, err := share.NewRoot(eds) require.NoError(t, err) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) randNamespace := dah.RowRoots[(len(dah.RowRoots)-1)/2][:share.NamespaceSize] - emptyShares, err := client.RequestND(ctx, &dah, randNamespace, server.host.ID()) + emptyShares, err := client.RequestND(ctx, dah, randNamespace, server.host.ID()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) From 9957618d71ad0299f36a28ec640e80e1c88277cb Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 7 Sep 2023 13:02:55 +0300 Subject: [PATCH 179/388] improvement!(blob/service): add options to submit (#2630) --- blob/service.go | 22 ++++++++++++++++++++-- cmd/celestia/blob.go | 2 +- nodebuilder/blob/blob.go | 8 ++++---- nodebuilder/blob/mocks/api.go | 11 +++++------ nodebuilder/node/admin.go | 2 +- nodebuilder/tests/api_test.go | 2 +- nodebuilder/tests/blob_test.go | 2 +- 7 files changed, 33 insertions(+), 16 deletions(-) diff --git a/blob/service.go b/blob/service.go index 31cd259efc..05139cdfb5 100644 --- a/blob/service.go +++ b/blob/service.go @@ -51,14 +51,32 @@ func NewService( } } +// SubmitOptions contains the information about fee and gasLimit price in order to configure the Submit request. +type SubmitOptions struct { + Fee int64 + GasLimit uint64 +} + +// DefaultSubmitOptions creates a default fee and gas price values. +func DefaultSubmitOptions() *SubmitOptions { + return &SubmitOptions{ + Fee: -1, + GasLimit: 0, + } +} + // Submit sends PFB transaction and reports the height in which it was included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. // Handles gas estimation and fee calculation. -func (s *Service) Submit(ctx context.Context, blobs []*Blob) (uint64, error) { +func (s *Service) Submit(ctx context.Context, blobs []*Blob, options *SubmitOptions) (uint64, error) { log.Debugw("submitting blobs", "amount", len(blobs)) - resp, err := s.blobSubmitter.SubmitPayForBlob(ctx, types.OneInt().Neg(), 0, blobs) + if options == nil { + options = DefaultSubmitOptions() + } + + resp, err := s.blobSubmitter.SubmitPayForBlob(ctx, types.NewInt(options.Fee), options.GasLimit, blobs) if err != nil { return 0, err } diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 8ab130c24d..0d09268257 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -118,7 +118,7 @@ var submitCmd = &cobra.Command{ return fmt.Errorf("error creating a blob:%v", err) } - height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}) + height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}, nil) response := struct { Height uint64 `json:"height"` diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index aae502824c..5e29d3b90c 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -16,7 +16,7 @@ type Module interface { // Submit sends Blobs and reports the height in which they were included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. - Submit(_ context.Context, _ []*blob.Blob) (height uint64, _ error) + Submit(_ context.Context, _ []*blob.Blob, _ *blob.SubmitOptions) (height uint64, _ error) // Get retrieves the blob by commitment under the given namespace and height. Get(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Blob, error) // GetAll returns all blobs under the given namespaces and height. @@ -30,7 +30,7 @@ type Module interface { type API struct { Internal struct { - Submit func(context.Context, []*blob.Blob) (uint64, error) `perm:"write"` + Submit func(context.Context, []*blob.Blob, *blob.SubmitOptions) (uint64, error) `perm:"write"` Get func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Blob, error) `perm:"read"` GetAll func(context.Context, uint64, []share.Namespace) ([]*blob.Blob, error) `perm:"read"` GetProof func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Proof, error) `perm:"read"` @@ -38,8 +38,8 @@ type API struct { } } -func (api *API) Submit(ctx context.Context, blobs []*blob.Blob) (uint64, error) { - return api.Internal.Submit(ctx, blobs) +func (api *API) Submit(ctx context.Context, blobs []*blob.Blob, options *blob.SubmitOptions) (uint64, error) { + return api.Internal.Submit(ctx, blobs, options) } func (api *API) Get( diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index 5cd34b74b6..6a994f4d7c 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - blob "github.com/celestiaorg/celestia-node/blob" share "github.com/celestiaorg/celestia-node/share" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -98,16 +97,16 @@ func (mr *MockModuleMockRecorder) Included(arg0, arg1, arg2, arg3, arg4 interfac } // Submit mocks base method. -func (m *MockModule) Submit(arg0 context.Context, arg1 []*blob.Blob) (uint64, error) { +func (m *MockModule) Submit(arg0 context.Context, arg1 []*blob.Blob, arg2 *blob.SubmitOptions) (uint64, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Submit", arg0, arg1) + ret := m.ctrl.Call(m, "Submit", arg0, arg1, arg2) ret0, _ := ret[0].(uint64) ret1, _ := ret[1].(error) return ret0, ret1 } // Submit indicates an expected call of Submit. -func (mr *MockModuleMockRecorder) Submit(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockModuleMockRecorder) Submit(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockModule)(nil).Submit), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockModule)(nil).Submit), arg0, arg1, arg2) } diff --git a/nodebuilder/node/admin.go b/nodebuilder/node/admin.go index c6c97625ef..03c9cf1b52 100644 --- a/nodebuilder/node/admin.go +++ b/nodebuilder/node/admin.go @@ -10,7 +10,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/authtoken" ) -const APIVersion = "v0.2.1" +const APIVersion = "v0.2.2" type module struct { tp Type diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 3a66c4e58c..0e225668bb 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -119,7 +119,7 @@ func TestBlobRPC(t *testing.T) { ) require.NoError(t, err) - height, err := client.Blob.Submit(ctx, []*blob.Blob{newBlob}) + height, err := client.Blob.Submit(ctx, []*blob.Blob{newBlob}, nil) require.NoError(t, err) require.True(t, height != 0) } diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index a9e3a0464b..80fee071fb 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -55,7 +55,7 @@ func TestBlobModule(t *testing.T) { lightNode := sw.NewNodeWithConfig(node.Light, lightCfg) require.NoError(t, lightNode.Start(ctx)) - height, err := fullNode.BlobServ.Submit(ctx, blobs) + height, err := fullNode.BlobServ.Submit(ctx, blobs, nil) require.NoError(t, err) _, err = fullNode.HeaderServ.WaitForHeight(ctx, height) From d803cf847cfac3640b1272b3accc7f5940068888 Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Viet Date: Thu, 7 Sep 2023 13:41:03 +0200 Subject: [PATCH 180/388] chore!: bump mocha chain with new bootstrappers and genesis hash (#2668) ## Overview - [x] Genesis hash [taken here](https://github.com/celestiaorg/networks/blob/master/mocha-4/genesis_hash.txt) - [x] Bootstrappers list updated - [x] Genesis hash updated - [x] Chain-ID updated ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- nodebuilder/p2p/bootstrap.go | 7 ++++--- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 3 files changed, 6 insertions(+), 5 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index 0718d8cf59..47b39eed31 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -44,9 +44,10 @@ var bootstrapList = map[Network][]string{ "/dns4/da-full-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWRByRF67a2kVM2j4MP5Po3jgTw7H2iL2Spu8aUwPkrRfP", }, Mocha: { - "/dns4/bootstr-mocha-1.celestia-mocha.com/tcp/2121/p2p/12D3KooWDRSJMbH3PS4dRDa11H7Tk615aqTUgkeEKz4pwd4sS6fN", - "/dns4/bootstr-mocha-2.celestia-mocha.com/tcp/2121/p2p/12D3KooWEk7cxtjQCC7kC84Uhs2j6dAHjdbwYnPcvUAqmj6Zsry2", - "/dns4/bootstr-mocha-3.celestia-mocha.com/tcp/2121/p2p/12D3KooWBE4QcFXZzENf2VRo6Y5LBvp9gzmpYRHKCvgGzEYj7Hdn", + "/dns4/da-bridge-mocha-4.celestia-mocha.com/tcp/2121/p2p/12D3KooWCBAbQbJSpCpCGKzqz3rAN4ixYbc63K68zJg9aisuAajg", + "/dns4/da-bridge-mocha-4-2.celestia-mocha.com/tcp/2121/p2p/12D3KooWK6wJkScGQniymdWtBwBuU36n6BRXp9rCDDUD6P5gJr3G", + "/dns4/da-full-1-mocha-4.celestia-mocha.com/tcp/2121/p2p/12D3KooWCUHPLqQXZzpTx1x3TAsdn3vYmTNDhzg66yG8hqoxGGN8", + "/dns4/da-full-2-mocha-4.celestia-mocha.com/tcp/2121/p2p/12D3KooWR6SHsXPkkvhCRn6vp1RqSefgaT1X1nMNvrVjU2o3GoYy", }, BlockspaceRace: { "/dns4/bootstr-incent-3.celestia.tools/tcp/2121/p2p/12D3KooWNzdKcHagtvvr6qtjcPTAdCN6ZBiBLH8FBHbihxqu4GZx", diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index 31ea0b0ae9..5066db5177 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -24,7 +24,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ Arabica: "5904E55478BA4B3002EE885621E007A2A6A2399662841912219AECD5D5CBE393", - Mocha: "79A97034D569C4199A867439B1B7B77D4E1E1D9697212755E1CE6D920CDBB541", + Mocha: "B93BBE20A0FBFDF955811B6420F8433904664D45DB4BF51022BE4200C1A1680D", BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", Private: "", } diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index 29e0218bb2..b1f9be8401 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -14,7 +14,7 @@ const ( // Arabica testnet. See: celestiaorg/networks. Arabica Network = "arabica-10" // Mocha testnet. See: celestiaorg/networks. - Mocha Network = "mocha-3" + Mocha Network = "mocha-4" // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. BlockspaceRace Network = "blockspacerace-0" // Private can be used to set up any private network, including local testing setups. From a3d4cd571aefa8763389e571fe43e74d6bc26a50 Mon Sep 17 00:00:00 2001 From: Manav Aggarwal Date: Thu, 7 Sep 2023 08:10:25 -0400 Subject: [PATCH 181/388] feat(headertest): Validate exact error types in header tests (#2644) ## Overview Closes: #2645 ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [x] Linked issues closed with keywords --------- Co-authored-by: Hlib Kanunnikov --- header/header.go | 15 ++++++++++++--- header/headertest/verify_test.go | 19 ++++++++----------- 2 files changed, 20 insertions(+), 14 deletions(-) diff --git a/header/header.go b/header/header.go index 1129284817..9778684b7d 100644 --- a/header/header.go +++ b/header/header.go @@ -3,6 +3,7 @@ package header import ( "bytes" "encoding/json" + "errors" "fmt" "time" @@ -162,6 +163,12 @@ func (eh *ExtendedHeader) Validate() error { return nil } +var ( + ErrValidatorHashMismatch = errors.New("validator hash mismatch") + ErrLastHeaderHashMismatch = errors.New("last header hash mismatch") + ErrVerifyCommitLightTrustingFailed = errors.New("commit light trusting verification failed") +) + // Verify validates given untrusted Header against trusted ExtendedHeader. func (eh *ExtendedHeader) Verify(untrst *ExtendedHeader) error { isAdjacent := eh.Height()+1 == untrst.Height() @@ -170,7 +177,8 @@ func (eh *ExtendedHeader) Verify(untrst *ExtendedHeader) error { // Check the validator hashes are the same if !bytes.Equal(untrst.ValidatorsHash, eh.NextValidatorsHash) { return &libhead.VerifyError{ - Reason: fmt.Errorf("expected old header next validators (%X) to match those from new header (%X)", + Reason: fmt.Errorf("%w: expected (%X), but got (%X)", + ErrValidatorHashMismatch, eh.NextValidatorsHash, untrst.ValidatorsHash, ), @@ -179,7 +187,8 @@ func (eh *ExtendedHeader) Verify(untrst *ExtendedHeader) error { if !bytes.Equal(untrst.LastHeader(), eh.Hash()) { return &libhead.VerifyError{ - Reason: fmt.Errorf("expected new header to point to last header hash (%X), but got %X)", + Reason: fmt.Errorf("%w: expected (%X), but got (%X)", + ErrLastHeaderHashMismatch, eh.Hash(), untrst.LastHeader(), ), @@ -191,7 +200,7 @@ func (eh *ExtendedHeader) Verify(untrst *ExtendedHeader) error { if err := eh.ValidatorSet.VerifyCommitLightTrusting(eh.ChainID(), untrst.Commit, light.DefaultTrustLevel); err != nil { return &libhead.VerifyError{ - Reason: err, + Reason: fmt.Errorf("%w: %w", ErrVerifyCommitLightTrustingFailed, err), SoftFailure: true, } } diff --git a/header/headertest/verify_test.go b/header/headertest/verify_test.go index 7ef16afc8d..82795ca5ff 100644 --- a/header/headertest/verify_test.go +++ b/header/headertest/verify_test.go @@ -1,6 +1,7 @@ package headertest import ( + "errors" "strconv" "testing" @@ -15,17 +16,17 @@ func TestVerify(t *testing.T) { trusted, untrustedAdj, untrustedNonAdj := h[0], h[1], h[2] tests := []struct { prepare func() *header.ExtendedHeader - err bool + err error }{ { prepare: func() *header.ExtendedHeader { return untrustedAdj }, - err: false, + err: nil, }, { prepare: func() *header.ExtendedHeader { return untrustedNonAdj }, - err: false, + err: nil, }, { prepare: func() *header.ExtendedHeader { @@ -33,7 +34,7 @@ func TestVerify(t *testing.T) { untrusted.ValidatorsHash = tmrand.Bytes(32) return &untrusted }, - err: true, + err: header.ErrValidatorHashMismatch, }, { prepare: func() *header.ExtendedHeader { @@ -41,7 +42,7 @@ func TestVerify(t *testing.T) { untrusted.RawHeader.LastBlockID.Hash = tmrand.Bytes(32) return &untrusted }, - err: true, + err: header.ErrLastHeaderHashMismatch, }, { prepare: func() *header.ExtendedHeader { @@ -49,18 +50,14 @@ func TestVerify(t *testing.T) { untrusted.Commit = NewTestSuite(t, 2).Commit(RandRawHeader(t)) return &untrusted }, - err: true, + err: header.ErrVerifyCommitLightTrustingFailed, }, } for i, test := range tests { t.Run(strconv.Itoa(i), func(t *testing.T) { err := trusted.Verify(test.prepare()) - if test.err { - assert.Error(t, err) - } else { - assert.NoError(t, err) - } + assert.ErrorIs(t, errors.Unwrap(err), test.err) }) } } From b1a6349ca839d838740319f2a19818a7481f087e Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Thu, 7 Sep 2023 07:10:46 -0500 Subject: [PATCH 182/388] chore: bump celestia-app to v1.0.0-rc14 (#2662) ## Overview This PR bumps celestia-app to [v1.0.0-rc14](https://github.com/celestiaorg/celestia-app/releases/tag/v1.0.0-rc14). Per Upgrade Notice in release notes, this change is not breaking. It only contains changes to the Makefile and spec/doc backports. There is technically two non breaking changes to the code as well, but the first is an arguably redundant check on the first block (which all networks are already past), and the second is a non breaking bump to dependencies. ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- go.mod | 6 +++--- go.sum | 33 +++++++++++++++++++++++++-------- 2 files changed, 28 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 0890bf64f1..7521ed74d6 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc13 + github.com/celestiaorg/celestia-app v1.0.0-rc14 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.3.1 @@ -135,7 +135,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/go-ethereum v1.12.0 // indirect + github.com/ethereum/go-ethereum v1.12.2 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -189,7 +189,7 @@ require ( github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect - github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c // indirect + github.com/holiman/uint256 v1.2.3 // indirect github.com/huin/goupnp v1.2.0 // indirect github.com/iancoleman/orderedmap v0.1.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect diff --git a/go.sum b/go.sum index 7d9c2fba70..cea5e19c5b 100644 --- a/go.sum +++ b/go.sum @@ -316,6 +316,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= +github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -357,8 +359,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc13 h1:BM9lBJw+RcAiFIUmVDd3XBYyXV9rKJBi8mDqc2wok1o= -github.com/celestiaorg/celestia-app v1.0.0-rc13/go.mod h1:JYu6i1NxJw26TVZ+XSllUdnw0Fw3nGNk5f3wm6RIcys= +github.com/celestiaorg/celestia-app v1.0.0-rc14 h1:kC3ljQzxBjQM1s8aw0dOnrn5pNbrfyfd+lS155S7QAc= +github.com/celestiaorg/celestia-app v1.0.0-rc14/go.mod h1:6aD4sFGQIe5XPYejLDKrDKsNWlDu9/P3e54pCLPywNk= github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28 h1:2efXQaggLFknz0wQufr4nUEz5G7pSVHS1j7NuJDsvII= github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28/go.mod h1:++dNzzzjP9jYg+NopN9G8sg1HEZ58lv1TPtg71evZ0E= github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 h1:dDfoQJOlVNj4HufJ1lBLTo2k3/L/255MIiKmEQziDmw= @@ -447,8 +449,12 @@ github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4 github.com/confio/ics23/go v0.9.0/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= +github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= +github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= +github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= +github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= @@ -497,6 +503,8 @@ github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHH github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= +github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= +github.com/crate-crypto/go-kzg-4844 v0.3.0/go.mod h1:SBP7ikXEgDnUPONgm33HtuDZEDtWa3L4QtN1ocJSEQ4= github.com/creachadair/taskgroup v0.3.2 h1:zlfutDS+5XG40AOxcHDSThxKzns8Tnr9jnr6VqkYlkM= github.com/creachadair/taskgroup v0.3.2/go.mod h1:wieWwecHVzsidg2CsUnFinW1faVN4+kq+TDlRJQ0Wbk= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= @@ -594,9 +602,11 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= +github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= +github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= -github.com/ethereum/go-ethereum v1.12.0 h1:bdnhLPtqETd4m3mS8BGMNvBTf36bO5bx/hxE2zljOa0= -github.com/ethereum/go-ethereum v1.12.0/go.mod h1:/oo2X/dZLJjf2mJ6YT9wcWxa4nNJDBKDBU6sFIpx1Gs= +github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= +github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= @@ -980,11 +990,13 @@ github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2p github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 h1:aSVUgRRRtOrZOC1fYmY9gV0e9z/Iu+xNVSASWjsuyGU= github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3/go.mod h1:5PC6ZNPde8bBqU/ewGZig35+UIZtw9Ytxez8/q5ZyFE= +github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7 h1:3JQNjnMRil1yD0IfZKHF9GxxWKDJGj8I0IqOUol//sw= +github.com/holiman/billy v0.0.0-20230718173358-1c7e68d277a7/go.mod h1:5GuXa7vkL8u9FkFuWdVvfR5ix8hRB7DbOAaYULamFpc= github.com/holiman/bloomfilter/v2 v2.0.3 h1:73e0e/V0tCydx14a0SCYS/EWCxgwLZ18CZcZKVu0fao= github.com/holiman/bloomfilter/v2 v2.0.3/go.mod h1:zpoh+gs7qcpqrHr3dB55AMiJwo0iURXE7ZOP9L9hSkA= github.com/holiman/uint256 v1.2.0/go.mod h1:y4ga/t+u+Xwd7CpDgZESaRcWy0I7XMlTMA25ApIH5Jw= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c h1:DZfsyhDK1hnSS5lH8l+JggqzEleHteTYfutAiVlSUM8= -github.com/holiman/uint256 v1.2.2-0.20230321075855-87b91420868c/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= +github.com/holiman/uint256 v1.2.3 h1:K8UWO1HUJpRMXBxbmaY1Y8IAMZC/RsKB+ArEnnK4l5o= +github.com/holiman/uint256 v1.2.3/go.mod h1:SC8Ryt4n+UBbPbIBKaG9zbbDlp4jOru9xFZmPzLUTxw= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= @@ -1769,6 +1781,8 @@ github.com/mitchellh/mapstructure v1.5.0 h1:jeMsZIYE/09sWLaz43PL7Gy6RuMjD2eJVyua github.com/mitchellh/mapstructure v1.5.0/go.mod h1:bFUtVrKA4DC2yAKiSyO/QUcy7e+RRV2QTWOzhPopBRo= github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjUEN1uBnDo34A= github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= +github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= +github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -2222,6 +2236,8 @@ github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcU github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= +github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= @@ -2277,8 +2293,8 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa h1:5SqCsI/2Qya2bCzK15ozrqo2sZxkh0FHynJZOTVoV6Q= -github.com/urfave/cli/v2 v2.17.2-0.20221006022127-8f469abc00aa/go.mod h1:1CNUng3PtjQMtRzJO4FMXBQvkGtuYRxxiR9xMa7jMwI= +github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= +github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -3281,6 +3297,7 @@ rsc.io/binaryregexp v0.2.0/go.mod h1:qTv7/COck+e2FymRvadv62gMdZztPaShugOCi3I+8D8 rsc.io/pdf v0.1.1/go.mod h1:n8OzWcQ6Sp37PL01nO98y4iUCRdTGarVfzxY20ICaU4= rsc.io/quote/v3 v3.1.0/go.mod h1:yEA65RcK8LyAZtP9Kv3t0HmxON59tX3rD+tICJqUlj0= rsc.io/sampler v1.3.0/go.mod h1:T1hPZKmBbMNahiBKFy5HrXp6adAjACjK9JXDnKaTXpA= +rsc.io/tmplfunc v0.0.3 h1:53XFQh69AfOa8Tw0Jm7t+GV7KZhOi6jzsCzTtKbMvzU= rsc.io/tmplfunc v0.0.3/go.mod h1:AG3sTPzElb1Io3Yg4voV9AGZJuleGAwaVRxL9M49PhA= sigs.k8s.io/yaml v1.1.0/go.mod h1:UJmg0vDUVViEyp3mgSv9WPwZCDxu4rQW1olrI1uml+o= sigs.k8s.io/yaml v1.3.0 h1:a2VclLzOGrwOHDiV8EfBGhvjHvP46CtW5j6POvhYGGo= From 3f9e88b471169027f8a0ea3b60ff095f2b15f861 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Fri, 8 Sep 2023 10:28:16 +0200 Subject: [PATCH 183/388] chore: bump libp2p (#2676) Closes #2674 --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index 7521ed74d6..bbf68e5b35 100644 --- a/go.mod +++ b/go.mod @@ -38,11 +38,11 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car v0.6.2 - github.com/libp2p/go-libp2p v0.30.0 - github.com/libp2p/go-libp2p-kad-dht v0.25.0 + github.com/libp2p/go-libp2p v0.31.0 + github.com/libp2p/go-libp2p-kad-dht v0.25.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 - github.com/libp2p/go-libp2p-routing-helpers v0.7.1 + github.com/libp2p/go-libp2p-routing-helpers v0.7.2 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 github.com/multiformats/go-multiaddr v0.11.0 @@ -165,7 +165,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect - github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b // indirect + github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect @@ -269,12 +269,12 @@ require ( github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect github.com/prometheus/client_model v0.4.0 // indirect - github.com/prometheus/common v0.42.0 // indirect - github.com/prometheus/procfs v0.10.1 // indirect + github.com/prometheus/common v0.44.0 // indirect + github.com/prometheus/procfs v0.11.1 // indirect github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-20 v0.3.2 // indirect - github.com/quic-go/quic-go v0.37.6 // indirect + github.com/quic-go/qtls-go1-20 v0.3.3 // indirect + github.com/quic-go/quic-go v0.38.1 // indirect github.com/quic-go/webtransport-go v0.5.3 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect diff --git a/go.sum b/go.sum index cea5e19c5b..ab86f8bdb0 100644 --- a/go.sum +++ b/go.sum @@ -862,8 +862,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b h1:h9U78+dx9a4BKdQkBBos92HalKpaGKHrp+3Uo6yTodo= -github.com/google/pprof v0.0.0-20230817174616-7a8ec2ada47b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= +github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= @@ -1411,8 +1411,8 @@ github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBx github.com/libp2p/go-libp2p v0.23.4/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= github.com/libp2p/go-libp2p v0.25.0/go.mod h1:vXHmFpcfl+xIGN4qW58Bw3a0/SKGAesr5/T4IuJHE3o= github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= -github.com/libp2p/go-libp2p v0.30.0 h1:9EZwFtJPFBcs/yJTnP90TpN1hgrT/EsFfM+OZuwV87U= -github.com/libp2p/go-libp2p v0.30.0/go.mod h1:nr2g5V7lfftwgiJ78/HrID+pwvayLyqKCEirT2Y3Byg= +github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= +github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= @@ -1469,8 +1469,8 @@ github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQO github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= github.com/libp2p/go-libp2p-kad-dht v0.21.0/go.mod h1:Bhm9diAFmc6qcWAr084bHNL159srVZRKADdp96Qqd1I= -github.com/libp2p/go-libp2p-kad-dht v0.25.0 h1:T2SXQ/VlXTQVLChWY/+OyOsmGMRJvB5kiR+eJt7jtvI= -github.com/libp2p/go-libp2p-kad-dht v0.25.0/go.mod h1:P6fz+J+u4tPigvS5J0kxQ1isksqAhmXiS/pNaEw/nFI= +github.com/libp2p/go-libp2p-kad-dht v0.25.1 h1:ofFNrf6MMEy4vi3R1VbJ7LOcTn3Csh0cDcaWHTxtWNA= +github.com/libp2p/go-libp2p-kad-dht v0.25.1/go.mod h1:6za56ncRHYXX4Nc2vn8z7CZK0P4QiMcrn77acKLM2Oo= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= @@ -1514,8 +1514,8 @@ github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3qu github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= -github.com/libp2p/go-libp2p-routing-helpers v0.7.1 h1:kc0kWCZecbBPAiFEHhxfGJZPqjg1g9zV+X+ovR4Tmnc= -github.com/libp2p/go-libp2p-routing-helpers v0.7.1/go.mod h1:cHStPSRC/wgbfpb5jYdMP7zaSmc2wWcb1mkzNr6AR8o= +github.com/libp2p/go-libp2p-routing-helpers v0.7.2 h1:xJMFyhQ3Iuqnk9Q2dYE1eUTzsah7NLw3Qs2zjUV78T0= +github.com/libp2p/go-libp2p-routing-helpers v0.7.2/go.mod h1:cN4mJAD/7zfPKXBcs9ze31JGYAZgzdABEm+q/hkswb8= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= @@ -2054,8 +2054,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.42.0 h1:EKsfXEYo4JpWMHH5cg+KOUWeuJSov1Id8zGR8eeI1YM= -github.com/prometheus/common v0.42.0/go.mod h1:xBwqVerjNdUDjgODMpudtOMwlOwf2SaTr1yjz4b7Zbc= +github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= +github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -2067,8 +2067,8 @@ github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.10.1 h1:kYK1Va/YMlutzCGazswoHKo//tZVlFpKYh+PymziUAg= -github.com/prometheus/procfs v0.10.1/go.mod h1:nwNm2aOCAYw8uTR/9bWRREkZFxAUcWzPHWJq+XBB/FM= +github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= +github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= @@ -2081,11 +2081,11 @@ github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1 github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/qtls-go1-20 v0.3.2 h1:rRgN3WfnKbyik4dBV8A6girlJVxGand/d+jVKbQq5GI= -github.com/quic-go/qtls-go1-20 v0.3.2/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= +github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= -github.com/quic-go/quic-go v0.37.6 h1:2IIUmQzT5YNxAiaPGjs++Z4hGOtIR0q79uS5qE9ccfY= -github.com/quic-go/quic-go v0.37.6/go.mod h1:YsbH1r4mSHPJcLF4k4zruUkLBqctEMBDR6VPvcYjIsU= +github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= +github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= github.com/quic-go/webtransport-go v0.5.1/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= From 4f047c252761dcc9e4b07f86f03f1cf84bc0e2e3 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Fri, 8 Sep 2023 15:09:55 +0200 Subject: [PATCH 184/388] fix(modshare): start peer manager after Syncer (#2677) Fixes panic: ``` panic: invalid type received %!s() goroutine 602 [running]: github.com/celestiaorg/go-header/p2p.(*subscription[...]).NextHeader(0x103fbeaa0?, {0x103fdcd50, 0x14000723310?}) /Users/rford/go/pkg/mod/github.com/celestiaorg/go-header@v0.3.1/p2p/subscription.go:44 +0x2ac github.com/celestiaorg/celestia-node/share/p2p/peers.(*Manager).subscribeHeader(0x1400158c960, {0x103fdcd50, 0x14000723310}, {0x103fbe928?, 0x14000676250?}) /Users/rford/Programming/Celestia/celestia-node/share/p2p/peers/manager.go:299 +0xc0 created by github.com/celestiaorg/celestia-node/share/p2p/peers.(*Manager).Start in goroutine 262 /Users/rford/Programming/Celestia/celestia-node/share/p2p/peers/manager.go:180 +0x410 ``` --- nodebuilder/share/module.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index f7a84bf526..35f8813111 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -9,6 +9,7 @@ import ( "go.uber.org/fx" libhead "github.com/celestiaorg/go-header" + "github.com/celestiaorg/go-header/sync" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" @@ -151,6 +152,9 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option connGater *conngater.BasicConnectionGater, shrexSub *shrexsub.PubSub, headerSub libhead.Subscriber[*header.ExtendedHeader], + // we must ensure Syncer is started before PeerManager + // so that Syncer registers header validator before PeerManager subscribes to headers + _ *sync.Syncer[*header.ExtendedHeader], ) (*peers.Manager, error) { return peers.NewManager( params, From c6504537de96c5e52c65391b7d9e7a65a9bef26f Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 8 Sep 2023 16:17:59 +0300 Subject: [PATCH 185/388] feat(cmd/blob): add fee and gasLimit flags (#2669) --- cmd/celestia/blob.go | 27 +++++++++++++++++++++++++-- 1 file changed, 25 insertions(+), 2 deletions(-) diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 0d09268257..6d397525f5 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -14,7 +14,12 @@ import ( "github.com/celestiaorg/celestia-node/share" ) -var base64Flag bool +var ( + base64Flag bool + + fee int64 + gasLimit uint64 +) func init() { blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) @@ -31,6 +36,20 @@ func init() { false, "printed blob's data as a base64 string", ) + + submitCmd.PersistentFlags().Int64Var( + &fee, + "fee", + -1, + "specifies fee for blob submission", + ) + + submitCmd.PersistentFlags().Uint64Var( + &gasLimit, + "gas.limit", + 0, + "specifies max gas for the blob submission", + ) } var blobCmd = &cobra.Command{ @@ -118,7 +137,11 @@ var submitCmd = &cobra.Command{ return fmt.Errorf("error creating a blob:%v", err) } - height, err := client.Blob.Submit(cmd.Context(), []*blob.Blob{parsedBlob}, nil) + height, err := client.Blob.Submit( + cmd.Context(), + []*blob.Blob{parsedBlob}, + &blob.SubmitOptions{Fee: fee, GasLimit: gasLimit}, + ) response := struct { Height uint64 `json:"height"` From a770ca15a11456f438e447dbe5d690bdfe10f803 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:11:24 +0200 Subject: [PATCH 186/388] chore(nodebuilder/p2p): BlockTime from 15 sec --> 10 sec (#2692) Block times will be closer to 10 seconds on average than 15, so we should shift this to make calculations based on block time in our codebase more accurate. This will also fix issues with syncing `arabica-10` --- das/options.go | 2 +- nodebuilder/p2p/network.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/das/options.go b/das/options.go index 665fc5a7d6..e38e488c04 100644 --- a/das/options.go +++ b/das/options.go @@ -52,7 +52,7 @@ func DefaultParameters() Parameters { ConcurrencyLimit: concurrencyLimit, BackgroundStoreInterval: 10 * time.Minute, SampleFrom: 1, - // SampleTimeout = block time * max amount of catchup workers + // SampleTimeout = approximate block time (with a bit of wiggle room) * max amount of catchup workers SampleTimeout: 15 * time.Second * time.Duration(concurrencyLimit), } } diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index b1f9be8401..a6197aa321 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -21,7 +21,7 @@ const ( Private Network = "private" // BlockTime is a network block time. // TODO @renaynay @Wondertan (#790) - BlockTime = time.Second * 15 + BlockTime = time.Second * 10 ) // Network is a type definition for DA network run by Celestia Node. From 6cc54e7eed7b27d2ede56ab64c9d84be4689084a Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 11 Sep 2023 15:30:23 +0200 Subject: [PATCH 187/388] refactor(nodebuilder/header): get rid of `InitStore` (#2678) Not needed anymore --- nodebuilder/header/constructors.go | 25 +++++++++++++++++-------- nodebuilder/header/module.go | 14 -------------- nodebuilder/header/module_test.go | 9 +++++++-- nodebuilder/testing.go | 4 ++-- 4 files changed, 26 insertions(+), 26 deletions(-) diff --git a/nodebuilder/header/constructors.go b/nodebuilder/header/constructors.go index 984d551434..be4bcbd427 100644 --- a/nodebuilder/header/constructors.go +++ b/nodebuilder/header/constructors.go @@ -3,6 +3,7 @@ package header import ( "context" + "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" "github.com/libp2p/go-libp2p/core/peerstore" @@ -63,7 +64,7 @@ func newP2PExchange[H libhead.Header[H]]( func newSyncer[H libhead.Header[H]]( ex libhead.Exchange[H], fservice libfraud.Service[H], - store InitStore[H], + store libhead.Store[H], sub libhead.Subscriber[H], cfg Config, ) (*sync.Syncer[H], *modfraud.ServiceBreaker[*sync.Syncer[H], H], error) { @@ -82,18 +83,19 @@ func newSyncer[H libhead.Header[H]]( }, nil } -// InitStore is a type representing initialized header store. -// NOTE: It is needed to ensure that Store is always initialized before Syncer is started. -type InitStore[H libhead.Header[H]] libhead.Store[H] - // newInitStore constructs an initialized store func newInitStore[H libhead.Header[H]]( lc fx.Lifecycle, cfg Config, net modp2p.Network, - s libhead.Store[H], + ds datastore.Batching, ex libhead.Exchange[H], -) (InitStore[H], error) { +) (libhead.Store[H], error) { + s, err := store.NewStore[H](ds, store.WithParams(cfg.Store)) + if err != nil { + return nil, err + } + trustedHash, err := cfg.trustedHash(net) if err != nil { return nil, err @@ -101,7 +103,14 @@ func newInitStore[H libhead.Header[H]]( lc.Append(fx.Hook{ OnStart: func(ctx context.Context) error { - return store.Init(ctx, s, ex, trustedHash) + err = store.Init[H](ctx, s, ex, trustedHash) + if err != nil { + return err + } + return s.Start(ctx) + }, + OnStop: func(ctx context.Context) error { + return s.Stop(ctx) }, }) diff --git a/nodebuilder/header/module.go b/nodebuilder/header/module.go index eaee6b047f..557cbcfab7 100644 --- a/nodebuilder/header/module.go +++ b/nodebuilder/header/module.go @@ -11,7 +11,6 @@ import ( libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/go-header/p2p" - "github.com/celestiaorg/go-header/store" "github.com/celestiaorg/go-header/sync" "github.com/celestiaorg/celestia-node/header" @@ -31,19 +30,6 @@ func ConstructModule[H libhead.Header[H]](tp node.Type, cfg *Config) fx.Option { fx.Supply(*cfg), fx.Error(cfgErr), fx.Provide(newHeaderService), - fx.Provide(fx.Annotate( - func(ds datastore.Batching) (libhead.Store[H], error) { - return store.NewStore[H](ds, store.WithParams(cfg.Store)) - }, - fx.OnStart(func(ctx context.Context, str libhead.Store[H]) error { - s := str.(*store.Store[H]) - return s.Start(ctx) - }), - fx.OnStop(func(ctx context.Context, str libhead.Store[H]) error { - s := str.(*store.Store[H]) - return s.Stop(ctx) - }), - )), fx.Provide(newInitStore[H]), fx.Provide(func(subscriber *p2p.Subscriber[H]) libhead.Subscriber[H] { return subscriber diff --git a/nodebuilder/header/module_test.go b/nodebuilder/header/module_test.go index c31cf546d8..23ca41050f 100644 --- a/nodebuilder/header/module_test.go +++ b/nodebuilder/header/module_test.go @@ -36,8 +36,13 @@ func TestConstructModule_StoreParams(t *testing.T) { app := fxtest.New(t, fx.Supply(modp2p.Private), - fx.Provide(func() datastore.Batching { - return datastore.NewMapDatastore() + fx.Supply(modp2p.Bootstrappers{}), + fx.Provide(context.Background), + fx.Provide(libp2p.New), + fx.Provide(conngater.NewBasicConnectionGater), + fx.Provide(func() (datastore.Batching, datastore.Datastore) { + ds := datastore.NewMapDatastore() + return ds, ds }), ConstructModule[*header.ExtendedHeader](node.Light, &cfg), fx.Invoke( diff --git a/nodebuilder/testing.go b/nodebuilder/testing.go index 36f2c2f47f..8d49772aef 100644 --- a/nodebuilder/testing.go +++ b/nodebuilder/testing.go @@ -9,12 +9,12 @@ import ( "go.uber.org/fx" apptypes "github.com/celestiaorg/celestia-app/x/blob/types" + libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/celestia-node/core" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/libs/fxutil" - modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -48,7 +48,7 @@ func TestNodeWithConfig(t *testing.T, tp node.Type, cfg *Config, opts ...fx.Opti // temp dir for the eds store FIXME: Should be in mem fx.Replace(node.StorePath(t.TempDir())), // avoid requesting trustedPeer during initialization - fxutil.ReplaceAs(headertest.NewStore(t), new(modhead.InitStore[*header.ExtendedHeader])), + fxutil.ReplaceAs(headertest.NewStore(t), new(libhead.Store[*header.ExtendedHeader])), ) // in fact, we don't need core.Client in tests, but Bridge requires is a valid one From 8ca777e1a3f04b7d0494e3bae8a1c1bcf68c5d8c Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Viet Date: Mon, 11 Sep 2023 16:19:44 +0200 Subject: [PATCH 188/388] chore: fix unknown in ghcr registry (#2693) Closes #2412 --- Dockerfile | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/Dockerfile b/Dockerfile index 1f1705c2f3..bebd61a644 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,4 +1,4 @@ -FROM docker.io/golang:1.21-alpine3.18 as builder +FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 as builder # hadolint ignore=DL3018 RUN apk update && apk add --no-cache \ @@ -15,7 +15,7 @@ COPY . . RUN make build && make cel-key -FROM docker.io/alpine:3.18.3 +FROM --platform=$BUILDPLATFORM docker.io/alpine:3.18.3 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 @@ -30,6 +30,8 @@ ENV P2P_NETWORK mocha # hadolint ignore=DL3018 RUN apk update && apk add --no-cache \ bash \ + curl \ + jq \ # Creates a user with $UID and $GID=$UID && adduser ${USER_NAME} \ -D \ From 3c8b9c003cc961c01bfab2e8924f079056e68ebb Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Viet Date: Tue, 12 Sep 2023 10:50:59 +0200 Subject: [PATCH 189/388] chore: bump docker ci to v0.2.3 (#2696) --- .github/workflows/docker-build-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 2ce01c81df..c2589ff141 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -18,6 +18,6 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.2 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.3 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile From 9b9d8d5f3d899137e144be214320ded0d92fb775 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 09:32:26 +0000 Subject: [PATCH 190/388] chore(deps): Bump actions/checkout from 3 to 4 (#2690) --- .github/workflows/ci_release.yml | 6 +++--- .github/workflows/go-ci.yml | 10 +++++----- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index e10a032681..e653b2653a 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -32,14 +32,14 @@ jobs: yamllint: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.2 markdown-lint: name: Markdown Lint runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-node@v3 with: node-version: 18 @@ -57,7 +57,7 @@ jobs: if: ${{ github.event_name == 'workflow_dispatch' }} permissions: "write-all" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: Version Release uses: celestiaorg/.github/.github/actions/version-release@v0.2.2 with: diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index f68825bec1..288a9cfb2f 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -16,7 +16,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: @@ -32,7 +32,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - uses: actions/setup-go@v4 with: @@ -48,7 +48,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: set up go uses: actions/setup-go@v4 @@ -69,7 +69,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: set up go uses: actions/setup-go@v4 @@ -84,7 +84,7 @@ jobs: runs-on: ubuntu-latest steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - name: set up go uses: actions/setup-go@v4 From 4b8cd1265a8c9e2961de64dd99c410e8d1ea9cc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:04:15 +0200 Subject: [PATCH 191/388] chore(deps): Bump golang.org/x/crypto from 0.12.0 to 0.13.0 (#2688) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.12.0 to 0.13.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.12.0&new-version=0.13.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index bbf68e5b35..0dafd16aed 100644 --- a/go.mod +++ b/go.mod @@ -68,10 +68,10 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 go.uber.org/zap v1.25.0 - golang.org/x/crypto v0.12.0 + golang.org/x/crypto v0.13.0 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 - golang.org/x/text v0.12.0 + golang.org/x/text v0.13.0 google.golang.org/grpc v1.57.0 google.golang.org/protobuf v1.31.0 ) @@ -315,8 +315,8 @@ require ( golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.8.0 // indirect - golang.org/x/sys v0.11.0 // indirect - golang.org/x/term v0.11.0 // indirect + golang.org/x/sys v0.12.0 // indirect + golang.org/x/term v0.12.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect diff --git a/go.sum b/go.sum index ab86f8bdb0..0d1546da5c 100644 --- a/go.sum +++ b/go.sum @@ -2505,8 +2505,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.12.0 h1:tFM/ta59kqch6LlvYnPa0yx5a83cL2nHflFhYKvv9Yk= -golang.org/x/crypto v0.12.0/go.mod h1:NF0Gs7EO5K4qLn+Ylc+fih8BSTeIjAP05siRnAh98yw= +golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= +golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2851,8 +2851,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.11.0 h1:eG7RXZHdqOJ1i+0lgLgCpSXAp6M3LYlAo6osgSi0xOM= -golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2861,8 +2861,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.11.0 h1:F9tnn/DA/Im8nCwm+fX+1/eBwi4qFjRT++MhtVC4ZX0= -golang.org/x/term v0.11.0/go.mod h1:zC9APTIj3jG3FdV/Ons+XE1riIZXG4aZ4GTHiPZJPIU= +golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= +golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2877,8 +2877,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.12.0 h1:k+n5B8goJNdU7hSvEtMUz3d1Q6D/XW4COJSJR6fN0mc= -golang.org/x/text v0.12.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= +golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 5a47b1a125625d4f4a1fda0855d27a4a05c5e128 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:11:48 +0000 Subject: [PATCH 192/388] chore(deps): Bump github.com/ipfs/go-block-format from 0.1.2 to 0.2.0 (#2682) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/go-block-format](https://github.com/ipfs/go-block-format) from 0.1.2 to 0.2.0.
Release notes

Sourced from github.com/ipfs/go-block-format's releases.

v0.2.0

What's Changed

New Contributors

Full Changelog: https://github.com/ipfs/go-block-format/compare/v0.1.2...v0.2.0

Commits
  • f2d9400 v0.2.0 bump
  • ee6647c chore!: go@1.20, update deps, uci, remove gx
  • ffa6dd1 build(deps): bump golang.org/x/sys
  • 441d112 chore: Update .github/workflows/stale.yml [skip ci]
  • 1f4e90a chore: Update .github/workflows/stale.yml [skip ci]
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/go-block-format&package-manager=go_modules&previous-version=0.1.2&new-version=0.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 0dafd16aed..a8cc31e5a0 100644 --- a/go.mod +++ b/go.mod @@ -31,7 +31,7 @@ require ( github.com/hashicorp/golang-lru v1.0.2 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.12.0 - github.com/ipfs/go-block-format v0.1.2 + github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-cbor v0.1.0 diff --git a/go.sum b/go.sum index 0d1546da5c..30eadc04c4 100644 --- a/go.sum +++ b/go.sum @@ -1054,8 +1054,9 @@ github.com/ipfs/go-block-format v0.0.1/go.mod h1:DK/YYcsSUIVAFNwo/KZCdIIbpN0ROH/ github.com/ipfs/go-block-format v0.0.2/go.mod h1:AWR46JfpcObNfg3ok2JHDUfdiHRgWhJgCQF+KIgOPJY= github.com/ipfs/go-block-format v0.0.3/go.mod h1:4LmD4ZUw0mhO+JSKdpWwrzATiEfM7WWgQ8H5l6P8MVk= github.com/ipfs/go-block-format v0.1.1/go.mod h1:+McEIT+g52p+zz5xGAABGSOKrzmrdX97bc0USBdWPUs= -github.com/ipfs/go-block-format v0.1.2 h1:GAjkfhVx1f4YTODS6Esrj1wt2HhrtwTnhEr+DyPUaJo= github.com/ipfs/go-block-format v0.1.2/go.mod h1:mACVcrxarQKstUU3Yf/RdwbC4DzPV6++rO2a3d+a/KE= +github.com/ipfs/go-block-format v0.2.0 h1:ZqrkxBA2ICbDRbK8KJs/u0O3dlp6gmAuuXUJNiW1Ycs= +github.com/ipfs/go-block-format v0.2.0/go.mod h1:+jpL11nFx5A/SPpsoBn6Bzkra/zaArfSmsknbPMYgzM= github.com/ipfs/go-blockservice v0.1.0/go.mod h1:hzmMScl1kXHg3M2BjTymbVPjv627N7sYcvYaKbop39M= github.com/ipfs/go-blockservice v0.1.4/go.mod h1:OTZhFpkgY48kNzbgyvcexW9cHrpjBYIjSR0KoDOFOLU= github.com/ipfs/go-blockservice v0.2.1/go.mod h1:k6SiwmgyYgs4M/qt+ww6amPeUH9EISLRBnvUurKJhi8= From 5ed602c5277543920ed133e922530bd6861e675a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:31:40 +0200 Subject: [PATCH 193/388] chore(deps): Bump github.com/libp2p/go-libp2p-routing-helpers from 0.7.2 to 0.7.3 (#2687) Bumps [github.com/libp2p/go-libp2p-routing-helpers](https://github.com/libp2p/go-libp2p-routing-helpers) from 0.7.2 to 0.7.3.
Commits
  • b207376 chore: release v0.7.3
  • 95acf72 nit: invert if
  • ee65248 fix: for getValueOrErrorParallel do not return values if they come with errors
  • aefde64 test: add test to make sure we return not found when we get errors back with ...
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/libp2p/go-libp2p-routing-helpers&package-manager=go_modules&previous-version=0.7.2&new-version=0.7.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a8cc31e5a0..05e175366b 100644 --- a/go.mod +++ b/go.mod @@ -42,7 +42,7 @@ require ( github.com/libp2p/go-libp2p-kad-dht v0.25.1 github.com/libp2p/go-libp2p-pubsub v0.9.3 github.com/libp2p/go-libp2p-record v0.2.0 - github.com/libp2p/go-libp2p-routing-helpers v0.7.2 + github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 github.com/multiformats/go-multiaddr v0.11.0 diff --git a/go.sum b/go.sum index 30eadc04c4..b12350ca11 100644 --- a/go.sum +++ b/go.sum @@ -1515,8 +1515,8 @@ github.com/libp2p/go-libp2p-record v0.2.0 h1:oiNUOCWno2BFuxt3my4i1frNrt7PerzB3qu github.com/libp2p/go-libp2p-record v0.2.0/go.mod h1:I+3zMkvvg5m2OcSdoL0KPljyJyvNDFGKX7QdlpYUcwk= github.com/libp2p/go-libp2p-resource-manager v0.2.1/go.mod h1:K+eCkiapf+ey/LADO4TaMpMTP9/Qde/uLlrnRqV4PLQ= github.com/libp2p/go-libp2p-routing-helpers v0.4.0/go.mod h1:dYEAgkVhqho3/YKxfOEGdFMIcWfAFNlZX8iAIihYA2E= -github.com/libp2p/go-libp2p-routing-helpers v0.7.2 h1:xJMFyhQ3Iuqnk9Q2dYE1eUTzsah7NLw3Qs2zjUV78T0= -github.com/libp2p/go-libp2p-routing-helpers v0.7.2/go.mod h1:cN4mJAD/7zfPKXBcs9ze31JGYAZgzdABEm+q/hkswb8= +github.com/libp2p/go-libp2p-routing-helpers v0.7.3 h1:u1LGzAMVRK9Nqq5aYDVOiq/HaB93U9WWczBzGyAC5ZY= +github.com/libp2p/go-libp2p-routing-helpers v0.7.3/go.mod h1:cN4mJAD/7zfPKXBcs9ze31JGYAZgzdABEm+q/hkswb8= github.com/libp2p/go-libp2p-secio v0.1.0/go.mod h1:tMJo2w7h3+wN4pgU2LSYeiKPrfqBgkOsdiKK77hE7c8= github.com/libp2p/go-libp2p-secio v0.2.0/go.mod h1:2JdZepB8J5V9mBp79BmwsaPQhRPNN2NrnB2lKQcdy6g= github.com/libp2p/go-libp2p-secio v0.2.1/go.mod h1:cWtZpILJqkqrSkiYcDBh5lA3wbT2Q+hz3rJQq3iftD8= From 3f5ca0734f2d70aae7a5cf6a2962672f60bdb65b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 12:34:52 +0200 Subject: [PATCH 194/388] chore(deps): Bump go.opentelemetry.io/contrib/instrumentation/runtime from 0.42.0 to 0.43.0 (#2685) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 05e175366b..65397d401d 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 - go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 go.opentelemetry.io/otel v1.17.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 diff --git a/go.sum b/go.sum index b12350ca11..a3fe98b643 100644 --- a/go.sum +++ b/go.sum @@ -2374,8 +2374,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0 h1:EbmAUG9hEAMXyfWEasIt2kmh/WmXUznUksChApTgBGc= -go.opentelemetry.io/contrib/instrumentation/runtime v0.42.0/go.mod h1:rD9feqRYP24P14t5kmhNMqsqm1jvKmpx2H2rKVw52V8= +go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 h1:NunhgxcK14rU7Hw2gKtV6uCSyohkXPisqneRFjnZNKQ= +go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0/go.mod h1:rwb7icgpDjIhhHqv1qPGw6dDjAdAR7IKAe4PQdzBbsg= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= From bf31c4f0271441fd9ef80f7ac168d1ff4de383d0 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 12 Sep 2023 10:44:39 +0000 Subject: [PATCH 195/388] chore(deps): Bump go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp from 1.16.0 to 1.17.0 (#2683) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 5 ++--- go.sum | 10 ++++------ 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 65397d401d..bb65be4a9d 100644 --- a/go.mod +++ b/go.mod @@ -59,8 +59,8 @@ require ( go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 go.opentelemetry.io/otel v1.17.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0 go.opentelemetry.io/otel/metric v1.17.0 go.opentelemetry.io/otel/sdk v1.17.0 go.opentelemetry.io/otel/sdk/metric v0.40.0 @@ -306,7 +306,6 @@ require ( github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect diff --git a/go.sum b/go.sum index a3fe98b643..e29fef53dc 100644 --- a/go.sum +++ b/go.sum @@ -2382,16 +2382,14 @@ go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+n go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0 h1:eU0ffpYuEY7eQ75K+nKr9CI5KcY8h+GPk/9DDlEO1NI= -go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.17.0/go.mod h1:9P5RK5JS2sjKepuCkqFwPp3etwV/57E0eigLw18Mn1k= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 h1:MZbjiZeMmn5wFMORhozpouGKDxj9POHTuU5UA8msBQk= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0/go.mod h1:C7tOYVCJmrDTCwxNny0MuUtnDIR3032vFHYke0F2ZrU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 h1:SZaSbubADNhH2Gxm+1GaZ/cFsGiYefZoodMMX79AOd4= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0/go.mod h1:N65FzQDfQH7NY7umgb0U+7ypGKVYKwwE24L6KXT4OA8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0 h1:cbsD4cUcviQGXdw8+bo5x2wazq10SKz8hEbtCRPcU78= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.16.0/go.mod h1:JgXSGah17croqhJfhByOLVY719k1emAXC8MVhCIJlRs= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0 h1:iqjq9LAB8aK++sKVcELezzn655JnBNdsDhghU4G/So8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.16.0/go.mod h1:hGXzO5bhhSHZnKvrDaXB82Y9DRFour0Nz/KrBh7reWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 h1:U5GYackKpVKlPrd/5gKMlrTlP2dCESAAFU682VCpieY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0/go.mod h1:aFsJfCEnLzEu9vRRAcUiB/cpRTbVsNdF3OHSPpdjxZQ= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0 h1:kvWMtSUNVylLVrOE4WLUmBtgziYoCIYUNSpTYtMzVJI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0/go.mod h1:SExUrRYIXhDgEKG4tkiQovd2HTaELiHUsuK08s5Nqx4= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= From 28bd438d365f57f4e8f28b282fe8d9115f5329b3 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 13 Sep 2023 10:23:46 +0100 Subject: [PATCH 196/388] Stale issue github workflow (#2667) --- .github/workflows/stale.yml | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) create mode 100644 .github/workflows/stale.yml diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml new file mode 100644 index 0000000000..d1088e5f33 --- /dev/null +++ b/.github/workflows/stale.yml @@ -0,0 +1,18 @@ +name: 'Close stale issues and PRs' +on: + schedule: + - cron: '30 8 * * *' + +jobs: + stale: + runs-on: ubuntu-latest + steps: + - uses: actions/stale@v8 + with: + repo-token: ${{ secrets.GITHUB_TOKEN }} + days-before-stale: 60 + days-before-close: 14 + days-before-issue-stale: 90 + days-before-issue-close: 21 + exempt-issue-labels: 'keep-open' + start-date: '2023-09-01T00:00:00Z' From 9260a8b697ec770e6be873947923cab516921c90 Mon Sep 17 00:00:00 2001 From: Ryan Date: Wed, 13 Sep 2023 11:43:34 +0200 Subject: [PATCH 197/388] refactor(swamp): use RPC client instead of service pointers (#2699) Replaces #2356 First step of #2337 --- nodebuilder/node.go | 2 + nodebuilder/tests/api_test.go | 48 +++++++++++--------- nodebuilder/tests/blob_test.go | 19 ++++---- nodebuilder/tests/fraud_test.go | 12 +++-- nodebuilder/tests/nd_test.go | 22 ++++++--- nodebuilder/tests/p2p_test.go | 39 ++++++++++++---- nodebuilder/tests/reconstruct_test.go | 39 ++++++++++------ nodebuilder/tests/sync_test.go | 64 ++++++++++++++++----------- 8 files changed, 158 insertions(+), 87 deletions(-) diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 58a182276c..3e6950a6ae 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -6,6 +6,7 @@ import ( "fmt" "strings" + "github.com/cristalhq/jwt" "github.com/ipfs/boxo/blockservice" "github.com/ipfs/boxo/exchange" logging "github.com/ipfs/go-log/v2" @@ -48,6 +49,7 @@ type Node struct { Network p2p.Network Bootstrappers p2p.Bootstrappers Config *Config + AdminSigner jwt.Signer // rpc components RPCServer *rpc.Server // not optional diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 0e225668bb..2fd4b2d3da 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -13,11 +13,27 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" + "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) +func getAdminClient(ctx context.Context, nd *nodebuilder.Node, t *testing.T) *client.Client { + t.Helper() + + signer := nd.AdminSigner + listenAddr := "ws://" + nd.RPCServer.ListenAddr() + + jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + require.NoError(t, err) + + client, err := client.NewClient(ctx, listenAddr, jwt) + require.NoError(t, err) + + return client +} + func TestNodeModule(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) @@ -66,26 +82,20 @@ func TestGetByHeight(t *testing.T) { err := bridge.Start(ctx) require.NoError(t, err) - adminPerms := []auth.Permission{"public", "read", "write", "admin"} - jwt, err := bridge.AdminServ.AuthNew(ctx, adminPerms) - require.NoError(t, err) - - bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() - client, err := client.NewClient(ctx, bridgeAddr, jwt) - require.NoError(t, err) + rpcClient := getAdminClient(ctx, bridge, t) // let a few blocks be produced - _, err = client.Header.WaitForHeight(ctx, 3) + _, err = rpcClient.Header.WaitForHeight(ctx, 3) require.NoError(t, err) - networkHead, err := client.Header.NetworkHead(ctx) + networkHead, err := rpcClient.Header.NetworkHead(ctx) require.NoError(t, err) - _, err = client.Header.GetByHeight(ctx, networkHead.Height()+1) + _, err = rpcClient.Header.GetByHeight(ctx, networkHead.Height()+1) require.Nil(t, err, "Requesting syncer.Head()+1 shouldn't return an error") - networkHead, err = client.Header.NetworkHead(ctx) + networkHead, err = rpcClient.Header.NetworkHead(ctx) require.NoError(t, err) - _, err = client.Header.GetByHeight(ctx, networkHead.Height()+2) + _, err = rpcClient.Header.GetByHeight(ctx, networkHead.Height()+2) require.ErrorContains(t, err, "given height is from the future") } @@ -101,13 +111,7 @@ func TestBlobRPC(t *testing.T) { err := bridge.Start(ctx) require.NoError(t, err) - adminPerms := []auth.Permission{"public", "read", "write", "admin"} - jwt, err := bridge.AdminServ.AuthNew(ctx, adminPerms) - require.NoError(t, err) - - bridgeAddr := "http://" + bridge.RPCServer.ListenAddr() - client, err := client.NewClient(ctx, bridgeAddr, jwt) - require.NoError(t, err) + rpcClient := getAdminClient(ctx, bridge, t) appBlobs, err := blobtest.GenerateV0Blobs([]int{8}, false) require.NoError(t, err) @@ -119,7 +123,7 @@ func TestBlobRPC(t *testing.T) { ) require.NoError(t, err) - height, err := client.Blob.Submit(ctx, []*blob.Blob{newBlob}, nil) + height, err := rpcClient.Blob.Submit(ctx, []*blob.Blob{newBlob}, nil) require.NoError(t, err) require.True(t, height != 0) } @@ -147,9 +151,11 @@ func TestHeaderSubscription(t *testing.T) { err = light.Start(ctx) require.NoError(t, err) + lightClient := getAdminClient(ctx, light, t) + // subscribe to headers via the light node's RPC header subscription subctx, subcancel := context.WithCancel(ctx) - sub, err := light.HeaderServ.Subscribe(subctx) + sub, err := lightClient.Header.Subscribe(subctx) require.NoError(t, err) // listen for 5 headers for i := 0; i < 5; i++ { diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 80fee071fb..2078fbfa74 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -55,12 +55,15 @@ func TestBlobModule(t *testing.T) { lightNode := sw.NewNodeWithConfig(node.Light, lightCfg) require.NoError(t, lightNode.Start(ctx)) - height, err := fullNode.BlobServ.Submit(ctx, blobs, nil) + fullClient := getAdminClient(ctx, fullNode, t) + lightClient := getAdminClient(ctx, lightNode, t) + + height, err := fullClient.Blob.Submit(ctx, blobs, nil) require.NoError(t, err) - _, err = fullNode.HeaderServ.WaitForHeight(ctx, height) + _, err = fullClient.Header.WaitForHeight(ctx, height) require.NoError(t, err) - _, err = lightNode.HeaderServ.WaitForHeight(ctx, height) + _, err = lightClient.Header.WaitForHeight(ctx, height) require.NoError(t, err) var test = []struct { @@ -70,7 +73,7 @@ func TestBlobModule(t *testing.T) { { name: "Get", doFn: func(t *testing.T) { - blob1, err := fullNode.BlobServ.Get(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) + blob1, err := fullClient.Blob.Get(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) require.Equal(t, blobs[0], blob1) }, @@ -78,7 +81,7 @@ func TestBlobModule(t *testing.T) { { name: "GetAll", doFn: func(t *testing.T) { - newBlobs, err := fullNode.BlobServ.GetAll(ctx, height, []share.Namespace{blobs[0].Namespace()}) + newBlobs, err := fullClient.Blob.GetAll(ctx, height, []share.Namespace{blobs[0].Namespace()}) require.NoError(t, err) require.Len(t, newBlobs, len(appBlobs0)) require.True(t, bytes.Equal(blobs[0].Commitment, newBlobs[0].Commitment)) @@ -88,10 +91,10 @@ func TestBlobModule(t *testing.T) { { name: "Included", doFn: func(t *testing.T) { - proof, err := fullNode.BlobServ.GetProof(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) + proof, err := fullClient.Blob.GetProof(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) - included, err := lightNode.BlobServ.Included( + included, err := lightClient.Blob.Included( ctx, height, blobs[0].Namespace(), @@ -114,7 +117,7 @@ func TestBlobModule(t *testing.T) { ) require.NoError(t, err) - b, err := fullNode.BlobServ.Get(ctx, height, newBlob.Namespace(), newBlob.Commitment) + b, err := fullClient.Blob.Get(ctx, height, newBlob.Namespace(), newBlob.Commitment) assert.Nil(t, b) require.Error(t, err) require.ErrorIs(t, err, blob.ErrBlobNotFound) diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index 95c702c0c0..cd5be80517 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -89,9 +89,11 @@ func TestFraudProofHandling(t *testing.T) { err = full.Start(ctx) require.NoError(t, err) + fullClient := getAdminClient(ctx, full, t) + // 5. subCtx, subCancel := context.WithCancel(ctx) - subscr, err := full.FraudServ.Subscribe(subCtx, byzantine.BadEncoding) + subscr, err := fullClient.Fraud.Subscribe(subCtx, byzantine.BadEncoding) require.NoError(t, err) select { case p := <-subscr: @@ -108,7 +110,7 @@ func TestFraudProofHandling(t *testing.T) { // lifecycles of each Module. // 6. syncCtx, syncCancel := context.WithTimeout(context.Background(), blockTime*5) - _, err = full.HeaderServ.WaitForHeight(syncCtx, 15) + _, err = fullClient.Header.WaitForHeight(syncCtx, 15) require.ErrorIs(t, err, context.DeadlineExceeded) syncCancel() @@ -118,10 +120,11 @@ func TestFraudProofHandling(t *testing.T) { lnStore := nodebuilder.MockStore(t, cfg) light := sw.NewNodeWithStore(node.Light, lnStore) require.NoError(t, light.Start(ctx)) + lightClient := getAdminClient(ctx, light, t) // 8. subCtx, subCancel = context.WithCancel(ctx) - subscr, err = light.FraudServ.Subscribe(subCtx, byzantine.BadEncoding) + subscr, err = lightClient.Fraud.Subscribe(subCtx, byzantine.BadEncoding) require.NoError(t, err) select { case p := <-subscr: @@ -135,7 +138,8 @@ func TestFraudProofHandling(t *testing.T) { // 9. fN := sw.NewNodeWithStore(node.Full, store) require.Error(t, fN.Start(ctx)) - proofs, err := fN.FraudServ.Get(ctx, byzantine.BadEncoding) + fNClient := getAdminClient(ctx, fN, t) + proofs, err := fNClient.Fraud.Get(ctx, byzantine.BadEncoding) require.NoError(t, err) require.NotNil(t, proofs) diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index cb02e2a178..64d672cddc 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -45,6 +45,9 @@ func TestShrexNDFromLights(t *testing.T) { err = light.Start(ctx) require.NoError(t, err) + bridgeClient := getAdminClient(ctx, bridge, t) + lightClient := getAdminClient(ctx, light, t) + // wait for chain to be filled require.NoError(t, <-fillDn) @@ -54,7 +57,7 @@ func TestShrexNDFromLights(t *testing.T) { // the block that actually has transactions. We can get this data from the // response returned by FillBlock. for i := 16; i < blocks; i++ { - h, err := bridge.HeaderServ.GetByHeight(ctx, uint64(i)) + h, err := bridgeClient.Header.GetByHeight(ctx, uint64(i)) require.NoError(t, err) reqCtx, cancel := context.WithTimeout(ctx, time.Second*5) @@ -62,9 +65,9 @@ func TestShrexNDFromLights(t *testing.T) { // ensure to fetch random namespace (not the reserved namespace) namespace := h.DAH.RowRoots[1][:share.NamespaceSize] - expected, err := bridge.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + expected, err := bridgeClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) require.NoError(t, err) - got, err := light.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + got, err := lightClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) require.NoError(t, err) require.True(t, len(got[0].Shares) > 0) @@ -113,12 +116,15 @@ func TestShrexNDFromLightsWithBadFulls(t *testing.T) { require.NoError(t, startFullNodes(ctx, fulls...)) require.NoError(t, light.Start(ctx)) + bridgeClient := getAdminClient(ctx, bridge, t) + lightClient := getAdminClient(ctx, light, t) + // wait for chain to fill up require.NoError(t, <-fillDn) // first 2 blocks are not filled with data for i := 3; i < blocks; i++ { - h, err := bridge.HeaderServ.GetByHeight(ctx, uint64(i)) + h, err := bridgeClient.Header.GetByHeight(ctx, uint64(i)) require.NoError(t, err) if len(h.DAH.RowRoots) != bsize*2 { @@ -133,16 +139,18 @@ func TestShrexNDFromLightsWithBadFulls(t *testing.T) { // ensure to fetch random namespace (not the reserved namespace) namespace := h.DAH.RowRoots[1][:share.NamespaceSize] - expected, err := bridge.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + expected, err := bridgeClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) require.NoError(t, err) require.True(t, len(expected[0].Shares) > 0) // choose a random full to test - gotFull, err := fulls[len(fulls)/2].ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + fN := fulls[len(fulls)/2] + fnClient := getAdminClient(ctx, fN, t) + gotFull, err := fnClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) require.NoError(t, err) require.True(t, len(gotFull[0].Shares) > 0) - gotLight, err := light.ShareServ.GetSharesByNamespace(reqCtx, h.DAH, namespace) + gotLight, err := lightClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) require.NoError(t, err) require.True(t, len(gotLight[0].Shares) > 0) diff --git a/nodebuilder/tests/p2p_test.go b/nodebuilder/tests/p2p_test.go index 083712dfdd..d05846f40c 100644 --- a/nodebuilder/tests/p2p_test.go +++ b/nodebuilder/tests/p2p_test.go @@ -46,7 +46,10 @@ func TestBridgeNodeAsBootstrapper(t *testing.T) { require.NoError(t, nd.Start(ctx)) assert.Equal(t, *addr, nd.Bootstrappers[0]) // ensure that node is actually connected to BN - assert.True(t, nd.Host.Network().Connectedness(addr.ID) == network.Connected) + client := getAdminClient(ctx, nd, t) + connectedenss, err := client.P2P.Connectedness(ctx, addr.ID) + require.NoError(t, err) + assert.Equal(t, connectedenss, network.Connected) } } @@ -102,15 +105,22 @@ func TestFullDiscoveryViaBootstrapper(t *testing.T) { for index := range nodes { require.NoError(t, nodes[index].Start(ctx)) assert.Equal(t, *bootstrapper, nodes[index].Bootstrappers[0]) - assert.True(t, nodes[index].Host.Network().Connectedness(bootstrapper.ID) == network.Connected) + // ensure that node is actually connected to BN + client := getAdminClient(ctx, nodes[index], t) + connectedness, err := client.P2P.Connectedness(ctx, bootstrapper.ID) + require.NoError(t, err) + assert.Equal(t, connectedness, network.Connected) } for { if ctx.Err() != nil { t.Fatal(ctx.Err()) } - if light.Host.Network().Connectedness(host.InfoFromHost(full.Host).ID) == network.Connected { - // LN discovered FN successfully and is now connected + // LN discovered FN successfully and is now connected + client := getAdminClient(ctx, light, t) + connectedness, err := client.P2P.Connectedness(ctx, host.InfoFromHost(full.Host).ID) + require.NoError(t, err) + if connectedness == network.Connected { break } } @@ -158,11 +168,19 @@ func TestRestartNodeDiscovery(t *testing.T) { for index := 0; index < numFulls; index++ { nodes[index] = sw.NewNodeWithConfig(node.Full, fullCfg, nodesConfig) require.NoError(t, nodes[index].Start(ctx)) - assert.True(t, nodes[index].Host.Network().Connectedness(bridgeAddr.ID) == network.Connected) + client := getAdminClient(ctx, nodes[index], t) + connectedness, err := client.P2P.Connectedness(ctx, bridgeAddr.ID) + require.NoError(t, err) + assert.Equal(t, connectedness, network.Connected) } // ensure FNs are connected to each other - require.True(t, nodes[0].Host.Network().Connectedness(nodes[1].Host.ID()) == network.Connected) + fullClient1 := getAdminClient(ctx, nodes[0], t) + fullClient2 := getAdminClient(ctx, nodes[1], t) + + connectedness, err := fullClient1.P2P.Connectedness(ctx, nodes[1].Host.ID()) + require.NoError(t, err) + assert.Equal(t, connectedness, network.Connected) // disconnect the FNs sw.Disconnect(t, nodes[0], nodes[1]) @@ -175,8 +193,13 @@ func TestRestartNodeDiscovery(t *testing.T) { // ensure that the FN with disabled discovery is discovered by both of the // running FNs that have discovery enabled - require.True(t, nodes[0].Host.Network().Connectedness(disabledDiscoveryFN.Host.ID()) == network.Connected) - require.True(t, nodes[1].Host.Network().Connectedness(disabledDiscoveryFN.Host.ID()) == network.Connected) + connectedness, err = fullClient1.P2P.Connectedness(ctx, disabledDiscoveryFN.Host.ID()) + require.NoError(t, err) + assert.Equal(t, connectedness, network.Connected) + + connectedness, err = fullClient2.P2P.Connectedness(ctx, disabledDiscoveryFN.Host.ID()) + require.NoError(t, err) + assert.Equal(t, connectedness, network.Connected) } func setTimeInterval(cfg *nodebuilder.Config, interval time.Duration) { diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index d8640c5249..3c2e8b1f83 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -53,10 +53,11 @@ func TestFullReconstructFromBridge(t *testing.T) { bridge := sw.NewBridgeNode() err := bridge.Start(ctx) require.NoError(t, err) + bridgeClient := getAdminClient(ctx, bridge, t) // TODO: This is required to avoid flakes coming from unfinished retry // mechanism for the same peer in go-header - _, err = bridge.HeaderServ.WaitForHeight(ctx, uint64(blocks)) + _, err = bridgeClient.Header.WaitForHeight(ctx, uint64(blocks)) require.NoError(t, err) cfg := nodebuilder.DefaultConfig(node.Full) @@ -65,17 +66,18 @@ func TestFullReconstructFromBridge(t *testing.T) { full := sw.NewNodeWithConfig(node.Full, cfg) err = full.Start(ctx) require.NoError(t, err) + fullClient := getAdminClient(ctx, full, t) errg, bctx := errgroup.WithContext(ctx) for i := 1; i <= blocks+1; i++ { i := i errg.Go(func() error { - h, err := full.HeaderServ.WaitForHeight(bctx, uint64(i)) + h, err := fullClient.Header.WaitForHeight(bctx, uint64(i)) if err != nil { return err } - return full.ShareServ.SharesAvailable(bctx, h.DAH) + return fullClient.Share.SharesAvailable(bctx, h.DAH) }) } require.NoError(t, <-fillDn) @@ -106,10 +108,11 @@ func TestFullReconstructFromFulls(t *testing.T) { sw.SetBootstrapper(t, bridge) require.NoError(t, bridge.Start(ctx)) + bridgeClient := getAdminClient(ctx, bridge, t) // TODO: This is required to avoid flakes coming from unfinished retry // mechanism for the same peer in go-header - _, err := bridge.HeaderServ.WaitForHeight(ctx, uint64(blocks)) + _, err := bridgeClient.Header.WaitForHeight(ctx, uint64(blocks)) require.NoError(t, err) lights1 := make([]*nodebuilder.Node, lnodes/2) @@ -175,6 +178,9 @@ func TestFullReconstructFromFulls(t *testing.T) { require.NoError(t, full1.Start(ctx)) require.NoError(t, full2.Start(ctx)) + fullClient1 := getAdminClient(ctx, full1, t) + fullClient2 := getAdminClient(ctx, full2, t) + // Form topology for i := 0; i < lnodes/2; i++ { // Separate light nodes into two subnetworks @@ -198,17 +204,17 @@ func TestFullReconstructFromFulls(t *testing.T) { sw.Disconnect(t, full1, bridge) sw.Disconnect(t, full2, bridge) - h, err := full1.HeaderServ.WaitForHeight(ctx, uint64(10+blocks-1)) + h, err := fullClient1.Header.WaitForHeight(ctx, uint64(10+blocks-1)) require.NoError(t, err) // Ensure that the full nodes cannot reconstruct before being connected to each other ctxErr, cancelErr := context.WithTimeout(ctx, time.Second*30) errg, errCtx = errgroup.WithContext(ctxErr) errg.Go(func() error { - return full1.ShareServ.SharesAvailable(errCtx, h.DAH) + return fullClient1.Share.SharesAvailable(errCtx, h.DAH) }) errg.Go(func() error { - return full2.ShareServ.SharesAvailable(errCtx, h.DAH) + return fullClient1.Share.SharesAvailable(errCtx, h.DAH) }) require.Error(t, errg.Wait()) cancelErr() @@ -218,13 +224,13 @@ func TestFullReconstructFromFulls(t *testing.T) { errg, bctx := errgroup.WithContext(ctx) for i := 10; i < blocks+11; i++ { - h, err := full1.HeaderServ.WaitForHeight(bctx, uint64(i)) + h, err := fullClient1.Header.WaitForHeight(bctx, uint64(i)) require.NoError(t, err) errg.Go(func() error { - return full1.ShareServ.SharesAvailable(bctx, h.DAH) + return fullClient1.Share.SharesAvailable(bctx, h.DAH) }) errg.Go(func() error { - return full2.ShareServ.SharesAvailable(bctx, h.DAH) + return fullClient2.Share.SharesAvailable(bctx, h.DAH) }) } @@ -278,12 +284,14 @@ func TestFullReconstructFromLights(t *testing.T) { } bootstrapper := sw.NewNodeWithConfig(node.Full, cfg) require.NoError(t, bootstrapper.Start(ctx)) - require.NoError(t, bridge.Start(ctx)) bootstrapperAddr := host.InfoFromHost(bootstrapper.Host) + require.NoError(t, bridge.Start(ctx)) + bridgeClient := getAdminClient(ctx, bridge, t) + // TODO: This is required to avoid flakes coming from unfinished retry // mechanism for the same peer in go-header - _, err = bridge.HeaderServ.WaitForHeight(ctx, uint64(blocks)) + _, err = bridgeClient.Header.WaitForHeight(ctx, uint64(blocks)) require.NoError(t, err) cfg = nodebuilder.DefaultConfig(node.Full) @@ -313,8 +321,11 @@ func TestFullReconstructFromLights(t *testing.T) { return light.Start(errCtx) }) } + require.NoError(t, errg.Wait()) require.NoError(t, full.Start(ctx)) + fullClient := getAdminClient(ctx, full, t) + for i := 0; i < lnodes; i++ { select { case <-ctx.Done(): @@ -328,12 +339,12 @@ func TestFullReconstructFromLights(t *testing.T) { for i := 1; i <= blocks+1; i++ { i := i errg.Go(func() error { - h, err := full.HeaderServ.WaitForHeight(bctx, uint64(i)) + h, err := fullClient.Header.WaitForHeight(bctx, uint64(i)) if err != nil { return err } - return full.ShareServ.SharesAvailable(bctx, h.DAH) + return fullClient.Share.SharesAvailable(bctx, h.DAH) }) } require.NoError(t, <-fillDn) diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index 65db1332ff..0bb0e1c757 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -56,8 +56,9 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { // start bridge and wait for it to sync to 20 err := bridge.Start(ctx) require.NoError(t, err) + bridgeClient := getAdminClient(ctx, bridge, t) - h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) + h, err := bridgeClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) @@ -68,16 +69,17 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { // start light node and wait for it to sync 20 blocks err = light.Start(ctx) require.NoError(t, err) - h, err = light.HeaderServ.WaitForHeight(ctx, numBlocks) + lightClient := getAdminClient(ctx, light, t) + h, err = lightClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check that the light node has also sampled over the block at height 20 - err = light.ShareServ.SharesAvailable(ctx, h.DAH) + err = lightClient.Share.SharesAvailable(ctx, h.DAH) assert.NoError(t, err) // wait until the entire chain (up to network head) has been sampled - err = light.DASer.WaitCatchUp(ctx) + err = lightClient.DAS.WaitCatchUp(ctx) require.NoError(t, err) }) @@ -87,16 +89,17 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { // let full node sync 20 blocks err = full.Start(ctx) require.NoError(t, err) - h, err = full.HeaderServ.WaitForHeight(ctx, numBlocks) + fullClient := getAdminClient(ctx, full, t) + h, err = fullClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check to ensure the full node can sync the 20th block's data - err = full.ShareServ.SharesAvailable(ctx, h.DAH) + err = fullClient.Share.SharesAvailable(ctx, h.DAH) assert.NoError(t, err) // wait for full node to sync up the blocks from genesis -> network head. - err = full.DASer.WaitCatchUp(ctx) + err = fullClient.DAS.WaitCatchUp(ctx) require.NoError(t, err) }) @@ -144,7 +147,8 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { // start bridge and wait for it to sync to 20 err := bridge.Start(ctx) require.NoError(t, err) - h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) + bridgeClient := getAdminClient(ctx, bridge, t) + h, err := bridgeClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) @@ -155,16 +159,17 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { // start light node and wait for it to sync 20 blocks err = light.Start(ctx) require.NoError(t, err) - h, err = light.HeaderServ.WaitForHeight(ctx, numBlocks) + lightClient := getAdminClient(ctx, light, t) + h, err = lightClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check that the light node has also sampled over the block at height 20 - err = light.ShareServ.SharesAvailable(ctx, h.DAH) + err = lightClient.Share.SharesAvailable(ctx, h.DAH) assert.NoError(t, err) // wait until the entire chain (up to network head) has been sampled - err = light.DASer.WaitCatchUp(ctx) + err = lightClient.DAS.WaitCatchUp(ctx) require.NoError(t, err) }) @@ -174,16 +179,17 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { // let full node sync 20 blocks err = full.Start(ctx) require.NoError(t, err) - h, err = full.HeaderServ.WaitForHeight(ctx, numBlocks) + fullClient := getAdminClient(ctx, full, t) + h, err = fullClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check to ensure the full node can sync the 20th block's data - err = full.ShareServ.SharesAvailable(ctx, h.DAH) + err = fullClient.Share.SharesAvailable(ctx, h.DAH) assert.NoError(t, err) // wait for full node to sync up the blocks from genesis -> network head. - err = full.DASer.WaitCatchUp(ctx) + err = fullClient.DAS.WaitCatchUp(ctx) require.NoError(t, err) }) } @@ -219,7 +225,8 @@ func TestSyncStartStopLightWithBridge(t *testing.T) { // and let bridge node sync up 20 blocks err := bridge.Start(ctx) require.NoError(t, err) - h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) + bridgeClient := getAdminClient(ctx, bridge, t) + h, err := bridgeClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) @@ -228,7 +235,8 @@ func TestSyncStartStopLightWithBridge(t *testing.T) { // start light node and let it sync to 20 err = light.Start(ctx) require.NoError(t, err) - h, err = light.HeaderServ.WaitForHeight(ctx, numBlocks) + lightClient := getAdminClient(ctx, light, t) + h, err = lightClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) require.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) @@ -239,7 +247,7 @@ func TestSyncStartStopLightWithBridge(t *testing.T) { // ensure when light node comes back up, it can sync the remainder of the chain it // missed while sleeping - h, err = light.HeaderServ.WaitForHeight(ctx, 40) + h, err = lightClient.Header.WaitForHeight(ctx, 40) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, 40)) @@ -284,7 +292,8 @@ func TestSyncLightAgainstFull(t *testing.T) { // start a bridge node and wait for it to sync up 20 blocks err := bridge.Start(ctx) require.NoError(t, err) - h, err := bridge.HeaderServ.WaitForHeight(ctx, numBlocks) + bridgeClient := getAdminClient(ctx, bridge, t) + h, err := bridgeClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) @@ -293,9 +302,10 @@ func TestSyncLightAgainstFull(t *testing.T) { // start FN and wait for it to sync up to head of BN err = full.Start(ctx) require.NoError(t, err) - bridgeHead, err := bridge.HeaderServ.LocalHead(ctx) + fullClient := getAdminClient(ctx, full, t) + bridgeHead, err := bridgeClient.Header.LocalHead(ctx) require.NoError(t, err) - _, err = full.HeaderServ.WaitForHeight(ctx, bridgeHead.Height()) + _, err = fullClient.Header.WaitForHeight(ctx, bridgeHead.Height()) require.NoError(t, err) assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) @@ -315,9 +325,10 @@ func TestSyncLightAgainstFull(t *testing.T) { // start LN and wait for it to sync up to network head against the head of the FN err = light.Start(ctx) require.NoError(t, err) - fullHead, err := full.HeaderServ.LocalHead(ctx) + lightClient := getAdminClient(ctx, light, t) + fullHead, err := fullClient.Header.LocalHead(ctx) require.NoError(t, err) - _, err = light.HeaderServ.WaitForHeight(ctx, fullHead.Height()) + _, err = lightClient.Header.WaitForHeight(ctx, fullHead.Height()) require.NoError(t, err) // wait for the core block filling process to exit @@ -359,7 +370,8 @@ func TestSyncLightWithTrustedPeers(t *testing.T) { // let it sync to network head err := bridge.Start(ctx) require.NoError(t, err) - _, err = bridge.HeaderServ.WaitForHeight(ctx, numBlocks) + bridgeClient := getAdminClient(ctx, bridge, t) + _, err = bridgeClient.Header.WaitForHeight(ctx, numBlocks) require.NoError(t, err) // create a FN with BN as trusted peer @@ -368,7 +380,8 @@ func TestSyncLightWithTrustedPeers(t *testing.T) { // let FN sync to network head err = full.Start(ctx) require.NoError(t, err) - err = full.HeaderServ.SyncWait(ctx) + fullClient := getAdminClient(ctx, full, t) + err = fullClient.Header.SyncWait(ctx) require.NoError(t, err) // add full node as a bootstrapper for the suite @@ -380,7 +393,8 @@ func TestSyncLightWithTrustedPeers(t *testing.T) { // let LN sync to network head err = light.Start(ctx) require.NoError(t, err) - err = light.HeaderServ.SyncWait(ctx) + lightClient := getAdminClient(ctx, light, t) + err = lightClient.Header.SyncWait(ctx) require.NoError(t, err) // wait for the core block filling process to exit From 25ac0020e73a80a2ed3b27cf7d6a5d550995cbe9 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Wed, 13 Sep 2023 08:34:54 -0400 Subject: [PATCH 198/388] Goreleaser (#2661) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit ## Overview **UPDATE** This PR adds building binaries to the release process by using [goreleaser](https://goreleaser.com/). The CI release process is largely unchanged. Releases can be trigger by either manually triggering the `ci_release` workflow, or they can be triggered by pushing a version tag. If the team has not been using the CI to generate releases, so I added a `make goreleaser-release` command to build the binaries for the release locally so that they can be manually uploaded. Result of running `make goreleaser-release`: ``` build └── goreleaser ├── artifacts.json ├── celestia-node_Darwin_arm64.tar.gz ├── celestia-node_Darwin_x86_64.tar.gz ├── celestia-node_Linux_arm64.tar.gz ├── celestia-node_Linux_x86_64.tar.gz ├── celestia-node_darwin_amd64_v1 │   └── celestia ├── celestia-node_darwin_arm64 │   └── celestia ├── celestia-node_linux_amd64_v1 │   └── celestia ├── celestia-node_linux_arm64 │   └── celestia ├── checksums.txt ├── config.yaml └── metadata.json ``` The `.goreleaser.yaml` file is 90% stock generated from `goreleaser init`. The celestia node specific items are: ```yaml builds: - main: ./cmd/celestia binary: celestia env: # NOTE: goreleaser doesn't fully support CGO natively. If CGO is needed # for any node features, this should be removed and a workaround might # need to be created. # REF: https://goreleaser.com/limitations/cgo/ - CGO_ENABLED=0 - VersioningPath={{ "github.com/celestiaorg/celestia-node/nodebuilder/node" }} goos: - linux - darwin goarch: - amd64 - arm64 ldflags: # Ref: https://goreleaser.com/customization/templates/#common-fields # # .CommitDate is used to help with reproducible builds, ensuring that the # same date is always used # # .FullCommit is git commit hash goreleaser is using for the release # # .Version is the version being released - -X "{{ .Env.VersioningPath }}.buildTime={{ .CommitDate }}" - -X "{{ .Env.VersioningPath }}.lastCommit={{ .FullCommit }}" - -X "{{ .Env.VersioningPath }}.semanticVersion={{ .Version }}" dist: ./build/goreleaser ``` For building locally, i added a `make goreleaser-build` command. The binaries are put into a `build/goreleaser` directory. The `make goreleaser` command lists the `goreleaser` commands and also checks the version as a way to verify you have `goreleaser` installed. Result of running `make goreleaser-build`: ``` build └── goreleaser ├── artifacts.json ├── celestia-node_darwin_amd64_v1 │ └── celestia ├── config.yaml └── metadata.json ``` Successful github action run: https://github.com/MSevey/celestia-node/actions/runs/6123729144/job/16622244813 Example release generated: https://github.com/MSevey/celestia-node/releases Created #2445 as a follow up discussion how to add signing. --------- Co-authored-by: Ismail Khoffi Co-authored-by: ramin --- .github/workflows/ci_release.yml | 42 +++++++++++++++++++++--- .goreleaser.yaml | 55 ++++++++++++++++++++++++++++++++ .yamllint.yml | 9 ++++++ Makefile | 17 ++++++++++ 4 files changed, 118 insertions(+), 5 deletions(-) create mode 100644 .goreleaser.yaml create mode 100644 .yamllint.yml diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index e653b2653a..841db11bfa 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -50,19 +50,51 @@ jobs: go-ci: uses: ./.github/workflows/go-ci.yml - # Make a release if this is a manually trigger job, i.e. workflow_dispatch - release: + # If this was a workflow dispatch event, we need to generate and push a tag + # for goreleaser to grab + version_bump: needs: [hadolint, yamllint, markdown-lint, go-ci] runs-on: ubuntu-latest - if: ${{ github.event_name == 'workflow_dispatch' }} permissions: "write-all" steps: - uses: actions/checkout@v4 + - name: Bump version and push tag + # Placing the if condition here is a workaround for needing to block + # on this step during workflow dispatch events but the step not + # needing to run on tags. If we had the if condition on the full + # version_bump section, it would skip and not run, which would result + # in goreleaser not running either. + if: ${{ github.event_name == 'workflow_dispatch' }} + uses: mathieudutour/github-tag-action@v6.0 + - name: Version Release uses: celestiaorg/.github/.github/actions/version-release@v0.2.2 with: - github-token: ${{secrets.GITHUB_TOKEN}} - version-bump: ${{inputs.version}} + github_token: ${{ secrets.GITHUB_TOKEN }} + default_bump: ${{ inputs.version }} + + # Generate the release with goreleaser to include pre-built binaries + goreleaser: + needs: version_bump + runs-on: ubuntu-latest + if: | + github.event_name == 'workflow_dispatch' || + (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) + permissions: "write-all" + steps: + - uses: actions/checkout@v3 + - run: git fetch --force --tags + - uses: actions/setup-go@v4 + with: + go-version: 1.21 + # Generate the binaries and release + - uses: goreleaser/goreleaser-action@v4 + with: + distribution: goreleaser + version: latest + args: release --clean + env: + GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} # TODO: permission issue, but not worth fixing as this should be refactored # into the celestiaorg/.github repo, at which point any permission issues will diff --git a/.goreleaser.yaml b/.goreleaser.yaml new file mode 100644 index 0000000000..70336dabc0 --- /dev/null +++ b/.goreleaser.yaml @@ -0,0 +1,55 @@ +# This is an example .goreleaser.yml file with some sensible defaults. +# Make sure to check the documentation at https://goreleaser.com +before: + hooks: + - go mod tidy +builds: + - main: ./cmd/celestia + binary: celestia + env: + # NOTE: goreleaser doesn't fully support CGO natively. If CGO is needed + # for any node features, this should be removed and a workaround might + # need to be created. + # REF: https://goreleaser.com/limitations/cgo/ + - CGO_ENABLED=0 + - VersioningPath={{ "github.com/celestiaorg/celestia-node/nodebuilder/node" }} + goos: + - linux + - darwin + goarch: + - amd64 + - arm64 + ldflags: + # Ref: https://goreleaser.com/customization/templates/#common-fields + # + # .CommitDate is used to help with reproducible builds, ensuring that the + # same date is always used + # + # .FullCommit is git commit hash goreleaser is using for the release + # + # .Version is the version being released + - -X "{{ .Env.VersioningPath }}.buildTime={{ .CommitDate }}" + - -X "{{ .Env.VersioningPath }}.lastCommit={{ .FullCommit }}" + - -X "{{ .Env.VersioningPath }}.semanticVersion={{ .Version }}" +dist: ./build/goreleaser +archives: + - format: tar.gz + # this name template makes the OS and Arch compatible with the results of + # uname. + name_template: >- + {{ .ProjectName }}_ + {{- title .Os }}_ + {{- if eq .Arch "amd64" }}x86_64 + {{- else if eq .Arch "386" }}i386 + {{- else }}{{ .Arch }}{{ end }} + {{- if .Arm }}v{{ .Arm }}{{ end }} +checksum: + name_template: "checksums.txt" +snapshot: + name_template: "{{ incpatch .Version }}-next" +changelog: + sort: asc + filters: + exclude: + - "^docs:" + - "^test:" diff --git a/.yamllint.yml b/.yamllint.yml new file mode 100644 index 0000000000..cd2a9e8293 --- /dev/null +++ b/.yamllint.yml @@ -0,0 +1,9 @@ +--- +# Built from docs https://yamllint.readthedocs.io/en/stable/configuration.html +extends: default + +rules: + # 120 chars should be enough, but don't fail if a line is longer + line-length: + max: 120 + level: warning diff --git a/Makefile b/Makefile index 30b0c18be3..d90075431a 100644 --- a/Makefile +++ b/Makefile @@ -189,3 +189,20 @@ telemetry-infra-up: telemetry-infra-down: PWD="${DIR_FULLPATH}/docker/telemetry" docker-compose -f ./docker/telemetry/docker-compose.yml down .PHONY: telemetry-infra-down + +## goreleaser: List Goreleaser commands and checks if GoReleaser is installed. +goreleaser: Makefile + @echo " Choose a goreleaser command to run:" + @sed -n 's/^## goreleaser/goreleaser/p' $< | column -t -s ':' | sed -e 's/^/ /' + @goreleaser --version +.PHONY: goreleaser + +## goreleaser-build: Builds the celestia binary using GoReleaser for your local OS. +goreleaser-build: + goreleaser build --snapshot --clean --single-target +.PHONY: goreleaser-build + +## goreleaser-release: Builds the release celestia binaries as defined in .goreleaser.yaml. This requires there be a git tag for the release in the local git history. +goreleaser-release: + goreleaser release --clean --fail-fast --skip-publish +.PHONY: goreleaser-release From 7c1f8f30ef62ebb8ee3782aab68019d66a04a26b Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 14 Sep 2023 19:46:47 +0200 Subject: [PATCH 199/388] fix(shrexeds): close stream (#2705) --- share/p2p/shrexeds/client.go | 1 + 1 file changed, 1 insertion(+) diff --git a/share/p2p/shrexeds/client.go b/share/p2p/shrexeds/client.go index 387594186e..7602bb5fb0 100644 --- a/share/p2p/shrexeds/client.go +++ b/share/p2p/shrexeds/client.go @@ -89,6 +89,7 @@ func (c *Client) doRequest( if err != nil { return nil, fmt.Errorf("failed to open stream: %w", err) } + defer stream.Close() c.setStreamDeadlines(ctx, stream) From 4d9858599052fca6dcc8841bd0a12430cd333266 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 18 Sep 2023 16:26:45 +0800 Subject: [PATCH 200/388] feat(share/eds): Rework accessor cache (#2612) **Motivation** Previously, all accessors went into cache. They were closed only when they were evicted from it and there was no way to manually close one. Now, when we need to cache accessors only in some cases and in some not to, this approach will not work. Enabling control of accessor close required major cache rework. New cache features: - Old cache didn't allow to close and release of an accessor. Now, Instead of returning just a reader/blockstore, cache returns object that composed of builder and `io.Closer` (Accessor interface). This allows caller to have an ability to close and release accessor once it is done with reading from it. - Items stored in cache also provide io.Closer`, but it is noop. Lifecycle of cached items is controlled by cache itself. For now objects are released right after they are evicted from cache, but it could be changed to smarter release based on current readers count. Will be implemented as separate issue. - Lazy blockstore builder. Old cache created blockstore every time accessor was acquired, even when only reader was used and blockstore was not needed. Creating blockstore is costly operation that involves building index, so this was reworked to create blockstore, only when it is requested and store it for subsequent requests. - adds cache unit tests. - adds metrics for cache methods to track cache hits/miss. - general cleanup of unused methods/logic. It will be easier for reviewer to start from `store.go` and `blockstore.go` and then go into new `cache` pkg --- go.mod | 4 +- nodebuilder/store_test.go | 1 + share/eds/accessor_cache.go | 178 ------------------ share/eds/blockstore.go | 63 ++++--- share/eds/blockstore_test.go | 3 + share/eds/cache/accessor_cache.go | 173 +++++++++++++++++ share/eds/cache/accessor_cache_test.go | 246 +++++++++++++++++++++++++ share/eds/cache/cache.go | 49 +++++ share/eds/cache/metrics.go | 66 +++++++ share/eds/cache/noop.go | 31 ++++ share/eds/metrics.go | 24 +-- share/eds/ods_test.go | 6 + share/eds/store.go | 148 ++++++++------- share/eds/store_test.go | 154 ++++++++++++---- share/eds/utils.go | 44 +++++ share/getters/store.go | 10 + share/p2p/shrexeds/server.go | 9 +- 17 files changed, 876 insertions(+), 333 deletions(-) delete mode 100644 share/eds/accessor_cache.go create mode 100644 share/eds/cache/accessor_cache.go create mode 100644 share/eds/cache/accessor_cache_test.go create mode 100644 share/eds/cache/cache.go create mode 100644 share/eds/cache/metrics.go create mode 100644 share/eds/cache/noop.go create mode 100644 share/eds/utils.go diff --git a/go.mod b/go.mod index bb65be4a9d..d3a5771c23 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-retryablehttp v0.7.4 - github.com/hashicorp/golang-lru v1.0.2 + github.com/hashicorp/golang-lru/v2 v2.0.5 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-block-format v0.2.0 @@ -185,8 +185,8 @@ require ( github.com/hashicorp/go-multierror v1.1.1 // indirect github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect + github.com/hashicorp/golang-lru v1.0.2 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect - github.com/hashicorp/golang-lru/v2 v2.0.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/holiman/uint256 v1.2.3 // indirect diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index 8a39849060..8ea56a280f 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -154,6 +154,7 @@ func TestStoreRestart(t *testing.T) { require.NoError(t, err) _, err = eds.ReadEDS(ctx, odsReader, h) require.NoError(t, err) + require.NoError(t, edsReader.Close()) } } diff --git a/share/eds/accessor_cache.go b/share/eds/accessor_cache.go deleted file mode 100644 index 9f70178be6..0000000000 --- a/share/eds/accessor_cache.go +++ /dev/null @@ -1,178 +0,0 @@ -package eds - -import ( - "context" - "errors" - "fmt" - "reflect" - "sync" - - "github.com/filecoin-project/dagstore" - "github.com/filecoin-project/dagstore/shard" - lru "github.com/hashicorp/golang-lru" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/metric" -) - -var ( - defaultCacheSize = 128 - errCacheMiss = errors.New("accessor not found in blockstore cache") -) - -// accessorWithBlockstore is the value that we store in the blockstore cache -type accessorWithBlockstore struct { - sa *dagstore.ShardAccessor - // blockstore is stored separately because each access to the blockstore over the shard accessor - // reopens the underlying CAR. - bs dagstore.ReadBlockstore -} - -type blockstoreCache struct { - // stripedLocks prevents simultaneous RW access to the blockstore cache for a shard. Instead - // of using only one lock or one lock per key, we stripe the shard keys across 256 locks. 256 is - // chosen because it 0-255 is the range of values we get looking at the last byte of the key. - stripedLocks [256]sync.Mutex - // caches the blockstore for a given shard for shard read affinity i.e. - // further reads will likely be from the same shard. Maps (shard key -> blockstore). - cache *lru.Cache - - metrics *cacheMetrics -} - -func newBlockstoreCache(cacheSize int) (*blockstoreCache, error) { - bc := &blockstoreCache{} - // instantiate the blockstore cache - bslru, err := lru.NewWithEvict(cacheSize, bc.evictFn()) - if err != nil { - return nil, fmt.Errorf("failed to instantiate blockstore cache: %w", err) - } - bc.cache = bslru - return bc, nil -} - -func (bc *blockstoreCache) evictFn() func(_ interface{}, val interface{}) { - return func(_ interface{}, val interface{}) { - // ensure we close the blockstore for a shard when it's evicted so dagstore can gc it. - abs, ok := val.(*accessorWithBlockstore) - if !ok { - panic(fmt.Sprintf( - "casting value from cache to accessorWithBlockstore: %s", - reflect.TypeOf(val), - )) - } - - err := abs.sa.Close() - if err != nil { - log.Errorf("couldn't close accessor after cache eviction: %s", err) - } - bc.metrics.observeEvicted(err != nil) - } -} - -func (bc *blockstoreCache) Remove(key shard.Key) bool { - lk := &bc.stripedLocks[shardKeyToStriped(key)] - lk.Lock() - defer lk.Unlock() - - return bc.cache.Remove(key) -} - -// Get retrieves the blockstore for a given shard key from the cache. If the blockstore is not in -// the cache, it returns an errCacheMiss -func (bc *blockstoreCache) Get(shardContainingCid shard.Key) (*accessorWithBlockstore, error) { - lk := &bc.stripedLocks[shardKeyToStriped(shardContainingCid)] - lk.Lock() - defer lk.Unlock() - - return bc.unsafeGet(shardContainingCid) -} - -func (bc *blockstoreCache) unsafeGet(shardContainingCid shard.Key) (*accessorWithBlockstore, error) { - // We've already ensured that the given shard has the cid/multihash we are looking for. - val, ok := bc.cache.Get(shardContainingCid) - if !ok { - return nil, errCacheMiss - } - - accessor, ok := val.(*accessorWithBlockstore) - if !ok { - panic(fmt.Sprintf( - "casting value from cache to accessorWithBlockstore: %s", - reflect.TypeOf(val), - )) - } - return accessor, nil -} - -// Add adds a blockstore for a given shard key to the cache. -func (bc *blockstoreCache) Add( - shardContainingCid shard.Key, - accessor *dagstore.ShardAccessor, -) (*accessorWithBlockstore, error) { - lk := &bc.stripedLocks[shardKeyToStriped(shardContainingCid)] - lk.Lock() - defer lk.Unlock() - - return bc.unsafeAdd(shardContainingCid, accessor) -} - -func (bc *blockstoreCache) unsafeAdd( - shardContainingCid shard.Key, - accessor *dagstore.ShardAccessor, -) (*accessorWithBlockstore, error) { - blockStore, err := accessor.Blockstore() - if err != nil { - return nil, fmt.Errorf("failed to get blockstore from accessor: %w", err) - } - - newAccessor := &accessorWithBlockstore{ - bs: blockStore, - sa: accessor, - } - bc.cache.Add(shardContainingCid, newAccessor) - return newAccessor, nil -} - -// shardKeyToStriped returns the index of the lock to use for a given shard key. We use the last -// byte of the shard key as the pseudo-random index. -func shardKeyToStriped(sk shard.Key) byte { - return sk.String()[len(sk.String())-1] -} - -type cacheMetrics struct { - evictedCounter metric.Int64Counter -} - -func (bc *blockstoreCache) withMetrics() error { - evictedCounter, err := meter.Int64Counter("eds_blockstore_cache_evicted_counter", - metric.WithDescription("eds blockstore cache evicted event counter")) - if err != nil { - return err - } - - cacheSize, err := meter.Int64ObservableGauge("eds_blockstore_cache_size", - metric.WithDescription("total amount of items in blockstore cache"), - ) - if err != nil { - return err - } - - callback := func(ctx context.Context, observer metric.Observer) error { - observer.ObserveInt64(cacheSize, int64(bc.cache.Len())) - return nil - } - _, err = meter.RegisterCallback(callback, cacheSize) - if err != nil { - return err - } - bc.metrics = &cacheMetrics{evictedCounter: evictedCounter} - return nil -} - -func (m *cacheMetrics) observeEvicted(failed bool) { - if m == nil { - return - } - m.evictedCounter.Add(context.Background(), 1, metric.WithAttributes( - attribute.Bool(failedKey, failed))) -} diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go index 2abe4219f1..349d6f58ba 100644 --- a/share/eds/blockstore.go +++ b/share/eds/blockstore.go @@ -5,14 +5,14 @@ import ( "errors" "fmt" - "github.com/filecoin-project/dagstore" bstore "github.com/ipfs/boxo/blockstore" dshelp "github.com/ipfs/boxo/datastore/dshelp" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" ipld "github.com/ipfs/go-ipld-format" + + "github.com/celestiaorg/celestia-node/share/eds/cache" ) var _ bstore.Blockstore = (*blockstore)(nil) @@ -32,18 +32,9 @@ var ( // implementation to allow for the blockstore operations to be routed to the underlying stores. type blockstore struct { store *Store - cache *blockstoreCache ds datastore.Batching } -func newBlockstore(store *Store, cache *blockstoreCache, ds datastore.Batching) *blockstore { - return &blockstore{ - store: store, - cache: cache, - ds: namespace.Wrap(ds, blockstoreCacheKey), - } -} - func (bs *blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { @@ -63,6 +54,11 @@ func (bs *blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { func (bs *blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) { blockstr, err := bs.getReadOnlyBlockstore(ctx, cid) + if err == nil { + defer closeAndLog("blockstore", blockstr) + return blockstr.Get(ctx, cid) + } + if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { k := dshelp.MultihashToDsKey(cid.Hash()) blockData, err := bs.ds.Get(ctx, k) @@ -72,15 +68,18 @@ func (bs *blockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error // nmt's GetNode expects an ipld.ErrNotFound when a cid is not found. return nil, ipld.ErrNotFound{Cid: cid} } - if err != nil { - log.Debugf("failed to get blockstore for cid %s: %s", cid, err) - return nil, err - } - return blockstr.Get(ctx, cid) + + log.Debugf("failed to get blockstore for cid %s: %s", cid, err) + return nil, err } func (bs *blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { blockstr, err := bs.getReadOnlyBlockstore(ctx, cid) + if err == nil { + defer closeAndLog("blockstore", blockstr) + return blockstr.GetSize(ctx, cid) + } + if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { k := dshelp.MultihashToDsKey(cid.Hash()) size, err := bs.ds.GetSize(ctx, k) @@ -90,10 +89,9 @@ func (bs *blockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { // nmt's GetSize expects an ipld.ErrNotFound when a cid is not found. return 0, ipld.ErrNotFound{Cid: cid} } - if err != nil { - return 0, err - } - return blockstr.GetSize(ctx, cid) + + log.Debugf("failed to get size for cid %s: %s", cid, err) + return 0, err } func (bs *blockstore) DeleteBlock(ctx context.Context, cid cid.Cid) error { @@ -139,7 +137,7 @@ func (bs *blockstore) HashOnRead(bool) { } // getReadOnlyBlockstore finds the underlying blockstore of the shard that contains the given CID. -func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (dagstore.ReadBlockstore, error) { +func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (*BlockstoreCloser, error) { keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) if errors.Is(err, datastore.ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { return nil, ErrNotFound @@ -148,11 +146,28 @@ func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (d return nil, fmt.Errorf("failed to find shards containing multihash: %w", err) } - // a share can exist in multiple EDSes, so just take the first one. + // check if cache contains any of accessors shardKey := keys[0] - accessor, err := bs.store.getCachedAccessor(ctx, shardKey) + if accessor, err := bs.store.cache.Get(shardKey); err == nil { + return blockstoreCloser(accessor) + } + + // load accessor to the cache and use it as blockstoreCloser + accessor, err := bs.store.cache.GetOrLoad(ctx, shardKey, bs.store.getAccessor) if err != nil { return nil, fmt.Errorf("failed to get accessor for shard %s: %w", shardKey, err) } - return accessor.bs, nil + return blockstoreCloser(accessor) +} + +// blockstoreCloser constructs new BlockstoreCloser from cache.Accessor +func blockstoreCloser(ac cache.Accessor) (*BlockstoreCloser, error) { + bs, err := ac.Blockstore() + if err != nil { + return nil, fmt.Errorf("eds/store: failed to get blockstore: %w", err) + } + return &BlockstoreCloser{ + ReadBlockstore: bs, + Closer: ac, + }, nil } diff --git a/share/eds/blockstore_test.go b/share/eds/blockstore_test.go index 745797bf42..d9dbf7ed30 100644 --- a/share/eds/blockstore_test.go +++ b/share/eds/blockstore_test.go @@ -37,6 +37,9 @@ func TestBlockstore_Operations(t *testing.T) { topLevelBS := edsStore.Blockstore() carBS, err := edsStore.CARBlockstore(ctx, dah.Hash()) require.NoError(t, err) + defer func() { + require.NoError(t, carBS.Close()) + }() root, err := edsStore.GetDAH(ctx, dah.Hash()) require.NoError(t, err) diff --git a/share/eds/cache/accessor_cache.go b/share/eds/cache/accessor_cache.go new file mode 100644 index 0000000000..ff955f8c45 --- /dev/null +++ b/share/eds/cache/accessor_cache.go @@ -0,0 +1,173 @@ +package cache + +import ( + "context" + "fmt" + "io" + "sync" + + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/dagstore/shard" + lru "github.com/hashicorp/golang-lru/v2" +) + +var _ Cache = (*AccessorCache)(nil) + +// AccessorCache implements the Cache interface using an LRU cache backend. +type AccessorCache struct { + // The name is a prefix that will be used for cache metrics if they are enabled. + name string + // stripedLocks prevents simultaneous RW access to the blockstore cache for a shard. Instead + // of using only one lock or one lock per key, we stripe the shard keys across 256 locks. 256 is + // chosen because it 0-255 is the range of values we get looking at the last byte of the key. + stripedLocks [256]sync.Mutex + // Caches the blockstore for a given shard for shard read affinity, i.e., further reads will likely + // be from the same shard. Maps (shard key -> blockstore). + cache *lru.Cache[shard.Key, *accessorWithBlockstore] + + metrics *metrics +} + +// accessorWithBlockstore is the value that we store in the blockstore Cache. It implements the +// Accessor interface. +type accessorWithBlockstore struct { + sync.RWMutex + shardAccessor Accessor + // The blockstore is stored separately because each access to the blockstore over the shard + // accessor reopens the underlying CAR. + bs dagstore.ReadBlockstore +} + +// Blockstore implements the Blockstore of the Accessor interface. It creates the blockstore on the +// first request and reuses the created instance for all subsequent requests. +func (s *accessorWithBlockstore) Blockstore() (dagstore.ReadBlockstore, error) { + s.Lock() + defer s.Unlock() + var err error + if s.bs == nil { + s.bs, err = s.shardAccessor.Blockstore() + } + return s.bs, err +} + +// Reader returns a new copy of the reader to read data. +func (s *accessorWithBlockstore) Reader() io.Reader { + return s.shardAccessor.Reader() +} + +func NewAccessorCache(name string, cacheSize int) (*AccessorCache, error) { + bc := &AccessorCache{ + name: name, + } + // Instantiate the blockstore Cache. + bslru, err := lru.NewWithEvict[shard.Key, *accessorWithBlockstore](cacheSize, bc.evictFn()) + if err != nil { + return nil, fmt.Errorf("failed to instantiate blockstore cache: %w", err) + } + bc.cache = bslru + return bc, nil +} + +// evictFn will be invoked when an item is evicted from the cache. +func (bc *AccessorCache) evictFn() func(shard.Key, *accessorWithBlockstore) { + return func(_ shard.Key, abs *accessorWithBlockstore) { + err := abs.shardAccessor.Close() + if err != nil { + bc.metrics.observeEvicted(true) + log.Errorf("couldn't close accessor after cache eviction: %s", err) + return + } + bc.metrics.observeEvicted(false) + } +} + +// Get retrieves the Accessor for a given shard key from the Cache. If the Accessor is not in +// the Cache, it returns an errCacheMiss. +func (bc *AccessorCache) Get(key shard.Key) (Accessor, error) { + lk := &bc.stripedLocks[shardKeyToStriped(key)] + lk.Lock() + defer lk.Unlock() + + accessor, err := bc.get(key) + if err != nil { + bc.metrics.observeGet(false) + return nil, err + } + bc.metrics.observeGet(true) + return newCloser(accessor), nil +} + +func (bc *AccessorCache) get(key shard.Key) (*accessorWithBlockstore, error) { + abs, ok := bc.cache.Get(key) + if !ok { + return nil, errCacheMiss + } + return abs, nil +} + +// GetOrLoad attempts to get an item from the cache, and if not found, invokes +// the provided loader function to load it. +func (bc *AccessorCache) GetOrLoad( + ctx context.Context, + key shard.Key, + loader func(context.Context, shard.Key) (Accessor, error), +) (Accessor, error) { + lk := &bc.stripedLocks[shardKeyToStriped(key)] + lk.Lock() + defer lk.Unlock() + + abs, err := bc.get(key) + if err == nil { + bc.metrics.observeGet(true) + return newCloser(abs), nil + } + + // accessor not found in cache, so load new one using loader + accessor, err := loader(ctx, key) + if err != nil { + return nil, fmt.Errorf("unable to load accessor: %w", err) + } + + abs = &accessorWithBlockstore{ + shardAccessor: accessor, + } + + // Create a new accessor first to increment the reference count in it, so it cannot get evicted + // from the inner lru cache before it is used. + ac := newCloser(abs) + bc.cache.Add(key, abs) + return ac, nil +} + +// Remove removes the Accessor for a given key from the cache. +func (bc *AccessorCache) Remove(key shard.Key) error { + // The cache will call evictFn on removal, where accessor close will be called. + bc.cache.Remove(key) + return nil +} + +// EnableMetrics enables metrics for the cache. +func (bc *AccessorCache) EnableMetrics() error { + var err error + bc.metrics, err = newMetrics(bc) + return err +} + +// accessorCloser is a temporary object before reference counting is implemented. +type accessorCloser struct { + *accessorWithBlockstore + io.Closer +} + +func newCloser(abs *accessorWithBlockstore) *accessorCloser { + return &accessorCloser{ + accessorWithBlockstore: abs, + Closer: io.NopCloser(nil), + } +} + +// shardKeyToStriped returns the index of the lock to use for a given shard key. We use the last +// byte of the shard key as the pseudo-random index. +func shardKeyToStriped(sk shard.Key) byte { + return sk.String()[len(sk.String())-1] +} diff --git a/share/eds/cache/accessor_cache_test.go b/share/eds/cache/accessor_cache_test.go new file mode 100644 index 0000000000..5e928e85cc --- /dev/null +++ b/share/eds/cache/accessor_cache_test.go @@ -0,0 +1,246 @@ +package cache + +import ( + "bytes" + "context" + "errors" + "io" + "testing" + "time" + + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/dagstore/shard" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + "github.com/stretchr/testify/require" +) + +func TestAccessorCache(t *testing.T) { + t.Run("add / get item from cache", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{ + data: []byte("test_data"), + } + loaded, err := cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + + // check if item exists + got, err := cache.Get(key) + require.NoError(t, err) + + l, err := io.ReadAll(loaded.Reader()) + require.NoError(t, err) + require.Equal(t, mock.data, l) + g, err := io.ReadAll(got.Reader()) + require.NoError(t, err) + require.Equal(t, mock.data, g) + }) + + t.Run("get blockstore from accessor", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{} + accessor, err := cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + + // check if item exists + _, err = cache.Get(key) + require.NoError(t, err) + + // blockstore should be created only after first request + require.Equal(t, 0, mock.returnedBs) + + // try to get blockstore + _, err = accessor.Blockstore() + require.NoError(t, err) + + // second call to blockstore should return same blockstore + _, err = accessor.Blockstore() + require.NoError(t, err) + require.Equal(t, 1, mock.returnedBs) + }) + + t.Run("remove an item", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{} + ac, err := cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + err = ac.Close() + require.NoError(t, err) + + err = cache.Remove(key) + require.NoError(t, err) + + // accessor should be closed on removal + mock.checkClosed(t, true) + + // check if item exists + _, err = cache.Get(key) + require.ErrorIs(t, err, errCacheMiss) + }) + + t.Run("successive reads should read the same data", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{data: []byte("test")} + accessor, err := cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + + loaded, err := io.ReadAll(accessor.Reader()) + require.NoError(t, err) + require.Equal(t, mock.data, loaded) + + for i := 0; i < 2; i++ { + accessor, err = cache.Get(key) + require.NoError(t, err) + got, err := io.ReadAll(accessor.Reader()) + require.NoError(t, err) + require.Equal(t, mock.data, got) + } + }) + + t.Run("removed by eviction", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{} + ac1, err := cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + err = ac1.Close() + require.NoError(t, err) + + // add second item + key2 := shard.KeyFromString("key2") + ac2, err := cache.GetOrLoad(ctx, key2, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + err = ac2.Close() + require.NoError(t, err) + + // accessor should be closed on removal by eviction + mock.checkClosed(t, true) + + // check if item evicted + _, err = cache.Get(key) + require.ErrorIs(t, err, errCacheMiss) + }) + + t.Run("close on accessor is noop", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{} + _, err = cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + + // check if item exists + accessor, err := cache.Get(key) + require.NoError(t, err) + require.NotNil(t, accessor) + + // close on returned accessor should not close inner reader + err = accessor.Close() + require.NoError(t, err) + + // check that close was not performed on inner accessor + mock.checkClosed(t, false) + }) +} + +type mockAccessor struct { + data []byte + isClosed bool + returnedBs int +} + +func (m *mockAccessor) Reader() io.Reader { + return bytes.NewBuffer(m.data) +} + +func (m *mockAccessor) Blockstore() (dagstore.ReadBlockstore, error) { + if m.returnedBs > 0 { + return nil, errors.New("blockstore already returned") + } + m.returnedBs++ + return rbsMock{}, nil +} + +func (m *mockAccessor) Close() error { + if m.isClosed { + return errors.New("already closed") + } + m.isClosed = true + return nil +} + +func (m *mockAccessor) checkClosed(t *testing.T, expected bool) { + // item will be removed in background, so give it some time to settle + time.Sleep(time.Millisecond * 100) + require.Equal(t, expected, m.isClosed) +} + +// rbsMock is a dagstore.ReadBlockstore mock +type rbsMock struct{} + +func (r rbsMock) Has(context.Context, cid.Cid) (bool, error) { + panic("implement me") +} + +func (r rbsMock) Get(_ context.Context, _ cid.Cid) (blocks.Block, error) { + panic("implement me") +} + +func (r rbsMock) GetSize(context.Context, cid.Cid) (int, error) { + panic("implement me") +} + +func (r rbsMock) AllKeysChan(context.Context) (<-chan cid.Cid, error) { + panic("implement me") +} + +func (r rbsMock) HashOnRead(bool) { + panic("implement me") +} diff --git a/share/eds/cache/cache.go b/share/eds/cache/cache.go new file mode 100644 index 0000000000..13e207d7c0 --- /dev/null +++ b/share/eds/cache/cache.go @@ -0,0 +1,49 @@ +package cache + +import ( + "context" + "errors" + "io" + + "github.com/filecoin-project/dagstore" + "github.com/filecoin-project/dagstore/shard" + logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel" +) + +var ( + log = logging.Logger("share/eds/cache") + meter = otel.Meter("eds_store_cache") +) + +var ( + errCacheMiss = errors.New("accessor not found in blockstore cache") +) + +// Cache is an interface that defines the basic Cache operations. +type Cache interface { + // Get retrieves an item from the Cache. + Get(shard.Key) (Accessor, error) + + // GetOrLoad attempts to get an item from the Cache and, if not found, invokes + // the provided loader function to load it into the Cache. + GetOrLoad( + ctx context.Context, + key shard.Key, + loader func(context.Context, shard.Key) (Accessor, error), + ) (Accessor, error) + + // Remove removes an item from Cache. + Remove(shard.Key) error + + // EnableMetrics enables metrics in Cache + EnableMetrics() error +} + +// Accessor is a interface type returned by cache, that allows to read raw data by reader or create +// readblockstore +type Accessor interface { + Blockstore() (dagstore.ReadBlockstore, error) + Reader() io.Reader + io.Closer +} diff --git a/share/eds/cache/metrics.go b/share/eds/cache/metrics.go new file mode 100644 index 0000000000..21e52fec10 --- /dev/null +++ b/share/eds/cache/metrics.go @@ -0,0 +1,66 @@ +package cache + +import ( + "context" + + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/metric" +) + +const ( + cacheFoundKey = "found" + failedKey = "failed" +) + +type metrics struct { + getCounter metric.Int64Counter + evictedCounter metric.Int64Counter +} + +func newMetrics(bc *AccessorCache) (*metrics, error) { + evictedCounter, err := meter.Int64Counter("eds_blockstore_cache_"+bc.name+"_evicted_counter", + metric.WithDescription("eds blockstore cache evicted event counter")) + if err != nil { + return nil, err + } + + getCounter, err := meter.Int64Counter("eds_blockstore_cache_"+bc.name+"_get_counter", + metric.WithDescription("eds blockstore cache evicted event counter")) + if err != nil { + return nil, err + } + + cacheSize, err := meter.Int64ObservableGauge("eds_blockstore_cache_"+bc.name+"_size", + metric.WithDescription("total amount of items in blockstore cache"), + ) + if err != nil { + return nil, err + } + + callback := func(ctx context.Context, observer metric.Observer) error { + observer.ObserveInt64(cacheSize, int64(bc.cache.Len())) + return nil + } + _, err = meter.RegisterCallback(callback, cacheSize) + + return &metrics{ + getCounter: getCounter, + evictedCounter: evictedCounter, + }, err +} + +func (m *metrics) observeEvicted(failed bool) { + if m == nil { + return + } + m.evictedCounter.Add(context.Background(), 1, metric.WithAttributes( + attribute.Bool(failedKey, failed))) +} + +func (m *metrics) observeGet(found bool) { + if m == nil { + return + } + m.getCounter.Add(context.Background(), 1, metric.WithAttributes( + attribute.Bool(cacheFoundKey, found))) +} diff --git a/share/eds/cache/noop.go b/share/eds/cache/noop.go new file mode 100644 index 0000000000..2af94feb1b --- /dev/null +++ b/share/eds/cache/noop.go @@ -0,0 +1,31 @@ +package cache + +import ( + "context" + + "github.com/filecoin-project/dagstore/shard" +) + +var _ Cache = (*NoopCache)(nil) + +// NoopCache implements noop version of Cache interface +type NoopCache struct{} + +func (n NoopCache) Get(shard.Key) (Accessor, error) { + return nil, errCacheMiss +} + +func (n NoopCache) GetOrLoad( + context.Context, shard.Key, + func(context.Context, shard.Key) (Accessor, error), +) (Accessor, error) { + return nil, nil +} + +func (n NoopCache) Remove(shard.Key) error { + return nil +} + +func (n NoopCache) EnableMetrics() error { + return nil +} diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 9d4b2a53ef..8f87643f17 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -12,7 +12,6 @@ import ( const ( failedKey = "failed" sizeKey = "eds_size" - cachedKey = "cached" putResultKey = "result" putOK putResult = "ok" @@ -45,7 +44,6 @@ type metrics struct { getTime metric.Float64Histogram hasTime metric.Float64Histogram listTime metric.Float64Histogram - getAccessorTime metric.Float64Histogram shardFailureCount metric.Int64Counter @@ -102,12 +100,6 @@ func (s *Store) WithMetrics() error { return err } - getAccessorTime, err := meter.Float64Histogram("eds_store_get_accessor_time_histogram", - metric.WithDescription("eds store get accessor time histogram(s)")) - if err != nil { - return err - } - shardFailureCount, err := meter.Int64Counter("eds_store_shard_failure_counter", metric.WithDescription("eds store OpShardFail counter")) if err != nil { @@ -132,7 +124,7 @@ func (s *Store) WithMetrics() error { return err } - if err = s.cache.withMetrics(); err != nil { + if err = s.cache.EnableMetrics(); err != nil { return err } @@ -160,7 +152,6 @@ func (s *Store) WithMetrics() error { getTime: getTime, hasTime: hasTime, listTime: listTime, - getAccessorTime: getAccessorTime, shardFailureCount: shardFailureCount, longOpTime: longOpTime, gcTime: gcTime, @@ -299,16 +290,3 @@ func (m *metrics) observeList(ctx context.Context, dur time.Duration, failed boo m.listTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) } - -func (m *metrics) observeGetAccessor(ctx context.Context, dur time.Duration, cached, failed bool) { - if m == nil { - return - } - if ctx.Err() != nil { - ctx = context.Background() - } - - m.getAccessorTime.Record(ctx, dur.Seconds(), metric.WithAttributes( - attribute.Bool(cachedKey, cached), - attribute.Bool(failedKey, failed))) -} diff --git a/share/eds/ods_test.go b/share/eds/ods_test.go index 5b6ed5568b..0f7c69e708 100644 --- a/share/eds/ods_test.go +++ b/share/eds/ods_test.go @@ -32,6 +32,9 @@ func TestODSReader(t *testing.T) { // get CAR reader from store r, err := edsStore.GetCAR(ctx, dah.Hash()) assert.NoError(t, err) + defer func() { + require.NoError(t, r.Close()) + }() // create ODSReader wrapper based on car reader to limit reads to ODS only odsR, err := ODSReader(r) @@ -81,6 +84,9 @@ func TestODSReaderReconstruction(t *testing.T) { // get CAR reader from store r, err := edsStore.GetCAR(ctx, dah.Hash()) assert.NoError(t, err) + defer func() { + require.NoError(t, r.Close()) + }() // create ODSReader wrapper based on car reader to limit reads to ODS only odsR, err := ODSReader(r) diff --git a/share/eds/store.go b/share/eds/store.go index 974f147292..e8caf4c35a 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -26,6 +26,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/cache" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -38,6 +39,8 @@ const ( // shards that are currently available but inactive, or errored. // We don't use transient files right now, so GC is turned off by default. defaultGCInterval = 0 + + defaultCacheSize = 128 ) var ErrNotFound = errors.New("eds not found in store") @@ -52,8 +55,8 @@ type Store struct { dgstr *dagstore.DAGStore mounts *mount.Registry - cache *blockstoreCache - bs bstore.Blockstore + bs *blockstore + cache cache.Cache carIdx index.FullIndexRepo invertedIdx *simpleInvertedIndex @@ -111,9 +114,9 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { return nil, fmt.Errorf("failed to create DAGStore: %w", err) } - cache, err := newBlockstoreCache(defaultCacheSize) + accessorCache, err := cache.NewAccessorCache("cache", defaultCacheSize) if err != nil { - return nil, fmt.Errorf("failed to create blockstore cache: %w", err) + return nil, fmt.Errorf("failed to create recent blocks cache: %w", err) } store := &Store{ @@ -124,9 +127,9 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { gcInterval: defaultGCInterval, mounts: r, shardFailures: failureChan, - cache: cache, + cache: accessorCache, } - store.bs = newBlockstore(store, cache, ds) + store.bs = newBlockstore(store, ds) return store, nil } @@ -177,7 +180,6 @@ func (s *Store) gc(ctx context.Context) { } s.lastGCResult.Store(res) } - } } @@ -237,7 +239,7 @@ func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext if err != nil { return err } - defer f.Close() + defer closeAndLog("car file", f) // save encoded eds into buffer mount := &inMemoryOnceMount{ @@ -261,17 +263,35 @@ func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext return fmt.Errorf("failed to initiate shard registration: %w", err) } + var result dagstore.ShardResult select { + case result = <-ch: case <-ctx.Done(): - // if context finished before result was received, track result in separate goroutine + // if the context finished before the result was received, track the result in a separate goroutine go trackLateResult("put", ch, s.metrics, time.Minute*5) return ctx.Err() - case result := <-ch: - if result.Error != nil { - return fmt.Errorf("failed to register shard: %w", result.Error) - } - return nil } + + if result.Error != nil { + return fmt.Errorf("failed to register shard: %w", result.Error) + } + + // the accessor returned in the result will be nil, so the shard needs to be acquired first to + // become available in the cache. It might take some time, and the result should not affect the put + // operation, so do it in a goroutine + // TODO: Ideally, only recent blocks should be put in the cache, but there is no way right now to + // check such a condition. + go func() { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer cancel() + _, err := s.cache.GetOrLoad(ctx, result.Key, s.getAccessor) + if err != nil { + log.Warnw("unable to put accessor to recent blocks accessors cache", "err", err) + return + } + }() + + return nil } // waitForResult waits for a result from the res channel for a maximum duration specified by @@ -292,7 +312,7 @@ func trackLateResult(opName string, res <-chan dagstore.ShardResult, metrics *me } if result.Error != nil { metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpFailed) - log.Errorf("failed to register shard after context expired: %v ago, err: %w", time.Since(tnow), result.Error) + log.Errorf("failed to register shard after context expired: %v ago, err: %s", time.Since(tnow), result.Error) return } metrics.observeLongOp(context.Background(), opName, time.Since(tnow), longOpOK) @@ -309,7 +329,7 @@ func trackLateResult(opName string, res <-chan dagstore.ShardResult, metrics *me // // The shard is cached in the Store, so subsequent calls to GetCAR with the same root will use the // same reader. The cache is responsible for closing the underlying reader. -func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.Reader, error) { +func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.ReadCloser, error) { ctx, span := tracer.Start(ctx, "store/get-car") tnow := time.Now() r, err := s.getCAR(ctx, root) @@ -318,13 +338,21 @@ func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.Reader, err return r, err } -func (s *Store) getCAR(ctx context.Context, root share.DataHash) (io.Reader, error) { - key := root.String() - accessor, err := s.getCachedAccessor(ctx, shard.KeyFromString(key)) +func (s *Store) getCAR(ctx context.Context, root share.DataHash) (io.ReadCloser, error) { + key := shard.KeyFromString(root.String()) + accessor, err := s.cache.Get(key) + if err == nil { + return newReadCloser(accessor), nil + } + // If the accessor is not found in the cache, create a new one from dagstore. We don't put the + // accessor in the cache here because getCAR is used by shrex-eds. There is a lower probability, + // compared to other cache put triggers, that the same block will be requested again soon. + shardAccessor, err := s.getAccessor(ctx, key) if err != nil { return nil, fmt.Errorf("failed to get accessor: %w", err) } - return accessor.sa.Reader(), nil + + return newReadCloser(shardAccessor), nil } // Blockstore returns an IPFS blockstore providing access to individual shares/nodes of all EDS @@ -342,25 +370,31 @@ func (s *Store) Blockstore() bstore.Blockstore { func (s *Store) CARBlockstore( ctx context.Context, root share.DataHash, -) (dagstore.ReadBlockstore, error) { +) (*BlockstoreCloser, error) { ctx, span := tracer.Start(ctx, "store/car-blockstore") tnow := time.Now() - r, err := s.carBlockstore(ctx, root) + cbs, err := s.carBlockstore(ctx, root) s.metrics.observeCARBlockstore(ctx, time.Since(tnow), err != nil) utils.SetStatusAndEnd(span, err) - return r, err + return cbs, err } func (s *Store) carBlockstore( ctx context.Context, root share.DataHash, -) (dagstore.ReadBlockstore, error) { +) (*BlockstoreCloser, error) { key := shard.KeyFromString(root.String()) - accessor, err := s.getCachedAccessor(ctx, key) + accessor, err := s.cache.Get(key) + if err == nil { + return blockstoreCloser(accessor) + } + + // if the accessor is not found in the cache, create a new one from dagstore + sa, err := s.getAccessor(ctx, key) if err != nil { - return nil, fmt.Errorf("eds/store: failed to get accessor: %w", err) + return nil, fmt.Errorf("failed to get accessor: %w", err) } - return accessor.bs, nil + return blockstoreCloser(sa) } // GetDAH returns the DataAvailabilityHeader for the EDS identified by DataHash. @@ -374,13 +408,13 @@ func (s *Store) GetDAH(ctx context.Context, root share.DataHash) (*share.Root, e } func (s *Store) getDAH(ctx context.Context, root share.DataHash) (*share.Root, error) { - key := shard.KeyFromString(root.String()) - accessor, err := s.getCachedAccessor(ctx, key) + r, err := s.getCAR(ctx, root) if err != nil { - return nil, fmt.Errorf("eds/store: failed to get accessor: %w", err) + return nil, fmt.Errorf("eds/store: failed to get CAR file: %w", err) } + defer closeAndLog("car reader", r) - carHeader, err := carv1.ReadHeader(bufio.NewReader(accessor.sa.Reader())) + carHeader, err := carv1.ReadHeader(bufio.NewReader(r)) if err != nil { return nil, fmt.Errorf("eds/store: failed to read car header: %w", err) } @@ -405,7 +439,7 @@ func dahFromCARHeader(carHeader *carv1.CarHeader) *share.Root { } } -func (s *Store) getAccessor(ctx context.Context, key shard.Key) (*dagstore.ShardAccessor, error) { +func (s *Store) getAccessor(ctx context.Context, key shard.Key) (cache.Accessor, error) { ch := make(chan dagstore.ShardResult, 1) err := s.dgstr.AcquireShard(ctx, key, ch, dagstore.AcquireOpts{}) if err != nil { @@ -427,33 +461,6 @@ func (s *Store) getAccessor(ctx context.Context, key shard.Key) (*dagstore.Shard } } -func (s *Store) getCachedAccessor(ctx context.Context, key shard.Key) (*accessorWithBlockstore, error) { - lk := &s.cache.stripedLocks[shardKeyToStriped(key)] - lk.Lock() - defer lk.Unlock() - - tnow := time.Now() - accessor, err := s.cache.unsafeGet(key) - if err != nil && err != errCacheMiss { - log.Errorf("unexpected error while reading key from bs cache %s: %s", key, err) - } - if accessor != nil { - s.metrics.observeGetAccessor(ctx, time.Since(tnow), true, false) - return accessor, nil - } - - // wasn't found in cache, so acquire it and add to cache - shardAccessor, err := s.getAccessor(ctx, key) - if err != nil { - s.metrics.observeGetAccessor(ctx, time.Since(tnow), false, err != nil) - return nil, err - } - - a, err := s.cache.unsafeAdd(key, shardAccessor) - s.metrics.observeGetAccessor(ctx, time.Since(tnow), false, err != nil) - return a, err -} - // Remove removes EDS from Store by the given share.Root hash and cleans up all // the indexing. func (s *Store) Remove(ctx context.Context, root share.DataHash) error { @@ -466,14 +473,13 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) error { } func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { - key := root.String() - - // Remove from accessor cache, so that existing readers are closed and - // DestroyShard can be executed. - s.cache.Remove(shard.KeyFromString(key)) - + key := shard.KeyFromString(root.String()) + // remove open links to accessor from cache + if err := s.cache.Remove(key); err != nil { + log.Warnw("remove accessor from cache", "err", err) + } ch := make(chan dagstore.ShardResult, 1) - err = s.dgstr.DestroyShard(ctx, shard.KeyFromString(key), ch, dagstore.DestroyOpts{}) + err = s.dgstr.DestroyShard(ctx, key, ch, dagstore.DestroyOpts{}) if err != nil { return fmt.Errorf("failed to initiate shard destruction: %w", err) } @@ -488,7 +494,7 @@ func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { return ctx.Err() } - dropped, err := s.carIdx.DropFullIndex(shard.KeyFromString(key)) + dropped, err := s.carIdx.DropFullIndex(key) if !dropped { log.Warnf("failed to drop index for %s", key) } @@ -496,7 +502,7 @@ func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { return fmt.Errorf("failed to drop index for %s: %w", key, err) } - err = os.Remove(s.basepath + blocksPath + key) + err = os.Remove(s.basepath + blocksPath + root.String()) if err != nil { return fmt.Errorf("failed to remove CAR file: %w", err) } @@ -522,11 +528,13 @@ func (s *Store) get(ctx context.Context, root share.DataHash) (eds *rsmt2d.Exten utils.SetStatusAndEnd(span, err) }() - f, err := s.GetCAR(ctx, root) + r, err := s.getCAR(ctx, root) if err != nil { return nil, fmt.Errorf("failed to get CAR file: %w", err) } - eds, err = ReadEDS(ctx, f, root) + defer closeAndLog("car reader", r) + + eds, err = ReadEDS(ctx, r, root) if err != nil { return nil, fmt.Errorf("failed to read EDS from CAR file: %w", err) } diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 21239e320a..0d5283e2f2 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -2,6 +2,7 @@ package eds import ( "context" + "io" "os" "sync" "testing" @@ -9,6 +10,7 @@ import ( "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" + "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/ipld/go-car" @@ -19,6 +21,7 @@ import ( "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/cache" "github.com/celestiaorg/celestia-node/share/eds/edstest" ) @@ -72,6 +75,9 @@ func TestEDSStore(t *testing.T) { r, err := edsStore.GetCAR(ctx, dah.Hash()) assert.NoError(t, err) + defer func() { + require.NoError(t, r.Close()) + }() carReader, err := car.NewCarReader(r) assert.NoError(t, err) @@ -107,6 +113,9 @@ func TestEDSStore(t *testing.T) { _, err = os.Stat(edsStore.basepath + blocksPath + dah.String()) assert.NoError(t, err) + // accessor will be registered in cache async on put, so give it some time to settle + time.Sleep(time.Millisecond * 100) + err = edsStore.Remove(ctx, dah.Hash()) assert.NoError(t, err) @@ -143,6 +152,13 @@ func TestEDSStore(t *testing.T) { err = os.Remove(path) assert.NoError(t, err) + // accessor will be registered in cache async on put, so give it some time to settle + time.Sleep(time.Millisecond * 100) + + // remove non-failed accessor from cache + err = edsStore.cache.Remove(shard.KeyFromString(dah.String())) + assert.NoError(t, err) + _, err = edsStore.GetCAR(ctx, dah.Hash()) assert.Error(t, err) @@ -177,22 +193,18 @@ func TestEDSStore(t *testing.T) { assert.True(t, ok) }) - t.Run("BlockstoreCache", func(t *testing.T) { + t.Run("RecentBlocksCache", func(t *testing.T) { eds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(t, err) - // key isnt in cache yet, so get returns errCacheMiss + // accessor will be registered in cache async on put, so give it some time to settle + time.Sleep(time.Millisecond * 100) + + // check, that the key is in the cache after put shardKey := shard.KeyFromString(dah.String()) _, err = edsStore.cache.Get(shardKey) - assert.ErrorIs(t, err, errCacheMiss) - - // now get it, so that the key is in the cache - _, err = edsStore.CARBlockstore(ctx, dah.Hash()) assert.NoError(t, err) - _, err = edsStore.cache.Get(shardKey) - assert.NoError(t, err, errCacheMiss) }) t.Run("List", func(t *testing.T) { @@ -256,6 +268,14 @@ func TestEDSStore_GC(t *testing.T) { err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(t, err) + // accessor will be registered in cache async on put, so give it some time to settle + time.Sleep(time.Millisecond * 100) + + // remove links to the shard from cache + key := shard.KeyFromString(share.DataHash(dah.Hash()).String()) + err = edsStore.cache.Remove(key) + require.NoError(t, err) + // doesn't exist yet assert.NotContains(t, edsStore.lastGCResult.Load().Shards, shardKey) @@ -281,20 +301,43 @@ func Test_BlockstoreCache(t *testing.T) { err = edsStore.Start(ctx) require.NoError(t, err) + // store eds to the store with noopCache to allow clean cache after put + swap := edsStore.cache + edsStore.cache = cache.NoopCache{} eds, dah := randomEDS(t) err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(t, err) - // key isnt in cache yet, so get returns errCacheMiss + // get any key from saved eds + bs, err := edsStore.carBlockstore(ctx, dah.Hash()) + require.NoError(t, err) + defer func() { + require.NoError(t, bs.Close()) + }() + keys, err := bs.AllKeysChan(ctx) + require.NoError(t, err) + var key cid.Cid + select { + case key = <-keys: + case <-ctx.Done(): + t.Fatal("context timeout") + } + + // swap back original cache + edsStore.cache = swap + + // key shouldn't be in cache yet, check for returned errCacheMiss shardKey := shard.KeyFromString(dah.String()) _, err = edsStore.cache.Get(shardKey) - assert.ErrorIs(t, err, errCacheMiss) + require.Error(t, err) - // now get it, so that the key is in the cache - _, err = edsStore.getCachedAccessor(ctx, shardKey) - assert.NoError(t, err) + // now get it from blockstore, to trigger storing to cache + _, err = edsStore.Blockstore().Get(ctx, key) + require.NoError(t, err) + + // should be no errCacheMiss anymore _, err = edsStore.cache.Get(shardKey) - assert.NoError(t, err, errCacheMiss) + require.NoError(t, err) } // Test_CachedAccessor verifies that the reader represented by a cached accessor can be read from @@ -312,26 +355,67 @@ func Test_CachedAccessor(t *testing.T) { err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(t, err) - shardKey := shard.KeyFromString(dah.String()) - // adds to cache - cachedAccessor, err := edsStore.getCachedAccessor(ctx, shardKey) - assert.NoError(t, err) - - // first read - carReader, err := car.NewCarReader(cachedAccessor.sa.Reader()) - assert.NoError(t, err) - firstBlock, err := carReader.Next() - assert.NoError(t, err) - - // second read - cachedAccessor, err = edsStore.getCachedAccessor(ctx, shardKey) - assert.NoError(t, err) - carReader, err = car.NewCarReader(cachedAccessor.sa.Reader()) - assert.NoError(t, err) - secondBlock, err := carReader.Next() - assert.NoError(t, err) - - assert.Equal(t, firstBlock, secondBlock) + // accessor will be registered in cache async on put, so give it some time to settle + time.Sleep(time.Millisecond * 100) + + // accessor should be in cache + cachedAccessor, err := edsStore.cache.Get(shard.KeyFromString(dah.String())) + require.NoError(t, err) + + // first read from cached accessor + firstBlock, err := io.ReadAll(cachedAccessor.Reader()) + require.NoError(t, err) + require.NoError(t, cachedAccessor.Close()) + + // second read from cached accessor + cachedAccessor, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) + require.NoError(t, err) + secondBlock, err := io.ReadAll(cachedAccessor.Reader()) + require.NoError(t, err) + require.NoError(t, cachedAccessor.Close()) + + require.Equal(t, firstBlock, secondBlock) +} + +// Test_CachedAccessor verifies that the reader represented by a accessor obtained directly from +// dagstore can be read from multiple times, without exhausting the underlying reader. +func Test_NotCachedAccessor(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + + edsStore, err := newStore(t) + require.NoError(t, err) + err = edsStore.Start(ctx) + require.NoError(t, err) + // replace cache with noopCache to + edsStore.cache = cache.NoopCache{} + + eds, dah := randomEDS(t) + err = edsStore.Put(ctx, dah.Hash(), eds) + require.NoError(t, err) + + // accessor will be registered in cache async on put, so give it some time to settle + time.Sleep(time.Millisecond * 100) + + // accessor should be in cache + _, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) + require.Error(t, err) + + // first read from direct accessor + carReader, err := edsStore.getCAR(ctx, dah.Hash()) + require.NoError(t, err) + firstBlock, err := io.ReadAll(carReader) + require.NoError(t, err) + require.NoError(t, carReader.Close()) + + // second read from direct accessor + carReader, err = edsStore.getCAR(ctx, dah.Hash()) + require.NoError(t, err) + secondBlock, err := io.ReadAll(carReader) + require.NoError(t, err) + require.NoError(t, carReader.Close()) + + require.Equal(t, firstBlock, secondBlock) } func BenchmarkStore(b *testing.B) { diff --git a/share/eds/utils.go b/share/eds/utils.go new file mode 100644 index 0000000000..e7b24a9aee --- /dev/null +++ b/share/eds/utils.go @@ -0,0 +1,44 @@ +package eds + +import ( + "io" + + "github.com/filecoin-project/dagstore" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" + + "github.com/celestiaorg/celestia-node/share/eds/cache" +) + +// readCloser is a helper struct, that combines io.Reader and io.Closer +type readCloser struct { + io.Reader + io.Closer +} + +// BlockstoreCloser represents a blockstore that can also be closed. It combines the functionality +// of a dagstore.ReadBlockstore with that of an io.Closer. +type BlockstoreCloser struct { + dagstore.ReadBlockstore + io.Closer +} + +func newReadCloser(ac cache.Accessor) io.ReadCloser { + return readCloser{ + ac.Reader(), + ac, + } +} + +func newBlockstore(store *Store, ds datastore.Batching) *blockstore { + return &blockstore{ + store: store, + ds: namespace.Wrap(ds, blockstoreCacheKey), + } +} + +func closeAndLog(name string, closer io.Closer) { + if err := closer.Close(); err != nil { + log.Warnw("closing "+name, "err", err) + } +} diff --git a/share/getters/store.go b/share/getters/store.go index 5eca956faa..415c9f047f 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -58,6 +58,11 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve blockstore: %w", err) } + defer func() { + if err := bs.Close(); err != nil { + log.Warnw("closing blockstore", "err", err) + } + }() // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) @@ -117,6 +122,11 @@ func (sg *StoreGetter) GetSharesByNamespace( if err != nil { return nil, fmt.Errorf("getter/store: failed to retrieve blockstore: %w", err) } + defer func() { + if err := bs.Close(); err != nil { + log.Warnw("closing blockstore", "err", err) + } + }() // wrap the read-only CAR blockstore in a getter blockGetter := eds.NewBlockGetter(bs) diff --git a/share/p2p/shrexeds/server.go b/share/p2p/shrexeds/server.go index fffa0e8152..11b99a3438 100644 --- a/share/p2p/shrexeds/server.go +++ b/share/p2p/shrexeds/server.go @@ -100,8 +100,15 @@ func (s *Server) handleStream(stream network.Stream) { // we do not close the reader, so that other requests will not need to re-open the file. // closing is handled by the LRU cache. edsReader, err := s.store.GetCAR(ctx, hash) - status := p2p_pb.Status_OK + var status p2p_pb.Status switch { + case err == nil: + defer func() { + if err := edsReader.Close(); err != nil { + log.Warnw("closing car reader", "err", err) + } + }() + status = p2p_pb.Status_OK case errors.Is(err, eds.ErrNotFound): logger.Warnw("server: request hash not found") s.metrics.ObserveRequests(ctx, 1, p2p.StatusNotFound) From dd15372963b17443417ccefa692e602991ca1015 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 14:21:01 +0200 Subject: [PATCH 201/388] chore(deps): Bump actions/checkout from 3 to 4 (#2718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/checkout](https://github.com/actions/checkout) from 3 to 4.
Release notes

Sourced from actions/checkout's releases.

v4.0.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v4.0.0

v3.6.0

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3.5.3...v3.6.0

v3.5.3

What's Changed

New Contributors

Full Changelog: https://github.com/actions/checkout/compare/v3...v3.5.3

v3.5.2

What's Changed

Full Changelog: https://github.com/actions/checkout/compare/v3.5.1...v3.5.2

v3.5.1

What's Changed

New Contributors

... (truncated)

Changelog

Sourced from actions/checkout's changelog.

Changelog

v4.0.0

v3.6.0

v3.5.3

v3.5.2

v3.5.1

v3.5.0

v3.4.0

v3.3.0

v3.2.0

v3.1.0

v3.0.2

v3.0.1

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/checkout&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 841db11bfa..6e9293cc38 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -82,7 +82,7 @@ jobs: (github.event_name == 'push' && contains(github.ref, 'refs/tags/')) permissions: "write-all" steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - run: git fetch --force --tags - uses: actions/setup-go@v4 with: From 319674645c05b7965543f23b403952b9e49ecdd4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:36:01 +0000 Subject: [PATCH 202/388] chore(deps): Bump go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp from 0.40.0 to 0.41.0 (#2713) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp](https://github.com/open-telemetry/opentelemetry-go) from 0.40.0 to 0.41.0.
Release notes

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's releases.

Release v1.19.0-rc.1/v0.42.0-rc.1

This is a release candidate for the v1.19.0/v0.42.0 release. That release is expected to include the v1 release of the OpenTelemetry Go metric SDK and will provide stability guarantees of that SDK. See our versioning policy for more information about these stability guarantees.

Changed

  • Allow '/' characters in metric instrument names. (#4501)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the Prometheus metric on every Collect if we know the scope is invalid. (#4499)

New Contributors

Full Changelog: https://github.com/open-telemetry/opentelemetry-go/compare/v1.18.0...v1.19.0-rc.1

Release v1.18.0/v0.41.0/v0.0.6

This release drops the compatibility guarantee of Go 1.19.

Added

  • Add WithProducer option in go.opentelemetry.op/otel/exporters/prometheus to restore the ability to register producers on the prometheus exporter's manual reader. (#4473)
  • Add IgnoreValue option in go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest to allow ignoring values when comparing metrics. (#4447)

Deprecated

  • The NewMetricExporter in go.opentelemetry.io/otel/bridge/opencensus was deprecated in v0.35.0 (#3541). The deprecation notice format for the function has been corrected to trigger Go documentation and build tooling. (#4470)

Removed

  • Removed the deprecated go.opentelemetry.io/otel/exporters/jaeger package. (#4467)
  • Removed the deprecated go.opentelemetry.io/otel/example/jaeger package. (#4467)
  • Removed the deprecated go.opentelemetry.io/otel/sdk/metric/aggregation package. (#4468)
  • Removed the deprecated internal packages in go.opentelemetry.io/otel/exporters/otlp and its sub-packages. (#4469)
  • Dropped guaranteed support for versions of Go less than 1.20. (#4481)

New Contributors

Full Changelog: https://github.com/open-telemetry/opentelemetry-go/compare/v1.17.0...v1.18.0

Release v1.17.0/v0.40.0/v0.5.0

Added

  • Export the ManualReader struct in go.opentelemetry.io/otel/sdk/metric. (#4244)
  • Export the PeriodicReader struct in go.opentelemetry.io/otel/sdk/metric. (#4244)

... (truncated)

Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's changelog.

[1.18.0/0.41.0/0.0.6] 2023-09-12

This release drops the compatibility guarantee of [Go 1.19].

Added

  • Add WithProducer option in go.opentelemetry.op/otel/exporters/prometheus to restore the ability to register producers on the prometheus exporter's manual reader. (#4473)
  • Add IgnoreValue option in go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest to allow ignoring values when comparing metrics. (#4447)

Changed

  • Use a TestingT interface instead of *testing.T struct in go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest. (#4483)

Deprecated

  • The NewMetricExporter in go.opentelemetry.io/otel/bridge/opencensus was deprecated in v0.35.0 (#3541). The deprecation notice format for the function has been corrected to trigger Go documentation and build tooling. (#4470)

Removed

  • Removed the deprecated go.opentelemetry.io/otel/exporters/jaeger package. (#4467)
  • Removed the deprecated go.opentelemetry.io/otel/example/jaeger package. (#4467)
  • Removed the deprecated go.opentelemetry.io/otel/sdk/metric/aggregation package. (#4468)
  • Removed the deprecated internal packages in go.opentelemetry.io/otel/exporters/otlp and its sub-packages. (#4469)
  • Dropped guaranteed support for versions of Go less than 1.20. (#4481)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp&package-manager=go_modules&previous-version=0.40.0&new-version=0.41.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Vlad <13818348+walldiss@users.noreply.github.com> --- go.mod | 18 +++++++++--------- go.sum | 36 ++++++++++++++++++------------------ 2 files changed, 27 insertions(+), 27 deletions(-) diff --git a/go.mod b/go.mod index d3a5771c23..8cda25d380 100644 --- a/go.mod +++ b/go.mod @@ -57,14 +57,14 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 - go.opentelemetry.io/otel v1.17.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 + go.opentelemetry.io/otel v1.18.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0 - go.opentelemetry.io/otel/metric v1.17.0 - go.opentelemetry.io/otel/sdk v1.17.0 - go.opentelemetry.io/otel/sdk/metric v0.40.0 - go.opentelemetry.io/otel/trace v1.17.0 + go.opentelemetry.io/otel/metric v1.18.0 + go.opentelemetry.io/otel/sdk v1.18.0 + go.opentelemetry.io/otel/sdk/metric v0.41.0 + go.opentelemetry.io/otel/trace v1.18.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 go.uber.org/zap v1.25.0 @@ -72,7 +72,7 @@ require ( golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 - google.golang.org/grpc v1.57.0 + google.golang.org/grpc v1.58.0 google.golang.org/protobuf v1.31.0 ) @@ -306,14 +306,14 @@ require ( github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.14.0 // indirect - golang.org/x/oauth2 v0.8.0 // indirect + golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/term v0.12.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect diff --git a/go.sum b/go.sum index e29fef53dc..354b14ac32 100644 --- a/go.sum +++ b/go.sum @@ -2380,12 +2380,12 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.17.0 h1:MW+phZ6WZ5/uk2nd93ANk/6yJ+dVrvNWUjGhnnFU5jM= -go.opentelemetry.io/otel v1.17.0/go.mod h1:I2vmBGtFaODIVMBSTPVDlJSzBDNf93k60E6Ft0nyjo0= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0 h1:MZbjiZeMmn5wFMORhozpouGKDxj9POHTuU5UA8msBQk= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.40.0/go.mod h1:C7tOYVCJmrDTCwxNny0MuUtnDIR3032vFHYke0F2ZrU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0 h1:SZaSbubADNhH2Gxm+1GaZ/cFsGiYefZoodMMX79AOd4= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.40.0/go.mod h1:N65FzQDfQH7NY7umgb0U+7ypGKVYKwwE24L6KXT4OA8= +go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= +go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0 h1:k0k7hFNDd8K4iOMJXj7s8sHaC4mhTlAeppRmZXLgZ6k= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 h1:iV3BOgW4fry1Riw9dwypigqlIYWXvSRVT2RJmblzo40= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0/go.mod h1:7PGzqlKrxIRmbj5tlNW0nTkYZ5fHXDgk6Fy8/KjR0CI= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 h1:U5GYackKpVKlPrd/5gKMlrTlP2dCESAAFU682VCpieY= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0/go.mod h1:aFsJfCEnLzEu9vRRAcUiB/cpRTbVsNdF3OHSPpdjxZQ= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0 h1:kvWMtSUNVylLVrOE4WLUmBtgziYoCIYUNSpTYtMzVJI= @@ -2393,20 +2393,20 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0/go.mod h go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.17.0 h1:iG6LGVz5Gh+IuO0jmgvpTB6YVrCGngi8QGm+pMd8Pdc= -go.opentelemetry.io/otel/metric v1.17.0/go.mod h1:h4skoxdZI17AxwITdmdZjjYJQH5nzijUUjm+wtPph5o= +go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= +go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.17.0 h1:FLN2X66Ke/k5Sg3V623Q7h7nt3cHXaW1FOvKKrW0IpE= -go.opentelemetry.io/otel/sdk v1.17.0/go.mod h1:U87sE0f5vQB7hwUoW98pW5Rz4ZDuCFBZFNUBlSgmDFQ= -go.opentelemetry.io/otel/sdk/metric v0.40.0 h1:qOM29YaGcxipWjL5FzpyZDpCYrDREvX0mVlmXdOjCHU= -go.opentelemetry.io/otel/sdk/metric v0.40.0/go.mod h1:dWxHtdzdJvg+ciJUKLTKwrMe5P6Dv3FyDbh8UkfgkVs= +go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY= +go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M= +go.opentelemetry.io/otel/sdk/metric v0.41.0 h1:c3sAt9/pQ5fSIUfl0gPtClV3HhE18DCVzByD33R/zsk= +go.opentelemetry.io/otel/sdk/metric v0.41.0/go.mod h1:PmOmSt+iOklKtIg5O4Vz9H/ttcRFSNTgii+E1KGyn1w= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.17.0 h1:/SWhSRHmDPOImIAetP1QAeMnZYiQXrTy4fMMYOdSKWQ= -go.opentelemetry.io/otel/trace v1.17.0/go.mod h1:I/4vKTgFclIsXRVucpH25X0mpFSczM7aHeaz0ZBLWjY= +go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= +go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= @@ -2687,8 +2687,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.8.0 h1:6dkIjl3j3LtZ/O3sTgZTMsLKSftL/B8Zgq4huOIIUu8= -golang.org/x/oauth2 v0.8.0/go.mod h1:yr7u4HXZRm1R1kBWqr/xKNqewf0plRYoB7sla+BCIXE= +golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= +golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -3209,8 +3209,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.57.0 h1:kfzNeI/klCGD2YPMUlaGNT3pxvYfga7smW3Vth8Zsiw= -google.golang.org/grpc v1.57.0/go.mod h1:Sd+9RMTACXwmub0zcNY2c4arhtrbBYD1AUHI/dt16Mo= +google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= +google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From fd90764a0e2b939036ddf5a9fdd488db6f3a1aa4 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 12:40:18 +0000 Subject: [PATCH 203/388] chore(deps): Bump github.com/hashicorp/golang-lru/v2 from 2.0.5 to 2.0.6 (#2711) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/hashicorp/golang-lru/v2](https://github.com/hashicorp/golang-lru) from 2.0.5 to 2.0.6.
Release notes

Sourced from github.com/hashicorp/golang-lru/v2's releases.

golang-lru 2.0.6

This release removes calling the eviction callback when the Add method is called with an item that is already in the cache (#154); it reverts PR #135 which caused issue #141.

What's Changed

Full Changelog: https://github.com/hashicorp/golang-lru/compare/v2.0.5...v2.0.6

Commits
  • 8f8764f test: Add tests for eviction callback
  • 0ed35fd Revert "call evict on replacing value (#135)"
  • 1e956f5 [COMPLIANCE] Add Copyright and License Headers
  • 3afaa48 Release update arc dependencies to v2.0.5
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/golang-lru/v2&package-manager=go_modules&previous-version=2.0.5&new-version=2.0.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 8cda25d380..74f49bec0e 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-retryablehttp v0.7.4 - github.com/hashicorp/golang-lru/v2 v2.0.5 + github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.12.0 github.com/ipfs/go-block-format v0.2.0 diff --git a/go.sum b/go.sum index 354b14ac32..8363974ccb 100644 --- a/go.sum +++ b/go.sum @@ -980,8 +980,8 @@ github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/golang-lru/v2 v2.0.5 h1:wW7h1TG88eUIJ2i69gaE3uNVtEPIagzhGvHgwfx2Vm4= -github.com/hashicorp/golang-lru/v2 v2.0.5/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= +github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= From 288e9925a90c5840db440aabc343e364beee8240 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 18 Sep 2023 21:10:50 +0800 Subject: [PATCH 204/388] feat(share/store/cache): Split accessor cache into recent and blockstore (#2656) Adds MultiCache. Replaces eds store cache with multicache containing 2 separate caches: - recently added edses (10 accessors, added on put only) - edses requested by ipld (128 accessors, added on request to blockstore from bitswap server) --- share/eds/blockstore.go | 31 +++++++++------------ share/eds/cache/doublecache.go | 51 ++++++++++++++++++++++++++++++++++ share/eds/store.go | 16 +++++++---- share/eds/store_test.go | 4 +-- share/eds/utils.go | 16 +++++++---- 5 files changed, 87 insertions(+), 31 deletions(-) create mode 100644 share/eds/cache/doublecache.go diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go index 349d6f58ba..9cbb3f4e8a 100644 --- a/share/eds/blockstore.go +++ b/share/eds/blockstore.go @@ -10,9 +10,8 @@ import ( blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" ipld "github.com/ipfs/go-ipld-format" - - "github.com/celestiaorg/celestia-node/share/eds/cache" ) var _ bstore.Blockstore = (*blockstore)(nil) @@ -35,6 +34,13 @@ type blockstore struct { ds datastore.Batching } +func newBlockstore(store *Store, ds datastore.Batching) *blockstore { + return &blockstore{ + store: store, + ds: namespace.Wrap(ds, blockstoreCacheKey), + } +} + func (bs *blockstore) Has(ctx context.Context, cid cid.Cid) (bool, error) { keys, err := bs.store.dgstr.ShardsContainingMultihash(ctx, cid.Hash()) if errors.Is(err, ErrNotFound) || errors.Is(err, ErrNotFoundInIndex) { @@ -146,28 +152,17 @@ func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (* return nil, fmt.Errorf("failed to find shards containing multihash: %w", err) } - // check if cache contains any of accessors + // check if either cache contains an accessor shardKey := keys[0] - if accessor, err := bs.store.cache.Get(shardKey); err == nil { + accessor, err := bs.store.cache.Get(shardKey) + if err == nil { return blockstoreCloser(accessor) } - // load accessor to the cache and use it as blockstoreCloser - accessor, err := bs.store.cache.GetOrLoad(ctx, shardKey, bs.store.getAccessor) + // load accessor to the blockstore cache and use it as blockstoreCloser + accessor, err = bs.store.cache.Second().GetOrLoad(ctx, shardKey, bs.store.getAccessor) if err != nil { return nil, fmt.Errorf("failed to get accessor for shard %s: %w", shardKey, err) } return blockstoreCloser(accessor) } - -// blockstoreCloser constructs new BlockstoreCloser from cache.Accessor -func blockstoreCloser(ac cache.Accessor) (*BlockstoreCloser, error) { - bs, err := ac.Blockstore() - if err != nil { - return nil, fmt.Errorf("eds/store: failed to get blockstore: %w", err) - } - return &BlockstoreCloser{ - ReadBlockstore: bs, - Closer: ac, - }, nil -} diff --git a/share/eds/cache/doublecache.go b/share/eds/cache/doublecache.go new file mode 100644 index 0000000000..a63eadee9e --- /dev/null +++ b/share/eds/cache/doublecache.go @@ -0,0 +1,51 @@ +package cache + +import ( + "errors" + + "github.com/filecoin-project/dagstore/shard" +) + +// DoubleCache represents a Cache that looks into multiple caches one by one. +type DoubleCache struct { + first, second Cache +} + +// NewDoubleCache creates a new DoubleCache with the provided caches. +func NewDoubleCache(first, second Cache) *DoubleCache { + return &DoubleCache{ + first: first, + second: second, + } +} + +// Get looks for an item in all the caches one by one and returns the Cache found item. +func (mc *DoubleCache) Get(key shard.Key) (Accessor, error) { + ac, err := mc.first.Get(key) + if err == nil { + return ac, nil + } + return mc.second.Get(key) +} + +// Remove removes an item from all underlying caches +func (mc *DoubleCache) Remove(key shard.Key) error { + err1 := mc.first.Remove(key) + err2 := mc.second.Remove(key) + return errors.Join(err1, err2) +} + +func (mc *DoubleCache) First() Cache { + return mc.first +} + +func (mc *DoubleCache) Second() Cache { + return mc.second +} + +func (mc *DoubleCache) EnableMetrics() error { + if err := mc.first.EnableMetrics(); err != nil { + return err + } + return mc.second.EnableMetrics() +} diff --git a/share/eds/store.go b/share/eds/store.go index e8caf4c35a..14df4a4bee 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -40,7 +40,8 @@ const ( // We don't use transient files right now, so GC is turned off by default. defaultGCInterval = 0 - defaultCacheSize = 128 + defaultRecentBlocksCacheSize = 10 + defaultBlockstoreCacheSize = 128 ) var ErrNotFound = errors.New("eds not found in store") @@ -56,7 +57,7 @@ type Store struct { mounts *mount.Registry bs *blockstore - cache cache.Cache + cache *cache.DoubleCache carIdx index.FullIndexRepo invertedIdx *simpleInvertedIndex @@ -114,11 +115,16 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { return nil, fmt.Errorf("failed to create DAGStore: %w", err) } - accessorCache, err := cache.NewAccessorCache("cache", defaultCacheSize) + recentBlocksCache, err := cache.NewAccessorCache("recent", defaultRecentBlocksCacheSize) if err != nil { return nil, fmt.Errorf("failed to create recent blocks cache: %w", err) } + blockstoreCache, err := cache.NewAccessorCache("blockstore", defaultBlockstoreCacheSize) + if err != nil { + return nil, fmt.Errorf("failed to create blockstore blocks cache: %w", err) + } + store := &Store{ basepath: basepath, dgstr: dagStore, @@ -127,7 +133,7 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { gcInterval: defaultGCInterval, mounts: r, shardFailures: failureChan, - cache: accessorCache, + cache: cache.NewDoubleCache(recentBlocksCache, blockstoreCache), } store.bs = newBlockstore(store, ds) return store, nil @@ -284,7 +290,7 @@ func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext go func() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - _, err := s.cache.GetOrLoad(ctx, result.Key, s.getAccessor) + _, err := s.cache.First().GetOrLoad(ctx, result.Key, s.getAccessor) if err != nil { log.Warnw("unable to put accessor to recent blocks accessors cache", "err", err) return diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 0d5283e2f2..b38a25c827 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -303,7 +303,7 @@ func Test_BlockstoreCache(t *testing.T) { // store eds to the store with noopCache to allow clean cache after put swap := edsStore.cache - edsStore.cache = cache.NoopCache{} + edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) eds, dah := randomEDS(t) err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(t, err) @@ -388,7 +388,7 @@ func Test_NotCachedAccessor(t *testing.T) { err = edsStore.Start(ctx) require.NoError(t, err) // replace cache with noopCache to - edsStore.cache = cache.NoopCache{} + edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) eds, dah := randomEDS(t) err = edsStore.Put(ctx, dah.Hash(), eds) diff --git a/share/eds/utils.go b/share/eds/utils.go index e7b24a9aee..3417a2aa62 100644 --- a/share/eds/utils.go +++ b/share/eds/utils.go @@ -1,11 +1,10 @@ package eds import ( + "fmt" "io" "github.com/filecoin-project/dagstore" - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/namespace" "github.com/celestiaorg/celestia-node/share/eds/cache" ) @@ -30,11 +29,16 @@ func newReadCloser(ac cache.Accessor) io.ReadCloser { } } -func newBlockstore(store *Store, ds datastore.Batching) *blockstore { - return &blockstore{ - store: store, - ds: namespace.Wrap(ds, blockstoreCacheKey), +// blockstoreCloser constructs new BlockstoreCloser from cache.Accessor +func blockstoreCloser(ac cache.Accessor) (*BlockstoreCloser, error) { + bs, err := ac.Blockstore() + if err != nil { + return nil, fmt.Errorf("eds/store: failed to get blockstore: %w", err) } + return &BlockstoreCloser{ + ReadBlockstore: bs, + Closer: ac, + }, nil } func closeAndLog(name string, closer io.Closer) { From eae39fd9fa4edff2b14a034acfc4c934eaac5cc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 13:51:55 +0000 Subject: [PATCH 205/388] chore(deps): Bump mathieudutour/github-tag-action from 6.0 to 6.1 (#2721) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [mathieudutour/github-tag-action](https://github.com/mathieudutour/github-tag-action) from 6.0 to 6.1.
Commits
  • fcfbdce check in prod dependencies
  • dbd80d2 version 6.1
  • a1afa11 Correctly parse create_annotated_tag (#137)
  • 3f9dffd Merge pull request #147 from JohnTitor/nodejs16
  • c4f56e7 Merge pull request #146 from JohnTitor/actions-core
  • 923acce Update @actions/core to v1.10.0
  • c5ababc Update to Node.js 16
  • e923a61 Merge pull request #142 from mathieudutour/dependabot/npm_and_yarn/actions/co...
  • 8bee2b2 Bump @​actions/core from 1.6.0 to 1.9.1
  • 92dc56e Merge pull request #124 from mathieudutour/dependabot/npm_and_yarn/node-fetch...
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=mathieudutour/github-tag-action&package-manager=github_actions&previous-version=6.0&new-version=6.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 6e9293cc38..7f72a24f13 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -65,7 +65,7 @@ jobs: # version_bump section, it would skip and not run, which would result # in goreleaser not running either. if: ${{ github.event_name == 'workflow_dispatch' }} - uses: mathieudutour/github-tag-action@v6.0 + uses: mathieudutour/github-tag-action@v6.1 - name: Version Release uses: celestiaorg/.github/.github/actions/version-release@v0.2.2 From c134ce74a640d54fbec20f9adf58c381cbb4804a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 13:59:55 +0000 Subject: [PATCH 206/388] chore(deps): Bump goreleaser/goreleaser-action from 4 to 5 (#2719) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [goreleaser/goreleaser-action](https://github.com/goreleaser/goreleaser-action) from 4 to 5.
Release notes

Sourced from goreleaser/goreleaser-action's releases.

v5.0.0

What's Changed

Full Changelog: https://github.com/goreleaser/goreleaser-action/compare/v4.6.0...v5.0.0

v4.6.0

Reverts the change to node20 runtime.

Full Changelog: https://github.com/goreleaser/goreleaser-action/compare/v4.5.0...v4.6.0

v4.5.0

What's Changed

New Contributors

Full Changelog: https://github.com/goreleaser/goreleaser-action/compare/v4.4.0...v4.5.0

v4.4.0

What's Changed

Full Changelog: https://github.com/goreleaser/goreleaser-action/compare/v4.3.0...v4.4.0

v4.3.0

What's Changed

New Contributors

Full Changelog: https://github.com/goreleaser/goreleaser-action/compare/v4.2.0...v4.3.0

... (truncated)

Commits
  • 7ec5c2b chore(deps): bump @​actions/core from 1.10.0 to 1.10.1 (#434)
  • 3529a65 chore: node 20 as default runtime (#432)
  • d2f6e33 chore(deps): bump crazy-max/ghaction-import-gpg from 5 to 6 (#433)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=goreleaser/goreleaser-action&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 7f72a24f13..cfe1bafd90 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -88,7 +88,7 @@ jobs: with: go-version: 1.21 # Generate the binaries and release - - uses: goreleaser/goreleaser-action@v4 + - uses: goreleaser/goreleaser-action@v5 with: distribution: goreleaser version: latest From 60eb760a43a203b3bb3a3df6886e35925c55ae88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 22:08:25 +0800 Subject: [PATCH 207/388] chore(deps): Bump celestiaorg/.github from 0.2.2 to 0.2.3 (#2720) Bumps [celestiaorg/.github](https://github.com/celestiaorg/.github) from 0.2.2 to 0.2.3.
Release notes

Sourced from celestiaorg/.github's releases.

Release v0.2.3

Fixes unknown/unknown in our ghcr.io packages

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=celestiaorg/.github&package-manager=github_actions&previous-version=0.2.2&new-version=0.2.3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Co-authored-by: Vlad <13818348+walldiss@users.noreply.github.com> --- .github/workflows/ci_release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index cfe1bafd90..271cab472f 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -25,7 +25,7 @@ on: jobs: # Dockerfile Linting hadolint: - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.2 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.3 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.2 + - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.3 markdown-lint: name: Markdown Lint @@ -68,7 +68,7 @@ jobs: uses: mathieudutour/github-tag-action@v6.1 - name: Version Release - uses: celestiaorg/.github/.github/actions/version-release@v0.2.2 + uses: celestiaorg/.github/.github/actions/version-release@v0.2.3 with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.version }} From f6d10560b3c3a586bd1dce784f1c1b508fe0789b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Sep 2023 15:23:18 +0000 Subject: [PATCH 208/388] chore(deps): Bump go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp from 1.17.0 to 1.18.0 (#2714) Bumps [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp](https://github.com/open-telemetry/opentelemetry-go) from 1.17.0 to 1.18.0.
Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp's changelog.

[1.18.0/0.41.0/0.0.6] 2023-09-12

This release drops the compatibility guarantee of [Go 1.19].

Added

  • Add WithProducer option in go.opentelemetry.op/otel/exporters/prometheus to restore the ability to register producers on the prometheus exporter's manual reader. (#4473)
  • Add IgnoreValue option in go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest to allow ignoring values when comparing metrics. (#4447)

Changed

  • Use a TestingT interface instead of *testing.T struct in go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest. (#4483)

Deprecated

  • The NewMetricExporter in go.opentelemetry.io/otel/bridge/opencensus was deprecated in v0.35.0 (#3541). The deprecation notice format for the function has been corrected to trigger Go documentation and build tooling. (#4470)

Removed

  • Removed the deprecated go.opentelemetry.io/otel/exporters/jaeger package. (#4467)
  • Removed the deprecated go.opentelemetry.io/otel/example/jaeger package. (#4467)
  • Removed the deprecated go.opentelemetry.io/otel/sdk/metric/aggregation package. (#4468)
  • Removed the deprecated internal packages in go.opentelemetry.io/otel/exporters/otlp and its sub-packages. (#4469)
  • Dropped guaranteed support for versions of Go less than 1.20. (#4481)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp&package-manager=go_modules&previous-version=1.17.0&new-version=1.18.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 74f49bec0e..dc7aeb1676 100644 --- a/go.mod +++ b/go.mod @@ -59,8 +59,8 @@ require ( go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 go.opentelemetry.io/otel v1.18.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 go.opentelemetry.io/otel/metric v1.18.0 go.opentelemetry.io/otel/sdk v1.18.0 go.opentelemetry.io/otel/sdk/metric v0.41.0 diff --git a/go.sum b/go.sum index 8363974ccb..2959221c98 100644 --- a/go.sum +++ b/go.sum @@ -2386,10 +2386,10 @@ go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0 h1:k0k7hFNDd8K4iOMJXj go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 h1:iV3BOgW4fry1Riw9dwypigqlIYWXvSRVT2RJmblzo40= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0/go.mod h1:7PGzqlKrxIRmbj5tlNW0nTkYZ5fHXDgk6Fy8/KjR0CI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0 h1:U5GYackKpVKlPrd/5gKMlrTlP2dCESAAFU682VCpieY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.17.0/go.mod h1:aFsJfCEnLzEu9vRRAcUiB/cpRTbVsNdF3OHSPpdjxZQ= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0 h1:kvWMtSUNVylLVrOE4WLUmBtgziYoCIYUNSpTYtMzVJI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.17.0/go.mod h1:SExUrRYIXhDgEKG4tkiQovd2HTaELiHUsuK08s5Nqx4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 h1:IAtl+7gua134xcV3NieDhJHjjOVeJhXAnYf/0hswjUY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0/go.mod h1:w+pXobnBzh95MNIkeIuAKcHe/Uu/CX2PKIvBP6ipKRA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 h1:6pu8ttx76BxHf+xz/H77AUZkPF3cwWzXqAUsXhVKI18= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0/go.mod h1:IOmXxPrxoxFMXdNy7lfDmE8MzE61YPcurbUm0SMjerI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= From f044c92591abf81c983f3152e16c88807a9dc4bf Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Tue, 19 Sep 2023 23:04:59 +0800 Subject: [PATCH 209/388] chore(dependabot): add otel group to dependabot (#2722) Adds dependabot rule, to reduce amount of otel dependancies PRs --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 96228eea15..26f1b4da15 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -19,6 +19,10 @@ updates: - renaynay labels: - kind:deps + groups: + otel: + patterns: + - "^go.opentelemetry.io/otel*" - package-ecosystem: docker directory: "/" schedule: From ebae9b909f83d9d20565270b8f0ad8e1bcfb8560 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 19 Sep 2023 15:16:30 +0000 Subject: [PATCH 210/388] chore(deps): Bump go.uber.org/zap from 1.25.0 to 1.26.0 (#2731) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [go.uber.org/zap](https://github.com/uber-go/zap) from 1.25.0 to 1.26.0.
Release notes

Sourced from go.uber.org/zap's releases.

v1.26.0

Enhancements:

  • #1297[]: Add Dict as a Field.
  • #1319[]: Add WithLazy method to Logger which lazily evaluates the structured context.
  • #1350[]: String encoding is much (~50%) faster now.

Thanks to @​hhk7734, @​jquirke, @​cdvr1993 for their contributions to this release.

#1297: uber-go/zap#1297 #1319: uber-go/zap#1319 #1350: uber-go/zap#1350

Changelog

Sourced from go.uber.org/zap's changelog.

1.26.0 (14 Sep 2023)

Enhancements:

  • #1297[]: Add Dict as a Field.
  • #1319[]: Add WithLazy method to Logger which lazily evaluates the structured context.
  • #1350[]: String encoding is much (~50%) faster now.

Thanks to @​hhk7734, @​jquirke, and @​cdvr1993 for their contributions to this release.

#1297: uber-go/zap#1297 #1319: uber-go/zap#1319 #1350: uber-go/zap#1350

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.uber.org/zap&package-manager=go_modules&previous-version=1.25.0&new-version=1.26.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index dc7aeb1676..40b61e9ef8 100644 --- a/go.mod +++ b/go.mod @@ -67,7 +67,7 @@ require ( go.opentelemetry.io/otel/trace v1.18.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 - go.uber.org/zap v1.25.0 + go.uber.org/zap v1.26.0 golang.org/x/crypto v0.13.0 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 diff --git a/go.sum b/go.sum index 2959221c98..baf55dcb88 100644 --- a/go.sum +++ b/go.sum @@ -2453,8 +2453,8 @@ go.uber.org/zap v1.21.0/go.mod h1:wjWOCqI0f2ZZrJF/UufIOkiC8ii6tm1iqIsLo76RfJw= go.uber.org/zap v1.22.0/go.mod h1:H4siCOZOrAolnUPJEkfaSjDqyP+BDS0DdDWzwcgt3+U= go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY= go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg= -go.uber.org/zap v1.25.0 h1:4Hvk6GtkucQ790dqmj7l1eEnRdKm3k3ZUrUMS2d5+5c= -go.uber.org/zap v1.25.0/go.mod h1:JIAUzQIH94IC4fOJQm7gMmBJP5k7wQfdcnYdPoEXJYk= +go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= +go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= go4.org v0.0.0-20180809161055-417644f6feb5/go.mod h1:MkTOUMDaeVYJUOUsaDXIhWPZYa1yOyC1qaOBpL57BhE= golang.org/x/build v0.0.0-20190111050920-041ab4dc3f9d/go.mod h1:OWs+y06UdEOHN4y+MfF/py+xQ/tYqIWW03b70/CG9Rw= golang.org/x/crypto v0.0.0-20170930174604-9419663f5a44/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= From 38c516fb7f07a05628c1a0df7f9b57b1859862f5 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 20 Sep 2023 09:56:11 +0200 Subject: [PATCH 211/388] chore(deps): Bump go.opentelemetry.io/contrib/instrumentation/runtime from 0.43.0 to 0.44.0 (#2732) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [go.opentelemetry.io/contrib/instrumentation/runtime](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.43.0 to 0.44.0.
Release notes

Sourced from go.opentelemetry.io/contrib/instrumentation/runtime's releases.

Release v1.19.0/v0.44.0/v0.13.0

Added

  • Add gcp.gce.instance.name and gcp.gce.instance.hostname resource attributes to go.opentelemetry.io/contrib/detectors/gcp. (#4263)

Changed

  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/ec2 have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/ecs have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/eks have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/lambda have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda have been upgraded to v1.21.0. (#4265)
    • The faas.execution attribute is now faas.invocation_id.
    • The faas.id attribute is now aws.lambda.invoked_arn.
  • The semantic conventions used by go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws have been upgraded to v1.21.0. (#4265)
  • The http.request.method attribute will only allow known HTTP methods from the metrics generated by go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#4277)

Removed

  • The high cardinality attributes net.sock.peer.addr, net.sock.peer.port, http.user_agent, enduser.id, and http.client_ip were removed from the metrics generated by go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#4277)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/astaxie/beego/otelbeego module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/go-kit/kit/otelkit module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/bradfitz/gomemcache/memcache/otelmemcache module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/gocql/gocql/otelgocql module is removed. (#4295)

New Contributors

Full Changelog: https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.18.0...v1.19.0

Release v1.18.0/v0.43.0/v0.12.0

Added

  • Add NewMiddleware function in go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#2964)
  • The go.opentelemetry.io/contrib/exporters/autoexport package to provide configuration of trace exporters with useful defaults and environment variable support. (#2753, #4100, #4130, #4132, #4134)
  • WithRouteTag in go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp adds HTTP route attribute to metrics. (#615)
  • Add WithSpanOptions option in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc. (#3768)
  • Add testing support for Go 1.21. (#4233)

Changed

  • Change interceptors in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc to disable SENT/RECEIVED events. Use WithMessageEvents() to turn back on. (#3964)

Fixed

  • AWS XRay Remote Sampling to cap quotaBalance to 1x quota in go.opentelemetry.io/contrib/samplers/aws/xray. (#3651, #3652)
  • Do not panic when the HTTP request has the "Expect: 100-continue" header in go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace. (#3892)

... (truncated)

Changelog

Sourced from go.opentelemetry.io/contrib/instrumentation/runtime's changelog.

[1.19.0/0.44.0/0.13.0] - 2023-09-12

Added

  • Add gcp.gce.instance.name and gcp.gce.instance.hostname resource attributes to go.opentelemetry.io/contrib/detectors/gcp. (#4263)

Changed

  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/ec2 have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/ecs have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/eks have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/lambda have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda have been upgraded to v1.21.0. (#4265)
    • The faas.execution attribute is now faas.invocation_id.
    • The faas.id attribute is now aws.lambda.invoked_arn.
  • The semantic conventions used by go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws have been upgraded to v1.21.0. (#4265)
  • The http.request.method attribute will only allow known HTTP methods from the metrics generated by go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#4277)

Removed

  • The high cardinality attributes net.sock.peer.addr, net.sock.peer.port, http.user_agent, enduser.id, and http.client_ip were removed from the metrics generated by go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#4277)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/astaxie/beego/otelbeego module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/go-kit/kit/otelkit module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/bradfitz/gomemcache/memcache/otelmemcache module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/gocql/gocql/otelgocql module is removed. (#4295)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/contrib/instrumentation/runtime&package-manager=go_modules&previous-version=0.43.0&new-version=0.44.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 40b61e9ef8..f0192607b3 100644 --- a/go.mod +++ b/go.mod @@ -56,7 +56,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 - go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0 go.opentelemetry.io/otel v1.18.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 diff --git a/go.sum b/go.sum index baf55dcb88..667c6399c5 100644 --- a/go.sum +++ b/go.sum @@ -2374,8 +2374,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0 h1:NunhgxcK14rU7Hw2gKtV6uCSyohkXPisqneRFjnZNKQ= -go.opentelemetry.io/contrib/instrumentation/runtime v0.43.0/go.mod h1:rwb7icgpDjIhhHqv1qPGw6dDjAdAR7IKAe4PQdzBbsg= +go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0 h1:TXu20nL4yYfJlQeqG/D3Ia6b0p2HZmLfJto9hqJTQ/c= +go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0/go.mod h1:tQ5gBnfjndV1su3+DiLuu6rnd9hBBzg4rkRILnjSNFg= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= From 591bc428179990f60c9bbd2a626ee82caa91322d Mon Sep 17 00:00:00 2001 From: hrt/derrandz Date: Wed, 20 Sep 2023 09:13:24 +0100 Subject: [PATCH 212/388] docs: Add share p2p documentation (#2008) ## Overview Self-explanatory ## How to Test Changes Launch: ``` $ godoc -http=:6060 ``` And browse the impacted packages. ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [x] Linked issues closed with keywords --------- Co-authored-by: ramin Co-authored-by: Ryan --- blob/service.go | 3 +- share/p2p/doc.go | 18 ++++++++++++ share/p2p/peers/doc.go | 52 +++++++++++++++++++++++++++++++++++ share/p2p/shrexeds/doc.go | 51 ++++++++++++++++++++++++++++++++++ share/p2p/shrexnd/doc.go | 43 +++++++++++++++++++++++++++++ share/p2p/shrexsub/doc.go | 58 +++++++++++++++++++++++++++++++++++++++ 6 files changed, 224 insertions(+), 1 deletion(-) create mode 100644 share/p2p/doc.go create mode 100644 share/p2p/peers/doc.go create mode 100644 share/p2p/shrexeds/doc.go create mode 100644 share/p2p/shrexnd/doc.go create mode 100644 share/p2p/shrexsub/doc.go diff --git a/blob/service.go b/blob/service.go index 05139cdfb5..2130904aa7 100644 --- a/blob/service.go +++ b/blob/service.go @@ -51,7 +51,8 @@ func NewService( } } -// SubmitOptions contains the information about fee and gasLimit price in order to configure the Submit request. +// SubmitOptions contains the information about fee and gasLimit price in order to configure the +// Submit request. type SubmitOptions struct { Fee int64 GasLimit uint64 diff --git a/share/p2p/doc.go b/share/p2p/doc.go new file mode 100644 index 0000000000..991ddf94db --- /dev/null +++ b/share/p2p/doc.go @@ -0,0 +1,18 @@ +// Package p2p provides p2p functionality that powers the share exchange protocols used by celestia-node. +// The available protocols are: +// +// - shrexsub : a floodsub-based pubsub protocol that is used to broadcast/subscribe to the event +// of new EDS in the network to peers. +// +// - shrexnd: a request/response protocol that is used to request shares by namespace or namespace data from peers. +// +// - shrexeds: a request/response protocol that is used to request extended data square shares from peers. +// This protocol exchanges the original data square in between the client and server, and it's up to the +// receiver to compute the extended data square. +// +// This package also defines a peer manager that is used to manage network peers that can be used to exchange +// shares. The peer manager is primarily responsible for providing peers to request shares from, +// and is primarily used by `getters.ShrexGetter` in share/getters/shrex.go. +// +// Find out more about each protocol in their respective sub-packages. +package p2p diff --git a/share/p2p/peers/doc.go b/share/p2p/peers/doc.go new file mode 100644 index 0000000000..bc1647eb42 --- /dev/null +++ b/share/p2p/peers/doc.go @@ -0,0 +1,52 @@ +// Package peers provides a peer manager that handles peer discovery and peer selection for the shrex getter. +// +// The peer manager is responsible for: +// - Discovering peers +// - Selecting peers for data retrieval +// - Validating peers +// - Blacklisting peers +// - Garbage collecting peers +// +// The peer manager is not responsible for: +// - Connecting to peers +// - Disconnecting from peers +// - Sending data to peers +// - Receiving data from peers +// +// The peer manager is a mechanism to store peers from shrexsub, a mechanism that +// handles "peer discovery" and "peer selection" by relying on a shrexsub subscription +// and header subscriptions, such that it listens for new headers and +// new shares and uses this information to pool peers by shares. +// +// This gives the peer manager an ability to block peers that gossip invalid shares, but also access a list of peers +// that are known to have been gossiping valid shares. +// The peers are then returned on request using a round-robin algorithm to return a different peer each time. +// If no peers are found, the peer manager will rely on full nodes retrieved from discovery. +// +// The peer manager is only concerned with recent heights, thus it retrieves peers that +// were active since `initialHeight`. +// The peer manager will also garbage collect peers such that it blacklists peers that +// have been active since `initialHeight` but have been found to be invalid. +// +// The peer manager is passed to the shrex getter and is used at request time to +// select peers for a given data hash for data retrieval. +// +// # Usage +// +// The peer manager is created using [NewManager] constructor: +// +// peerManager := peers.NewManager(headerSub, shrexSub, discovery, host, connGater, opts...) +// +// After creating the peer manager, it should be started to kick off listening and +// validation routines that enable peer selection and retrieval: +// +// err := peerManager.Start(ctx) +// +// The peer manager can be stopped at any time to stop all peer discovery and validation routines: +// +// err := peerManager.Stop(ctx) +// +// The peer manager can be used to select peers for a given datahash for shares retrieval: +// +// peer, err := peerManager.Peer(ctx, hash) +package peers diff --git a/share/p2p/shrexeds/doc.go b/share/p2p/shrexeds/doc.go new file mode 100644 index 0000000000..6bad3061f9 --- /dev/null +++ b/share/p2p/shrexeds/doc.go @@ -0,0 +1,51 @@ +// This package defines a protocol that is used to request +// extended data squares from peers in the network. +// +// This protocol is a request/response protocol that allows for sending requests for extended data squares by data root +// to the peers in the network and receiving a response containing the original data square(s), which is used +// to recompute the extended data square. +// +// The streams are established using the protocol ID: +// +// - "{networkID}/shrex/eds/v0.0.1" where networkID is the network ID of the network. (e.g. "arabica") +// +// When a peer receives a request for extended data squares, it will read +// the original data square from the EDS store by retrieving the underlying +// CARv1 file containing the full extended data square, but will limit reading +// to the original data square shares only. +// The client on the other hand will take care of computing the extended data squares from +// the original data square on receipt. +// +// # Usage +// +// To use a shrexeds client to request extended data squares from a peer, you must +// first create a new `shrexeds.Client` instance by: +// +// client, err := shrexeds.NewClient(params, host) +// +// where `params` is a `shrexeds.Parameters` instance and `host` is a `libp2p.Host` instance. +// +// To request extended data squares from a peer, you must first create a `Client.RequestEDS` instance by: +// +// eds, err := client.RequestEDS(ctx, dataHash, peer) +// +// where: +// - `ctx` is a `context.Context` instance, +// - `dataHash` is the data root of the extended data square and +// - `peer` is the peer ID of the peer to request the extended data square from. +// +// To use a shrexeds server to respond to requests for extended data squares from peers +// you must first create a new `shrexeds.Server` instance by: +// +// server, err := shrexeds.NewServer(params, host, store) +// +// where `params` is a [Parameters] instance, `host` is a libp2p.Host instance and `store` is a [eds.Store] instance. +// +// To start the server, you must call `Start` on the server: +// +// err := server.Start(ctx) +// +// To stop the server, you must call `Stop` on the server: +// +// err := server.Stop(ctx) +package shrexeds diff --git a/share/p2p/shrexnd/doc.go b/share/p2p/shrexnd/doc.go new file mode 100644 index 0000000000..74ba7397e8 --- /dev/null +++ b/share/p2p/shrexnd/doc.go @@ -0,0 +1,43 @@ +// This package defines a protocol that is used to request namespaced data from peers in the network. +// +// This protocol is a request/response protocol that sends a request for specific data that +// lives in a specific namespace ID and receives a response with the data. +// +// The streams are established using the protocol ID: +// +// - "{networkID}/shrex/nd/0.0.1" where networkID is the network ID of the network. (e.g. "arabica") +// +// The protocol uses protobuf to serialize and deserialize messages. +// +// # Usage +// +// To use a shrexnd client to request data from a peer, you must first create a new `shrexnd.Client` instance by: +// +// 1. Create a new client using `NewClient` and pass in the parameters of the protocol and the host: +// +// client, err := shrexnd.NewClient(params, host) +// +// 2. Request data from a peer by calling [Client.RequestND] on the client and +// pass in the context, the data root, the namespace ID and the peer ID: +// +// data, err := client.RequestND(ctx, dataRoot, peerID, namespaceID) +// +// where data is of type [share.NamespacedShares] +// +// To use a shrexnd server to respond to requests from peers, you must first create a new `shrexnd.Server` instance by: +// +// 1. Create a new server using `NewServer` and pass in the parameters of +// the protocol, the host, the store and store share getter: +// +// server, err := shrexnd.NewServer(params, host, store, storeShareGetter) +// +// where store is of type [share.Store] and storeShareGetter is of type [share.Getter] +// +// 2. Start the server by calling `Start` on the server: +// +// err := server.Start(ctx) +// +// 3. Stop the server by calling `Stop` on the server: +// +// err := server.Stop(ctx) +package shrexnd diff --git a/share/p2p/shrexsub/doc.go b/share/p2p/shrexsub/doc.go new file mode 100644 index 0000000000..95d08361a2 --- /dev/null +++ b/share/p2p/shrexsub/doc.go @@ -0,0 +1,58 @@ +// This package defines a protocol that is used to broadcast shares to peers over a pubsub network. +// +// This protocol runs on a rudimentary floodsub network is primarily a pubsub protocol +// that broadcasts and listens for shares over a pubsub topic. +// +// The pubsub topic used by this protocol is: +// +// "{networkID}/eds-sub/v0.1.0" +// +// where networkID is the network ID of the celestia-node that is running the protocol. (e.g. "arabica") +// +// # Usage +// +// To use this protocol, you must first create a new `shrexsub.PubSub` instance by: +// +// pubsub, err := shrexsub.NewPubSub(ctx, host, networkID) +// +// where host is the libp2p host that is running the protocol, and networkID is the network ID of the celestia-node +// that is running the protocol. +// +// After this, you can start the pubsub protocol by: +// +// err := pubsub.Start(ctx) +// +// Once you have started the `shrexsub.PubSub` instance, you can broadcast a share by: +// +// err := pubsub.Broadcast(ctx, notification) +// +// where `notification` is of type [shrexsub.Notification]. +// +// and `DataHash` is the hash of the share that you want to broadcast, and `Height` is the height of the share. +// +// You can also subscribe to the pubsub topic by: +// +// sub, err := pubsub.Subscribe(ctx) +// +// and then receive notifications by: +// +// for { +// select { +// case <-ctx.Done(): +// sub.Cancel() +// return +// case notification, err := <-sub.Next(): +// // handle notification or err +// } +// } +// +// You can also manipulate the received pubsub messages by using the [PubSub.AddValidator] method: +// +// pubsub.AddValidator(validator ValidatorFn) +// +// where `validator` is of type [shrexsub.ValidatorFn] and `Notification` is the same as above. +// +// You can also stop the pubsub protocol by: +// +// err := pubsub.Stop(ctx) +package shrexsub From 953a349f31b5320a60816ac3dc519269609bd158 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 21 Sep 2023 15:04:40 +0800 Subject: [PATCH 213/388] feat(share/eds/cache): Add accessor cache refs counter (#2723) Adds refs counting to accessor cache, to prevent situations when underlying accessor is closed while there are readers that uses it. --- das/options.go | 3 +- nodebuilder/blob/mocks/api.go | 3 +- share/eds/cache/accessor_cache.go | 127 +++++++++++++++++++++---- share/eds/cache/accessor_cache_test.go | 97 ++++++++++++++++++- share/eds/cache/metrics.go | 5 +- share/eds/cache/noop.go | 21 +++- share/eds/store.go | 9 +- share/eds/store_test.go | 21 ++-- 8 files changed, 249 insertions(+), 37 deletions(-) diff --git a/das/options.go b/das/options.go index e38e488c04..6af8a02174 100644 --- a/das/options.go +++ b/das/options.go @@ -52,7 +52,8 @@ func DefaultParameters() Parameters { ConcurrencyLimit: concurrencyLimit, BackgroundStoreInterval: 10 * time.Minute, SampleFrom: 1, - // SampleTimeout = approximate block time (with a bit of wiggle room) * max amount of catchup workers + // SampleTimeout = approximate block time (with a bit of wiggle room) * max amount of catchup + // workers SampleTimeout: 15 * time.Second * time.Duration(concurrencyLimit), } } diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index 6a994f4d7c..40bf299da1 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + blob "github.com/celestiaorg/celestia-node/blob" share "github.com/celestiaorg/celestia-node/share" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/share/eds/cache/accessor_cache.go b/share/eds/cache/accessor_cache.go index ff955f8c45..6f937818f8 100644 --- a/share/eds/cache/accessor_cache.go +++ b/share/eds/cache/accessor_cache.go @@ -2,15 +2,20 @@ package cache import ( "context" + "errors" "fmt" "io" "sync" + "sync/atomic" + "time" "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" lru "github.com/hashicorp/golang-lru/v2" ) +const defaultCloseTimeout = time.Minute + var _ Cache = (*AccessorCache)(nil) // AccessorCache implements the Cache interface using an LRU cache backend. @@ -36,6 +41,10 @@ type accessorWithBlockstore struct { // The blockstore is stored separately because each access to the blockstore over the shard // accessor reopens the underlying CAR. bs dagstore.ReadBlockstore + + done chan struct{} + refs atomic.Int32 + isClosed bool } // Blockstore implements the Blockstore of the Accessor interface. It creates the blockstore on the @@ -55,6 +64,51 @@ func (s *accessorWithBlockstore) Reader() io.Reader { return s.shardAccessor.Reader() } +func (s *accessorWithBlockstore) addRef() error { + s.Lock() + defer s.Unlock() + if s.isClosed { + // item is already closed and soon will be removed after all refs are released + return errCacheMiss + } + if s.refs.Add(1) == 1 { + // there were no refs previously and done channel was closed, reopen it by recreating + s.done = make(chan struct{}) + } + return nil +} + +func (s *accessorWithBlockstore) removeRef() { + s.Lock() + defer s.Unlock() + if s.refs.Add(-1) <= 0 { + close(s.done) + } +} + +func (s *accessorWithBlockstore) close() error { + s.Lock() + if s.isClosed { + s.Unlock() + // accessor will be closed by another goroutine + return nil + } + s.isClosed = true + done := s.done + s.Unlock() + + select { + case <-done: + case <-time.After(defaultCloseTimeout): + return fmt.Errorf("closing accessor, some readers didn't close the accessor within timeout,"+ + " amount left: %v", s.refs.Load()) + } + if err := s.shardAccessor.Close(); err != nil { + return fmt.Errorf("closing accessor: %w", err) + } + return nil +} + func NewAccessorCache(name string, cacheSize int) (*AccessorCache, error) { bc := &AccessorCache{ name: name, @@ -71,13 +125,16 @@ func NewAccessorCache(name string, cacheSize int) (*AccessorCache, error) { // evictFn will be invoked when an item is evicted from the cache. func (bc *AccessorCache) evictFn() func(shard.Key, *accessorWithBlockstore) { return func(_ shard.Key, abs *accessorWithBlockstore) { - err := abs.shardAccessor.Close() - if err != nil { - bc.metrics.observeEvicted(true) - log.Errorf("couldn't close accessor after cache eviction: %s", err) - return - } - bc.metrics.observeEvicted(false) + // we can release accessor from cache early, while it is being closed in parallel routine + go func() { + err := abs.close() + if err != nil { + bc.metrics.observeEvicted(true) + log.Errorf("couldn't close accessor after cache eviction: %s", err) + return + } + bc.metrics.observeEvicted(false) + }() } } @@ -94,7 +151,7 @@ func (bc *AccessorCache) Get(key shard.Key) (Accessor, error) { return nil, err } bc.metrics.observeGet(true) - return newCloser(accessor), nil + return newRefCloser(accessor) } func (bc *AccessorCache) get(key shard.Key) (*accessorWithBlockstore, error) { @@ -118,8 +175,12 @@ func (bc *AccessorCache) GetOrLoad( abs, err := bc.get(key) if err == nil { - bc.metrics.observeGet(true) - return newCloser(abs), nil + // return accessor, only of it is not closed yet + accessorWithRef, err := newRefCloser(abs) + if err == nil { + bc.metrics.observeGet(true) + return accessorWithRef, nil + } } // accessor not found in cache, so load new one using loader @@ -134,13 +195,27 @@ func (bc *AccessorCache) GetOrLoad( // Create a new accessor first to increment the reference count in it, so it cannot get evicted // from the inner lru cache before it is used. - ac := newCloser(abs) + accessorWithRef, err := newRefCloser(abs) + if err != nil { + return nil, err + } bc.cache.Add(key, abs) - return ac, nil + return accessorWithRef, nil } // Remove removes the Accessor for a given key from the cache. func (bc *AccessorCache) Remove(key shard.Key) error { + lk := &bc.stripedLocks[shardKeyToStriped(key)] + lk.Lock() + accessor, err := bc.get(key) + lk.Unlock() + if errors.Is(err, errCacheMiss) { + // item is not in cache + return nil + } + if err = accessor.close(); err != nil { + return err + } // The cache will call evictFn on removal, where accessor close will be called. bc.cache.Remove(key) return nil @@ -153,17 +228,31 @@ func (bc *AccessorCache) EnableMetrics() error { return err } -// accessorCloser is a temporary object before reference counting is implemented. -type accessorCloser struct { +// refCloser manages references to accessor from provided reader and removes the ref, when the +// Close is called +type refCloser struct { *accessorWithBlockstore - io.Closer + closeFn func() } -func newCloser(abs *accessorWithBlockstore) *accessorCloser { - return &accessorCloser{ - accessorWithBlockstore: abs, - Closer: io.NopCloser(nil), +// newRefCloser creates new refCloser +func newRefCloser(abs *accessorWithBlockstore) (*refCloser, error) { + if err := abs.addRef(); err != nil { + return nil, err } + + var closeOnce sync.Once + return &refCloser{ + accessorWithBlockstore: abs, + closeFn: func() { + closeOnce.Do(abs.removeRef) + }, + }, nil +} + +func (c *refCloser) Close() error { + c.closeFn() + return nil } // shardKeyToStriped returns the index of the lock to use for a given shard key. We use the last diff --git a/share/eds/cache/accessor_cache_test.go b/share/eds/cache/accessor_cache_test.go index 5e928e85cc..4f12301dee 100644 --- a/share/eds/cache/accessor_cache_test.go +++ b/share/eds/cache/accessor_cache_test.go @@ -162,7 +162,7 @@ func TestAccessorCache(t *testing.T) { require.ErrorIs(t, err, errCacheMiss) }) - t.Run("close on accessor is noop", func(t *testing.T) { + t.Run("close on accessor is not closing underlying accessor", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second) defer cancel() cache, err := NewAccessorCache("test", 1) @@ -181,13 +181,106 @@ func TestAccessorCache(t *testing.T) { require.NoError(t, err) require.NotNil(t, accessor) - // close on returned accessor should not close inner reader + // close on returned accessor should not close inner accessor err = accessor.Close() require.NoError(t, err) // check that close was not performed on inner accessor mock.checkClosed(t, false) }) + + t.Run("close on accessor should wait all readers to finish", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key := shard.KeyFromString("key") + mock := &mockAccessor{} + accessor1, err := cache.GetOrLoad(ctx, key, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock, nil + }) + require.NoError(t, err) + + // create second readers + accessor2, err := cache.Get(key) + require.NoError(t, err) + + // initialize close + done := make(chan struct{}) + go func() { + err = cache.Remove(key) + close(done) + }() + + // close on first reader and check that it is not enough to release the inner accessor + err = accessor1.Close() + require.NoError(t, err) + mock.checkClosed(t, false) + + // second close from same reader should not release accessor either + err = accessor1.Close() + require.NoError(t, err) + mock.checkClosed(t, false) + + // reads for item that is being evicted should result in errCacheMiss + _, err = cache.Get(key) + require.ErrorIs(t, err, errCacheMiss) + + // close second reader and wait for accessor to be closed + err = accessor2.Close() + require.NoError(t, err) + // wait until close is performed on accessor + select { + case <-done: + case <-ctx.Done(): + t.Fatal("timeout reached") + } + + // item will be removed + mock.checkClosed(t, true) + }) + + t.Run("slow reader should not block eviction", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second) + defer cancel() + cache, err := NewAccessorCache("test", 1) + require.NoError(t, err) + + // add accessor to the cache + key1 := shard.KeyFromString("key1") + mock1 := &mockAccessor{} + accessor1, err := cache.GetOrLoad(ctx, key1, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock1, nil + }) + require.NoError(t, err) + + // add second accessor, to trigger eviction of the first one + key2 := shard.KeyFromString("key2") + mock2 := &mockAccessor{} + accessor2, err := cache.GetOrLoad(ctx, key2, func(ctx context.Context, key shard.Key) (Accessor, error) { + return mock2, nil + }) + require.NoError(t, err) + + // first accessor should be evicted from cache + _, err = cache.Get(key1) + require.ErrorIs(t, err, errCacheMiss) + + // first accessor should not be closed before all refs are released by Close() is calls. + mock1.checkClosed(t, false) + + // after Close() is called on first accessor, it is free to get closed + err = accessor1.Close() + require.NoError(t, err) + mock1.checkClosed(t, true) + + // after Close called on second accessor, it should stay in cache (not closed) + err = accessor2.Close() + require.NoError(t, err) + mock2.checkClosed(t, false) + }) } type mockAccessor struct { diff --git a/share/eds/cache/metrics.go b/share/eds/cache/metrics.go index 21e52fec10..24056aa553 100644 --- a/share/eds/cache/metrics.go +++ b/share/eds/cache/metrics.go @@ -53,8 +53,9 @@ func (m *metrics) observeEvicted(failed bool) { if m == nil { return } - m.evictedCounter.Add(context.Background(), 1, metric.WithAttributes( - attribute.Bool(failedKey, failed))) + m.evictedCounter.Add(context.Background(), 1, + metric.WithAttributes( + attribute.Bool(failedKey, failed))) } func (m *metrics) observeGet(found bool) { diff --git a/share/eds/cache/noop.go b/share/eds/cache/noop.go index 2af94feb1b..0a1a39ec7e 100644 --- a/share/eds/cache/noop.go +++ b/share/eds/cache/noop.go @@ -2,7 +2,9 @@ package cache import ( "context" + "io" + "github.com/filecoin-project/dagstore" "github.com/filecoin-project/dagstore/shard" ) @@ -19,7 +21,7 @@ func (n NoopCache) GetOrLoad( context.Context, shard.Key, func(context.Context, shard.Key) (Accessor, error), ) (Accessor, error) { - return nil, nil + return NoopAccessor{}, nil } func (n NoopCache) Remove(shard.Key) error { @@ -29,3 +31,20 @@ func (n NoopCache) Remove(shard.Key) error { func (n NoopCache) EnableMetrics() error { return nil } + +var _ Accessor = (*NoopAccessor)(nil) + +// NoopAccessor implements noop version of Accessor interface +type NoopAccessor struct{} + +func (n NoopAccessor) Blockstore() (dagstore.ReadBlockstore, error) { + return nil, nil +} + +func (n NoopAccessor) Reader() io.Reader { + return nil +} + +func (n NoopAccessor) Close() error { + return nil +} diff --git a/share/eds/store.go b/share/eds/store.go index 14df4a4bee..c2685cabb2 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -122,7 +122,7 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { blockstoreCache, err := cache.NewAccessorCache("blockstore", defaultBlockstoreCacheSize) if err != nil { - return nil, fmt.Errorf("failed to create blockstore blocks cache: %w", err) + return nil, fmt.Errorf("failed to create blockstore cache: %w", err) } store := &Store{ @@ -290,11 +290,16 @@ func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext go func() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - _, err := s.cache.First().GetOrLoad(ctx, result.Key, s.getAccessor) + ac, err := s.cache.First().GetOrLoad(ctx, result.Key, s.getAccessor) if err != nil { log.Warnw("unable to put accessor to recent blocks accessors cache", "err", err) return } + + // need to close returned accessor to remove the reader reference + if err := ac.Close(); err != nil { + log.Warnw("unable to close accessor after loading", "err", err) + } }() return nil diff --git a/share/eds/store_test.go b/share/eds/store_test.go index b38a25c827..7a08c737c4 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -272,6 +272,7 @@ func TestEDSStore_GC(t *testing.T) { time.Sleep(time.Millisecond * 100) // remove links to the shard from cache + time.Sleep(time.Millisecond * 100) key := shard.KeyFromString(share.DataHash(dah.Hash()).String()) err = edsStore.cache.Remove(key) require.NoError(t, err) @@ -359,20 +360,22 @@ func Test_CachedAccessor(t *testing.T) { time.Sleep(time.Millisecond * 100) // accessor should be in cache - cachedAccessor, err := edsStore.cache.Get(shard.KeyFromString(dah.String())) + _, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) require.NoError(t, err) // first read from cached accessor - firstBlock, err := io.ReadAll(cachedAccessor.Reader()) + carReader, err := edsStore.getCAR(ctx, dah.Hash()) + require.NoError(t, err) + firstBlock, err := io.ReadAll(carReader) require.NoError(t, err) - require.NoError(t, cachedAccessor.Close()) + require.NoError(t, carReader.Close()) // second read from cached accessor - cachedAccessor, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) + carReader, err = edsStore.getCAR(ctx, dah.Hash()) require.NoError(t, err) - secondBlock, err := io.ReadAll(cachedAccessor.Reader()) + secondBlock, err := io.ReadAll(carReader) require.NoError(t, err) - require.NoError(t, cachedAccessor.Close()) + require.NoError(t, carReader.Close()) require.Equal(t, firstBlock, secondBlock) } @@ -397,18 +400,18 @@ func Test_NotCachedAccessor(t *testing.T) { // accessor will be registered in cache async on put, so give it some time to settle time.Sleep(time.Millisecond * 100) - // accessor should be in cache + // accessor should not be in cache _, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) require.Error(t, err) - // first read from direct accessor + // first read from direct accessor (not from cache) carReader, err := edsStore.getCAR(ctx, dah.Hash()) require.NoError(t, err) firstBlock, err := io.ReadAll(carReader) require.NoError(t, err) require.NoError(t, carReader.Close()) - // second read from direct accessor + // second read from direct accessor (not from cache) carReader, err = edsStore.getCAR(ctx, dah.Hash()) require.NoError(t, err) secondBlock, err := io.ReadAll(carReader) From 15b8bcc8e67274c5b7ff0a7cd83a11660c360ba7 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 21 Sep 2023 16:45:40 +0800 Subject: [PATCH 214/388] feat(share/eds/store): expose eds store params (#2724) Adds exported params to allow control of eds store configuration --- core/exchange_test.go | 3 +- libs/edssser/edssser.go | 3 +- nodebuilder/share/config.go | 5 ++++ nodebuilder/share/module.go | 2 +- nodebuilder/share/share_test.go | 3 +- nodebuilder/store_test.go | 10 +++---- nodebuilder/tests/fraud_test.go | 4 +-- nodebuilder/tests/swamp/swamp.go | 3 +- share/eds/store.go | 30 +++++++++----------- share/eds/store_options.go | 43 +++++++++++++++++++++++++++++ share/eds/store_test.go | 6 ++-- share/getters/getter_test.go | 11 ++++---- share/getters/shrex_test.go | 3 +- share/p2p/peers/options.go | 2 +- share/p2p/shrexeds/exchange_test.go | 4 +-- share/p2p/shrexnd/exchange_test.go | 4 +-- 16 files changed, 90 insertions(+), 46 deletions(-) create mode 100644 share/eds/store_options.go diff --git a/core/exchange_test.go b/core/exchange_test.go index 579c69fba6..8aa62593c8 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -43,7 +43,8 @@ func createCoreFetcher(t *testing.T, cfg *testnode.Config) (*BlockFetcher, testn func createStore(t *testing.T) *eds.Store { t.Helper() - store, err := eds.NewStore(t.TempDir(), ds_sync.MutexWrap(ds.NewMapDatastore())) + storeCfg := eds.DefaultParameters() + store, err := eds.NewStore(storeCfg, t.TempDir(), ds_sync.MutexWrap(ds.NewMapDatastore())) require.NoError(t, err) return store } diff --git a/libs/edssser/edssser.go b/libs/edssser/edssser.go index fd11b47fcf..34712b785a 100644 --- a/libs/edssser/edssser.go +++ b/libs/edssser/edssser.go @@ -38,7 +38,8 @@ type EDSsser struct { } func NewEDSsser(path string, datastore datastore.Batching, cfg Config) (*EDSsser, error) { - edsstore, err := eds.NewStore(path, datastore) + storeCfg := eds.DefaultParameters() + edsstore, err := eds.NewStore(storeCfg, path, datastore) if err != nil { return nil, err } diff --git a/nodebuilder/share/config.go b/nodebuilder/share/config.go index 7fd845a672..5d66ea7691 100644 --- a/nodebuilder/share/config.go +++ b/nodebuilder/share/config.go @@ -5,6 +5,7 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/share/availability/light" + "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" @@ -13,6 +14,9 @@ import ( // TODO: some params are pointers and other are not, Let's fix this. type Config struct { + // EDSStoreParams sets eds store configuration parameters + EDSStoreParams *eds.Parameters + UseShareExchange bool // ShrExEDSParams sets shrexeds client and server configuration parameters ShrExEDSParams *shrexeds.Parameters @@ -27,6 +31,7 @@ type Config struct { func DefaultConfig(tp node.Type) Config { cfg := Config{ + EDSStoreParams: eds.DefaultParameters(), Discovery: discovery.DefaultParameters(), ShrExEDSParams: shrexeds.DefaultParameters(), ShrExNDParams: shrexnd.DefaultParameters(), diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 35f8813111..15eb776551 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -113,7 +113,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option )), fx.Provide(fx.Annotate( func(path node.StorePath, ds datastore.Batching) (*eds.Store, error) { - return eds.NewStore(string(path), ds) + return eds.NewStore(cfg.EDSStoreParams, string(path), ds) }, fx.OnStart(func(ctx context.Context, store *eds.Store) error { err := store.Start(ctx) diff --git a/nodebuilder/share/share_test.go b/nodebuilder/share/share_test.go index 57119b5003..db170709db 100644 --- a/nodebuilder/share/share_test.go +++ b/nodebuilder/share/share_test.go @@ -17,9 +17,8 @@ func Test_EmptyCARExists(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - tmpDir := t.TempDir() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(tmpDir, ds) + edsStore, err := eds.NewStore(eds.DefaultParameters(), t.TempDir(), ds) require.NoError(t, err) err = edsStore.Start(ctx) require.NoError(t, err) diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index 8ea56a280f..bd179c1258 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -73,7 +73,7 @@ func BenchmarkStore(b *testing.B) { err := Init(*DefaultConfig(node.Full), dir, node.Full) require.NoError(b, err) - store := newStore(ctx, b, dir) + store := newStore(ctx, b, eds.DefaultParameters(), dir) size := 128 b.Run("enabled eds proof caching", func(b *testing.B) { b.StopTimer() @@ -128,7 +128,7 @@ func TestStoreRestart(t *testing.T) { err := Init(*DefaultConfig(node.Full), dir, node.Full) require.NoError(t, err) - store := newStore(ctx, t, dir) + store := newStore(ctx, t, eds.DefaultParameters(), dir) hashes := make([][]byte, blocks) for i := range hashes { @@ -145,7 +145,7 @@ func TestStoreRestart(t *testing.T) { // restart store store.stop(ctx, t) - store = newStore(ctx, t, dir) + store = newStore(ctx, t, eds.DefaultParameters(), dir) for _, h := range hashes { edsReader, err := store.edsStore.GetCAR(ctx, h) @@ -163,12 +163,12 @@ type store struct { edsStore *eds.Store } -func newStore(ctx context.Context, t require.TestingT, dir string) store { +func newStore(ctx context.Context, t require.TestingT, params *eds.Parameters, dir string) store { s, err := OpenStore(dir, nil) require.NoError(t, err) ds, err := s.Datastore() require.NoError(t, err) - edsStore, err := eds.NewStore(dir, ds) + edsStore, err := eds.NewStore(params, dir, ds) require.NoError(t, err) err = edsStore.Start(ctx) require.NoError(t, err) diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index cd5be80517..d708f6d454 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -55,9 +55,9 @@ func TestFraudProofHandling(t *testing.T) { set, val := sw.Validators(t) fMaker := headerfraud.NewFraudMaker(t, 10, []types.PrivValidator{val}, set) - tmpDir := t.TempDir() + storeCfg := eds.DefaultParameters() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(tmpDir, ds) + edsStore, err := eds.NewStore(storeCfg, t.TempDir(), ds) require.NoError(t, err) require.NoError(t, edsStore.Start(ctx)) t.Cleanup(func() { diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 4f07d96b51..55426cf70c 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -172,7 +172,8 @@ func (s *Swamp) setupGenesis() { // ensure core has surpassed genesis block s.WaitTillHeight(ctx, 2) - store, err := eds.NewStore(s.t.TempDir(), ds_sync.MutexWrap(ds.NewMapDatastore())) + ds := ds_sync.MutexWrap(ds.NewMapDatastore()) + store, err := eds.NewStore(eds.DefaultParameters(), s.t.TempDir(), ds) require.NoError(s.t, err) ex := core.NewExchange( diff --git a/share/eds/store.go b/share/eds/store.go index c2685cabb2..9fc90046fc 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -34,14 +34,6 @@ const ( blocksPath = "/blocks/" indexPath = "/index/" transientsPath = "/transients/" - - // GC performs DAG store garbage collection by reclaiming transient files of - // shards that are currently available but inactive, or errored. - // We don't use transient files right now, so GC is turned off by default. - defaultGCInterval = 0 - - defaultRecentBlocksCacheSize = 10 - defaultBlockstoreCacheSize = 128 ) var ErrNotFound = errors.New("eds not found in store") @@ -75,8 +67,12 @@ type Store struct { } // NewStore creates a new EDS Store under the given basepath and datastore. -func NewStore(basepath string, ds datastore.Batching) (*Store, error) { - err := setupPath(basepath) +func NewStore(params *Parameters, basePath string, ds datastore.Batching) (*Store, error) { + if err := params.Validate(); err != nil { + return nil, err + } + + err := setupPath(basePath) if err != nil { return nil, fmt.Errorf("failed to setup eds.Store directories: %w", err) } @@ -90,12 +86,12 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { return nil, fmt.Errorf("failed to register FS mount on the registry: %w", err) } - fsRepo, err := index.NewFSRepo(basepath + indexPath) + fsRepo, err := index.NewFSRepo(basePath + indexPath) if err != nil { return nil, fmt.Errorf("failed to create index repository: %w", err) } - invertedIdx, err := newSimpleInvertedIndex(basepath) + invertedIdx, err := newSimpleInvertedIndex(basePath) if err != nil { return nil, fmt.Errorf("failed to create index: %w", err) } @@ -103,7 +99,7 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { failureChan := make(chan dagstore.ShardResult) dagStore, err := dagstore.NewDAGStore( dagstore.Config{ - TransientsDir: basepath + transientsPath, + TransientsDir: basePath + transientsPath, IndexRepo: fsRepo, Datastore: ds, MountRegistry: r, @@ -115,22 +111,22 @@ func NewStore(basepath string, ds datastore.Batching) (*Store, error) { return nil, fmt.Errorf("failed to create DAGStore: %w", err) } - recentBlocksCache, err := cache.NewAccessorCache("recent", defaultRecentBlocksCacheSize) + recentBlocksCache, err := cache.NewAccessorCache("recent", params.RecentBlocksCacheSize) if err != nil { return nil, fmt.Errorf("failed to create recent blocks cache: %w", err) } - blockstoreCache, err := cache.NewAccessorCache("blockstore", defaultBlockstoreCacheSize) + blockstoreCache, err := cache.NewAccessorCache("blockstore", params.BlockstoreCacheSize) if err != nil { return nil, fmt.Errorf("failed to create blockstore cache: %w", err) } store := &Store{ - basepath: basepath, + basepath: basePath, dgstr: dagStore, carIdx: fsRepo, invertedIdx: invertedIdx, - gcInterval: defaultGCInterval, + gcInterval: params.GCInterval, mounts: r, shardFailures: failureChan, cache: cache.NewDoubleCache(recentBlocksCache, blockstoreCache), diff --git a/share/eds/store_options.go b/share/eds/store_options.go new file mode 100644 index 0000000000..e5e6ffa73d --- /dev/null +++ b/share/eds/store_options.go @@ -0,0 +1,43 @@ +package eds + +import ( + "fmt" + "time" +) + +type Parameters struct { + // GC performs DAG store garbage collection by reclaiming transient files of + // shards that are currently available but inactive, or errored. + // We don't use transient files right now, so GC is turned off by default. + GCInterval time.Duration + + // RecentBlocksCacheSize is the size of the cache for recent blocks. + RecentBlocksCacheSize int + + // BlockstoreCacheSize is the size of the cache for blockstore requested accessors. + BlockstoreCacheSize int +} + +// DefaultParameters returns the default configuration values for the EDS store parameters. +func DefaultParameters() *Parameters { + return &Parameters{ + GCInterval: 0, + RecentBlocksCacheSize: 10, + BlockstoreCacheSize: 128, + } +} + +func (p *Parameters) Validate() error { + if p.GCInterval < 0 { + return fmt.Errorf("eds: GC interval cannot be negative") + } + + if p.RecentBlocksCacheSize < 1 { + return fmt.Errorf("eds: recent blocks cache size must be positive") + } + + if p.BlockstoreCacheSize < 1 { + return fmt.Errorf("eds: blockstore cache size must be positive") + } + return nil +} diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 7a08c737c4..ce42a0e0b9 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -425,9 +425,8 @@ func BenchmarkStore(b *testing.B) { ctx, cancel := context.WithCancel(context.Background()) b.Cleanup(cancel) - tmpDir := b.TempDir() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := NewStore(tmpDir, ds) + edsStore, err := NewStore(DefaultParameters(), b.TempDir(), ds) require.NoError(b, err) err = edsStore.Start(ctx) require.NoError(b, err) @@ -469,9 +468,8 @@ func BenchmarkStore(b *testing.B) { func newStore(t *testing.T) (*Store, error) { t.Helper() - tmpDir := t.TempDir() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - return NewStore(tmpDir, ds) + return NewStore(DefaultParameters(), t.TempDir(), ds) } func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index bee7b3a963..f8d1d0a5f0 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -26,9 +26,9 @@ func TestTeeGetter(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - tmpDir := t.TempDir() + storeCfg := eds.DefaultParameters() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(tmpDir, ds) + edsStore, err := eds.NewStore(storeCfg, t.TempDir(), ds) require.NoError(t, err) err = edsStore.Start(ctx) @@ -82,8 +82,9 @@ func TestStoreGetter(t *testing.T) { t.Cleanup(cancel) tmpDir := t.TempDir() + storeCfg := eds.DefaultParameters() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(tmpDir, ds) + edsStore, err := eds.NewStore(storeCfg, tmpDir, ds) require.NoError(t, err) err = edsStore.Start(ctx) @@ -185,9 +186,9 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - tmpDir := t.TempDir() + storeCfg := eds.DefaultParameters() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(tmpDir, ds) + edsStore, err := eds.NewStore(storeCfg, t.TempDir(), ds) require.NoError(t, err) err = edsStore.Start(ctx) diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 38611ed0ef..9b97a4dd5a 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -190,9 +190,8 @@ func TestShrexGetter(t *testing.T) { func newStore(t *testing.T) (*eds.Store, error) { t.Helper() - tmpDir := t.TempDir() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - return eds.NewStore(tmpDir, ds) + return eds.NewStore(eds.DefaultParameters(), t.TempDir(), ds) } func generateTestEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root, share.Namespace) { diff --git a/share/p2p/peers/options.go b/share/p2p/peers/options.go index 97ec30df4a..2970dd2465 100644 --- a/share/p2p/peers/options.go +++ b/share/p2p/peers/options.go @@ -45,7 +45,7 @@ func (p *Parameters) Validate() error { return nil } -// DefaultParameters returns the default configuration values for the daser parameters +// DefaultParameters returns the default configuration values for the peer manager parameters func DefaultParameters() Parameters { return Parameters{ // PoolValidationTimeout's default value is based on the default daser sampling timeout of 1 minute. diff --git a/share/p2p/shrexeds/exchange_test.go b/share/p2p/shrexeds/exchange_test.go index e9305a96b9..9155be6dec 100644 --- a/share/p2p/shrexeds/exchange_test.go +++ b/share/p2p/shrexeds/exchange_test.go @@ -136,9 +136,9 @@ func TestExchange_RequestEDS(t *testing.T) { func newStore(t *testing.T) *eds.Store { t.Helper() - tmpDir := t.TempDir() + storeCfg := eds.DefaultParameters() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - store, err := eds.NewStore(tmpDir, ds) + store, err := eds.NewStore(storeCfg, t.TempDir(), ds) require.NoError(t, err) return store } diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 50324844b1..99a7e2f8fb 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -126,9 +126,9 @@ func (m notFoundGetter) GetSharesByNamespace( func newStore(t *testing.T) *eds.Store { t.Helper() - tmpDir := t.TempDir() + storeCfg := eds.DefaultParameters() ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - store, err := eds.NewStore(tmpDir, ds) + store, err := eds.NewStore(storeCfg, t.TempDir(), ds) require.NoError(t, err) return store } From 2b610335454c02fa26512a472e005cc57cdaaaf0 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 21 Sep 2023 10:51:28 +0200 Subject: [PATCH 215/388] deps!: bump nmt, breaking JSON RPC return types including nmt Proofs (#2728) Related: https://github.com/celestiaorg/nmt/pull/233 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index f0192607b3..0bee3a3181 100644 --- a/go.mod +++ b/go.mod @@ -15,7 +15,7 @@ require ( github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.3.1 github.com/celestiaorg/go-libp2p-messenger v0.2.0 - github.com/celestiaorg/nmt v0.19.0 + github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 github.com/cosmos/cosmos-sdk v0.46.14 github.com/cosmos/cosmos-sdk/api v0.1.0 diff --git a/go.sum b/go.sum index 667c6399c5..baae4c6cf5 100644 --- a/go.sum +++ b/go.sum @@ -377,8 +377,8 @@ github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= -github.com/celestiaorg/nmt v0.19.0 h1:9VXFeI/gt+q8h5HeCE0RjXJhOxsFzxJUjHrkvF9CMYE= -github.com/celestiaorg/nmt v0.19.0/go.mod h1:Oz15Ub6YPez9uJV0heoU4WpFctxazuIhKyUtaYNio7E= +github.com/celestiaorg/nmt v0.20.0 h1:9i7ultZ8Wv5ytt8ZRaxKQ5KOOMo4A2K2T/aPGjIlSas= +github.com/celestiaorg/nmt v0.20.0/go.mod h1:Oz15Ub6YPez9uJV0heoU4WpFctxazuIhKyUtaYNio7E= github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= github.com/celestiaorg/rsmt2d v0.11.0 h1:lcto/637WyTEZR3dLRoNvyuExfnUbxvdvKi3qz/2V4k= From b6ac8c2363706b542d83b3dd285ef4bd99d00b4f Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 21 Sep 2023 11:28:56 +0200 Subject: [PATCH 216/388] refactoring(full/availability): removing TeeGetter, storing via SharesAvailable (#2726) --- das/daser_test.go | 2 +- nodebuilder/share/constructors.go | 8 +- nodebuilder/tests/nd_test.go | 2 +- share/availability/cache/testing.go | 2 +- share/availability/full/availability.go | 23 ++++-- share/availability/full/availability_test.go | 19 ++++- share/availability/full/testing.go | 14 +++- share/getters/getter_test.go | 55 ------------- share/getters/store.go | 12 +-- share/getters/tee.go | 85 -------------------- share/ipld/corrupted_data_test.go | 5 +- 11 files changed, 61 insertions(+), 166 deletions(-) delete mode 100644 share/getters/tee.go diff --git a/das/daser_test.go b/das/daser_test.go index a14da325f2..6a3378cdb5 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -155,7 +155,7 @@ func TestDASer_stopsAfter_BEFP(t *testing.T) { ps, err := pubsub.NewGossipSub(ctx, net.Hosts()[0], pubsub.WithMessageSignaturePolicy(pubsub.StrictNoSign)) require.NoError(t, err) - avail := full.TestAvailability(getters.NewIPLDGetter(bServ)) + avail := full.TestAvailability(t, getters.NewIPLDGetter(bServ)) // 15 headers from the past and 15 future headers mockGet, sub, _ := createDASerSubcomponents(t, bServ, 15, 15) diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 0e8d17f208..28e0997163 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -92,7 +92,6 @@ func lightGetter( // by shrex the next time the data is retrieved (meaning shard recovery is // manual after corruption is detected). func bridgeGetter( - store *eds.Store, storeGetter *getters.StoreGetter, shrexGetter *getters.ShrexGetter, cfg Config, @@ -100,13 +99,12 @@ func bridgeGetter( var cascade []share.Getter cascade = append(cascade, storeGetter) if cfg.UseShareExchange { - cascade = append(cascade, getters.NewTeeGetter(shrexGetter, store)) + cascade = append(cascade, shrexGetter) } return getters.NewCascadeGetter(cascade) } func fullGetter( - store *eds.Store, storeGetter *getters.StoreGetter, shrexGetter *getters.ShrexGetter, ipldGetter *getters.IPLDGetter, @@ -115,8 +113,8 @@ func fullGetter( var cascade []share.Getter cascade = append(cascade, storeGetter) if cfg.UseShareExchange { - cascade = append(cascade, getters.NewTeeGetter(shrexGetter, store)) + cascade = append(cascade, shrexGetter) } - cascade = append(cascade, getters.NewTeeGetter(ipldGetter, store)) + cascade = append(cascade, ipldGetter) return getters.NewCascadeGetter(cascade) } diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index 64d672cddc..8149ea9fe7 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -204,7 +204,7 @@ func replaceShareGetter() fx.Option { ) share.Getter { cascade := make([]share.Getter, 0, 2) cascade = append(cascade, storeGetter) - cascade = append(cascade, getters.NewTeeGetter(shrexGetter, store)) + cascade = append(cascade, shrexGetter) return getters.NewCascadeGetter(cascade) }, )) diff --git a/share/availability/cache/testing.go b/share/availability/cache/testing.go index 00a1520114..989bee522c 100644 --- a/share/availability/cache/testing.go +++ b/share/availability/cache/testing.go @@ -32,7 +32,7 @@ func FullAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availabilit store := dssync.MutexWrap(ds.NewMapDatastore()) getter := getters.NewIPLDGetter(bServ) avail := NewShareAvailability( - full.TestAvailability(getter), + full.TestAvailability(t, getter), store, ) return avail, availability_test.RandFillBS(t, n, bServ) diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index fb8dead839..51d9fd8518 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -3,12 +3,15 @@ package full import ( "context" "errors" + "fmt" + "github.com/filecoin-project/dagstore" logging "github.com/ipfs/go-log/v2" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/discovery" ) @@ -63,13 +66,15 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro } // a hack to avoid loading the whole EDS in mem if we store it already. - if fa.store != nil { - if ok, _ := fa.store.Has(ctx, root.Hash()); ok { - return nil - } + if ok, _ := fa.store.Has(ctx, root.Hash()); ok { + return nil } - _, err := fa.getter.GetEDS(ctx, root) + adder := ipld.NewProofsAdder(len(root.RowRoots)) + ctx = ipld.CtxWithProofsAdder(ctx, adder) + defer adder.Purge() + + eds, err := fa.getter.GetEDS(ctx, root) if err != nil { if errors.Is(err, context.Canceled) { return err @@ -79,8 +84,14 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro if errors.Is(err, share.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) { return share.ErrNotAvailable } + return err } - return err + + err = fa.store.Put(ctx, root.Hash(), eds) + if err != nil && !errors.Is(err, dagstore.ErrShardExists) { + return fmt.Errorf("full availability: failed to store eds: %w", err) + } + return nil } func (fa *ShareAvailability) ProbabilityOfAvailability(context.Context) float64 { diff --git a/share/availability/full/availability_test.go b/share/availability/full/availability_test.go index b6d217cd6b..a769c981c4 100644 --- a/share/availability/full/availability_test.go +++ b/share/availability/full/availability_test.go @@ -35,11 +35,26 @@ func TestSharesAvailable_Full(t *testing.T) { // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it getter, dah := GetterWithRandSquare(t, 16) - avail := TestAvailability(getter) + avail := TestAvailability(t, getter) err := avail.SharesAvailable(ctx, dah) assert.NoError(t, err) } +func TestSharesAvailable_StoresToEDSStore(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it + getter, dah := GetterWithRandSquare(t, 16) + avail := TestAvailability(t, getter) + err := avail.SharesAvailable(ctx, dah) + assert.NoError(t, err) + + has, err := avail.store.Has(ctx, dah.Hash()) + assert.NoError(t, err) + assert.True(t, has) +} + func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) { ctrl := gomock.NewController(t) getter := mocks.NewMockGetter(ctrl) @@ -49,7 +64,7 @@ func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) { eds := edstest.RandEDS(t, 4) dah, err := da.NewDataAvailabilityHeader(eds) require.NoError(t, err) - avail := TestAvailability(getter) + avail := TestAvailability(t, getter) errors := []error{share.ErrNotFound, context.DeadlineExceeded} for _, getterErr := range errors { diff --git a/share/availability/full/testing.go b/share/availability/full/testing.go index 4638fde1bc..a636b26ea6 100644 --- a/share/availability/full/testing.go +++ b/share/availability/full/testing.go @@ -1,14 +1,18 @@ package full import ( + "context" "testing" "time" + "github.com/ipfs/go-datastore" routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/p2p/discovery/routing" + "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/discovery" @@ -32,16 +36,20 @@ func RandNode(dn *availability_test.TestDagNet, squareSize int) (*availability_t func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { nd := dn.NewTestNode() nd.Getter = getters.NewIPLDGetter(nd.BlockService) - nd.Availability = TestAvailability(nd.Getter) + nd.Availability = TestAvailability(dn.T, nd.Getter) return nd } -func TestAvailability(getter share.Getter) *ShareAvailability { +func TestAvailability(t *testing.T, getter share.Getter) *ShareAvailability { disc := discovery.NewDiscovery( nil, routing.NewRoutingDiscovery(routinghelpers.Null{}), discovery.WithAdvertiseInterval(time.Second), discovery.WithPeersLimit(10), ) - return NewShareAvailability(nil, getter, disc) + store, err := eds.NewStore(eds.DefaultParameters(), t.TempDir(), datastore.NewMapDatastore()) + require.NoError(t, err) + err = store.Start(context.Background()) + require.NoError(t, err) + return NewShareAvailability(store, getter, disc) } diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index f8d1d0a5f0..bacb0a2c39 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -22,61 +22,6 @@ import ( "github.com/celestiaorg/celestia-node/share/sharetest" ) -func TestTeeGetter(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - storeCfg := eds.DefaultParameters() - ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) - edsStore, err := eds.NewStore(storeCfg, t.TempDir(), ds) - require.NoError(t, err) - - err = edsStore.Start(ctx) - require.NoError(t, err) - - bServ := ipld.NewMemBlockservice() - ig := NewIPLDGetter(bServ) - tg := NewTeeGetter(ig, edsStore) - - t.Run("TeesToEDSStore", func(t *testing.T) { - randEds, dah := randomEDS(t) - _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) - require.NoError(t, err) - - // eds store doesn't have the EDS yet - ok, err := edsStore.Has(ctx, dah.Hash()) - assert.False(t, ok) - assert.NoError(t, err) - - retrievedEDS, err := tg.GetEDS(ctx, dah) - require.NoError(t, err) - require.True(t, randEds.Equals(retrievedEDS)) - - // eds store now has the EDS and it can be retrieved - ok, err = edsStore.Has(ctx, dah.Hash()) - assert.True(t, ok) - assert.NoError(t, err) - finalEDS, err := edsStore.Get(ctx, dah.Hash()) - assert.NoError(t, err) - require.True(t, randEds.Equals(finalEDS)) - }) - - t.Run("ShardAlreadyExistsDoesntError", func(t *testing.T) { - randEds, dah := randomEDS(t) - _, err := ipld.ImportShares(ctx, randEds.Flattened(), bServ) - require.NoError(t, err) - - retrievedEDS, err := tg.GetEDS(ctx, dah) - require.NoError(t, err) - require.True(t, randEds.Equals(retrievedEDS)) - - // no error should be returned, even though the EDS identified by the DAH already exists - retrievedEDS, err = tg.GetEDS(ctx, dah) - require.NoError(t, err) - require.True(t, randEds.Equals(retrievedEDS)) - }) -} - func TestStoreGetter(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) diff --git a/share/getters/store.go b/share/getters/store.go index 415c9f047f..ad05cf8f37 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -132,11 +132,13 @@ func (sg *StoreGetter) GetSharesByNamespace( blockGetter := eds.NewBlockGetter(bs) shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) if errors.Is(err, ipld.ErrNodeNotFound) { - // IPLD node not found after the index pointed to this shard and the CAR blockstore has been - // opened successfully is a strong indicator of corruption. We remove the block on bridges - // and fulls and return share.ErrNotFound to ensure the data is retrieved by the next - // getter. Note that this recovery is manual and will only be restored by an RPC call to - // fetch the same datahash that was removed. + // IPLD node not found after the index pointed to this shard and the CAR + // blockstore has been opened successfully is a strong indicator of + // corruption. We remove the block on bridges and fulls and return + // share.ErrNotFound to ensure the data is retrieved by the next getter. + // Note that this recovery is manual and will only be restored by an RPC + // call to SharesAvailable that fetches the same datahash that was + // removed. err = sg.store.Remove(ctx, root.Hash()) if err != nil { log.Errorf("getter/store: failed to remove CAR after detected corruption: %w", err) diff --git a/share/getters/tee.go b/share/getters/tee.go deleted file mode 100644 index 9c89b2dec5..0000000000 --- a/share/getters/tee.go +++ /dev/null @@ -1,85 +0,0 @@ -package getters - -import ( - "context" - "errors" - "fmt" - - "github.com/filecoin-project/dagstore" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - - "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/eds" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -var _ share.Getter = (*TeeGetter)(nil) - -// TeeGetter is a share.Getter that wraps a getter and stores the results of GetEDS into an -// eds.Store. -type TeeGetter struct { - getter share.Getter - store *eds.Store -} - -// NewTeeGetter creates a new TeeGetter. -func NewTeeGetter(getter share.Getter, store *eds.Store) *TeeGetter { - return &TeeGetter{ - getter: getter, - store: store, - } -} - -func (tg *TeeGetter) GetShare(ctx context.Context, root *share.Root, row, col int) (share share.Share, err error) { - ctx, span := tracer.Start(ctx, "tee/get-share", trace.WithAttributes( - attribute.Int("row", row), - attribute.Int("col", col), - )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - return tg.getter.GetShare(ctx, root, row, col) -} - -func (tg *TeeGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d.ExtendedDataSquare, err error) { - ctx, span := tracer.Start(ctx, "tee/get-eds") - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - adder := ipld.NewProofsAdder(len(root.RowRoots)) - ctx = ipld.CtxWithProofsAdder(ctx, adder) - defer adder.Purge() - - eds, err = tg.getter.GetEDS(ctx, root) - if err != nil { - return nil, err - } - - err = tg.store.Put(ctx, root.Hash(), eds) - if err != nil && !errors.Is(err, dagstore.ErrShardExists) { - return nil, fmt.Errorf("getter/tee: failed to store eds: %w", err) - } - - return eds, nil -} - -func (tg *TeeGetter) GetSharesByNamespace( - ctx context.Context, - root *share.Root, - namespace share.Namespace, -) (shares share.NamespacedShares, err error) { - ctx, span := tracer.Start(ctx, "tee/get-shares-by-namespace", trace.WithAttributes( - attribute.String("namespace", namespace.String()), - )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - return tg.getter.GetSharesByNamespace(ctx, root, namespace) -} diff --git a/share/ipld/corrupted_data_test.go b/share/ipld/corrupted_data_test.go index df1d0d7888..58ddf785fb 100644 --- a/share/ipld/corrupted_data_test.go +++ b/share/ipld/corrupted_data_test.go @@ -26,7 +26,7 @@ func TestNamespaceHasher_CorruptedData(t *testing.T) { requestor := full.Node(net) provider, mockBS := availability_test.MockNode(t, net) - provider.Availability = full.TestAvailability(getters.NewIPLDGetter(provider.BlockService)) + provider.Availability = full.TestAvailability(t, getters.NewIPLDGetter(provider.BlockService)) net.ConnectAll() // before the provider starts attacking, we should be able to retrieve successfully. We pass a size @@ -38,7 +38,8 @@ func TestNamespaceHasher_CorruptedData(t *testing.T) { require.NoError(t, err) // clear the storage of the requester so that it must retrieve again, then start attacking - requestor.ClearStorage() + // we reinitialize the node to clear the eds store + requestor = full.Node(net) mockBS.Attacking = true getCtx, cancelGet = context.WithTimeout(ctx, sharesAvailableTimeout) t.Cleanup(cancelGet) From 9be8dab2af89f907c9bf5f648620a3221266fbe8 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 09:47:43 +0000 Subject: [PATCH 217/388] chore(deps): Bump google.golang.org/grpc from 1.58.0 to 1.58.1 (#2740) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.58.0 to 1.58.1.
Release notes

Sourced from google.golang.org/grpc's releases.

Release 1.58.1

Bug Fixes

  • grpc: fix a bug that was decrementing active RPC count too early for streaming RPCs; leading to channel moving to IDLE even though it had open streams
  • grpc: fix a bug where transports were not being closed upon channel entering IDLE
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.58.0&new-version=1.58.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0bee3a3181..5ba14005a0 100644 --- a/go.mod +++ b/go.mod @@ -72,7 +72,7 @@ require ( golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 - google.golang.org/grpc v1.58.0 + google.golang.org/grpc v1.58.1 google.golang.org/protobuf v1.31.0 ) diff --git a/go.sum b/go.sum index baae4c6cf5..1a926b5aa8 100644 --- a/go.sum +++ b/go.sum @@ -3209,8 +3209,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.58.0 h1:32JY8YpPMSR45K+c3o6b8VL73V+rR8k+DeMIr4vRH8o= -google.golang.org/grpc v1.58.0/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= +google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 303db05be2ea00093494551d0d51b758b22bb288 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Sep 2023 09:58:41 +0000 Subject: [PATCH 218/388] chore(deps): Bump github.com/ipfs/boxo from 0.12.0 to 0.13.0 (#2734) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/boxo](https://github.com/ipfs/boxo) from 0.12.0 to 0.13.0.
Release notes

Sourced from github.com/ipfs/boxo's releases.

v0.13.0

Added

  • ✨ The routing/http implements Delegated Peer Routing introduced in IPIP-417.

Changed

  • 🛠 The routing/http package received the following modifications:
    • Client GetIPNSRecord and PutIPNSRecord have been renamed to GetIPNS and PutIPNS, respectively. Similarly, the required function names in the server ContentRouter have also been updated.
    • ReadBitswapProviderRecord has been renamed to BitswapRecord and marked as deprecated. From now on, please use the protocol-agnostic PeerRecord for most use cases. The new Peer Schema has been introduced in IPIP-417.

Removed

  • 🛠 The routing/http package experienced following removals:
    • Server and client no longer support the experimental Provide method. ProvideBitswap is still usable, but marked as deprecated. A protocol-agnostic provide mechanism is being worked on in IPIP-378.
    • Server no longer exports FindProvidersPath and ProvidePath.

Fixed

  • The normalization of DNSLink identifiers in gateway has been corrected in the edge case where the value passed to the path component of the URL is already normalized.

What's Changed

Full Changelog: https://github.com/ipfs/boxo/compare/v0.12.0...v0.13.0

Changelog

Sourced from github.com/ipfs/boxo's changelog.

[v0.13.0]

Added

  • ✨ The routing/http implements Delegated Peer Routing introduced in IPIP-417.

Changed

  • 🛠 The routing/http package received the following modifications:
    • Client GetIPNSRecord and PutIPNSRecord have been renamed to GetIPNS and PutIPNS, respectively. Similarly, the required function names in the server ContentRouter have also been updated.
    • ReadBitswapProviderRecord has been renamed to BitswapRecord and marked as deprecated. From now on, please use the protocol-agnostic PeerRecord for most use cases. The new Peer Schema has been introduced in IPIP-417.

Removed

  • 🛠 The routing/http package experienced following removals:
    • Server and client no longer support the experimental Provide method. ProvideBitswap is still usable, but marked as deprecated. A protocol-agnostic provide mechanism is being worked on in IPIP-378.
    • Server no longer exports FindProvidersPath and ProvidePath.

Fixed

  • The normalization of DNSLink identifiers in gateway has been corrected in the edge case where the value passed to the path component of the URL is already normalized.

Security

Commits
  • 33ff595 Merge pull request #465 from ipfs/release-v0.13.0
  • f0ff3ce chore: bump to v0.13.0
  • 857bb53 docs: update changelog for v0.13.0
  • 68914ea fix(gateway): normalization of DNSLink inlining (#462)
  • 574df96 ci: remove obsolete protocol/cache-go-action (#460)
  • 1356946 feat(routing/http)!: delegated peer routing server and client, IPIP 417 (#422)
  • 79159c3 bitswap: mark TestSessionGetBlocks flaky
  • c135968 provider: mark TestOfflineRecordsThenOnlineRepublish flaky on windows
  • 38ecf0e bitswap: reenable tests
  • 303595b blockservice/test: passthrough blockservice options
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/boxo&package-manager=go_modules&previous-version=0.12.0&new-version=0.13.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 5ba14005a0..f10e76fd5e 100644 --- a/go.mod +++ b/go.mod @@ -30,7 +30,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.12.0 + github.com/ipfs/boxo v0.13.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 1a926b5aa8..dd72331b3e 100644 --- a/go.sum +++ b/go.sum @@ -1037,8 +1037,8 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.12.0 h1:AXHg/1ONZdRQHQLgG5JHsSC3XoE4DjCAMgK+asZvUcQ= -github.com/ipfs/boxo v0.12.0/go.mod h1:xAnfiU6PtxWCnRqu7dcXQ10bB5/kvI1kXRotuGqGBhg= +github.com/ipfs/boxo v0.13.0 h1:uzCQekieYS4PysbYYdodNmKLuqOdLjlUziXVZ19oDeQ= +github.com/ipfs/boxo v0.13.0/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= From 3426751af3216bb7db94cfcea6b51d45fa5ff9ee Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 22 Sep 2023 14:48:10 +0200 Subject: [PATCH 219/388] fix(full/availability): short-circuiting SharesAvailable on empty DAH (#2746) When we stopped wrapping full availability with cache availability we forgot to add a short circuit to full avail. This will save us a datastore hit through the eds store Has call. --- share/availability/full/availability.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index 51d9fd8518..be269077d7 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -57,6 +57,11 @@ func (fa *ShareAvailability) Stop(context.Context) error { // SharesAvailable reconstructs the data committed to the given Root by requesting // enough Shares from the network. func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Root) error { + // short-circuit if the given root is minimum DAH of an empty data square, to avoid datastore hit + if share.DataHash(root.Hash()).IsEmptyRoot() { + return nil + } + // we assume the caller of this method has already performed basic validation on the // given dah/root. If for some reason this has not happened, the node should panic. if err := root.ValidateBasic(); err != nil { From 5e22259e9e541539c1f7f470f7db89532b8d72be Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 25 Sep 2023 10:37:41 +0200 Subject: [PATCH 220/388] fix: improve ErrNoStateAccess message (#2752) --- nodebuilder/state/stub.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder/state/stub.go b/nodebuilder/state/stub.go index 94326fed5e..8d17d651dd 100644 --- a/nodebuilder/state/stub.go +++ b/nodebuilder/state/stub.go @@ -10,7 +10,7 @@ import ( "github.com/celestiaorg/celestia-node/state" ) -var ErrNoStateAccess = errors.New("node is running without state access") +var ErrNoStateAccess = errors.New("node is running without state access. run with --core.ip to resolve") // stubbedStateModule provides a stub for the state module to return // errors when state endpoints are accessed without a running connection From 4289e0975b3648b33a0333a02a3de935589c049d Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 25 Sep 2023 14:48:57 +0200 Subject: [PATCH 221/388] refactor(availability): merging cache.ShareAvailability and light.ShareAvailability (#2741) --- nodebuilder/share/constructors.go | 13 -- nodebuilder/share/module.go | 13 +- share/availability/cache/availability.go | 97 ------------- share/availability/cache/availability_test.go | 136 ------------------ share/availability/cache/testing.go | 39 ----- share/availability/light/availability.go | 59 +++++++- share/availability/light/availability_test.go | 66 ++++++++- share/availability/light/testing.go | 4 +- 8 files changed, 131 insertions(+), 296 deletions(-) delete mode 100644 share/availability/cache/availability.go delete mode 100644 share/availability/cache/availability_test.go delete mode 100644 share/availability/cache/testing.go diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 28e0997163..944b7f2d23 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -6,17 +6,13 @@ import ( "github.com/filecoin-project/dagstore" "github.com/ipfs/boxo/blockservice" - "github.com/ipfs/go-datastore" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/routing" routingdisc "github.com/libp2p/go-libp2p/p2p/discovery/routing" - "go.uber.org/fx" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/availability/cache" - "github.com/celestiaorg/celestia-node/share/availability/light" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/ipld" @@ -37,15 +33,6 @@ func newDiscovery(cfg Config) func(routing.ContentRouting, host.Host) *disc.Disc } } -// cacheAvailability wraps light availability with a cache for result sampling. -func cacheAvailability(lc fx.Lifecycle, ds datastore.Batching, avail *light.ShareAvailability) share.Availability { - ca := cache.NewShareAvailability(avail, ds) - lc.Append(fx.Hook{ - OnStop: ca.Close, - }) - return ca -} - func newModule(getter share.Getter, avail share.Availability) Module { return &module{getter, avail} } diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 15eb776551..b2e6eed4e5 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -214,10 +214,15 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option return nil } }), - fx.Provide(light.NewShareAvailability), - // cacheAvailability's lifecycle continues to use a fx hook, - // since the LC requires a cacheAvailability but the constructor returns a share.Availability - fx.Provide(cacheAvailability), + fx.Provide(fx.Annotate( + light.NewShareAvailability, + fx.OnStop(func(ctx context.Context, la *light.ShareAvailability) error { + return la.Close(ctx) + }), + )), + fx.Provide(func(avail *light.ShareAvailability) share.Availability { + return avail + }), ) default: panic("invalid node type") diff --git a/share/availability/cache/availability.go b/share/availability/cache/availability.go deleted file mode 100644 index a530d94d1d..0000000000 --- a/share/availability/cache/availability.go +++ /dev/null @@ -1,97 +0,0 @@ -package cache - -import ( - "bytes" - "context" - "sync" - - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/autobatch" - "github.com/ipfs/go-datastore/namespace" - logging "github.com/ipfs/go-log/v2" - - "github.com/celestiaorg/celestia-node/share" -) - -var ( - log = logging.Logger("share/cache") - - cacheAvailabilityPrefix = datastore.NewKey("sampling_result") - writeBatchSize = 2048 -) - -// ShareAvailability wraps a given share.Availability (whether it's light or full) -// and stores the results of a successful sampling routine over a given Root's hash -// to disk. -type ShareAvailability struct { - avail share.Availability - - // TODO(@Wondertan): Once we come to parallelized DASer, this lock becomes a contention point - // Related to #483 - dsLk sync.RWMutex - ds *autobatch.Datastore -} - -// NewShareAvailability wraps the given share.Availability with an additional datastore -// for sampling result caching. -func NewShareAvailability( - avail share.Availability, - ds datastore.Batching, -) *ShareAvailability { - ds = namespace.Wrap(ds, cacheAvailabilityPrefix) - autoDS := autobatch.NewAutoBatching(ds, writeBatchSize) - - return &ShareAvailability{ - avail: avail, - ds: autoDS, - } -} - -// SharesAvailable will store, upon success, the hash of the given Root to disk. -func (ca *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Root) error { - // short-circuit if the given root is minimum DAH of an empty data square - if isMinRoot(root) { - return nil - } - // do not sample over Root that has already been sampled - key := rootKey(root) - - ca.dsLk.RLock() - exists, err := ca.ds.Has(ctx, key) - ca.dsLk.RUnlock() - if err != nil || exists { - return err - } - - err = ca.avail.SharesAvailable(ctx, root) - if err != nil { - return err - } - - ca.dsLk.Lock() - err = ca.ds.Put(ctx, key, []byte{}) - ca.dsLk.Unlock() - if err != nil { - log.Errorw("storing root of successful SharesAvailable request to disk", "err", err) - } - return err -} - -func (ca *ShareAvailability) ProbabilityOfAvailability(ctx context.Context) float64 { - return ca.avail.ProbabilityOfAvailability(ctx) -} - -// Close flushes all queued writes to disk. -func (ca *ShareAvailability) Close(ctx context.Context) error { - return ca.ds.Flush(ctx) -} - -func rootKey(root *share.Root) datastore.Key { - return datastore.NewKey(root.String()) -} - -// isMinRoot returns whether the given root is a minimum (empty) -// DataAvailabilityHeader (DAH). -func isMinRoot(root *share.Root) bool { - return bytes.Equal(share.EmptyRoot().Hash(), root.Hash()) -} diff --git a/share/availability/cache/availability_test.go b/share/availability/cache/availability_test.go deleted file mode 100644 index 458ce5cc95..0000000000 --- a/share/availability/cache/availability_test.go +++ /dev/null @@ -1,136 +0,0 @@ -package cache - -import ( - "context" - "fmt" - "strconv" - "testing" - "time" - - "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/sync" - "github.com/stretchr/testify/assert" - "github.com/stretchr/testify/require" - - "github.com/celestiaorg/celestia-app/pkg/da" - - "github.com/celestiaorg/celestia-node/share" - availability_test "github.com/celestiaorg/celestia-node/share/availability/test" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -// TestCacheAvailability tests to ensure that the successful result of a -// sampling process is properly stored locally. -func TestCacheAvailability(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - fullLocalServ, dah0 := FullAvailabilityWithLocalRandSquare(t, 16) - lightLocalServ, dah1 := LightAvailabilityWithLocalRandSquare(t, 16) - - var tests = []struct { - avail share.Availability - root *share.Root - }{ - { - avail: fullLocalServ, - root: dah0, - }, - { - avail: lightLocalServ, - root: dah1, - }, - } - - for i, tt := range tests { - t.Run(strconv.Itoa(i), func(t *testing.T) { - ca := tt.avail.(*ShareAvailability) - // ensure the dah isn't yet in the cache - exists, err := ca.ds.Has(ctx, rootKey(tt.root)) - require.NoError(t, err) - assert.False(t, exists) - err = tt.avail.SharesAvailable(ctx, tt.root) - require.NoError(t, err) - // ensure the dah was stored properly - exists, err = ca.ds.Has(ctx, rootKey(tt.root)) - require.NoError(t, err) - assert.True(t, exists) - }) - } -} - -var invalidHeader = da.DataAvailabilityHeader{ - RowRoots: [][]byte{{1, 2}}, -} - -// TestCacheAvailability_Failed tests to make sure a failed -// sampling process is not stored. -func TestCacheAvailability_Failed(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second) - defer cancel() - - ca := NewShareAvailability(&dummyAvailability{}, sync.MutexWrap(datastore.NewMapDatastore())) - - err := ca.SharesAvailable(ctx, &invalidHeader) - require.Error(t, err) - // ensure the dah was NOT cached - exists, err := ca.ds.Has(ctx, rootKey(&invalidHeader)) - require.NoError(t, err) - assert.False(t, exists) -} - -// TestCacheAvailability_NoDuplicateSampling tests to ensure that -// if the Root was already sampled, it does not run a sampling routine -// again. -func TestCacheAvailability_NoDuplicateSampling(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - // create root to cache - root := availability_test.RandFillBS(t, 16, ipld.NewMemBlockservice()) - // wrap dummyAvailability with a datastore - ds := sync.MutexWrap(datastore.NewMapDatastore()) - ca := NewShareAvailability(&dummyAvailability{counter: 0}, ds) - // sample the root - err := ca.SharesAvailable(ctx, root) - require.NoError(t, err) - // ensure root was cached - exists, err := ca.ds.Has(ctx, rootKey(root)) - require.NoError(t, err) - assert.True(t, exists) - // call sampling routine over same root again and ensure no error is returned - // if an error was returned, that means duplicate sampling occurred - err = ca.SharesAvailable(ctx, root) - require.NoError(t, err) -} - -// TestCacheAvailability_MinRoot tests to make sure `SharesAvailable` will -// short circuit if the given root is a minimum DataAvailabilityHeader (minRoot). -func TestCacheAvailability_MinRoot(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - defer cancel() - - fullLocalAvail, _ := FullAvailabilityWithLocalRandSquare(t, 16) - err := fullLocalAvail.SharesAvailable(ctx, share.EmptyRoot()) - assert.NoError(t, err) -} - -type dummyAvailability struct { - counter int -} - -// SharesAvailable should only be called once, if called more than once, return -// error. -func (da *dummyAvailability) SharesAvailable(_ context.Context, root *share.Root) error { - if root == &invalidHeader { - return fmt.Errorf("invalid header") - } - if da.counter > 0 { - return fmt.Errorf("duplicate sampling process called") - } - da.counter++ - return nil -} - -func (da *dummyAvailability) ProbabilityOfAvailability(context.Context) float64 { - return 0 -} diff --git a/share/availability/cache/testing.go b/share/availability/cache/testing.go deleted file mode 100644 index 989bee522c..0000000000 --- a/share/availability/cache/testing.go +++ /dev/null @@ -1,39 +0,0 @@ -package cache - -import ( - "testing" - - ds "github.com/ipfs/go-datastore" - dssync "github.com/ipfs/go-datastore/sync" - - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/availability/full" - "github.com/celestiaorg/celestia-node/share/availability/light" - availability_test "github.com/celestiaorg/celestia-node/share/availability/test" - "github.com/celestiaorg/celestia-node/share/getters" - "github.com/celestiaorg/celestia-node/share/ipld" -) - -// LightAvailabilityWithLocalRandSquare wraps light.GetterWithRandSquare with cache availability -func LightAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availability, *share.Root) { - bServ := ipld.NewMemBlockservice() - store := dssync.MutexWrap(ds.NewMapDatastore()) - getter := getters.NewIPLDGetter(bServ) - avail := NewShareAvailability( - light.TestAvailability(getter), - store, - ) - return avail, availability_test.RandFillBS(t, n, bServ) -} - -// FullAvailabilityWithLocalRandSquare wraps full.GetterWithRandSquare with cache availability -func FullAvailabilityWithLocalRandSquare(t *testing.T, n int) (share.Availability, *share.Root) { - bServ := ipld.NewMemBlockservice() - store := dssync.MutexWrap(ds.NewMapDatastore()) - getter := getters.NewIPLDGetter(bServ) - avail := NewShareAvailability( - full.TestAvailability(t, getter), - store, - ) - return avail, availability_test.RandFillBS(t, n, bServ) -} diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index cc2e08129e..0fc073a2df 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -4,7 +4,11 @@ import ( "context" "errors" "math" + "sync" + "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/autobatch" + "github.com/ipfs/go-datastore/namespace" ipldFormat "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" @@ -12,7 +16,11 @@ import ( "github.com/celestiaorg/celestia-node/share/getters" ) -var log = logging.Logger("share/light") +var ( + log = logging.Logger("share/light") + cacheAvailabilityPrefix = datastore.NewKey("sampling_result") + writeBatchSize = 2048 +) // ShareAvailability implements share.Availability using Data Availability Sampling technique. // It is light because it does not require the downloading of all the data to verify @@ -21,30 +29,58 @@ var log = logging.Logger("share/light") type ShareAvailability struct { getter share.Getter params Parameters + + // TODO(@Wondertan): Once we come to parallelized DASer, this lock becomes a contention point + // Related to #483 + // TODO: Striped locks? :D + dsLk sync.RWMutex + ds *autobatch.Datastore } // NewShareAvailability creates a new light Availability. func NewShareAvailability( getter share.Getter, + ds datastore.Batching, opts ...Option, ) *ShareAvailability { params := DefaultParameters() + ds = namespace.Wrap(ds, cacheAvailabilityPrefix) + autoDS := autobatch.NewAutoBatching(ds, writeBatchSize) for _, opt := range opts { opt(¶ms) } - return &ShareAvailability{getter, params} + return &ShareAvailability{ + getter: getter, + params: params, + ds: autoDS, + } } // SharesAvailable randomly samples `params.SampleAmount` amount of Shares committed to the given // Root. This way SharesAvailable subjectively verifies that Shares are available. func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Root) error { - log.Debugw("Validate availability", "root", dah.String()) + // short-circuit if the given root is minimum DAH of an empty data square + if share.DataHash(dah.Hash()).IsEmptyRoot() { + return nil + } + + // do not sample over Root that has already been sampled + key := rootKey(dah) + + la.dsLk.RLock() + exists, err := la.ds.Has(ctx, key) + la.dsLk.RUnlock() + if err != nil || exists { + return err + } + + log.Debugw("validate availability", "root", dah.String()) // We assume the caller of this method has already performed basic validation on the // given dah/root. If for some reason this has not happened, the node should panic. if err := dah.ValidateBasic(); err != nil { - log.Errorw("Availability validation cannot be performed on a malformed DataAvailabilityHeader", + log.Errorw("availability validation cannot be performed on a malformed DataAvailabilityHeader", "err", err) panic(err) } @@ -95,6 +131,12 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Roo } } + la.dsLk.Lock() + err = la.ds.Put(ctx, key, []byte{}) + la.dsLk.Unlock() + if err != nil { + log.Errorw("storing root of successful SharesAvailable request to disk", "err", err) + } return nil } @@ -106,3 +148,12 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Roo func (la *ShareAvailability) ProbabilityOfAvailability(context.Context) float64 { return 1 - math.Pow(0.75, float64(la.params.SampleAmount)) } + +func rootKey(root *share.Root) datastore.Key { + return datastore.NewKey(root.String()) +} + +// Close flushes all queued writes to disk. +func (la *ShareAvailability) Close(ctx context.Context) error { + return la.ds.Flush(ctx) +} diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 549ea14ce5..1e6b99cda4 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -11,9 +11,69 @@ import ( "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/sharetest" ) +func TestSharesAvailableCaches(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + getter, dah := GetterWithRandSquare(t, 16) + avail := TestAvailability(getter) + + // cache doesn't have dah yet + has, err := avail.ds.Has(ctx, rootKey(dah)) + assert.NoError(t, err) + assert.False(t, has) + + err = avail.SharesAvailable(ctx, dah) + assert.NoError(t, err) + + // is now cached + has, err = avail.ds.Has(ctx, rootKey(dah)) + assert.NoError(t, err) + assert.True(t, has) +} + +func TestSharesAvailableHitsCache(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + getter, _ := GetterWithRandSquare(t, 16) + avail := TestAvailability(getter) + + bServ := ipld.NewMemBlockservice() + dah := availability_test.RandFillBS(t, 16, bServ) + + // blockstore doesn't actually have the dah + err := avail.SharesAvailable(ctx, dah) + require.Error(t, err) + + // cache doesn't have dah yet, since it errored + has, err := avail.ds.Has(ctx, rootKey(dah)) + assert.NoError(t, err) + assert.False(t, has) + + err = avail.ds.Put(ctx, rootKey(dah), []byte{}) + require.NoError(t, err) + + // should hit cache after putting + err = avail.SharesAvailable(ctx, dah) + require.NoError(t, err) +} + +func TestSharesAvailableEmptyRoot(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + defer cancel() + + getter, _ := GetterWithRandSquare(t, 16) + avail := TestAvailability(getter) + + err := avail.SharesAvailable(ctx, share.EmptyRoot()) + assert.NoError(t, err) +} + func TestSharesAvailable(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -28,10 +88,12 @@ func TestSharesAvailableFailed(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() + bServ := ipld.NewMemBlockservice() + dah := availability_test.RandFillBS(t, 16, bServ) + getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) - empty := share.EmptyRoot() - err := avail.SharesAvailable(ctx, empty) + err := avail.SharesAvailable(ctx, dah) assert.Error(t, err) } diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index 991d143a62..e68068fc25 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -4,6 +4,7 @@ import ( "testing" "github.com/ipfs/boxo/blockservice" + "github.com/ipfs/go-datastore" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" @@ -42,7 +43,8 @@ func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { } func TestAvailability(getter share.Getter) *ShareAvailability { - return NewShareAvailability(getter) + ds := datastore.NewMapDatastore() + return NewShareAvailability(getter, ds) } func SubNetNode(sn *availability_test.SubNet) *availability_test.TestNode { From 8602d6e51d00c018d4cc06b0a11db7a1c43b82f7 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 25 Sep 2023 20:52:47 +0800 Subject: [PATCH 222/388] refactor(share/discovery): decrease default Advertisement interval (#2758) Resolves https://github.com/celestiaorg/celestia-node/issues/2704 --- share/p2p/discovery/options.go | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/share/p2p/discovery/options.go b/share/p2p/discovery/options.go index 8a6a162e11..9d2735c50d 100644 --- a/share/p2p/discovery/options.go +++ b/share/p2p/discovery/options.go @@ -23,9 +23,8 @@ type Option func(*Parameters) // for the Discovery module func DefaultParameters() Parameters { return Parameters{ - PeersLimit: 5, - // based on https://github.com/libp2p/go-libp2p-kad-dht/pull/793 - AdvertiseInterval: time.Hour * 22, + PeersLimit: 5, + AdvertiseInterval: time.Hour, } } From 4c931473c10720b528c22c7c4b915ca4a1b4a76a Mon Sep 17 00:00:00 2001 From: Callum Waters Date: Mon, 25 Sep 2023 16:20:14 +0200 Subject: [PATCH 223/388] chore!: bump core and app (#2753) --- go.mod | 31 ++++++++++++++++---------- go.sum | 69 ++++++++++++++++++++++++++++++---------------------------- 2 files changed, 56 insertions(+), 44 deletions(-) diff --git a/go.mod b/go.mod index f10e76fd5e..0c85d4633d 100644 --- a/go.mod +++ b/go.mod @@ -1,8 +1,6 @@ module github.com/celestiaorg/celestia-node -go 1.21 - -toolchain go1.21.0 +go 1.21.1 require ( cosmossdk.io/errors v1.0.0 @@ -10,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc14 + github.com/celestiaorg/celestia-app v1.0.0-rc17 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.3.1 @@ -76,6 +74,19 @@ require ( google.golang.org/protobuf v1.31.0 ) +require ( + github.com/Microsoft/go-winio v0.6.1 // indirect + github.com/bits-and-blooms/bitset v1.5.0 // indirect + github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.1 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.10.0 // indirect + github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect + github.com/ethereum/c-kzg-4844 v0.3.1 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect + github.com/supranational/blst v0.3.11 // indirect + rsc.io/tmplfunc v0.0.3 // indirect +) + require ( cloud.google.com/go v0.110.6 // indirect cloud.google.com/go/compute v1.23.0 // indirect @@ -98,7 +109,6 @@ require ( github.com/bgentry/speakeasy v0.1.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect - github.com/celestiaorg/quantum-gravity-bridge v1.3.0 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -135,7 +145,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/go-ethereum v1.12.2 // indirect + github.com/ethereum/go-ethereum v1.13.1 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -190,7 +200,7 @@ require ( github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/holiman/uint256 v1.2.3 // indirect - github.com/huin/goupnp v1.2.0 // indirect + github.com/huin/goupnp v1.3.0 // indirect github.com/iancoleman/orderedmap v0.1.0 // indirect github.com/improbable-eng/grpc-web v0.15.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect @@ -296,8 +306,8 @@ require ( github.com/tendermint/go-amino v0.16.0 // indirect github.com/tendermint/tm-db v0.6.7 // indirect github.com/tidwall/btree v1.5.0 // indirect - github.com/tklauser/go-sysconf v0.3.10 // indirect - github.com/tklauser/numcpus v0.4.0 // indirect + github.com/tklauser/go-sysconf v0.3.12 // indirect + github.com/tklauser/numcpus v0.6.1 // indirect github.com/ulikunitz/xz v0.5.10 // indirect github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect @@ -325,7 +335,6 @@ require ( google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect gopkg.in/ini.v1 v1.67.0 // indirect - gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect @@ -339,5 +348,5 @@ replace ( github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28 ) diff --git a/go.sum b/go.sum index dd72331b3e..30a32a19f3 100644 --- a/go.sum +++ b/go.sum @@ -229,9 +229,8 @@ github.com/ChainSafe/go-schnorrkel v1.0.0 h1:3aDA67lAykLaG1y3AOjs88dMxC88PgUuHRr github.com/ChainSafe/go-schnorrkel v1.0.0/go.mod h1:dpzHYVxLZcp8pjlV+O+UR8K0Hp/z7vcchBSbMBEhCw4= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= +github.com/DataDog/zstd v1.5.0 h1:+K/VEwIAaPcHiMtQvpLD4lqW7f0Gk3xdYZmI1hD+CXo= github.com/DataDog/zstd v1.5.0/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= -github.com/DataDog/zstd v1.5.2 h1:vUG4lAyuPCXO0TLbXvPv7EB7cNK1QV/luu55UHLrrn8= -github.com/DataDog/zstd v1.5.2/go.mod h1:g4AWEaM3yOg3HYfnJ3YIawPnVdXJh9QME85blwSAmyw= github.com/Jorropo/jsync v1.0.1 h1:6HgRolFZnsdfzRUj+ImB9og1JYOxQoReSywkHOGSaUU= github.com/Jorropo/jsync v1.0.1/go.mod h1:jCOZj3vrBCri3bSU3ErUYvevKlnbssrXeCivybS5ABQ= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= @@ -239,8 +238,8 @@ github.com/Kubuxu/go-os-helper v0.0.1/go.mod h1:N8B+I7vPCT80IcP58r50u4+gEEcsZETF github.com/Masterminds/glide v0.13.2/go.mod h1:STyF5vcenH/rUqTEv+/hBXlSTo7KYwg2oc2f4tzPWic= github.com/Masterminds/semver v1.4.2/go.mod h1:MB6lktGJrhw8PrUyiEoblNEGEQ+RzHPF078ddwwvV3Y= github.com/Masterminds/vcs v1.13.0/go.mod h1:N09YCmOQr6RLxC6UNHzuVwAdodYbbnycGHSmwVJjcKA= -github.com/Microsoft/go-winio v0.6.0 h1:slsWYD/zyx7lCXoZVlvQrj0hPTM1HI4+v1sIda2yDvg= -github.com/Microsoft/go-winio v0.6.0/go.mod h1:cTAf44im0RAYeL23bpB+fzCyDH2MJiz2BO69KH/soAE= +github.com/Microsoft/go-winio v0.6.1 h1:9/kr64B9VUZrLm5YYwbGtUJnMgqWVOdUAXu6Migciow= +github.com/Microsoft/go-winio v0.6.1/go.mod h1:LRdKpFKfdobln8UmuiYcKPot9D2v6svN5+sAH+4kjUM= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5 h1:TngWCqHvy9oXAN6lEVMRuU21PR1EtLVZJmdB18Gu3Rw= github.com/Nvveen/Gotty v0.0.0-20120604004816-cd527374f1e5/go.mod h1:lmUJ/7eu/Q8D7ML55dXQrVaamCz2vxCfdQBasLZfHKk= github.com/OneOfOne/xxhash v1.2.2 h1:KMrpdQIwFcEqXDklaen+P1axHaj9BSKzvpUUfnHldSE= @@ -316,8 +315,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= -github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= +github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -359,10 +358,10 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc14 h1:kC3ljQzxBjQM1s8aw0dOnrn5pNbrfyfd+lS155S7QAc= -github.com/celestiaorg/celestia-app v1.0.0-rc14/go.mod h1:6aD4sFGQIe5XPYejLDKrDKsNWlDu9/P3e54pCLPywNk= -github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28 h1:2efXQaggLFknz0wQufr4nUEz5G7pSVHS1j7NuJDsvII= -github.com/celestiaorg/celestia-core v1.26.2-tm-v0.34.28/go.mod h1:++dNzzzjP9jYg+NopN9G8sg1HEZ58lv1TPtg71evZ0E= +github.com/celestiaorg/celestia-app v1.0.0-rc17 h1:ak9OIViNJCfDFklopPgxnjRqUx5oKPcctpGpmwmqJy4= +github.com/celestiaorg/celestia-app v1.0.0-rc17/go.mod h1:gDKOpD8xCWHw/PUdcTc0WFYf52Mq0etOFZ1oTkBlFkQ= +github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28 h1:BE7JFZ1SYpwM9OfL9cPcjlO5xjIbDPgdFkJNouyl6jA= +github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28/go.mod h1:1GT0RfdNqOXvyR3Hq4ROcNBknQNz9E6K5l3Cla9eFFk= github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 h1:dDfoQJOlVNj4HufJ1lBLTo2k3/L/255MIiKmEQziDmw= github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14/go.mod h1:kkdiHo/zG6ar80730+bG1owdMAQXrGp4utFu7mbfADo= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= @@ -379,8 +378,8 @@ github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.20.0 h1:9i7ultZ8Wv5ytt8ZRaxKQ5KOOMo4A2K2T/aPGjIlSas= github.com/celestiaorg/nmt v0.20.0/go.mod h1:Oz15Ub6YPez9uJV0heoU4WpFctxazuIhKyUtaYNio7E= -github.com/celestiaorg/quantum-gravity-bridge v1.3.0 h1:9zPIp7w1FWfkPnn16y3S4FpFLnQtS7rm81CUVcHEts0= -github.com/celestiaorg/quantum-gravity-bridge v1.3.0/go.mod h1:6WOajINTDEUXpSj5UZzod16UZ96ZVB/rFNKyM+Mt1gI= +github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.1 h1:xOBMoRYSh/hnsEW7OLI3bjSJXKlqILfZehjPEK/+3oI= +github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.1/go.mod h1:NziNBP12grIi+DH3TqIKUUPM2T6Hl9BTyxiK3+p8yeA= github.com/celestiaorg/rsmt2d v0.11.0 h1:lcto/637WyTEZR3dLRoNvyuExfnUbxvdvKi3qz/2V4k= github.com/celestiaorg/rsmt2d v0.11.0/go.mod h1:6Y580I3gVr0+OVFfW6m2JTwnCCmvW3WfbwSLfuT+HCA= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -430,14 +429,16 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/errors v1.9.1 h1:yFVvsI0VxmRShfawbt/laCIDy/mtTqqnvoNgiy5bEV8= -github.com/cockroachdb/errors v1.9.1/go.mod h1:2sxOtL2WIc096WSZqZ5h8fa17rdDq9HZOZLBCor4mBk= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= -github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811 h1:ytcWPaNPhNoGMWEhDvS3zToKcDpRsLuRolQJBVGdozk= -github.com/cockroachdb/pebble v0.0.0-20230209160836-829675f94811/go.mod h1:Nb5lgvnQ2+oGlE/EyZy4+2/CxRh9KfvCXnag1vtpxVM= -github.com/cockroachdb/redact v1.1.3 h1:AKZds10rFSIj7qADf0g46UixK8NNLwWTNdCIGS5wfSQ= -github.com/cockroachdb/redact v1.1.3/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= +github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= +github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= +github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= +github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= +github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= +github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= +github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= @@ -605,8 +606,8 @@ github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eE github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= -github.com/ethereum/go-ethereum v1.12.2 h1:eGHJ4ij7oyVqUQn48LBz3B7pvQ8sV0wGJiIE6gDq/6Y= -github.com/ethereum/go-ethereum v1.12.2/go.mod h1:1cRAEV+rp/xX0zraSCBnu9Py3HQ+geRMj3HdR+k0wfI= +github.com/ethereum/go-ethereum v1.13.1 h1:UF2FaUKPIy5jeZk3X06ait3y2Q4wI+vJ1l7+UARp+60= +github.com/ethereum/go-ethereum v1.13.1/go.mod h1:xHQKzwkHSl0gnSjZK1mWa06XEdm9685AHqhRknOzqGQ= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= @@ -655,8 +656,6 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= -github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= -github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= @@ -867,6 +866,7 @@ github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0Z github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -1002,8 +1002,8 @@ github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmK github.com/huin/goupnp v1.0.0/go.mod h1:n9v9KO1tAxYH82qOn+UTIFQDmx5n1Zxd/ClZDMX7Bnc= github.com/huin/goupnp v1.0.3-0.20220313090229-ca81a64b4204/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= github.com/huin/goupnp v1.0.3/go.mod h1:ZxNlw5WqJj6wSsRK5+YfflQGXYfccj5VgQsMNixHM7Y= -github.com/huin/goupnp v1.2.0 h1:uOKW26NG1hsSSbXIZ1IR7XP9Gjd1U8pnLaCMgntmkmY= -github.com/huin/goupnp v1.2.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= +github.com/huin/goupnp v1.3.0 h1:UvLUlWDNpoUdYzb2TCn+MuTWtcjXKSza2n6CBdQ0xXc= +github.com/huin/goupnp v1.3.0/go.mod h1:gnGPsThkYa7bFi/KWmEysQRf48l2dvR5bxr2OFckNX8= github.com/huin/goutil v0.0.0-20170803182201-1ca381bf3150/go.mod h1:PpLOETDnJ0o3iZrZfqZzyLl6l7F3c6L1oWn7OICBi6o= github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= github.com/iancoleman/orderedmap v0.1.0 h1:2orAxZBJsvimgEBmMWfXaFlzSG2fbQil5qzP3F6cCkg= @@ -1374,6 +1374,7 @@ github.com/kylelemons/godebug v1.1.0 h1:RPNrshWIDI6G2gRW9EHilWtl7Z6Sb1BR0xunSBf0 github.com/kylelemons/godebug v1.1.0/go.mod h1:9/0rRGxNHcop5bhtWyNeEfOS8JIWk580+fNqagV/RAw= github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4FW1e6jwpg= github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= +github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= @@ -1784,6 +1785,7 @@ github.com/mitchellh/pointerstructure v1.2.0 h1:O+i9nHnXS3l/9Wu7r4NrEdwA2VFTicjU github.com/mitchellh/pointerstructure v1.2.0/go.mod h1:BRAsLI5zgXmw97Lf6s25bs8ohIXc3tViBH44KcwB2g4= github.com/mmcloughlin/addchain v0.4.0 h1:SobOdjm2xLj1KkXN5/n0xTIWyZA2+s99UCY1iPfkHRY= github.com/mmcloughlin/addchain v0.4.0/go.mod h1:A86O+tHqZLMNO4w6ZZ4FlVQEadcoqkyU72HC5wJ4RlU= +github.com/mmcloughlin/profile v0.1.1/go.mod h1:IhHD7q1ooxgwTgjxQYkACGA77oFTDdFVejUS1/tS/qU= github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd h1:TRLaZ9cD/w8PVh93nsPXa1VrQ6jlwL5oN8l14QlcNfg= github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q= @@ -2268,11 +2270,11 @@ github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDW github.com/tinylib/msgp v1.1.5/go.mod h1:eQsjooMTnV42mHu917E26IogZ2930nFyBQdofk10Udg= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= github.com/tklauser/go-sysconf v0.3.5/go.mod h1:MkWzOF4RMCshBAMXuhXJs64Rte09mITnppBXY/rYEFI= -github.com/tklauser/go-sysconf v0.3.10 h1:IJ1AZGZRWbY8T5Vfk04D9WOA5WSejdflXxP03OUqALw= -github.com/tklauser/go-sysconf v0.3.10/go.mod h1:C8XykCvCb+Gn0oNCWPIlcb0RuglQTYaQ2hGm7jmxEFk= +github.com/tklauser/go-sysconf v0.3.12 h1:0QaGUFOdQaIVdPgfITYzaTegZvdCjmYO52cSFAEVmqU= +github.com/tklauser/go-sysconf v0.3.12/go.mod h1:Ho14jnntGE1fpdOqQEEaiKRpvIavV0hSfmBq8nJbHYI= github.com/tklauser/numcpus v0.2.2/go.mod h1:x3qojaO3uyYt0i56EW/VUYs7uBvdl2fkfZFu0T9wgjM= -github.com/tklauser/numcpus v0.4.0 h1:E53Dm1HjH1/R2/aoCtXtPgzmElmn51aOkhCFSuZq//o= -github.com/tklauser/numcpus v0.4.0/go.mod h1:1+UI3pD8NW14VMwdgJNJ1ESk2UnwhAnz5hMwiKKqXCQ= +github.com/tklauser/numcpus v0.6.1 h1:ng9scYS7az0Bk4OZLvrNXNSAO2Pxr1XXRAPyjhIx+Fk= +github.com/tklauser/numcpus v0.6.1/go.mod h1:1XfjsgE2zo8GVw7POkMbHENHzVg3GzmoZ9fESEdAacY= github.com/tmc/grpc-websocket-proxy v0.0.0-20170815181823-89b8d40f7ca8/go.mod h1:ncp9v5uamzpCO7NfCPTXjqaC+bZgJeR0sMTm6dMHP7U= github.com/ttacon/chalk v0.0.0-20160626202418-22c06c80ed31/go.mod h1:onvgF043R+lC5RZ8IT9rBXDaEDnpnw/Cl+HFiw+v/7Q= github.com/tv42/httpunix v0.0.0-20150427012821-b75d8614f926/go.mod h1:9ESjWnEqriFuLhtthL60Sar/7RFoluCcXsuvEwTV5KM= @@ -2294,8 +2296,8 @@ github.com/urfave/cli v1.22.2/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtX github.com/urfave/cli v1.22.10 h1:p8Fspmz3iTctJstry1PYS3HVdllxnEzTEsgIgtxTrCk= github.com/urfave/cli v1.22.10/go.mod h1:Gos4lmkARVdJ6EkW0WaNv/tZAAMe9V7XWyB60NtXRu0= github.com/urfave/cli/v2 v2.3.0/go.mod h1:LJmUH05zAU44vOAcrfzZQKsZbVcdbOG8rtL3/XcUArI= -github.com/urfave/cli/v2 v2.24.1 h1:/QYYr7g0EhwXEML8jO+8OYt5trPnLHS0p3mrgExJ5NU= -github.com/urfave/cli/v2 v2.24.1/go.mod h1:GHupkWPMM0M/sj1a2b4wUrWBPzazNrIjouW6fmdJLxc= +github.com/urfave/cli/v2 v2.25.7 h1:VAzn5oq403l5pHjc4OhD54+XGO9cdKVL/7lDjF+iKUs= +github.com/urfave/cli/v2 v2.25.7/go.mod h1:8qnjx1vcq5s2/wpsqoZFndg2CE5tNFyrTvS6SinrnYQ= github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/valyala/fasttemplate v1.0.1/go.mod h1:UQGH1tvbgY+Nz5t2n7tXsz52dQxojPUpymEIMZ47gx8= github.com/valyala/fasttemplate v1.2.1/go.mod h1:KHLXt3tVN2HBp8eijSv/kGJopbvo7S+qRAEEKiv+SiQ= @@ -2850,6 +2852,8 @@ golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.4.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -3247,7 +3251,6 @@ gopkg.in/ini.v1 v1.67.0 h1:Dgnx+6+nfE+IfzjUEISNeydPJh9AXNNsWbGP9KzCsOA= gopkg.in/ini.v1 v1.67.0/go.mod h1:pNLf8WUiyNEtQjuu5G5vTm06TEv9tsIgeAvK8hOrP4k= gopkg.in/natefinch/lumberjack.v2 v2.0.0 h1:1Lc07Kr7qY4U2YPouBjpCLxpiyxIVoxqXgkXLknAOE8= gopkg.in/natefinch/lumberjack.v2 v2.0.0/go.mod h1:l0ndWWf7gzL7RNwBG7wST/UCcT4T24xpD6X8LsfU/+k= -gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce h1:+JknDZhAj8YMt7GC73Ei8pv4MzjDUNPHgQWJdtMAaDU= gopkg.in/natefinch/npipe.v2 v2.0.0-20160621034901-c1b8fa8bdcce/go.mod h1:5AcXVHNjg+BDxry382+8OKon8SEWiKktQR07RKPsv1c= gopkg.in/olebedev/go-duktape.v3 v3.0.0-20200619000410-60c24ae608a6/go.mod h1:uAJfkITjFhyEEuUfm7bsmCZRbW5WRq8s9EY8HZ6hCns= gopkg.in/resty.v1 v1.12.0/go.mod h1:mDo4pnntr5jdWRML875a/NmxYqAlA73dVijT2AXvQQo= From 925961e427b5b825b2840bea024988c758113880 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:21:37 +0200 Subject: [PATCH 224/388] chore(deps): Bump github.com/ipfs/boxo from 0.13.0 to 0.13.1 (#2757) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/boxo](https://github.com/ipfs/boxo) from 0.13.0 to 0.13.1.
Release notes

Sourced from github.com/ipfs/boxo's releases.

v0.13.1

Added

  • An option DisableHTMLErrors has been added to gateway.Config. When this option is true, pretty HTML error pages for web browsers are disabled. Instead, a text/plain page with the raw error message as the body is returned.

What's Changed

Full Changelog: https://github.com/ipfs/boxo/compare/v0.13.0...v0.13.1

Changelog

Sourced from github.com/ipfs/boxo's changelog.

[v0.13.1]

Added

  • An option DisableHTMLErrors has been added to gateway.Config. When this option is true, pretty HTML error pages for web browsers are disabled. Instead, a text/plain page with the raw error message as the body is returned.

Changed

Removed

Fixed

Security

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/boxo&package-manager=go_modules&previous-version=0.13.0&new-version=0.13.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0c85d4633d..99105c50d7 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru/v2 v2.0.6 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.13.0 + github.com/ipfs/boxo v0.13.1 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 30a32a19f3..9021ba7c15 100644 --- a/go.sum +++ b/go.sum @@ -1037,8 +1037,8 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.0 h1:uzCQekieYS4PysbYYdodNmKLuqOdLjlUziXVZ19oDeQ= -github.com/ipfs/boxo v0.13.0/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.13.1 h1:nQ5oQzcMZR3oL41REJDcTbrvDvuZh3J9ckc9+ILeRQI= +github.com/ipfs/boxo v0.13.1/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= From cfdd9bee7677cf29d5c0393e947278c7c164e529 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:22:01 +0200 Subject: [PATCH 225/388] chore(deps): Bump google.golang.org/grpc from 1.58.1 to 1.58.2 (#2756) Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.58.1 to 1.58.2.
Release notes

Sourced from google.golang.org/grpc's releases.

Release 1.58.2

Bug Fixes

  • balancer/weighted_round_robin: fix ticker leak on update

    A new ticker is created every time there is an update of addresses or configuration, but was not properly stopped. This change stops the ticker when it is no longer needed.

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.58.1&new-version=1.58.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 99105c50d7..7c023ce604 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 - google.golang.org/grpc v1.58.1 + google.golang.org/grpc v1.58.2 google.golang.org/protobuf v1.31.0 ) diff --git a/go.sum b/go.sum index 9021ba7c15..3776182581 100644 --- a/go.sum +++ b/go.sum @@ -3213,8 +3213,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= -google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= +google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 49e6c246e5ab0246ccf473f9f316887c90bda8ce Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 27 Sep 2023 11:28:22 +0200 Subject: [PATCH 226/388] fix(share/p2p/peers): Prevent making pools for nil DataHash (#2761) Not critical, but if nil DataHash is broadcasted, node could create pool for empty datahash and just keep it around until garbage collected. Best to short-circuit instead and reject message. --- share/p2p/peers/manager_test.go | 14 +++-- share/p2p/shrexsub/pubsub.go | 5 +- share/p2p/shrexsub/pubsub_test.go | 100 +++++++++++++++++++++++++----- 3 files changed, 97 insertions(+), 22 deletions(-) diff --git a/share/p2p/peers/manager_test.go b/share/p2p/peers/manager_test.go index c60a737baa..1ad7e65c02 100644 --- a/share/p2p/peers/manager_test.go +++ b/share/p2p/peers/manager_test.go @@ -17,6 +17,7 @@ import ( "github.com/libp2p/go-libp2p/p2p/net/conngater" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/rand" libhead "github.com/celestiaorg/go-header" @@ -348,18 +349,18 @@ func TestIntegration(t *testing.T) { time.Sleep(time.Millisecond * 100) // broadcast from BN - peerHash := share.DataHash("peer1") + randHash := rand.Bytes(32) require.NoError(t, bnPubSub.Broadcast(ctx, shrexsub.Notification{ - DataHash: peerHash, + DataHash: randHash, Height: 1, })) // FN should get message - peerID, _, err := fnPeerManager.Peer(ctx, peerHash) + gotPeer, _, err := fnPeerManager.Peer(ctx, randHash) require.NoError(t, err) - // check that peerID matched bridge node - require.Equal(t, nw.Hosts()[0].ID(), peerID) + // check that gotPeer matched bridge node + require.Equal(t, nw.Hosts()[0].ID(), gotPeer) }) t.Run("get peer from discovery", func(t *testing.T) { @@ -488,7 +489,8 @@ func stopManager(t *testing.T, m *Manager) { func testHeader() *header.ExtendedHeader { return &header.ExtendedHeader{ RawHeader: header.RawHeader{ - Height: 1, + Height: 1, + DataHash: rand.Bytes(32), }, } } diff --git a/share/p2p/shrexsub/pubsub.go b/share/p2p/shrexsub/pubsub.go index 82ed256e33..ed713b4614 100644 --- a/share/p2p/shrexsub/pubsub.go +++ b/share/p2p/shrexsub/pubsub.go @@ -102,8 +102,9 @@ func (v ValidatorFn) validate(ctx context.Context, p peer.ID, msg *pubsub.Messag DataHash: pbmsg.DataHash, Height: pbmsg.Height, } - if n.DataHash.IsEmptyRoot() { - // we don't send empty EDS data hashes, but If someone sent it to us - do hard reject + if n.Height == 0 || n.DataHash.IsEmptyRoot() || n.DataHash.Validate() != nil { + // hard reject malicious height (height 0 does not exist) and + // empty/invalid datahashes return pubsub.ValidationReject } return v(ctx, p, n) diff --git a/share/p2p/shrexsub/pubsub_test.go b/share/p2p/shrexsub/pubsub_test.go index 4198b64379..dcf85515e8 100644 --- a/share/p2p/shrexsub/pubsub_test.go +++ b/share/p2p/shrexsub/pubsub_test.go @@ -6,8 +6,10 @@ import ( "time" pubsub "github.com/libp2p/go-libp2p-pubsub" + "github.com/libp2p/go-libp2p/core/peer" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/libs/rand" pb "github.com/celestiaorg/celestia-node/share/p2p/shrexsub/pb" ) @@ -23,6 +25,13 @@ func TestPubSub(t *testing.T) { pSub2, err := NewPubSub(ctx, h.Hosts()[1], "test") require.NoError(t, err) + err = pSub2.AddValidator( + func(ctx context.Context, p peer.ID, n Notification) pubsub.ValidationResult { + // only testing shrexsub validation here + return pubsub.ValidationAccept + }, + ) + require.NoError(t, err) require.NoError(t, pSub1.Start(ctx)) require.NoError(t, pSub2.Start(ctx)) @@ -30,23 +39,86 @@ func TestPubSub(t *testing.T) { subs, err := pSub2.Subscribe() require.NoError(t, err) - notification := Notification{ - DataHash: []byte("data"), - Height: 1, + var tests = []struct { + name string + notif Notification + errExpected bool + }{ + { + name: "vaild height, valid hash", + notif: Notification{ + Height: 1, + DataHash: rand.Bytes(32), + }, + errExpected: false, + }, + { + name: "valid height, invalid hash (<32 bytes)", + notif: Notification{ + Height: 2, + DataHash: rand.Bytes(20), + }, + errExpected: true, + }, + { + name: "valid height, invalid hash (>32 bytes)", + notif: Notification{ + Height: 2, + DataHash: rand.Bytes(64), + }, + errExpected: true, + }, + { + name: "invalid height, valid hash", + notif: Notification{ + Height: 0, + DataHash: rand.Bytes(32), + }, + errExpected: true, + }, + { + name: "invalid height, nil hash", + notif: Notification{ + Height: 0, + DataHash: nil, + }, + errExpected: true, + }, + { + name: "valid height, nil hash", + notif: Notification{ + Height: 30, + DataHash: nil, + }, + errExpected: true, + }, } - msg := pb.RecentEDSNotification{ - Height: notification.Height, - DataHash: notification.DataHash, - } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + msg := pb.RecentEDSNotification{ + Height: tt.notif.Height, + DataHash: tt.notif.DataHash, + } + data, err := msg.Marshal() + require.NoError(t, err) - data, err := msg.Marshal() - require.NoError(t, err) + err = pSub1.topic.Publish(ctx, data, pubsub.WithReadiness(pubsub.MinTopicSize(1))) + require.NoError(t, err) - err = pSub1.topic.Publish(ctx, data, pubsub.WithReadiness(pubsub.MinTopicSize(1))) - require.NoError(t, err) + reqCtx, reqCtxCancel := context.WithTimeout(context.Background(), time.Millisecond*100) + defer reqCtxCancel() + + got, err := subs.Next(reqCtx) + if tt.errExpected { + require.Error(t, err) + require.ErrorIs(t, err, context.DeadlineExceeded) + return + } + require.NoError(t, err) + require.NoError(t, err) + require.Equal(t, tt.notif, got) + }) + } - got, err := subs.Next(ctx) - require.NoError(t, err) - require.Equal(t, notification, got) } From 849cb6757f60841ab3c6fd5f9564186df81c5405 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 28 Sep 2023 11:27:51 +0100 Subject: [PATCH 227/388] ignore dependabot in issue-label-automation/automate-labels workflow (#2763) ## Overview - Ignores triggering the issue-label-automation workflow when a workflow is triggered by dependabot[bot]. - why?: It always fails as the "check external contributor" action doesn't find a legitimate user - this seemed simplest way vs attempting to modify the imported action to allow configuration to ignore dependabot or similar - also introduces some minor/nit formatting with a whitespace change ## Checklist - [n/a] New and updated code has appropriate documentation - [n/a] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [n/a] Visual proof for any user facing features like CLI or documentation updates - [n/a] Linked issues closed with keywords --- .github/workflows/issue-label-automation.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/issue-label-automation.yml b/.github/workflows/issue-label-automation.yml index 4b24bca99c..965dbe8e9f 100644 --- a/.github/workflows/issue-label-automation.yml +++ b/.github/workflows/issue-label-automation.yml @@ -5,12 +5,15 @@ on: types: [opened] issues: types: [opened] + jobs: automate-labels: runs-on: ubuntu-latest + if: ${{ github.actor != 'dependabot[bot]' }} permissions: issues: write pull-requests: write + steps: - name: Check for External Contributor uses: tspascoal/get-user-teams-membership@v2 From bb9658a510e16a97d0a00782e69ff807a951c047 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 2 Oct 2023 20:29:40 +0800 Subject: [PATCH 228/388] test(share/eds/cache): benchmark eds cache eviction performance (#2778) Adds benchmarks to ensure cache overload performance. 1. Measures time to fully load accessor from disk to cache 2. Measures time it takes for multiple parallel ipld readers (clients) to sample eds directly from store, when there is not enough cache slots. Results shows it takes ~12s to sample for parallel 10 readers. --- share/eds/store_test.go | 58 +++++++++++++++++++++++++++++ share/getters/getter_test.go | 71 ++++++++++++++++++++++++++++++++++++ 2 files changed, 129 insertions(+) diff --git a/share/eds/store_test.go b/share/eds/store_test.go index ce42a0e0b9..7052533555 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -18,11 +18,13 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" + dsbadger "github.com/celestiaorg/go-ds-badger4" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/cache" "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" ) func TestEDSStore(t *testing.T) { @@ -465,6 +467,62 @@ func BenchmarkStore(b *testing.B) { }) } +// BenchmarkCacheEviction benchmarks the time it takes to load a block to the cache, when the +// cache size is set to 1. This forces cache eviction on every read. +// BenchmarkCacheEviction-10/128 384 3533586 ns/op (~3ms) +func BenchmarkCacheEviction(b *testing.B) { + const ( + blocks = 4 + size = 128 + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + b.Cleanup(cancel) + + dir := b.TempDir() + ds, err := dsbadger.NewDatastore(dir, &dsbadger.DefaultOptions) + require.NoError(b, err) + + newStore := func(params *Parameters) *Store { + edsStore, err := NewStore(params, dir, ds) + require.NoError(b, err) + err = edsStore.Start(ctx) + require.NoError(b, err) + return edsStore + } + edsStore := newStore(DefaultParameters()) + + // generate EDSs and store them + cids := make([]cid.Cid, blocks) + for i := range cids { + eds := edstest.RandEDS(b, size) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) + err = edsStore.Put(ctx, dah.Hash(), eds) + require.NoError(b, err) + + // store cids for read loop later + cids[i] = ipld.MustCidFromNamespacedSha256(dah.RowRoots[0]) + } + + // restart store to clear cache + require.NoError(b, edsStore.Stop(ctx)) + + // set BlockstoreCacheSize to 1 to force eviction on every read + params := DefaultParameters() + params.BlockstoreCacheSize = 1 + bstore := newStore(params).Blockstore() + + // start benchmark + b.ResetTimer() + for i := 0; i < b.N; i++ { + h := cids[i%blocks] + // every read will trigger eviction + _, err := bstore.Get(ctx, h) + require.NoError(b, err) + } +} + func newStore(t *testing.T) (*Store, error) { t.Helper() diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index bacb0a2c39..6c75ddf83f 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -3,6 +3,7 @@ package getters import ( "context" "os" + "sync" "testing" "time" @@ -12,7 +13,9 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" + dsbadger "github.com/celestiaorg/go-ds-badger4" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" @@ -217,6 +220,74 @@ func TestIPLDGetter(t *testing.T) { }) } +// BenchmarkIPLDGetterOverBusyCache benchmarks the performance of the IPLDGetter when the +// cache size of the underlying blockstore is less than the number of blocks being requested in +// parallel. This is to ensure performance doesn't degrade when the cache is being frequently +// evicted. +// BenchmarkIPLDGetterOverBusyCache-10/128 1 12460428417 ns/op (~12s) +func BenchmarkIPLDGetterOverBusyCache(b *testing.B) { + const ( + blocks = 10 + size = 128 + ) + + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + b.Cleanup(cancel) + + dir := b.TempDir() + ds, err := dsbadger.NewDatastore(dir, &dsbadger.DefaultOptions) + require.NoError(b, err) + + newStore := func(params *eds.Parameters) *eds.Store { + edsStore, err := eds.NewStore(params, dir, ds) + require.NoError(b, err) + err = edsStore.Start(ctx) + require.NoError(b, err) + return edsStore + } + edsStore := newStore(eds.DefaultParameters()) + + // generate EDSs and store them + hashes := make([]da.DataAvailabilityHeader, blocks) + for i := range hashes { + eds := edstest.RandEDS(b, size) + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(b, err) + err = edsStore.Put(ctx, dah.Hash(), eds) + require.NoError(b, err) + + // store cids for read loop later + hashes[i] = dah + } + + // restart store to clear cache + require.NoError(b, edsStore.Stop(ctx)) + + // set BlockstoreCacheSize to 1 to force eviction on every read + params := eds.DefaultParameters() + params.BlockstoreCacheSize = 1 + edsStore = newStore(params) + bstore := edsStore.Blockstore() + bserv := ipld.NewBlockservice(bstore, offline.Exchange(bstore)) + + // start client + getter := NewIPLDGetter(bserv) + + // request blocks in parallel + b.ResetTimer() + g := sync.WaitGroup{} + g.Add(blocks) + for _, h := range hashes { + h := h + go func() { + defer g.Done() + _, err := getter.GetEDS(ctx, &h) + require.NoError(b, err) + }() + } + g.Wait() +} + func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { eds := edstest.RandEDS(t, 4) dah, err := share.NewRoot(eds) From 20c7476a5fbc47d9cf1569277e293632fba9579b Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 2 Oct 2023 21:03:16 +0800 Subject: [PATCH 229/388] chore(deps): fix dependabot otel grouping (#2795) Fixes dependabot regex to not use`^`, since it seems to not support anything but `*` wildcard --- .github/dependabot.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 26f1b4da15..4ab88e1d32 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -22,7 +22,7 @@ updates: groups: otel: patterns: - - "^go.opentelemetry.io/otel*" + - "go.opentelemetry.io/otel*" - package-ecosystem: docker directory: "/" schedule: From 9e0d02169a556e7a306e96c2dadd7fd6badfccbd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 2 Oct 2023 16:17:59 +0100 Subject: [PATCH 230/388] chore(deps): Bump go.opentelemetry.io/otel/exporters/otlp/otlptrace from 1.18.0 to 1.19.0 (#2784) Bumps [go.opentelemetry.io/otel/exporters/otlp/otlptrace](https://github.com/open-telemetry/opentelemetry-go) from 1.18.0 to 1.19.0.
Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlptrace's changelog.

[1.19.0/0.42.0/0.0.7] 2023-09-28

This release contains the first stable release of the OpenTelemetry Go [metric SDK]. Our project stability guarantees now apply to the go.opentelemetry.io/otel/sdk/metric package. See our versioning policy for more information about these stability guarantees.

Added

  • Add the "Roll the dice" getting started application example in go.opentelemetry.io/otel/example/dice. (#4539)
  • The WithWriter and WithPrettyPrint options to go.opentelemetry.io/otel/exporters/stdout/stdoutmetric to set a custom io.Writer, and allow displaying the output in human-readable JSON. (#4507)

Changed

  • Allow '/' characters in metric instrument names. (#4501)
  • The exporter in go.opentelemetry.io/otel/exporters/stdout/stdoutmetric does not prettify its output by default anymore. (#4507)
  • Upgrade gopkg.io/yaml from v2 to v3 in go.opentelemetry.io/otel/schema. (#4535)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the Prometheus metric on every Collect if we know the scope is invalid. (#4499)

Removed

  • Remove "go.opentelemetry.io/otel/bridge/opencensus".NewMetricExporter, which is replaced by NewMetricProducer. (#4566)

[1.19.0-rc.1/0.42.0-rc.1] 2023-09-14

This is a release candidate for the v1.19.0/v0.42.0 release. That release is expected to include the v1 release of the OpenTelemetry Go metric SDK and will provide stability guarantees of that SDK. See our versioning policy for more information about these stability guarantees.

Changed

  • Allow '/' characters in metric instrument names. (#4501)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the prometheus metric on every Collect if we know the scope is invalid. (#4499)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/otel/exporters/otlp/otlptrace&package-manager=go_modules&previous-version=1.18.0&new-version=1.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 2 files changed, 15 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 7c023ce604..bcd7f2aab4 100644 --- a/go.mod +++ b/go.mod @@ -55,14 +55,14 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0 - go.opentelemetry.io/otel v1.18.0 + go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 - go.opentelemetry.io/otel/metric v1.18.0 - go.opentelemetry.io/otel/sdk v1.18.0 + go.opentelemetry.io/otel/metric v1.19.0 + go.opentelemetry.io/otel/sdk v1.19.0 go.opentelemetry.io/otel/sdk/metric v0.41.0 - go.opentelemetry.io/otel/trace v1.18.0 + go.opentelemetry.io/otel/trace v1.19.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 3776182581..236dd50f4f 100644 --- a/go.sum +++ b/go.sum @@ -2382,33 +2382,33 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= -go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0 h1:k0k7hFNDd8K4iOMJXj7s8sHaC4mhTlAeppRmZXLgZ6k= go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 h1:iV3BOgW4fry1Riw9dwypigqlIYWXvSRVT2RJmblzo40= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0/go.mod h1:7PGzqlKrxIRmbj5tlNW0nTkYZ5fHXDgk6Fy8/KjR0CI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 h1:IAtl+7gua134xcV3NieDhJHjjOVeJhXAnYf/0hswjUY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0/go.mod h1:w+pXobnBzh95MNIkeIuAKcHe/Uu/CX2PKIvBP6ipKRA= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 h1:6pu8ttx76BxHf+xz/H77AUZkPF3cwWzXqAUsXhVKI18= go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0/go.mod h1:IOmXxPrxoxFMXdNy7lfDmE8MzE61YPcurbUm0SMjerI= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= -go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY= -go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= go.opentelemetry.io/otel/sdk/metric v0.41.0 h1:c3sAt9/pQ5fSIUfl0gPtClV3HhE18DCVzByD33R/zsk= go.opentelemetry.io/otel/sdk/metric v0.41.0/go.mod h1:PmOmSt+iOklKtIg5O4Vz9H/ttcRFSNTgii+E1KGyn1w= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= -go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= From 403a39e139a8cf3feccb666d777a368eb32ff25a Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Thu, 5 Oct 2023 07:21:24 -0500 Subject: [PATCH 231/388] chore: bump celestia-app, celestia-core, and the cosmos-sdk (#2801) bumps app, core, and the sdk. leaving as a draft for now since we still have to cut the official app version after we get enough approves. The review process can start for this tho --- go.mod | 12 ++++++------ go.sum | 24 ++++++++++++------------ 2 files changed, 18 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index bcd7f2aab4..702d964619 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc17 + github.com/celestiaorg/celestia-app v1.0.0-rc18 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.3.1 @@ -77,7 +77,7 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect github.com/bits-and-blooms/bitset v1.5.0 // indirect - github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.1 // indirect + github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.10.0 // indirect github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect @@ -116,7 +116,7 @@ require ( github.com/cockroachdb/apd/v2 v2.0.2 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect github.com/cometbft/cometbft-db v0.7.0 // indirect - github.com/confio/ics23/go v0.9.0 // indirect + github.com/confio/ics23/go v0.9.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect @@ -145,7 +145,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/go-ethereum v1.13.1 // indirect + github.com/ethereum/go-ethereum v1.13.2 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect @@ -343,10 +343,10 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28 + github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 ) diff --git a/go.sum b/go.sum index 236dd50f4f..0ee3078273 100644 --- a/go.sum +++ b/go.sum @@ -358,12 +358,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc17 h1:ak9OIViNJCfDFklopPgxnjRqUx5oKPcctpGpmwmqJy4= -github.com/celestiaorg/celestia-app v1.0.0-rc17/go.mod h1:gDKOpD8xCWHw/PUdcTc0WFYf52Mq0etOFZ1oTkBlFkQ= -github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28 h1:BE7JFZ1SYpwM9OfL9cPcjlO5xjIbDPgdFkJNouyl6jA= -github.com/celestiaorg/celestia-core v1.27.0-tm-v0.34.28/go.mod h1:1GT0RfdNqOXvyR3Hq4ROcNBknQNz9E6K5l3Cla9eFFk= -github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14 h1:dDfoQJOlVNj4HufJ1lBLTo2k3/L/255MIiKmEQziDmw= -github.com/celestiaorg/cosmos-sdk v1.18.0-sdk-v0.46.14/go.mod h1:kkdiHo/zG6ar80730+bG1owdMAQXrGp4utFu7mbfADo= +github.com/celestiaorg/celestia-app v1.0.0-rc18 h1:d2+KYPr4ri10aOtnXih3hOUd6ap94c/Hv46JFzpWzy0= +github.com/celestiaorg/celestia-app v1.0.0-rc18/go.mod h1:Q375ijriqMv8PjKmMujNs2WzMAsgxjpbeUoQt+AZre0= +github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= +github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= +github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 h1:c4cMVLU2bGTesZW1ZVgeoCB++gOOJTF3OvBsqBvo6n0= +github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14/go.mod h1:D5y5Exw0bJkcDv9fvYDiZfZrDV1b6+xsFyiungxrCsU= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= @@ -378,8 +378,8 @@ github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4/go.mod h1:fzuHnhzj1pUygGz+1ZkB3uQbEUL4htqCGJ4Qs2LwMZA= github.com/celestiaorg/nmt v0.20.0 h1:9i7ultZ8Wv5ytt8ZRaxKQ5KOOMo4A2K2T/aPGjIlSas= github.com/celestiaorg/nmt v0.20.0/go.mod h1:Oz15Ub6YPez9uJV0heoU4WpFctxazuIhKyUtaYNio7E= -github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.1 h1:xOBMoRYSh/hnsEW7OLI3bjSJXKlqILfZehjPEK/+3oI= -github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.1/go.mod h1:NziNBP12grIi+DH3TqIKUUPM2T6Hl9BTyxiK3+p8yeA= +github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2 h1:Q8nr5SAtDW5gocrBwqwDJcSS/JedqU58WwQA2SP+nXw= +github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2/go.mod h1:s/LzLUw0WeYPJ6qdk4q46jKLOq7rc9Z5Mdrxtfpcigw= github.com/celestiaorg/rsmt2d v0.11.0 h1:lcto/637WyTEZR3dLRoNvyuExfnUbxvdvKi3qz/2V4k= github.com/celestiaorg/rsmt2d v0.11.0/go.mod h1:6Y580I3gVr0+OVFfW6m2JTwnCCmvW3WfbwSLfuT+HCA= github.com/cenkalti/backoff v2.2.1+incompatible h1:tNowT99t7UNflLxfYYSlKYsBpXdEet03Pg2g16Swow4= @@ -446,8 +446,8 @@ github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZ github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M= github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo= github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0= -github.com/confio/ics23/go v0.9.0 h1:cWs+wdbS2KRPZezoaaj+qBleXgUk5WOQFMP3CQFGTr4= -github.com/confio/ics23/go v0.9.0/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= +github.com/confio/ics23/go v0.9.1 h1:3MV46eeWwO3xCauKyAtuAdJYMyPnnchW4iLr2bTw6/U= +github.com/confio/ics23/go v0.9.1/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/bavard v0.1.8-0.20210915155054-088da2f7f54a/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/YjhQ= @@ -606,8 +606,8 @@ github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eE github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= -github.com/ethereum/go-ethereum v1.13.1 h1:UF2FaUKPIy5jeZk3X06ait3y2Q4wI+vJ1l7+UARp+60= -github.com/ethereum/go-ethereum v1.13.1/go.mod h1:xHQKzwkHSl0gnSjZK1mWa06XEdm9685AHqhRknOzqGQ= +github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw= +github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c h1:8ISkoahWXwZR41ois5lSJBSVw4D0OV19Ht/JSTzvSv0= github.com/facebookgo/ensure v0.0.0-20200202191622-63f1cf65ac4c/go.mod h1:Yg+htXGokKKdzcwhuNDwVvN+uBxDGXJ7G/VN1d8fa64= github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojtoVVWjGfOF9635RETekkoH6Cc9SX0A= From e6e840225c1b71d8dacf1150938364671bdb7858 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 4 Sep 2023 16:21:00 +0300 Subject: [PATCH 232/388] feat(cmd/rpc): add commands for the p2p module (#2599) --- cmd/celestia/blob.go | 58 ++--- cmd/celestia/p2p.go | 554 +++++++++++++++++++++++++++++++++++++++++++ cmd/celestia/rpc.go | 23 ++ 3 files changed, 591 insertions(+), 44 deletions(-) create mode 100644 cmd/celestia/p2p.go diff --git a/cmd/celestia/blob.go b/cmd/celestia/blob.go index 6d397525f5..9550f2c13a 100644 --- a/cmd/celestia/blob.go +++ b/cmd/celestia/blob.go @@ -2,9 +2,7 @@ package main import ( "encoding/base64" - "encoding/json" "fmt" - "os" "reflect" "strconv" @@ -85,8 +83,11 @@ var getCmd = &cobra.Command{ blob, err := client.Blob.Get(cmd.Context(), height, namespace, commitment) - printOutput(blob, err) - return nil + formatter := formatData + if base64Flag || err != nil { + formatter = nil + } + return printOutput(blob, err, formatter) }, } @@ -112,8 +113,11 @@ var getAllCmd = &cobra.Command{ blobs, err := client.Blob.GetAll(cmd.Context(), height, []share.Namespace{namespace}) - printOutput(blobs, err) - return nil + formatter := formatData + if base64Flag || err != nil { + formatter = nil + } + return printOutput(blobs, err, formatter) }, } @@ -150,9 +154,7 @@ var submitCmd = &cobra.Command{ Height: height, Commitment: parsedBlob.Commitment, } - - printOutput(response, err) - return nil + return printOutput(response, err, nil) }, } @@ -182,35 +184,10 @@ var getProofCmd = &cobra.Command{ } proof, err := client.Blob.GetProof(cmd.Context(), height, namespace, commitment) - - printOutput(proof, err) - return nil + return printOutput(proof, err, nil) }, } -func printOutput(data interface{}, err error) { - if err != nil { - data = err - } - - if !base64Flag && err == nil { - data = formatData(data) - } - - resp := struct { - Result interface{} `json:"result"` - }{ - Result: data, - } - - bytes, err := json.MarshalIndent(resp, "", " ") - if err != nil { - fmt.Fprintln(os.Stderr, err) - os.Exit(1) - } - fmt.Fprintln(os.Stdout, string(bytes)) -} - func formatData(data interface{}) interface{} { type tempBlob struct { Namespace []byte `json:"namespace"` @@ -220,11 +197,7 @@ func formatData(data interface{}) interface{} { } if reflect.TypeOf(data).Kind() == reflect.Slice { - blobs, ok := data.([]*blob.Blob) - if !ok { - return data - } - + blobs := data.([]*blob.Blob) result := make([]tempBlob, len(blobs)) for i, b := range blobs { result[i] = tempBlob{ @@ -237,10 +210,7 @@ func formatData(data interface{}) interface{} { return result } - b, ok := data.(*blob.Blob) - if !ok { - return data - } + b := data.(*blob.Blob) return tempBlob{ Namespace: b.Namespace(), Data: string(b.Data), diff --git a/cmd/celestia/p2p.go b/cmd/celestia/p2p.go new file mode 100644 index 0000000000..d0b6549f6c --- /dev/null +++ b/cmd/celestia/p2p.go @@ -0,0 +1,554 @@ +package main + +import ( + "github.com/libp2p/go-libp2p/core/metrics" + "github.com/libp2p/go-libp2p/core/network" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/libp2p/go-libp2p/core/protocol" + ma2 "github.com/multiformats/go-multiaddr" + "github.com/spf13/cobra" +) + +type peerInfo struct { + ID string `json:"id"` + PeerAddr []string `json:"peer_addr"` +} + +func init() { + p2pCmd.AddCommand(infoCmd, + peersCmd, + peerInfoCmd, + connectCmd, + closePeerCmd, + connectednessCmd, + natStatusCmd, + blockPeerCmd, + unblockPeerCmd, + blockedPeersCmd, + protectCmd, + unprotectCmd, + protectedCmd, + bandwidthStatsCmd, + peerBandwidthCmd, + bandwidthForProtocolCmd, + pubsubPeersCmd, + ) +} + +var p2pCmd = &cobra.Command{ + Use: "p2p [command]", + Short: "Allows interaction with the P2P Module via JSON-RPC", + Args: cobra.NoArgs, +} + +var infoCmd = &cobra.Command{ + Use: "info", + Short: "Gets the node's peer info (peer id and multiaddresses)", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + info, err := client.P2P.Info(cmd.Context()) + + formatter := func(data interface{}) interface{} { + peerAdd := data.(peer.AddrInfo) + ma := make([]string, len(info.Addrs)) + for i := range peerAdd.Addrs { + ma[i] = peerAdd.Addrs[i].String() + } + + return peerInfo{ + ID: peerAdd.ID.String(), + PeerAddr: ma, + } + } + return printOutput(info, err, formatter) + }, +} + +var peersCmd = &cobra.Command{ + Use: "peers", + Short: "Lists the peers we are connected to", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + result, err := client.P2P.Peers(cmd.Context()) + peers := make([]string, len(result)) + for i, peer := range result { + peers[i] = peer.String() + } + + formatter := func(data interface{}) interface{} { + conPeers := data.([]string) + return struct { + Peers []string `json:"peers"` + }{ + Peers: conPeers, + } + } + return printOutput(peers, err, formatter) + }, +} + +var peerInfoCmd = &cobra.Command{ + Use: "peer-info [param]", + Short: "Gets PeerInfo for a given peer", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + info, err := client.P2P.PeerInfo(cmd.Context(), pid) + formatter := func(data interface{}) interface{} { + peerAdd := data.(peer.AddrInfo) + ma := make([]string, len(info.Addrs)) + for i := range peerAdd.Addrs { + ma[i] = peerAdd.Addrs[i].String() + } + + return peerInfo{ + ID: peerAdd.ID.String(), + PeerAddr: ma, + } + } + return printOutput(info, err, formatter) + }, +} + +var connectCmd = &cobra.Command{ + Use: "connect [peer.ID, address]", + Short: "Establishes a connection with the given peer", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + ma, err := ma2.NewMultiaddr(args[1]) + if err != nil { + return err + } + + peerInfo := peer.AddrInfo{ + ID: pid, + Addrs: []ma2.Multiaddr{ma}, + } + + err = client.P2P.Connect(cmd.Context(), peerInfo) + if err != nil { + return printOutput(nil, err, nil) + } + return connectednessCmd.RunE(cmd, args) + }, +} + +var closePeerCmd = &cobra.Command{ + Use: "close-peer [peer.ID]", + Short: "Closes the connection with the given peer", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + err = client.P2P.ClosePeer(cmd.Context(), pid) + if err != nil { + return printOutput(nil, err, nil) + } + return connectednessCmd.RunE(cmd, args) + }, +} + +var connectednessCmd = &cobra.Command{ + Use: "connectedness [peer.ID]", + Short: "Checks the connection state between current and given peers", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + con, err := client.P2P.Connectedness(cmd.Context(), pid) + + formatter := func(data interface{}) interface{} { + conn := data.(network.Connectedness) + return struct { + ConnectionState string `json:"connection_state"` + }{ + ConnectionState: conn.String(), + } + } + return printOutput(con, err, formatter) + }, +} + +var natStatusCmd = &cobra.Command{ + Use: "nat-status", + Short: "Gets the currrent NAT status", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + r, err := client.P2P.NATStatus(cmd.Context()) + + formatter := func(data interface{}) interface{} { + rr := data.(network.Reachability) + return struct { + Reachability string `json:"reachability"` + }{ + Reachability: rr.String(), + } + } + return printOutput(r, err, formatter) + }, +} + +var blockPeerCmd = &cobra.Command{ + Use: "block-peer [peer.ID]", + Short: "Blocks the given peer", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + err = client.P2P.BlockPeer(cmd.Context(), pid) + + formatter := func(data interface{}) interface{} { + err, ok := data.(error) + blocked := false + if !ok { + blocked = true + } + return struct { + Blocked bool `json:"blocked"` + Peer string `json:"peer"` + Reason error `json:"reason,omitempty"` + }{ + Blocked: blocked, + Peer: args[0], + Reason: err, + } + } + return printOutput(err, nil, formatter) + }, +} + +var unblockPeerCmd = &cobra.Command{ + Use: "unblock-peer [peer.ID]", + Short: "Unblocks the given peer", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + err = client.P2P.UnblockPeer(cmd.Context(), pid) + + formatter := func(data interface{}) interface{} { + err, ok := data.(error) + unblocked := false + if !ok { + unblocked = true + } + + return struct { + Unblocked bool `json:"unblocked"` + Peer string `json:"peer"` + Reason error `json:"reason,omitempty"` + }{ + Unblocked: unblocked, + Peer: args[0], + Reason: err, + } + } + return printOutput(err, nil, formatter) + }, +} + +var blockedPeersCmd = &cobra.Command{ + Use: "blocked-peers", + Short: "Lists the node's blocked peers", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + list, err := client.P2P.ListBlockedPeers(cmd.Context()) + + pids := make([]string, len(list)) + for i, peer := range list { + pids[i] = peer.String() + } + + formatter := func(data interface{}) interface{} { + peers := data.([]string) + return struct { + Peers []string `json:"peers"` + }{ + Peers: peers, + } + } + return printOutput(pids, err, formatter) + }, +} + +var protectCmd = &cobra.Command{ + Use: "protect [peer.ID, tag]", + Short: "Protects the given peer from being pruned by the given tag", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + err = client.P2P.Protect(cmd.Context(), pid, args[1]) + + formatter := func(data interface{}) interface{} { + err, ok := data.(error) + protected := false + if !ok { + protected = true + } + return struct { + Protected bool `json:"protected"` + Peer string `json:"peer"` + Reason error `json:"reason,omitempty"` + }{ + Protected: protected, + Peer: args[0], + Reason: err, + } + } + return printOutput(err, nil, formatter) + }, +} + +var unprotectCmd = &cobra.Command{ + Use: "unprotect [peer.ID, tag]", + Short: "Removes a protection that may have been placed on a peer, under the specified tag." + + "The return value indicates whether the peer continues to be protected after this call, by way of a different tag", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + _, err = client.P2P.Unprotect(cmd.Context(), pid, args[1]) + + formatter := func(data interface{}) interface{} { + err, ok := data.(error) + unprotected := false + if !ok { + unprotected = true + } + return struct { + Unprotected bool `json:"unprotected"` + Peer string `json:"peer"` + Reason error `json:"reason,omitempty"` + }{ + Unprotected: unprotected, + Peer: args[0], + Reason: err, + } + } + return printOutput(err, nil, formatter) + }, +} + +var protectedCmd = &cobra.Command{ + Use: "protected [peer.ID, tag]", + Short: "Ensures that a given peer is protected under a specific tag", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + result, err := client.P2P.IsProtected(cmd.Context(), pid, args[1]) + return printOutput(result, err, nil) + }, +} + +type bandwidthStats struct { + TotalIn int64 `json:"total_in"` + TotalOut int64 `json:"total_out"` + RateIn float64 `json:"rate_in"` + RateOut float64 `json:"rate_out"` +} + +var bandwidthStatsCmd = &cobra.Command{ + Use: "bandwidth-stats", + Short: "Get stats struct with bandwidth metrics for all data sent/" + + "received by the local peer, regardless of protocol or remote peer IDs", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + result, err := client.P2P.BandwidthStats(cmd.Context()) + + formatter := func(data interface{}) interface{} { + stats := data.(metrics.Stats) + return bandwidthStats{ + TotalIn: stats.TotalIn, + TotalOut: stats.TotalOut, + RateIn: stats.RateIn, + RateOut: stats.RateOut, + } + } + return printOutput(result, err, formatter) + }, +} + +var peerBandwidthCmd = &cobra.Command{ + Use: "peer-bandwidth [peer.ID]", + Short: "Gets stats struct with bandwidth metrics associated with the given peer.ID", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + pid, err := peer.Decode(args[0]) + if err != nil { + return err + } + + result, err := client.P2P.BandwidthForPeer(cmd.Context(), pid) + + formatter := func(data interface{}) interface{} { + stats := data.(metrics.Stats) + return bandwidthStats{ + TotalIn: stats.TotalIn, + TotalOut: stats.TotalOut, + RateIn: stats.RateIn, + RateOut: stats.RateOut, + } + } + return printOutput(result, err, formatter) + }, +} + +var bandwidthForProtocolCmd = &cobra.Command{ + Use: "protocol-bandwidth [protocol.ID]", + Short: "Gets stats struct with bandwidth metrics associated with the given protocol.ID", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + result, err := client.P2P.BandwidthForProtocol(cmd.Context(), protocol.ID(args[0])) + + formatter := func(data interface{}) interface{} { + stats := data.(metrics.Stats) + return bandwidthStats{ + TotalIn: stats.TotalIn, + TotalOut: stats.TotalOut, + RateIn: stats.RateIn, + RateOut: stats.RateOut, + } + } + return printOutput(result, err, formatter) + }, +} + +var pubsubPeersCmd = &cobra.Command{ + Use: "pubsub-peers [topic]", + Short: "Lists the peers we are connected to in the given topic", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + result, err := client.P2P.PubSubPeers(cmd.Context(), args[0]) + peers := make([]string, len(result)) + + for i, peer := range result { + peers[i] = peer.String() + } + + formatter := func(data interface{}) interface{} { + conPeers := data.([]string) + return struct { + Peers []string `json:"peers"` + }{ + Peers: conPeers, + } + } + return printOutput(peers, err, formatter) + }, +} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index c263496b26..cc5ca059f7 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -62,6 +62,7 @@ func init() { ) rpcCmd.AddCommand(logCmd, logModuleCmd) rpcCmd.AddCommand(blobCmd) + rpcCmd.AddCommand(p2pCmd) rootCmd.AddCommand(rpcCmd) } @@ -400,3 +401,25 @@ func rpcClient(ctx context.Context) (*client.Client, error) { } return client, nil } + +func printOutput(data interface{}, err error, formatData func(interface{}) interface{}) error { + switch { + case err != nil: + data = err + case formatData != nil: + data = formatData(data) + } + + resp := struct { + Result interface{} `json:"result"` + }{ + Result: data, + } + + bytes, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + fmt.Fprintln(os.Stdout, string(bytes)) + return nil +} From ae7a1d813c824d1725b888d7ca4bc7264dfe13f6 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 5 Sep 2023 12:49:38 +0300 Subject: [PATCH 233/388] feat(rpc): add daser cmd (#2651) --- cmd/celestia/das.go | 27 +++++++++++++++++++++++++++ cmd/celestia/rpc.go | 1 + 2 files changed, 28 insertions(+) create mode 100644 cmd/celestia/das.go diff --git a/cmd/celestia/das.go b/cmd/celestia/das.go new file mode 100644 index 0000000000..58c4a8362b --- /dev/null +++ b/cmd/celestia/das.go @@ -0,0 +1,27 @@ +package main + +import "github.com/spf13/cobra" + +func init() { + dasCmd.AddCommand(samplingStatsCmd) +} + +var dasCmd = &cobra.Command{ + Use: "das [command]", + Short: "Allows to interact with the Daser via JSON-RPC", + Args: cobra.NoArgs, +} + +var samplingStatsCmd = &cobra.Command{ + Use: "sampling-stats", + Short: "Returns the current statistics over the DA sampling process", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + stats, err := client.DAS.SamplingStats(cmd.Context()) + return printOutput(stats, err, nil) + }, +} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index cc5ca059f7..c5d0a71343 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -63,6 +63,7 @@ func init() { rpcCmd.AddCommand(logCmd, logModuleCmd) rpcCmd.AddCommand(blobCmd) rpcCmd.AddCommand(p2pCmd) + rpcCmd.AddCommand(dasCmd) rootCmd.AddCommand(rpcCmd) } From c89ada18e986f7c9d16f2d65d817c428e870de5d Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 6 Sep 2023 16:19:53 +0300 Subject: [PATCH 234/388] cmd(rpc): add header cmd (#2658) --- cmd/celestia/header.go | 109 +++++++++++++++++++++++++++++++++++++++++ cmd/celestia/rpc.go | 1 + 2 files changed, 110 insertions(+) create mode 100644 cmd/celestia/header.go diff --git a/cmd/celestia/header.go b/cmd/celestia/header.go new file mode 100644 index 0000000000..1d8dc6d01b --- /dev/null +++ b/cmd/celestia/header.go @@ -0,0 +1,109 @@ +package main + +import ( + "encoding/hex" + "fmt" + "strconv" + + "github.com/spf13/cobra" +) + +func init() { + headerCmd.AddCommand( + localHeadCmd, + networkHeadCmd, + getByHashCmd, + getByHeightCmd, + syncStateCmd, + ) +} + +var headerCmd = &cobra.Command{ + Use: "header [command]", + Short: "Allows interaction with the Header Module via JSON-RPC", + Args: cobra.NoArgs, +} + +var localHeadCmd = &cobra.Command{ + Use: "local-head", + Short: "Returns the ExtendedHeader from the chain head.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + header, err := client.Header.LocalHead(cmd.Context()) + return printOutput(header, err, nil) + }, +} + +var networkHeadCmd = &cobra.Command{ + Use: "network-head", + Short: "Provides the Syncer's view of the current network head.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + header, err := client.Header.NetworkHead(cmd.Context()) + return printOutput(header, err, nil) + }, +} + +var getByHashCmd = &cobra.Command{ + Use: "get-by-hash", + Short: "Returns the header of the given hash from the node's header store.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + hash, err := hex.DecodeString(args[0]) + if err != nil { + return fmt.Errorf("error decoding a hash: expected a hex encoded string:%v", err) + } + header, err := client.Header.GetByHash(cmd.Context(), hash) + return printOutput(header, err, nil) + }, +} + +var getByHeightCmd = &cobra.Command{ + Use: "get-by-height", + Short: "Returns the ExtendedHeader at the given height if it is currently available.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + height, err := strconv.ParseUint(args[0], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a height:%v", err) + } + + header, err := client.Header.GetByHeight(cmd.Context(), height) + return printOutput(header, err, nil) + }, +} + +var syncStateCmd = &cobra.Command{ + Use: "sync-state", + Short: "Returns the current state of the header Syncer.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + header, err := client.Header.SyncState(cmd.Context()) + return printOutput(header, err, nil) + }, +} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index c5d0a71343..4425aa9ae5 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -64,6 +64,7 @@ func init() { rpcCmd.AddCommand(blobCmd) rpcCmd.AddCommand(p2pCmd) rpcCmd.AddCommand(dasCmd) + rpcCmd.AddCommand(headerCmd) rootCmd.AddCommand(rpcCmd) } From e21654401bbb80a9a8d672f84e74b414f4474d8c Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 13 Sep 2023 12:24:32 +0300 Subject: [PATCH 235/388] feat(rpc): add share cmd (#2664) --- cmd/celestia/rpc.go | 1 + cmd/celestia/share.go | 196 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 197 insertions(+) create mode 100644 cmd/celestia/share.go diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 4425aa9ae5..1516211c3e 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -65,6 +65,7 @@ func init() { rpcCmd.AddCommand(p2pCmd) rpcCmd.AddCommand(dasCmd) rpcCmd.AddCommand(headerCmd) + rpcCmd.AddCommand(shareCmd) rootCmd.AddCommand(rpcCmd) } diff --git a/cmd/celestia/share.go b/cmd/celestia/share.go new file mode 100644 index 0000000000..2f84871d80 --- /dev/null +++ b/cmd/celestia/share.go @@ -0,0 +1,196 @@ +package main + +import ( + "encoding/hex" + "encoding/json" + "strconv" + + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-app/pkg/da" + + "github.com/celestiaorg/celestia-node/share" +) + +func init() { + shareCmd.AddCommand( + sharesAvailableCmd, + probabilityOfAvailabilityCmd, + getSharesByNamespaceCmd, + getShare, + getEDS, + ) +} + +var shareCmd = &cobra.Command{ + Use: "share [command]", + Short: "Allows interaction with the Share Module via JSON-RPC", + Args: cobra.NoArgs, +} + +var sharesAvailableCmd = &cobra.Command{ + Use: "available", + Short: "Subjectively validates if Shares committed to the given Root are available on the Network.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + raw, err := parseJSON(args[0]) + if err != nil { + return err + } + + root := da.MinDataAvailabilityHeader() + err = json.Unmarshal(raw, &root) + if err != nil { + return err + } + + err = client.Share.SharesAvailable(cmd.Context(), &root) + formatter := func(data interface{}) interface{} { + err, ok := data.(error) + available := false + if !ok { + available = true + } + return struct { + Available bool `json:"available"` + Hash []byte `json:"dah_hash"` + Reason error `json:"reason,omitempty"` + }{ + Available: available, + Hash: []byte(args[0]), + Reason: err, + } + } + return printOutput(err, nil, formatter) + }, +} + +var probabilityOfAvailabilityCmd = &cobra.Command{ + Use: "availability", + Short: "Calculates the probability of the data square being available based on the number of samples collected.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + prob := client.Share.ProbabilityOfAvailability(cmd.Context()) + return printOutput(prob, nil, nil) + }, +} + +var getSharesByNamespaceCmd = &cobra.Command{ + Use: "get-by-namespace [dah, namespace]", + Short: "Gets all shares from an EDS within the given namespace.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + raw, err := parseJSON(args[0]) + if err != nil { + return err + } + + root := da.MinDataAvailabilityHeader() + err = json.Unmarshal(raw, &root) + if err != nil { + return err + } + + ns, err := parseV0Namespace(args[1]) + if err != nil { + return err + } + + shares, err := client.Share.GetSharesByNamespace(cmd.Context(), &root, ns) + return printOutput(shares, err, nil) + }, +} + +var getShare = &cobra.Command{ + Use: "get-share [dah, row, col]", + Short: "Gets a Share by coordinates in EDS.", + Args: cobra.ExactArgs(3), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + raw, err := parseJSON(args[0]) + if err != nil { + return err + } + + root := da.MinDataAvailabilityHeader() + err = json.Unmarshal(raw, &root) + if err != nil { + return err + } + + row, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return err + } + + col, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return err + } + + s, err := client.Share.GetShare(cmd.Context(), &root, int(row), int(col)) + + formatter := func(data interface{}) interface{} { + sh, ok := data.(share.Share) + if !ok { + return data + } + + ns := hex.EncodeToString(share.GetNamespace(sh)) + + return struct { + Namespace string `json:"namespace"` + Data []byte `json:"data"` + }{ + Namespace: ns, + Data: share.GetData(sh), + } + } + return printOutput(s, err, formatter) + }, +} + +var getEDS = &cobra.Command{ + Use: "get-eds [dah]", + Short: "Gets the full EDS identified by the given root", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + raw, err := parseJSON(args[0]) + if err != nil { + return err + } + + root := da.MinDataAvailabilityHeader() + err = json.Unmarshal(raw, &root) + if err != nil { + return err + } + + shares, err := client.Share.GetEDS(cmd.Context(), &root) + return printOutput(shares, err, nil) + }, +} From c38fcb7e8453c79280e6267e29454a03c9fa02de Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 15 Sep 2023 13:40:54 +0300 Subject: [PATCH 236/388] feat(rpc): add admin cmd (#2701) --- cmd/celestia/admin.go | 108 ++++++++++++++++++++++++++++++++++++++++++ cmd/celestia/logs.go | 48 ------------------- cmd/celestia/rpc.go | 3 +- 3 files changed, 109 insertions(+), 50 deletions(-) create mode 100644 cmd/celestia/admin.go delete mode 100644 cmd/celestia/logs.go diff --git a/cmd/celestia/admin.go b/cmd/celestia/admin.go new file mode 100644 index 0000000000..a3665e0aa1 --- /dev/null +++ b/cmd/celestia/admin.go @@ -0,0 +1,108 @@ +package main + +import ( + "context" + "errors" + "strings" + + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/spf13/cobra" +) + +func init() { + nodeCmd.AddCommand(nodeInfoCmd, logCmd, verifyCmd, authCmd) + rootCmd.AddCommand(nodeCmd) +} + +var nodeCmd = &cobra.Command{ + Use: "node [command]", + Short: "Allows administrating running node.", + Args: cobra.NoArgs, + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + rpcClient, err := newRPCClient(cmd.Context()) + if err != nil { + return err + } + + ctx := context.WithValue(cmd.Context(), rpcClientKey{}, rpcClient) + cmd.SetContext(ctx) + return nil + }, +} + +var nodeInfoCmd = &cobra.Command{ + Use: "info", + Args: cobra.NoArgs, + Short: "Returns administrative information about the node.", + RunE: func(c *cobra.Command, args []string) error { + client, err := rpcClient(c.Context()) + if err != nil { + return err + } + info, err := client.Node.Info(c.Context()) + return printOutput(info, err, nil) + }, +} + +var logCmd = &cobra.Command{ + Use: "log-level", + Args: cobra.MinimumNArgs(1), + Short: "Allows to set log level for module to in format :" + + "`DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL and their lower-case forms`.\n" + + "To set all modules to a particular level `*:` should be passed", + RunE: func(c *cobra.Command, args []string) error { + client, err := rpcClient(c.Context()) + if err != nil { + return err + } + + for _, ll := range args { + params := strings.Split(ll, ":") + if len(params) != 2 { + return errors.New("cmd: log-level arg must be in form :," + + "e.g. pubsub:debug") + } + + if err = client.Node.LogLevelSet(c.Context(), params[0], params[1]); err != nil { + return err + } + } + return nil + }, +} + +var verifyCmd = &cobra.Command{ + Use: "permissions", + Args: cobra.ExactArgs(1), + Short: "Returns the permissions assigned to the given token.", + + RunE: func(c *cobra.Command, args []string) error { + client, err := rpcClient(c.Context()) + if err != nil { + return err + } + + perms, err := client.Node.AuthVerify(c.Context(), args[0]) + return printOutput(perms, err, nil) + }, +} + +var authCmd = &cobra.Command{ + Use: "set-permissions", + Args: cobra.MinimumNArgs(1), + Short: "Signs and returns a new token with the given permissions.", + RunE: func(c *cobra.Command, args []string) error { + client, err := rpcClient(c.Context()) + if err != nil { + return err + } + + perms := make([]auth.Permission, len(args)) + for i, p := range args { + perms[i] = (auth.Permission)(p) + } + + result, err := client.Node.AuthNew(c.Context(), perms) + return printOutput(result, err, nil) + }, +} diff --git a/cmd/celestia/logs.go b/cmd/celestia/logs.go deleted file mode 100644 index ac302ff6dd..0000000000 --- a/cmd/celestia/logs.go +++ /dev/null @@ -1,48 +0,0 @@ -package main - -import ( - "fmt" - "strings" - - "github.com/spf13/cobra" - - "github.com/celestiaorg/celestia-node/cmd" -) - -var logCmd = &cobra.Command{ - Use: cmd.LogLevelFlag, - Args: cobra.ExactArgs(1), - Short: "Allows to set log level for all modules to " + - "`DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL and their lower-case forms`", - - RunE: func(c *cobra.Command, args []string) error { - client, err := rpcClient(c.Context()) - if err != nil { - return err - } - return client.Node.LogLevelSet(c.Context(), "*", args[0]) - }, -} - -var logModuleCmd = &cobra.Command{ - Use: cmd.LogLevelModuleFlag, - Args: cobra.MinimumNArgs(1), - Short: "Allows to set log level for a particular module in format :", - RunE: func(c *cobra.Command, args []string) error { - client, err := rpcClient(c.Context()) - if err != nil { - return err - } - for _, ll := range args { - params := strings.Split(ll, ":") - if len(params) != 2 { - return fmt.Errorf("cmd: %s arg must be in form :,"+ - "e.g. pubsub:debug", cmd.LogLevelModuleFlag) - } - if err = client.Node.LogLevelSet(c.Context(), params[0], params[1]); err != nil { - return err - } - } - return nil - }, -} diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 1516211c3e..a0dea6e0dd 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -23,7 +23,7 @@ import ( "github.com/celestiaorg/celestia-node/state" ) -const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" //nolint:gosec +const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" var requestURL string var authTokenFlag string @@ -60,7 +60,6 @@ func init() { false, "Print JSON-RPC request along with the response", ) - rpcCmd.AddCommand(logCmd, logModuleCmd) rpcCmd.AddCommand(blobCmd) rpcCmd.AddCommand(p2pCmd) rpcCmd.AddCommand(dasCmd) From cc02d71d9e10a2b747bdb6ea4796c6a0f87d4c4e Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 15 Sep 2023 13:46:41 +0300 Subject: [PATCH 237/388] feat(rpc): add state cmd (#2691) --- cmd/celestia/rpc.go | 1 + cmd/celestia/state.go | 424 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 425 insertions(+) create mode 100644 cmd/celestia/state.go diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index a0dea6e0dd..e7e3c0d5be 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -65,6 +65,7 @@ func init() { rpcCmd.AddCommand(dasCmd) rpcCmd.AddCommand(headerCmd) rpcCmd.AddCommand(shareCmd) + rpcCmd.AddCommand(stateCmd) rootCmd.AddCommand(rpcCmd) } diff --git a/cmd/celestia/state.go b/cmd/celestia/state.go new file mode 100644 index 0000000000..2f3d1e138a --- /dev/null +++ b/cmd/celestia/state.go @@ -0,0 +1,424 @@ +package main + +import ( + "fmt" + "strconv" + + "cosmossdk.io/math" + "github.com/cosmos/cosmos-sdk/types" + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/state" +) + +func init() { + stateCmd.AddCommand( + accountAddressCmd, + balanceCmd, + balanceForAddressCmd, + transferCmd, + submitTxCmd, + submitPFBCmd, + cancelUnbondingDelegationCmd, + beginRedelegateCmd, + undelegateCmd, + delegateCmd, + queryDelegationCmd, + queryUnbondingCmd, + queryRedelegationCmd, + ) +} + +var stateCmd = &cobra.Command{ + Use: "state [command]", + Short: "Allows interaction with the State Module via JSON-RPC", + Args: cobra.NoArgs, +} + +var accountAddressCmd = &cobra.Command{ + Use: "account-address", + Short: "Retrieves the address of the node's account/signer.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + address, err := client.State.AccountAddress(cmd.Context()) + return printOutput(address, err, nil) + }, +} + +var balanceCmd = &cobra.Command{ + Use: "balance", + Short: "Retrieves the Celestia coin balance for the node's account/signer and verifies it against " + + "the corresponding block's AppHash.", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + balance, err := client.State.Balance(cmd.Context()) + return printOutput(balance, err, nil) + }, +} + +var balanceForAddressCmd = &cobra.Command{ + Use: "balance-for-address [address]", + Short: "Retrieves the Celestia coin balance for the given address and verifies the returned balance against " + + "the corresponding block's AppHash.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + balance, err := client.State.BalanceForAddress(cmd.Context(), addr) + return printOutput(balance, err, nil) + }, +} + +var transferCmd = &cobra.Command{ + Use: "transfer [address] [amount] [fee] [gasLimit]", + Short: "Sends the given amount of coins from default wallet of the node to the given account address.", + Args: cobra.ExactArgs(4), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + amount, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return fmt.Errorf("error parsing an amount:%v", err) + } + fee, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + gasLimit, err := strconv.ParseUint(args[3], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a gas limit:%v", err) + } + + txResponse, err := client.State.Transfer( + cmd.Context(), + addr.Address.(state.AccAddress), + math.NewInt(amount), + math.NewInt(fee), gasLimit, + ) + return printOutput(txResponse, err, nil) + }, +} + +var submitTxCmd = &cobra.Command{ + Use: "submit-tx [tx]", + Short: "Submits the given transaction/message to the Celestia network and blocks until the tx is included in a block.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + rawTx, err := decodeToBytes(args[0]) + if err != nil { + return fmt.Errorf("failed to decode tx: %v", err) + } + txResponse, err := client.State.SubmitTx( + cmd.Context(), + rawTx, + ) + return printOutput(txResponse, err, nil) + }, +} + +var submitPFBCmd = &cobra.Command{ + Use: "submit-pfb [namespace] [data] [fee] [gasLim]", + Short: "Allows to submit PFBs", + Args: cobra.ExactArgs(4), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + namespace, err := parseV0Namespace(args[0]) + if err != nil { + return fmt.Errorf("error parsing a namespace:%v", err) + } + + fee, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + + gasLimit, err := strconv.ParseUint(args[3], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a gasLim:%v", err) + } + + parsedBlob, err := blob.NewBlobV0(namespace, []byte(args[1])) + if err != nil { + return fmt.Errorf("error creating a blob:%v", err) + } + + txResp, err := client.State.SubmitPayForBlob(cmd.Context(), types.NewInt(fee), gasLimit, []*blob.Blob{parsedBlob}) + return printOutput(txResp, err, nil) + }, +} + +var cancelUnbondingDelegationCmd = &cobra.Command{ + Use: "cancel-unbonding-delegation [address] [amount] [height] [fee] [gasLimit]", + Short: "Cancels a user's pending undelegation from a validator.", + Args: cobra.ExactArgs(5), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + amount, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return fmt.Errorf("error parsing an amount:%v", err) + } + + height, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + + fee, err := strconv.ParseInt(args[3], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + + gasLimit, err := strconv.ParseUint(args[4], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a gas limit:%v", err) + } + + txResponse, err := client.State.CancelUnbondingDelegation( + cmd.Context(), + addr.Address.(state.ValAddress), + math.NewInt(amount), + math.NewInt(height), + math.NewInt(fee), + gasLimit, + ) + return printOutput(txResponse, err, nil) + }, +} + +var beginRedelegateCmd = &cobra.Command{ + Use: "begin-redelegate [srcAddress] [dstAddress] [amount] [fee] [gasLimit]", + Short: "Sends a user's delegated tokens to a new validator for redelegation", + Args: cobra.ExactArgs(5), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + srcAddr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + dstAddr, err := parseAddressFromString(args[1]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + amount, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return fmt.Errorf("error parsing an amount:%v", err) + } + + fee, err := strconv.ParseInt(args[3], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + gasLimit, err := strconv.ParseUint(args[4], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a gas limit:%v", err) + } + + txResponse, err := client.State.BeginRedelegate( + cmd.Context(), + srcAddr.Address.(state.ValAddress), + dstAddr.Address.(state.ValAddress), + math.NewInt(amount), + math.NewInt(fee), + gasLimit, + ) + return printOutput(txResponse, err, nil) + }, +} + +var undelegateCmd = &cobra.Command{ + Use: "undelegate [valAddress] [amount] [fee] [gasLimit]", + Short: "Undelegates a user's delegated tokens, unbonding them from the current validator.", + Args: cobra.ExactArgs(4), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + amount, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return fmt.Errorf("error parsing an amount:%v", err) + } + fee, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + gasLimit, err := strconv.ParseUint(args[3], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a gas limit:%v", err) + } + + txResponse, err := client.State.Undelegate( + cmd.Context(), + addr.Address.(state.ValAddress), + math.NewInt(amount), + math.NewInt(fee), + gasLimit, + ) + return printOutput(txResponse, err, nil) + }, +} + +var delegateCmd = &cobra.Command{ + Use: "delegate [valAddress] [amount] [fee] [gasLimit]", + Short: "Sends a user's liquid tokens to a validator for delegation.", + Args: cobra.ExactArgs(4), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + amount, err := strconv.ParseInt(args[1], 10, 64) + if err != nil { + return fmt.Errorf("error parsing an amount:%v", err) + } + + fee, err := strconv.ParseInt(args[2], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a fee:%v", err) + } + + gasLimit, err := strconv.ParseUint(args[3], 10, 64) + if err != nil { + return fmt.Errorf("error parsing a gas limit:%v", err) + } + + txResponse, err := client.State.Delegate( + cmd.Context(), + addr.Address.(state.ValAddress), + math.NewInt(amount), + math.NewInt(fee), + gasLimit, + ) + return printOutput(txResponse, err, nil) + }, +} + +var queryDelegationCmd = &cobra.Command{ + Use: "get-delegation [valAddress]", + Short: "Retrieves the delegation information between a delegator and a validator.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + balance, err := client.State.QueryDelegation(cmd.Context(), addr.Address.(state.ValAddress)) + fmt.Println(balance) + fmt.Println(err) + return printOutput(balance, err, nil) + }, +} + +var queryUnbondingCmd = &cobra.Command{ + Use: "get-unbonding [valAddress]", + Short: "Retrieves the unbonding status between a delegator and a validator.", + Args: cobra.ExactArgs(1), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + addr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing an address:%v", err) + } + + response, err := client.State.QueryUnbonding(cmd.Context(), addr.Address.(state.ValAddress)) + return printOutput(response, err, nil) + }, +} + +var queryRedelegationCmd = &cobra.Command{ + Use: "get-redelegations [srcAddress] [dstAddress]", + Short: "Retrieves the status of the redelegations between a delegator and a validator.", + Args: cobra.ExactArgs(2), + RunE: func(cmd *cobra.Command, args []string) error { + client, err := rpcClient(cmd.Context()) + if err != nil { + return err + } + + srcAddr, err := parseAddressFromString(args[0]) + if err != nil { + return fmt.Errorf("error parsing a src address:%v", err) + } + + dstAddr, err := parseAddressFromString(args[1]) + if err != nil { + return fmt.Errorf("error parsing a dst address:%v", err) + } + + response, err := client.State.QueryRedelegations( + cmd.Context(), + srcAddr.Address.(state.ValAddress), + dstAddr.Address.(state.ValAddress), + ) + return printOutput(response, err, nil) + }, +} From 0d081eb134f4e8cc5ec2dbccc844e78bdced35e2 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 25 Sep 2023 13:43:00 +0300 Subject: [PATCH 238/388] refactor(cmd): extract rpc cmds (#2706) --- cmd/celestia/bridge.go | 2 +- cmd/celestia/cmd_test.go | 20 + cmd/celestia/das.go | 27 -- cmd/celestia/full.go | 2 +- cmd/celestia/light.go | 2 +- cmd/celestia/rpc.go | 452 ++---------------- cmd/celestia/util.go | 68 --- cmd/rpc.go | 58 +++ cmd/util.go | 127 +++++ cmd/{celestia/rpc_test.go => util_test.go} | 4 +- .../celestia => nodebuilder/blob/cmd}/blob.go | 52 +- nodebuilder/das/cmd/das.go | 34 ++ .../header/cmd}/header.go | 40 +- .../admin.go => nodebuilder/node/cmd/node.go | 55 +-- {cmd/celestia => nodebuilder/p2p/cmd}/p2p.go | 110 +++-- .../share/cmd}/share.go | 47 +- .../state/cmd}/state.go | 108 +++-- 17 files changed, 521 insertions(+), 687 deletions(-) delete mode 100644 cmd/celestia/das.go delete mode 100644 cmd/celestia/util.go create mode 100644 cmd/rpc.go create mode 100644 cmd/util.go rename cmd/{celestia/rpc_test.go => util_test.go} (97%) rename {cmd/celestia => nodebuilder/blob/cmd}/blob.go (77%) create mode 100644 nodebuilder/das/cmd/das.go rename {cmd/celestia => nodebuilder/header/cmd}/header.go (68%) rename cmd/celestia/admin.go => nodebuilder/node/cmd/node.go (61%) rename {cmd/celestia => nodebuilder/p2p/cmd}/p2p.go (79%) rename {cmd/celestia => nodebuilder/share/cmd}/share.go (76%) rename {cmd/celestia => nodebuilder/state/cmd}/state.go (80%) diff --git a/cmd/celestia/bridge.go b/cmd/celestia/bridge.go index fb5066e5d4..c0e2ab0d1a 100644 --- a/cmd/celestia/bridge.go +++ b/cmd/celestia/bridge.go @@ -42,6 +42,6 @@ var bridgeCmd = &cobra.Command{ Args: cobra.NoArgs, Short: "Manage your Bridge node", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return persistentPreRunEnv(cmd, node.Bridge, args) + return cmdnode.PersistentPreRunEnv(cmd, node.Bridge, args) }, } diff --git a/cmd/celestia/cmd_test.go b/cmd/celestia/cmd_test.go index 503548b708..9c26489e14 100644 --- a/cmd/celestia/cmd_test.go +++ b/cmd/celestia/cmd_test.go @@ -128,3 +128,23 @@ func TestBridge(t *testing.T) { }) */ } + +func parseSignatureForHelpstring(methodSig reflect.StructField) string { + simplifiedSignature := "(" + in, out := methodSig.Type.NumIn(), methodSig.Type.NumOut() + for i := 1; i < in; i++ { + simplifiedSignature += methodSig.Type.In(i).String() + if i != in-1 { + simplifiedSignature += ", " + } + } + simplifiedSignature += ") -> (" + for i := 0; i < out-1; i++ { + simplifiedSignature += methodSig.Type.Out(i).String() + if i != out-2 { + simplifiedSignature += ", " + } + } + simplifiedSignature += ")" + return simplifiedSignature +} diff --git a/cmd/celestia/das.go b/cmd/celestia/das.go deleted file mode 100644 index 58c4a8362b..0000000000 --- a/cmd/celestia/das.go +++ /dev/null @@ -1,27 +0,0 @@ -package main - -import "github.com/spf13/cobra" - -func init() { - dasCmd.AddCommand(samplingStatsCmd) -} - -var dasCmd = &cobra.Command{ - Use: "das [command]", - Short: "Allows to interact with the Daser via JSON-RPC", - Args: cobra.NoArgs, -} - -var samplingStatsCmd = &cobra.Command{ - Use: "sampling-stats", - Short: "Returns the current statistics over the DA sampling process", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) - if err != nil { - return err - } - stats, err := client.DAS.SamplingStats(cmd.Context()) - return printOutput(stats, err, nil) - }, -} diff --git a/cmd/celestia/full.go b/cmd/celestia/full.go index 912de0bca8..8baff1080e 100644 --- a/cmd/celestia/full.go +++ b/cmd/celestia/full.go @@ -46,6 +46,6 @@ var fullCmd = &cobra.Command{ Args: cobra.NoArgs, Short: "Manage your Full node", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return persistentPreRunEnv(cmd, node.Full, args) + return cmdnode.PersistentPreRunEnv(cmd, node.Full, args) }, } diff --git a/cmd/celestia/light.go b/cmd/celestia/light.go index 9c63945445..553660c5d3 100644 --- a/cmd/celestia/light.go +++ b/cmd/celestia/light.go @@ -46,6 +46,6 @@ var lightCmd = &cobra.Command{ Args: cobra.NoArgs, Short: "Manage your Light node", PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return persistentPreRunEnv(cmd, node.Light, args) + return cmdnode.PersistentPreRunEnv(cmd, node.Light, args) }, } diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index e7e3c0d5be..8cf51fda09 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -1,428 +1,40 @@ package main import ( - "bytes" - "context" - "encoding/base64" - "encoding/hex" - "encoding/json" - "errors" - "fmt" - "io" - "log" - "net/http" - "os" - "reflect" - "strconv" - "strings" - - "github.com/spf13/cobra" - - "github.com/celestiaorg/celestia-node/api/rpc/client" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/state" + "github.com/celestiaorg/celestia-node/cmd" + blob "github.com/celestiaorg/celestia-node/nodebuilder/blob/cmd" + das "github.com/celestiaorg/celestia-node/nodebuilder/das/cmd" + header "github.com/celestiaorg/celestia-node/nodebuilder/header/cmd" + node "github.com/celestiaorg/celestia-node/nodebuilder/node/cmd" + p2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p/cmd" + share "github.com/celestiaorg/celestia-node/nodebuilder/share/cmd" + state "github.com/celestiaorg/celestia-node/nodebuilder/state/cmd" ) -const authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" - -var requestURL string -var authTokenFlag string -var printRequest bool - -type jsonRPCRequest struct { - ID int64 `json:"id"` - JSONRPC string `json:"jsonrpc"` - Method string `json:"method"` - Params []interface{} `json:"params"` -} - -type outputWithRequest struct { - Request jsonRPCRequest - Response json.RawMessage -} - func init() { - rpcCmd.PersistentFlags().StringVar( - &requestURL, - "url", - "http://localhost:26658", - "Request URL", - ) - rpcCmd.PersistentFlags().StringVar( - &authTokenFlag, - "auth", - "", - "Authorization token (if not provided, the "+authEnvKey+" environment variable will be used)", + blob.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + das.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + header.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + p2p.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + share.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + state.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + node.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) + + blob.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + das.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + header.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + p2p.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + share.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + state.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + node.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + + rootCmd.AddCommand( + blob.Cmd, + das.Cmd, + header.Cmd, + p2p.Cmd, + share.Cmd, + state.Cmd, + node.Cmd, ) - rpcCmd.PersistentFlags().BoolVar( - &printRequest, - "print-request", - false, - "Print JSON-RPC request along with the response", - ) - rpcCmd.AddCommand(blobCmd) - rpcCmd.AddCommand(p2pCmd) - rpcCmd.AddCommand(dasCmd) - rpcCmd.AddCommand(headerCmd) - rpcCmd.AddCommand(shareCmd) - rpcCmd.AddCommand(stateCmd) - rootCmd.AddCommand(rpcCmd) -} - -var rpcCmd = &cobra.Command{ - Use: "rpc [namespace] [method] [params...]", - Short: "Send JSON-RPC request", - Args: cobra.MinimumNArgs(2), - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - rpcClient, err := newRPCClient(cmd.Context()) - if err != nil { - return err - } - - ctx := context.WithValue(cmd.Context(), rpcClientKey{}, rpcClient) - cmd.SetContext(ctx) - return nil - }, - PersistentPostRunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) - if err != nil { - return err - } - - client.Close() - return nil - }, - ValidArgsFunction: func(cmd *cobra.Command, args []string, toComplete string) ([]string, cobra.ShellCompDirective) { - modules := client.Modules - if len(args) == 0 { - // get keys from modules (map[string]interface{}) - var keys []string - for k := range modules { - keys = append(keys, k) - } - return keys, cobra.ShellCompDirectiveNoFileComp - } else if len(args) == 1 { - // get methods from module - module := modules[args[0]] - methods := reflect.VisibleFields(reflect.TypeOf(module).Elem()) - var methodNames []string - for _, m := range methods { - methodNames = append(methodNames, m.Name+"\t"+parseSignatureForHelpstring(m)) - } - return methodNames, cobra.ShellCompDirectiveNoFileComp - } - return nil, cobra.ShellCompDirectiveNoFileComp - }, - Run: func(cmd *cobra.Command, args []string) { - namespace := args[0] - method := args[1] - params := parseParams(method, args[2:]) - - sendJSONRPCRequest(namespace, method, params) - }, -} - -func parseParams(method string, params []string) []interface{} { - parsedParams := make([]interface{}, len(params)) - validateParamsFn := func(has, want int) error { - if has != want { - return fmt.Errorf("rpc: invalid amount of params. has=%d, want=%d", has, want) - } - return nil - } - switch method { - case "GetSharesByNamespace": - if err := validateParamsFn(len(params), 2); err != nil { - panic(err) - } - // 1. Share Root - root, err := parseJSON(params[0]) - if err != nil { - panic(fmt.Errorf("couldn't parse share root as json: %v", err)) - } - parsedParams[0] = root - // 2. Namespace - namespace, err := parseV0Namespace(params[1]) - if err != nil { - panic(fmt.Sprintf("Error parsing namespace: %v", err)) - } - parsedParams[1] = namespace - return parsedParams - case "QueryDelegation", "QueryUnbonding", "BalanceForAddress": - var err error - if err = validateParamsFn(len(params), 2); err != nil { - panic(err) - } - parsedParams[0], err = parseAddressFromString(params[0]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - return parsedParams - case "QueryRedelegations": - var err error - parsedParams[0], err = parseAddressFromString(params[0]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - parsedParams[1], err = parseAddressFromString(params[1]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - return parsedParams - case "Transfer", "Delegate", "Undelegate": - // 1. Address - var err error - if err = validateParamsFn(len(params), 4); err != nil { - panic(err) - } - parsedParams[0], err = parseAddressFromString(params[0]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - // 2. Amount + Fee - parsedParams[1] = params[1] - parsedParams[2] = params[2] - // 3. GasLimit (uint64) - num, err := strconv.ParseUint(params[3], 10, 64) - if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") - } - parsedParams[3] = num - return parsedParams - case "CancelUnbondingDelegation": - // 1. Validator Address - var err error - if err = validateParamsFn(len(params), 5); err != nil { - panic(err) - } - parsedParams[0], err = parseAddressFromString(params[0]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - // 2. Amount + Height + Fee - parsedParams[1] = params[1] - parsedParams[2] = params[2] - parsedParams[3] = params[3] - // 4. GasLimit (uint64) - num, err := strconv.ParseUint(params[4], 10, 64) - if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") - } - parsedParams[4] = num - return parsedParams - case "BeginRedelegate": - // 1. Source Validator Address - var err error - if err = validateParamsFn(len(params), 5); err != nil { - panic(err) - } - parsedParams[0], err = parseAddressFromString(params[0]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - // 2. Destination Validator Address - parsedParams[1], err = parseAddressFromString(params[1]) - if err != nil { - panic(fmt.Errorf("error parsing address: %w", err)) - } - // 2. Amount + Fee - parsedParams[2] = params[2] - parsedParams[3] = params[3] - // 4. GasLimit (uint64) - num, err := strconv.ParseUint(params[4], 10, 64) - if err != nil { - panic("Error parsing gas limit: uint64 could not be parsed.") - } - parsedParams[4] = num - return parsedParams - default: - } - - for i, param := range params { - if param[0] == '{' || param[0] == '[' { - rawJSON, err := parseJSON(param) - if err != nil { - parsedParams[i] = param - } else { - parsedParams[i] = rawJSON - } - } else { - // try to parse arguments as numbers before adding them as strings - num, err := strconv.ParseInt(param, 10, 64) - if err == nil { - parsedParams[i] = num - continue - } - parsedParams[i] = param - } - } - return parsedParams -} - -func sendJSONRPCRequest(namespace, method string, params []interface{}) { - url := requestURL - request := jsonRPCRequest{ - ID: 1, - JSONRPC: "2.0", - Method: fmt.Sprintf("%s.%s", namespace, method), - Params: params, - } - - requestBody, err := json.Marshal(request) - if err != nil { - log.Fatalf("Error marshaling JSON-RPC request: %v", err) - } - - req, err := http.NewRequest("POST", url, bytes.NewBuffer(requestBody)) - if err != nil { - log.Fatalf("Error creating JSON-RPC request: %v", err) - } - - req.Header.Set("Content-Type", "application/json") - - authToken := authTokenFlag - if authToken == "" { - authToken = os.Getenv(authEnvKey) - } - if authToken != "" { - req.Header.Set("Authorization", "Bearer "+authToken) - } - - client := &http.Client{} - resp, err := client.Do(req) - if err != nil { - log.Fatalf("Error sending JSON-RPC request: %v", err) - } - defer resp.Body.Close() - - responseBody, err := io.ReadAll(resp.Body) - if err != nil { - log.Fatalf("Error reading response body: %v", err) //nolint:gocritic - } - - rawResponseJSON, err := parseJSON(string(responseBody)) - if err != nil { - log.Fatalf("Error parsing JSON-RPC response: %v", err) - } - if printRequest { - output, err := json.MarshalIndent(outputWithRequest{ - Request: request, - Response: rawResponseJSON, - }, "", " ") - if err != nil { - panic(fmt.Sprintf("Error marshaling JSON-RPC response: %v", err)) - } - fmt.Println(string(output)) - return - } - - output, err := json.MarshalIndent(rawResponseJSON, "", " ") - if err != nil { - panic(fmt.Sprintf("Error marshaling JSON-RPC response: %v", err)) - } - fmt.Println(string(output)) -} - -func parseAddressFromString(addrStr string) (state.Address, error) { - var address state.Address - err := address.UnmarshalJSON([]byte(addrStr)) - if err != nil { - return address, err - } - return address, nil -} - -func parseSignatureForHelpstring(methodSig reflect.StructField) string { - simplifiedSignature := "(" - in, out := methodSig.Type.NumIn(), methodSig.Type.NumOut() - for i := 1; i < in; i++ { - simplifiedSignature += methodSig.Type.In(i).String() - if i != in-1 { - simplifiedSignature += ", " - } - } - simplifiedSignature += ") -> (" - for i := 0; i < out-1; i++ { - simplifiedSignature += methodSig.Type.Out(i).String() - if i != out-2 { - simplifiedSignature += ", " - } - } - simplifiedSignature += ")" - return simplifiedSignature -} - -// parseV0Namespace parses a namespace from a base64 or hex string. The param -// is expected to be the user-specified portion of a v0 namespace ID (i.e. the -// last 10 bytes). -func parseV0Namespace(param string) (share.Namespace, error) { - userBytes, err := decodeToBytes(param) - if err != nil { - return nil, err - } - - // if the namespace ID is <= 10 bytes, left pad it with 0s - return share.NewBlobNamespaceV0(userBytes) -} - -// decodeToBytes decodes a Base64 or hex input string into a byte slice. -func decodeToBytes(param string) ([]byte, error) { - if strings.HasPrefix(param, "0x") { - decoded, err := hex.DecodeString(param[2:]) - if err != nil { - return nil, fmt.Errorf("error decoding namespace ID: %w", err) - } - return decoded, nil - } - // otherwise, it's just a base64 string - decoded, err := base64.StdEncoding.DecodeString(param) - if err != nil { - return nil, fmt.Errorf("error decoding namespace ID: %w", err) - } - return decoded, nil -} - -func parseJSON(param string) (json.RawMessage, error) { - var raw json.RawMessage - err := json.Unmarshal([]byte(param), &raw) - return raw, err -} - -func newRPCClient(ctx context.Context) (*client.Client, error) { - if authTokenFlag == "" { - authTokenFlag = os.Getenv(authEnvKey) - } - return client.NewClient(ctx, requestURL, authTokenFlag) -} - -type rpcClientKey struct{} - -func rpcClient(ctx context.Context) (*client.Client, error) { - client, ok := ctx.Value(rpcClientKey{}).(*client.Client) - if !ok { - return nil, errors.New("rpc client was not set") - } - return client, nil -} - -func printOutput(data interface{}, err error, formatData func(interface{}) interface{}) error { - switch { - case err != nil: - data = err - case formatData != nil: - data = formatData(data) - } - - resp := struct { - Result interface{} `json:"result"` - }{ - Result: data, - } - - bytes, err := json.MarshalIndent(resp, "", " ") - if err != nil { - return err - } - fmt.Fprintln(os.Stdout, string(bytes)) - return nil } diff --git a/cmd/celestia/util.go b/cmd/celestia/util.go deleted file mode 100644 index a38860d1f7..0000000000 --- a/cmd/celestia/util.go +++ /dev/null @@ -1,68 +0,0 @@ -package main - -import ( - "github.com/spf13/cobra" - - cmdnode "github.com/celestiaorg/celestia-node/cmd" - "github.com/celestiaorg/celestia-node/nodebuilder/core" - "github.com/celestiaorg/celestia-node/nodebuilder/gateway" - "github.com/celestiaorg/celestia-node/nodebuilder/header" - "github.com/celestiaorg/celestia-node/nodebuilder/node" - "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/nodebuilder/rpc" - "github.com/celestiaorg/celestia-node/nodebuilder/state" -) - -func persistentPreRunEnv(cmd *cobra.Command, nodeType node.Type, _ []string) error { - var ( - ctx = cmd.Context() - err error - ) - - ctx = cmdnode.WithNodeType(ctx, nodeType) - - parsedNetwork, err := p2p.ParseNetwork(cmd) - if err != nil { - return err - } - ctx = cmdnode.WithNetwork(ctx, parsedNetwork) - - // loads existing config into the environment - ctx, err = cmdnode.ParseNodeFlags(ctx, cmd, cmdnode.Network(ctx)) - if err != nil { - return err - } - - cfg := cmdnode.NodeConfig(ctx) - - err = p2p.ParseFlags(cmd, &cfg.P2P) - if err != nil { - return err - } - - err = core.ParseFlags(cmd, &cfg.Core) - if err != nil { - return err - } - - if nodeType != node.Bridge { - err = header.ParseFlags(cmd, &cfg.Header) - if err != nil { - return err - } - } - - ctx, err = cmdnode.ParseMiscFlags(ctx, cmd) - if err != nil { - return err - } - - rpc.ParseFlags(cmd, &cfg.RPC) - gateway.ParseFlags(cmd, &cfg.Gateway) - state.ParseFlags(cmd, &cfg.State) - - // set config - ctx = cmdnode.WithNodeConfig(ctx, &cfg) - cmd.SetContext(ctx) - return nil -} diff --git a/cmd/rpc.go b/cmd/rpc.go new file mode 100644 index 0000000000..513967f0ed --- /dev/null +++ b/cmd/rpc.go @@ -0,0 +1,58 @@ +package cmd + +import ( + "context" + "errors" + "os" + + "github.com/spf13/cobra" + + rpc "github.com/celestiaorg/celestia-node/api/rpc/client" +) + +const ( + // defaultRPCAddress is a default address to dial to + defaultRPCAddress = "http://localhost:26658" + authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" //nolint:gosec +) + +var ( + requestURL string + authTokenFlag string +) + +func InitURLFlag() (*string, string, string, string) { + return &requestURL, "url", defaultRPCAddress, "Request URL" +} + +func InitAuthTokenFlag() (*string, string, string, string) { + return &authTokenFlag, + "token", + "", + "Authorization token (if not provided, the " + authEnvKey + " environment variable will be used)" +} + +func InitClient(cmd *cobra.Command, _ []string) error { + if authTokenFlag == "" { + authTokenFlag = os.Getenv(authEnvKey) + } + + client, err := rpc.NewClient(cmd.Context(), requestURL, authTokenFlag) + if err != nil { + return err + } + + ctx := context.WithValue(cmd.Context(), rpcClientKey{}, client) + cmd.SetContext(ctx) + return nil +} + +type rpcClientKey struct{} + +func ParseClientFromCtx(ctx context.Context) (*rpc.Client, error) { + client, ok := ctx.Value(rpcClientKey{}).(*rpc.Client) + if !ok { + return nil, errors.New("rpc client was not set") + } + return client, nil +} diff --git a/cmd/util.go b/cmd/util.go new file mode 100644 index 0000000000..625685fe0b --- /dev/null +++ b/cmd/util.go @@ -0,0 +1,127 @@ +package cmd + +import ( + "encoding/base64" + "encoding/hex" + "encoding/json" + "fmt" + "os" + "strings" + + "github.com/spf13/cobra" + + "github.com/celestiaorg/celestia-node/nodebuilder/core" + "github.com/celestiaorg/celestia-node/nodebuilder/gateway" + "github.com/celestiaorg/celestia-node/nodebuilder/header" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + rpc_cfg "github.com/celestiaorg/celestia-node/nodebuilder/rpc" + "github.com/celestiaorg/celestia-node/nodebuilder/state" + "github.com/celestiaorg/celestia-node/share" +) + +func PrintOutput(data interface{}, err error, formatData func(interface{}) interface{}) error { + switch { + case err != nil: + data = err + case formatData != nil: + data = formatData(data) + } + + resp := struct { + Result interface{} `json:"result"` + }{ + Result: data, + } + + bytes, err := json.MarshalIndent(resp, "", " ") + if err != nil { + return err + } + fmt.Fprintln(os.Stdout, string(bytes)) + return nil +} + +// ParseV0Namespace parses a namespace from a base64 or hex string. The param +// is expected to be the user-specified portion of a v0 namespace ID (i.e. the +// last 10 bytes). +func ParseV0Namespace(param string) (share.Namespace, error) { + userBytes, err := DecodeToBytes(param) + if err != nil { + return nil, err + } + + // if the namespace ID is <= 10 bytes, left pad it with 0s + return share.NewBlobNamespaceV0(userBytes) +} + +// DecodeToBytes decodes a Base64 or hex input string into a byte slice. +func DecodeToBytes(param string) ([]byte, error) { + if strings.HasPrefix(param, "0x") { + decoded, err := hex.DecodeString(param[2:]) + if err != nil { + return nil, fmt.Errorf("error decoding namespace ID: %w", err) + } + return decoded, nil + } + // otherwise, it's just a base64 string + decoded, err := base64.StdEncoding.DecodeString(param) + if err != nil { + return nil, fmt.Errorf("error decoding namespace ID: %w", err) + } + return decoded, nil +} + +func PersistentPreRunEnv(cmd *cobra.Command, nodeType node.Type, _ []string) error { + var ( + ctx = cmd.Context() + err error + ) + + ctx = WithNodeType(ctx, nodeType) + + parsedNetwork, err := p2p.ParseNetwork(cmd) + if err != nil { + return err + } + ctx = WithNetwork(ctx, parsedNetwork) + + // loads existing config into the environment + ctx, err = ParseNodeFlags(ctx, cmd, Network(ctx)) + if err != nil { + return err + } + + cfg := NodeConfig(ctx) + + err = p2p.ParseFlags(cmd, &cfg.P2P) + if err != nil { + return err + } + + err = core.ParseFlags(cmd, &cfg.Core) + if err != nil { + return err + } + + if nodeType != node.Bridge { + err = header.ParseFlags(cmd, &cfg.Header) + if err != nil { + return err + } + } + + ctx, err = ParseMiscFlags(ctx, cmd) + if err != nil { + return err + } + + rpc_cfg.ParseFlags(cmd, &cfg.RPC) + gateway.ParseFlags(cmd, &cfg.Gateway) + state.ParseFlags(cmd, &cfg.State) + + // set config + ctx = WithNodeConfig(ctx, &cfg) + cmd.SetContext(ctx) + return nil +} diff --git a/cmd/celestia/rpc_test.go b/cmd/util_test.go similarity index 97% rename from cmd/celestia/rpc_test.go rename to cmd/util_test.go index 53087646a7..b6e245f3e2 100644 --- a/cmd/celestia/rpc_test.go +++ b/cmd/util_test.go @@ -1,4 +1,4 @@ -package main +package cmd import ( "testing" @@ -69,7 +69,7 @@ func Test_parseNamespaceID(t *testing.T) { for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { - got, err := parseV0Namespace(tc.param) + got, err := ParseV0Namespace(tc.param) if tc.wantErr { assert.Error(t, err) return diff --git a/cmd/celestia/blob.go b/nodebuilder/blob/cmd/blob.go similarity index 77% rename from cmd/celestia/blob.go rename to nodebuilder/blob/cmd/blob.go index 9550f2c13a..1c5ec1ad50 100644 --- a/cmd/celestia/blob.go +++ b/nodebuilder/blob/cmd/blob.go @@ -1,4 +1,4 @@ -package main +package cmd import ( "encoding/base64" @@ -9,6 +9,7 @@ import ( "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/blob" + cmdnode "github.com/celestiaorg/celestia-node/cmd" "github.com/celestiaorg/celestia-node/share" ) @@ -20,7 +21,7 @@ var ( ) func init() { - blobCmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) + Cmd.AddCommand(getCmd, getAllCmd, submitCmd, getProofCmd) getCmd.PersistentFlags().BoolVar( &base64Flag, @@ -28,6 +29,7 @@ func init() { false, "printed blob's data a base64 string", ) + getAllCmd.PersistentFlags().BoolVar( &base64Flag, "base64", @@ -50,28 +52,30 @@ func init() { ) } -var blobCmd = &cobra.Command{ - Use: "blob [command]", - Short: "Allows to interact with the Blob Service via JSON-RPC", - Args: cobra.NoArgs, +var Cmd = &cobra.Command{ + Use: "blob [command]", + Short: "Allows to interact with the Blob Service via JSON-RPC", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, } var getCmd = &cobra.Command{ - Use: "get [height, namespace, commitment]", + Use: "get [height] [namespace] [commitment]", Args: cobra.ExactArgs(3), Short: "Returns the blob for the given namespace by commitment at a particular height.", RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return fmt.Errorf("error parsing a height:%v", err) } - namespace, err := parseV0Namespace(args[1]) + namespace, err := cmdnode.ParseV0Namespace(args[1]) if err != nil { return fmt.Errorf("error parsing a namespace:%v", err) } @@ -87,51 +91,52 @@ var getCmd = &cobra.Command{ if base64Flag || err != nil { formatter = nil } - return printOutput(blob, err, formatter) + return cmdnode.PrintOutput(blob, err, formatter) }, } var getAllCmd = &cobra.Command{ - Use: "get-all [height, namespace]", + Use: "get-all [height] [namespace]", Args: cobra.ExactArgs(2), Short: "Returns all blobs for the given namespace at a particular height.", RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return fmt.Errorf("error parsing a height:%v", err) } - namespace, err := parseV0Namespace(args[1]) + namespace, err := cmdnode.ParseV0Namespace(args[1]) if err != nil { return fmt.Errorf("error parsing a namespace:%v", err) } blobs, err := client.Blob.GetAll(cmd.Context(), height, []share.Namespace{namespace}) - formatter := formatData if base64Flag || err != nil { formatter = nil } - return printOutput(blobs, err, formatter) + return cmdnode.PrintOutput(blobs, err, formatter) }, } var submitCmd = &cobra.Command{ - Use: "submit [namespace, blobData]", + Use: "submit [namespace] [blobData]", Args: cobra.ExactArgs(2), Short: "Submit the blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() - namespace, err := parseV0Namespace(args[0]) + namespace, err := cmdnode.ParseV0Namespace(args[0]) if err != nil { return fmt.Errorf("error parsing a namespace:%v", err) } @@ -154,26 +159,27 @@ var submitCmd = &cobra.Command{ Height: height, Commitment: parsedBlob.Commitment, } - return printOutput(response, err, nil) + return cmdnode.PrintOutput(response, err, nil) }, } var getProofCmd = &cobra.Command{ - Use: "get-proof [height, namespace, commitment]", + Use: "get-proof [height] [namespace] [commitment]", Args: cobra.ExactArgs(3), Short: "Retrieves the blob in the given namespaces at the given height by commitment and returns its Proof.", RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { return fmt.Errorf("error parsing a height:%v", err) } - namespace, err := parseV0Namespace(args[1]) + namespace, err := cmdnode.ParseV0Namespace(args[1]) if err != nil { return fmt.Errorf("error parsing a namespace:%v", err) } @@ -184,7 +190,7 @@ var getProofCmd = &cobra.Command{ } proof, err := client.Blob.GetProof(cmd.Context(), height, namespace, commitment) - return printOutput(proof, err, nil) + return cmdnode.PrintOutput(proof, err, nil) }, } diff --git a/nodebuilder/das/cmd/das.go b/nodebuilder/das/cmd/das.go new file mode 100644 index 0000000000..7512861ac3 --- /dev/null +++ b/nodebuilder/das/cmd/das.go @@ -0,0 +1,34 @@ +package cmd + +import ( + "github.com/spf13/cobra" + + cmdnode "github.com/celestiaorg/celestia-node/cmd" +) + +func init() { + Cmd.AddCommand(samplingStatsCmd) +} + +var Cmd = &cobra.Command{ + Use: "das [command]", + Short: "Allows to interact with the Daser via JSON-RPC", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, +} + +var samplingStatsCmd = &cobra.Command{ + Use: "sampling-stats", + Short: "Returns the current statistics over the DA sampling process", + Args: cobra.NoArgs, + RunE: func(cmd *cobra.Command, args []string) error { + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) + if err != nil { + return err + } + defer client.Close() + + stats, err := client.DAS.SamplingStats(cmd.Context()) + return cmdnode.PrintOutput(stats, err, nil) + }, +} diff --git a/cmd/celestia/header.go b/nodebuilder/header/cmd/header.go similarity index 68% rename from cmd/celestia/header.go rename to nodebuilder/header/cmd/header.go index 1d8dc6d01b..b3bba1eb32 100644 --- a/cmd/celestia/header.go +++ b/nodebuilder/header/cmd/header.go @@ -1,4 +1,4 @@ -package main +package cmd import ( "encoding/hex" @@ -6,10 +6,12 @@ import ( "strconv" "github.com/spf13/cobra" + + cmdnode "github.com/celestiaorg/celestia-node/cmd" ) func init() { - headerCmd.AddCommand( + Cmd.AddCommand( localHeadCmd, networkHeadCmd, getByHashCmd, @@ -18,10 +20,11 @@ func init() { ) } -var headerCmd = &cobra.Command{ - Use: "header [command]", - Short: "Allows interaction with the Header Module via JSON-RPC", - Args: cobra.NoArgs, +var Cmd = &cobra.Command{ + Use: "header [command]", + Short: "Allows interaction with the Header Module via JSON-RPC", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, } var localHeadCmd = &cobra.Command{ @@ -29,13 +32,14 @@ var localHeadCmd = &cobra.Command{ Short: "Returns the ExtendedHeader from the chain head.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() header, err := client.Header.LocalHead(cmd.Context()) - return printOutput(header, err, nil) + return cmdnode.PrintOutput(header, err, nil) }, } @@ -44,13 +48,14 @@ var networkHeadCmd = &cobra.Command{ Short: "Provides the Syncer's view of the current network head.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() header, err := client.Header.NetworkHead(cmd.Context()) - return printOutput(header, err, nil) + return cmdnode.PrintOutput(header, err, nil) }, } @@ -59,17 +64,18 @@ var getByHashCmd = &cobra.Command{ Short: "Returns the header of the given hash from the node's header store.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() hash, err := hex.DecodeString(args[0]) if err != nil { return fmt.Errorf("error decoding a hash: expected a hex encoded string:%v", err) } header, err := client.Header.GetByHash(cmd.Context(), hash) - return printOutput(header, err, nil) + return cmdnode.PrintOutput(header, err, nil) }, } @@ -78,10 +84,11 @@ var getByHeightCmd = &cobra.Command{ Short: "Returns the ExtendedHeader at the given height if it is currently available.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() height, err := strconv.ParseUint(args[0], 10, 64) if err != nil { @@ -89,7 +96,7 @@ var getByHeightCmd = &cobra.Command{ } header, err := client.Header.GetByHeight(cmd.Context(), height) - return printOutput(header, err, nil) + return cmdnode.PrintOutput(header, err, nil) }, } @@ -98,12 +105,13 @@ var syncStateCmd = &cobra.Command{ Short: "Returns the current state of the header Syncer.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() header, err := client.Header.SyncState(cmd.Context()) - return printOutput(header, err, nil) + return cmdnode.PrintOutput(header, err, nil) }, } diff --git a/cmd/celestia/admin.go b/nodebuilder/node/cmd/node.go similarity index 61% rename from cmd/celestia/admin.go rename to nodebuilder/node/cmd/node.go index a3665e0aa1..a65727fb03 100644 --- a/cmd/celestia/admin.go +++ b/nodebuilder/node/cmd/node.go @@ -1,33 +1,24 @@ -package main +package cmd import ( - "context" "errors" "strings" "github.com/filecoin-project/go-jsonrpc/auth" "github.com/spf13/cobra" + + cmdnode "github.com/celestiaorg/celestia-node/cmd" ) func init() { - nodeCmd.AddCommand(nodeInfoCmd, logCmd, verifyCmd, authCmd) - rootCmd.AddCommand(nodeCmd) + Cmd.AddCommand(nodeInfoCmd, logCmd, verifyCmd, authCmd) } -var nodeCmd = &cobra.Command{ - Use: "node [command]", - Short: "Allows administrating running node.", - Args: cobra.NoArgs, - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - rpcClient, err := newRPCClient(cmd.Context()) - if err != nil { - return err - } - - ctx := context.WithValue(cmd.Context(), rpcClientKey{}, rpcClient) - cmd.SetContext(ctx) - return nil - }, +var Cmd = &cobra.Command{ + Use: "node [command]", + Short: "Allows administrating running node.", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, } var nodeInfoCmd = &cobra.Command{ @@ -35,26 +26,30 @@ var nodeInfoCmd = &cobra.Command{ Args: cobra.NoArgs, Short: "Returns administrative information about the node.", RunE: func(c *cobra.Command, args []string) error { - client, err := rpcClient(c.Context()) + client, err := cmdnode.ParseClientFromCtx(c.Context()) if err != nil { return err } + defer client.Close() + info, err := client.Node.Info(c.Context()) - return printOutput(info, err, nil) + return cmdnode.PrintOutput(info, err, nil) }, } var logCmd = &cobra.Command{ - Use: "log-level", - Args: cobra.MinimumNArgs(1), - Short: "Allows to set log level for module to in format :" + + Use: "log-level", + Args: cobra.MinimumNArgs(1), + Short: "Sets log level for module.", + Long: "Allows to set log level for module to in format :" + "`DEBUG, INFO, WARN, ERROR, DPANIC, PANIC, FATAL and their lower-case forms`.\n" + "To set all modules to a particular level `*:` should be passed", RunE: func(c *cobra.Command, args []string) error { - client, err := rpcClient(c.Context()) + client, err := cmdnode.ParseClientFromCtx(c.Context()) if err != nil { return err } + defer client.Close() for _, ll := range args { params := strings.Split(ll, ":") @@ -63,7 +58,7 @@ var logCmd = &cobra.Command{ "e.g. pubsub:debug") } - if err = client.Node.LogLevelSet(c.Context(), params[0], params[1]); err != nil { + if err := client.Node.LogLevelSet(c.Context(), params[0], params[1]); err != nil { return err } } @@ -77,13 +72,14 @@ var verifyCmd = &cobra.Command{ Short: "Returns the permissions assigned to the given token.", RunE: func(c *cobra.Command, args []string) error { - client, err := rpcClient(c.Context()) + client, err := cmdnode.ParseClientFromCtx(c.Context()) if err != nil { return err } + defer client.Close() perms, err := client.Node.AuthVerify(c.Context(), args[0]) - return printOutput(perms, err, nil) + return cmdnode.PrintOutput(perms, err, nil) }, } @@ -92,10 +88,11 @@ var authCmd = &cobra.Command{ Args: cobra.MinimumNArgs(1), Short: "Signs and returns a new token with the given permissions.", RunE: func(c *cobra.Command, args []string) error { - client, err := rpcClient(c.Context()) + client, err := cmdnode.ParseClientFromCtx(c.Context()) if err != nil { return err } + defer client.Close() perms := make([]auth.Permission, len(args)) for i, p := range args { @@ -103,6 +100,6 @@ var authCmd = &cobra.Command{ } result, err := client.Node.AuthNew(c.Context(), perms) - return printOutput(result, err, nil) + return cmdnode.PrintOutput(result, err, nil) }, } diff --git a/cmd/celestia/p2p.go b/nodebuilder/p2p/cmd/p2p.go similarity index 79% rename from cmd/celestia/p2p.go rename to nodebuilder/p2p/cmd/p2p.go index d0b6549f6c..5951595fa4 100644 --- a/cmd/celestia/p2p.go +++ b/nodebuilder/p2p/cmd/p2p.go @@ -1,4 +1,4 @@ -package main +package cmd import ( "github.com/libp2p/go-libp2p/core/metrics" @@ -7,6 +7,8 @@ import ( "github.com/libp2p/go-libp2p/core/protocol" ma2 "github.com/multiformats/go-multiaddr" "github.com/spf13/cobra" + + cmdnode "github.com/celestiaorg/celestia-node/cmd" ) type peerInfo struct { @@ -15,7 +17,7 @@ type peerInfo struct { } func init() { - p2pCmd.AddCommand(infoCmd, + Cmd.AddCommand(infoCmd, peersCmd, peerInfoCmd, connectCmd, @@ -35,10 +37,11 @@ func init() { ) } -var p2pCmd = &cobra.Command{ - Use: "p2p [command]", - Short: "Allows interaction with the P2P Module via JSON-RPC", - Args: cobra.NoArgs, +var Cmd = &cobra.Command{ + Use: "p2p [command]", + Short: "Allows interaction with the P2P Module via JSON-RPC", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, } var infoCmd = &cobra.Command{ @@ -46,10 +49,11 @@ var infoCmd = &cobra.Command{ Short: "Gets the node's peer info (peer id and multiaddresses)", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() info, err := client.P2P.Info(cmd.Context()) @@ -65,7 +69,7 @@ var infoCmd = &cobra.Command{ PeerAddr: ma, } } - return printOutput(info, err, formatter) + return cmdnode.PrintOutput(info, err, formatter) }, } @@ -74,10 +78,11 @@ var peersCmd = &cobra.Command{ Short: "Lists the peers we are connected to", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() result, err := client.P2P.Peers(cmd.Context()) peers := make([]string, len(result)) @@ -93,7 +98,7 @@ var peersCmd = &cobra.Command{ Peers: conPeers, } } - return printOutput(peers, err, formatter) + return cmdnode.PrintOutput(peers, err, formatter) }, } @@ -102,10 +107,11 @@ var peerInfoCmd = &cobra.Command{ Short: "Gets PeerInfo for a given peer", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -124,7 +130,7 @@ var peerInfoCmd = &cobra.Command{ PeerAddr: ma, } } - return printOutput(info, err, formatter) + return cmdnode.PrintOutput(info, err, formatter) }, } @@ -133,10 +139,11 @@ var connectCmd = &cobra.Command{ Short: "Establishes a connection with the given peer", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -155,7 +162,7 @@ var connectCmd = &cobra.Command{ err = client.P2P.Connect(cmd.Context(), peerInfo) if err != nil { - return printOutput(nil, err, nil) + return cmdnode.PrintOutput(nil, err, nil) } return connectednessCmd.RunE(cmd, args) }, @@ -166,10 +173,11 @@ var closePeerCmd = &cobra.Command{ Short: "Closes the connection with the given peer", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -178,7 +186,7 @@ var closePeerCmd = &cobra.Command{ err = client.P2P.ClosePeer(cmd.Context(), pid) if err != nil { - return printOutput(nil, err, nil) + return cmdnode.PrintOutput(nil, err, nil) } return connectednessCmd.RunE(cmd, args) }, @@ -189,10 +197,11 @@ var connectednessCmd = &cobra.Command{ Short: "Checks the connection state between current and given peers", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -209,7 +218,7 @@ var connectednessCmd = &cobra.Command{ ConnectionState: conn.String(), } } - return printOutput(con, err, formatter) + return cmdnode.PrintOutput(con, err, formatter) }, } @@ -218,10 +227,11 @@ var natStatusCmd = &cobra.Command{ Short: "Gets the currrent NAT status", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() r, err := client.P2P.NATStatus(cmd.Context()) @@ -233,7 +243,7 @@ var natStatusCmd = &cobra.Command{ Reachability: rr.String(), } } - return printOutput(r, err, formatter) + return cmdnode.PrintOutput(r, err, formatter) }, } @@ -242,10 +252,11 @@ var blockPeerCmd = &cobra.Command{ Short: "Blocks the given peer", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -270,7 +281,7 @@ var blockPeerCmd = &cobra.Command{ Reason: err, } } - return printOutput(err, nil, formatter) + return cmdnode.PrintOutput(err, nil, formatter) }, } @@ -279,10 +290,11 @@ var unblockPeerCmd = &cobra.Command{ Short: "Unblocks the given peer", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -308,7 +320,7 @@ var unblockPeerCmd = &cobra.Command{ Reason: err, } } - return printOutput(err, nil, formatter) + return cmdnode.PrintOutput(err, nil, formatter) }, } @@ -317,10 +329,11 @@ var blockedPeersCmd = &cobra.Command{ Short: "Lists the node's blocked peers", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() list, err := client.P2P.ListBlockedPeers(cmd.Context()) @@ -337,7 +350,7 @@ var blockedPeersCmd = &cobra.Command{ Peers: peers, } } - return printOutput(pids, err, formatter) + return cmdnode.PrintOutput(pids, err, formatter) }, } @@ -346,10 +359,11 @@ var protectCmd = &cobra.Command{ Short: "Protects the given peer from being pruned by the given tag", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -374,20 +388,22 @@ var protectCmd = &cobra.Command{ Reason: err, } } - return printOutput(err, nil, formatter) + return cmdnode.PrintOutput(err, nil, formatter) }, } var unprotectCmd = &cobra.Command{ - Use: "unprotect [peer.ID, tag]", - Short: "Removes a protection that may have been placed on a peer, under the specified tag." + + Use: "unprotect [peer.ID, tag]", + Short: "Removes protection from the given peer.", + Long: "Removes a protection that may have been placed on a peer, under the specified tag." + "The return value indicates whether the peer continues to be protected after this call, by way of a different tag", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -412,7 +428,7 @@ var unprotectCmd = &cobra.Command{ Reason: err, } } - return printOutput(err, nil, formatter) + return cmdnode.PrintOutput(err, nil, formatter) }, } @@ -421,10 +437,11 @@ var protectedCmd = &cobra.Command{ Short: "Ensures that a given peer is protected under a specific tag", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -432,7 +449,7 @@ var protectedCmd = &cobra.Command{ } result, err := client.P2P.IsProtected(cmd.Context(), pid, args[1]) - return printOutput(result, err, nil) + return cmdnode.PrintOutput(result, err, nil) }, } @@ -444,15 +461,17 @@ type bandwidthStats struct { } var bandwidthStatsCmd = &cobra.Command{ - Use: "bandwidth-stats", - Short: "Get stats struct with bandwidth metrics for all data sent/" + + Use: "bandwidth-stats", + Short: "Provides metrics for current peer.", + Long: "Get stats struct with bandwidth metrics for all data sent/" + "received by the local peer, regardless of protocol or remote peer IDs", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() result, err := client.P2P.BandwidthStats(cmd.Context()) @@ -465,7 +484,7 @@ var bandwidthStatsCmd = &cobra.Command{ RateOut: stats.RateOut, } } - return printOutput(result, err, formatter) + return cmdnode.PrintOutput(result, err, formatter) }, } @@ -474,10 +493,11 @@ var peerBandwidthCmd = &cobra.Command{ Short: "Gets stats struct with bandwidth metrics associated with the given peer.ID", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() pid, err := peer.Decode(args[0]) if err != nil { @@ -495,7 +515,7 @@ var peerBandwidthCmd = &cobra.Command{ RateOut: stats.RateOut, } } - return printOutput(result, err, formatter) + return cmdnode.PrintOutput(result, err, formatter) }, } @@ -504,10 +524,11 @@ var bandwidthForProtocolCmd = &cobra.Command{ Short: "Gets stats struct with bandwidth metrics associated with the given protocol.ID", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() result, err := client.P2P.BandwidthForProtocol(cmd.Context(), protocol.ID(args[0])) @@ -520,7 +541,7 @@ var bandwidthForProtocolCmd = &cobra.Command{ RateOut: stats.RateOut, } } - return printOutput(result, err, formatter) + return cmdnode.PrintOutput(result, err, formatter) }, } @@ -529,10 +550,11 @@ var pubsubPeersCmd = &cobra.Command{ Short: "Lists the peers we are connected to in the given topic", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() result, err := client.P2P.PubSubPeers(cmd.Context(), args[0]) peers := make([]string, len(result)) @@ -549,6 +571,6 @@ var pubsubPeersCmd = &cobra.Command{ Peers: conPeers, } } - return printOutput(peers, err, formatter) + return cmdnode.PrintOutput(peers, err, formatter) }, } diff --git a/cmd/celestia/share.go b/nodebuilder/share/cmd/share.go similarity index 76% rename from cmd/celestia/share.go rename to nodebuilder/share/cmd/share.go index 2f84871d80..fbf3e51db4 100644 --- a/cmd/celestia/share.go +++ b/nodebuilder/share/cmd/share.go @@ -1,4 +1,4 @@ -package main +package cmd import ( "encoding/hex" @@ -9,11 +9,12 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" + cmdnode "github.com/celestiaorg/celestia-node/cmd" "github.com/celestiaorg/celestia-node/share" ) func init() { - shareCmd.AddCommand( + Cmd.AddCommand( sharesAvailableCmd, probabilityOfAvailabilityCmd, getSharesByNamespaceCmd, @@ -22,10 +23,11 @@ func init() { ) } -var shareCmd = &cobra.Command{ - Use: "share [command]", - Short: "Allows interaction with the Share Module via JSON-RPC", - Args: cobra.NoArgs, +var Cmd = &cobra.Command{ + Use: "share [command]", + Short: "Allows interaction with the Share Module via JSON-RPC", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, } var sharesAvailableCmd = &cobra.Command{ @@ -33,10 +35,11 @@ var sharesAvailableCmd = &cobra.Command{ Short: "Subjectively validates if Shares committed to the given Root are available on the Network.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() raw, err := parseJSON(args[0]) if err != nil { @@ -66,7 +69,7 @@ var sharesAvailableCmd = &cobra.Command{ Reason: err, } } - return printOutput(err, nil, formatter) + return cmdnode.PrintOutput(err, nil, formatter) }, } @@ -75,13 +78,14 @@ var probabilityOfAvailabilityCmd = &cobra.Command{ Short: "Calculates the probability of the data square being available based on the number of samples collected.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() prob := client.Share.ProbabilityOfAvailability(cmd.Context()) - return printOutput(prob, nil, nil) + return cmdnode.PrintOutput(prob, nil, nil) }, } @@ -90,10 +94,11 @@ var getSharesByNamespaceCmd = &cobra.Command{ Short: "Gets all shares from an EDS within the given namespace.", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() raw, err := parseJSON(args[0]) if err != nil { @@ -106,13 +111,13 @@ var getSharesByNamespaceCmd = &cobra.Command{ return err } - ns, err := parseV0Namespace(args[1]) + ns, err := cmdnode.ParseV0Namespace(args[1]) if err != nil { return err } shares, err := client.Share.GetSharesByNamespace(cmd.Context(), &root, ns) - return printOutput(shares, err, nil) + return cmdnode.PrintOutput(shares, err, nil) }, } @@ -121,10 +126,11 @@ var getShare = &cobra.Command{ Short: "Gets a Share by coordinates in EDS.", Args: cobra.ExactArgs(3), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() raw, err := parseJSON(args[0]) if err != nil { @@ -165,7 +171,7 @@ var getShare = &cobra.Command{ Data: share.GetData(sh), } } - return printOutput(s, err, formatter) + return cmdnode.PrintOutput(s, err, formatter) }, } @@ -174,10 +180,11 @@ var getEDS = &cobra.Command{ Short: "Gets the full EDS identified by the given root", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() raw, err := parseJSON(args[0]) if err != nil { @@ -191,6 +198,12 @@ var getEDS = &cobra.Command{ } shares, err := client.Share.GetEDS(cmd.Context(), &root) - return printOutput(shares, err, nil) + return cmdnode.PrintOutput(shares, err, nil) }, } + +func parseJSON(param string) (json.RawMessage, error) { + var raw json.RawMessage + err := json.Unmarshal([]byte(param), &raw) + return raw, err +} diff --git a/cmd/celestia/state.go b/nodebuilder/state/cmd/state.go similarity index 80% rename from cmd/celestia/state.go rename to nodebuilder/state/cmd/state.go index 2f3d1e138a..be52b6d26a 100644 --- a/cmd/celestia/state.go +++ b/nodebuilder/state/cmd/state.go @@ -1,6 +1,7 @@ -package main +package cmd import ( + "encoding/hex" "fmt" "strconv" @@ -9,11 +10,12 @@ import ( "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/blob" + cmdnode "github.com/celestiaorg/celestia-node/cmd" "github.com/celestiaorg/celestia-node/state" ) func init() { - stateCmd.AddCommand( + Cmd.AddCommand( accountAddressCmd, balanceCmd, balanceForAddressCmd, @@ -30,10 +32,11 @@ func init() { ) } -var stateCmd = &cobra.Command{ - Use: "state [command]", - Short: "Allows interaction with the State Module via JSON-RPC", - Args: cobra.NoArgs, +var Cmd = &cobra.Command{ + Use: "state [command]", + Short: "Allows interaction with the State Module via JSON-RPC", + Args: cobra.NoArgs, + PersistentPreRunE: cmdnode.InitClient, } var accountAddressCmd = &cobra.Command{ @@ -41,13 +44,14 @@ var accountAddressCmd = &cobra.Command{ Short: "Retrieves the address of the node's account/signer.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() address, err := client.State.AccountAddress(cmd.Context()) - return printOutput(address, err, nil) + return cmdnode.PrintOutput(address, err, nil) }, } @@ -57,13 +61,14 @@ var balanceCmd = &cobra.Command{ "the corresponding block's AppHash.", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() balance, err := client.State.Balance(cmd.Context()) - return printOutput(balance, err, nil) + return cmdnode.PrintOutput(balance, err, nil) }, } @@ -73,10 +78,11 @@ var balanceForAddressCmd = &cobra.Command{ "the corresponding block's AppHash.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() addr, err := parseAddressFromString(args[0]) if err != nil { @@ -84,7 +90,7 @@ var balanceForAddressCmd = &cobra.Command{ } balance, err := client.State.BalanceForAddress(cmd.Context(), addr) - return printOutput(balance, err, nil) + return cmdnode.PrintOutput(balance, err, nil) }, } @@ -93,10 +99,11 @@ var transferCmd = &cobra.Command{ Short: "Sends the given amount of coins from default wallet of the node to the given account address.", Args: cobra.ExactArgs(4), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() addr, err := parseAddressFromString(args[0]) if err != nil { @@ -122,7 +129,7 @@ var transferCmd = &cobra.Command{ math.NewInt(amount), math.NewInt(fee), gasLimit, ) - return printOutput(txResponse, err, nil) + return cmdnode.PrintOutput(txResponse, err, nil) }, } @@ -131,20 +138,21 @@ var submitTxCmd = &cobra.Command{ Short: "Submits the given transaction/message to the Celestia network and blocks until the tx is included in a block.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() - rawTx, err := decodeToBytes(args[0]) + decoded, err := hex.DecodeString(args[0]) if err != nil { return fmt.Errorf("failed to decode tx: %v", err) } txResponse, err := client.State.SubmitTx( cmd.Context(), - rawTx, + decoded, ) - return printOutput(txResponse, err, nil) + return cmdnode.PrintOutput(txResponse, err, nil) }, } @@ -153,11 +161,13 @@ var submitPFBCmd = &cobra.Command{ Short: "Allows to submit PFBs", Args: cobra.ExactArgs(4), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } - namespace, err := parseV0Namespace(args[0]) + defer client.Close() + + namespace, err := cmdnode.ParseV0Namespace(args[0]) if err != nil { return fmt.Errorf("error parsing a namespace:%v", err) } @@ -177,8 +187,13 @@ var submitPFBCmd = &cobra.Command{ return fmt.Errorf("error creating a blob:%v", err) } - txResp, err := client.State.SubmitPayForBlob(cmd.Context(), types.NewInt(fee), gasLimit, []*blob.Blob{parsedBlob}) - return printOutput(txResp, err, nil) + txResp, err := client.State.SubmitPayForBlob( + cmd.Context(), + types.NewInt(fee), + gasLimit, + []*blob.Blob{parsedBlob}, + ) + return cmdnode.PrintOutput(txResp, err, nil) }, } @@ -187,10 +202,12 @@ var cancelUnbondingDelegationCmd = &cobra.Command{ Short: "Cancels a user's pending undelegation from a validator.", Args: cobra.ExactArgs(5), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() + addr, err := parseAddressFromString(args[0]) if err != nil { return fmt.Errorf("error parsing an address:%v", err) @@ -224,7 +241,7 @@ var cancelUnbondingDelegationCmd = &cobra.Command{ math.NewInt(fee), gasLimit, ) - return printOutput(txResponse, err, nil) + return cmdnode.PrintOutput(txResponse, err, nil) }, } @@ -233,10 +250,12 @@ var beginRedelegateCmd = &cobra.Command{ Short: "Sends a user's delegated tokens to a new validator for redelegation", Args: cobra.ExactArgs(5), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() + srcAddr, err := parseAddressFromString(args[0]) if err != nil { return fmt.Errorf("error parsing an address:%v", err) @@ -269,7 +288,7 @@ var beginRedelegateCmd = &cobra.Command{ math.NewInt(fee), gasLimit, ) - return printOutput(txResponse, err, nil) + return cmdnode.PrintOutput(txResponse, err, nil) }, } @@ -278,10 +297,12 @@ var undelegateCmd = &cobra.Command{ Short: "Undelegates a user's delegated tokens, unbonding them from the current validator.", Args: cobra.ExactArgs(4), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() + addr, err := parseAddressFromString(args[0]) if err != nil { return fmt.Errorf("error parsing an address:%v", err) @@ -307,7 +328,7 @@ var undelegateCmd = &cobra.Command{ math.NewInt(fee), gasLimit, ) - return printOutput(txResponse, err, nil) + return cmdnode.PrintOutput(txResponse, err, nil) }, } @@ -316,10 +337,11 @@ var delegateCmd = &cobra.Command{ Short: "Sends a user's liquid tokens to a validator for delegation.", Args: cobra.ExactArgs(4), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() addr, err := parseAddressFromString(args[0]) if err != nil { @@ -348,7 +370,7 @@ var delegateCmd = &cobra.Command{ math.NewInt(fee), gasLimit, ) - return printOutput(txResponse, err, nil) + return cmdnode.PrintOutput(txResponse, err, nil) }, } @@ -357,10 +379,11 @@ var queryDelegationCmd = &cobra.Command{ Short: "Retrieves the delegation information between a delegator and a validator.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() addr, err := parseAddressFromString(args[0]) if err != nil { @@ -368,9 +391,7 @@ var queryDelegationCmd = &cobra.Command{ } balance, err := client.State.QueryDelegation(cmd.Context(), addr.Address.(state.ValAddress)) - fmt.Println(balance) - fmt.Println(err) - return printOutput(balance, err, nil) + return cmdnode.PrintOutput(balance, err, nil) }, } @@ -379,10 +400,11 @@ var queryUnbondingCmd = &cobra.Command{ Short: "Retrieves the unbonding status between a delegator and a validator.", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() addr, err := parseAddressFromString(args[0]) if err != nil { @@ -390,7 +412,7 @@ var queryUnbondingCmd = &cobra.Command{ } response, err := client.State.QueryUnbonding(cmd.Context(), addr.Address.(state.ValAddress)) - return printOutput(response, err, nil) + return cmdnode.PrintOutput(response, err, nil) }, } @@ -399,10 +421,11 @@ var queryRedelegationCmd = &cobra.Command{ Short: "Retrieves the status of the redelegations between a delegator and a validator.", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { - client, err := rpcClient(cmd.Context()) + client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { return err } + defer client.Close() srcAddr, err := parseAddressFromString(args[0]) if err != nil { @@ -419,6 +442,15 @@ var queryRedelegationCmd = &cobra.Command{ srcAddr.Address.(state.ValAddress), dstAddr.Address.(state.ValAddress), ) - return printOutput(response, err, nil) + return cmdnode.PrintOutput(response, err, nil) }, } + +func parseAddressFromString(addrStr string) (state.Address, error) { + var address state.Address + err := address.UnmarshalJSON([]byte(addrStr)) + if err != nil { + return address, err + } + return address, nil +} From f6a1e9d89f21e24cbba48b76f8061f680ac815b5 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 26 Sep 2023 13:14:16 +0300 Subject: [PATCH 239/388] refactor(rpc/blob) extend docs in blob (#2755) --- nodebuilder/blob/cmd/blob.go | 17 ++++++++++++----- 1 file changed, 12 insertions(+), 5 deletions(-) diff --git a/nodebuilder/blob/cmd/blob.go b/nodebuilder/blob/cmd/blob.go index 1c5ec1ad50..f3510edfe1 100644 --- a/nodebuilder/blob/cmd/blob.go +++ b/nodebuilder/blob/cmd/blob.go @@ -41,15 +41,19 @@ func init() { &fee, "fee", -1, - "specifies fee for blob submission", + "specifies fee (in utia) for blob submission.\n"+ + "Fee will be automatically calculated if negative value is passed [optional]", ) submitCmd.PersistentFlags().Uint64Var( &gasLimit, "gas.limit", 0, - "specifies max gas for the blob submission", + "sets the amount of gas that is consumed during blob submission [optional]", ) + + // unset the default value to avoid users confusion + submitCmd.PersistentFlags().Lookup("fee").DefValue = "0" } var Cmd = &cobra.Command{ @@ -126,9 +130,12 @@ var getAllCmd = &cobra.Command{ } var submitCmd = &cobra.Command{ - Use: "submit [namespace] [blobData]", - Args: cobra.ExactArgs(2), - Short: "Submit the blob at the given namespace. Note: only one blob is allowed to submit through the RPC.", + Use: "submit [namespace] [blobData]", + Args: cobra.ExactArgs(2), + Short: "Submit the blob at the given namespace.\n" + + "Note:\n" + + "* only one blob is allowed to submit through the RPC.\n" + + "* fee and gas.limit params will be calculated automatically if they are not provided as arguments", RunE: func(cmd *cobra.Command, args []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { From f596e11b6f2239dfe16e891850c911f68bc75b13 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 2 Oct 2023 11:41:04 +0300 Subject: [PATCH 240/388] feat(cmd/rpc): add node store flag (#2762) --- cmd/auth.go | 82 +++++++++++++++++++++++---------------------- cmd/celestia/rpc.go | 22 ++++-------- cmd/flags_node.go | 2 +- cmd/rpc.go | 57 +++++++++++++++++++++++++++---- 4 files changed, 101 insertions(+), 62 deletions(-) diff --git a/cmd/auth.go b/cmd/auth.go index eb2000675e..3006526b15 100644 --- a/cmd/auth.go +++ b/cmd/auth.go @@ -25,59 +25,61 @@ func AuthCmd(fsets ...*flag.FlagSet) *cobra.Command { Short: "Signs and outputs a hex-encoded JWT token with the given permissions.", Long: "Signs and outputs a hex-encoded JWT token with the given permissions. NOTE: only use this command when " + "the node has already been initialized and started.", - RunE: newToken, - } + RunE: func(cmd *cobra.Command, args []string) error { + if len(args) != 1 { + return fmt.Errorf("must specify permissions") + } + permissions, err := convertToPerms(args[0]) + if err != nil { + return err + } - for _, set := range fsets { - cmd.Flags().AddFlagSet(set) - } - return cmd -} + ks, err := newKeystore(StorePath(cmd.Context())) + if err != nil { + return err -func newToken(cmd *cobra.Command, args []string) error { - if len(args) != 1 { - return fmt.Errorf("must specify permissions") - } + } - permissions, err := convertToPerms(args[0]) - if err != nil { - return err - } + key, err := ks.Get(nodemod.SecretName) + if err != nil { + if !errors.Is(err, keystore.ErrNotFound) { + return err + } + key, err = generateNewKey(ks) + if err != nil { + return err + } + } - expanded, err := homedir.Expand(filepath.Clean(StorePath(cmd.Context()))) - if err != nil { - return err - } - ks, err := keystore.NewFSKeystore(filepath.Join(expanded, "keys"), nil) - if err != nil { - return err + token, err := buildJWTToken(key.Body, permissions) + if err != nil { + return err + } + fmt.Printf("%s", token) + return nil + }, } - var key keystore.PrivKey - key, err = ks.Get(nodemod.SecretName) - if err != nil { - if !errors.Is(err, keystore.ErrNotFound) { - return err - } - // otherwise, generate and save new priv key - key, err = generateNewKey(ks) - if err != nil { - return err - } + for _, set := range fsets { + cmd.Flags().AddFlagSet(set) } + return cmd +} - signer, err := jwt.NewHS256(key.Body) +func newKeystore(path string) (keystore.Keystore, error) { + expanded, err := homedir.Expand(filepath.Clean(path)) if err != nil { - return err + return nil, err } + return keystore.NewFSKeystore(filepath.Join(expanded, "keys"), nil) +} - token, err := authtoken.NewSignedJWT(signer, permissions) +func buildJWTToken(body []byte, permissions []auth.Permission) (string, error) { + signer, err := jwt.NewHS256(body) if err != nil { - return err + return "", err } - - fmt.Printf("%s", token) - return nil + return authtoken.NewSignedJWT(signer, permissions) } func generateNewKey(ks keystore.Keystore) (keystore.PrivKey, error) { diff --git a/cmd/celestia/rpc.go b/cmd/celestia/rpc.go index 8cf51fda09..11e96c2e46 100644 --- a/cmd/celestia/rpc.go +++ b/cmd/celestia/rpc.go @@ -12,21 +12,13 @@ import ( ) func init() { - blob.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - das.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - header.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - p2p.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - share.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - state.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - node.Cmd.PersistentFlags().StringVar(cmd.InitURLFlag()) - - blob.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) - das.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) - header.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) - p2p.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) - share.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) - state.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) - node.Cmd.PersistentFlags().StringVar(cmd.InitAuthTokenFlag()) + blob.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) + das.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) + header.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) + p2p.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) + share.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) + state.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) + node.Cmd.PersistentFlags().AddFlagSet(cmd.RPCFlags()) rootCmd.AddCommand( blob.Cmd, diff --git a/cmd/flags_node.go b/cmd/flags_node.go index 8c73a06169..fe4981b6c6 100644 --- a/cmd/flags_node.go +++ b/cmd/flags_node.go @@ -15,7 +15,7 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/p2p" ) -var ( +const ( nodeStoreFlag = "node.store" nodeConfigFlag = "node.config" ) diff --git a/cmd/rpc.go b/cmd/rpc.go index 513967f0ed..7ea0173795 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -3,11 +3,15 @@ package cmd import ( "context" "errors" + "fmt" "os" "github.com/spf13/cobra" + flag "github.com/spf13/pflag" rpc "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/api/rpc/perms" + nodemod "github.com/celestiaorg/celestia-node/nodebuilder/node" ) const ( @@ -21,15 +25,26 @@ var ( authTokenFlag string ) -func InitURLFlag() (*string, string, string, string) { - return &requestURL, "url", defaultRPCAddress, "Request URL" -} +func RPCFlags() *flag.FlagSet { + fset := &flag.FlagSet{} + + fset.StringVar( + &requestURL, + "url", + defaultRPCAddress, + "Request URL", + ) -func InitAuthTokenFlag() (*string, string, string, string) { - return &authTokenFlag, + fset.StringVar( + &authTokenFlag, "token", "", - "Authorization token (if not provided, the " + authEnvKey + " environment variable will be used)" + "Authorization token (if not provided, the "+authEnvKey+" environment variable will be used)", + ) + + storeFlag := NodeFlags().Lookup(nodeStoreFlag) + fset.AddFlag(storeFlag) + return fset } func InitClient(cmd *cobra.Command, _ []string) error { @@ -37,6 +52,18 @@ func InitClient(cmd *cobra.Command, _ []string) error { authTokenFlag = os.Getenv(authEnvKey) } + if authTokenFlag == "" { + storePath := "" + if cmd.Flag(nodeStoreFlag).Changed { + storePath = cmd.Flag(nodeStoreFlag).Value.String() + } + token, err := getToken(storePath) + if err != nil { + return fmt.Errorf("cant get the access to the auth token: %v", err) + } + authTokenFlag = token + } + client, err := rpc.NewClient(cmd.Context(), requestURL, authTokenFlag) if err != nil { return err @@ -47,6 +74,24 @@ func InitClient(cmd *cobra.Command, _ []string) error { return nil } +func getToken(path string) (string, error) { + if path == "" { + return "", errors.New("root directory was not specified") + } + + ks, err := newKeystore(path) + if err != nil { + return "", err + } + + key, err := ks.Get(nodemod.SecretName) + if err != nil { + fmt.Printf("error getting the JWT secret: %v", err) + return "", err + } + return buildJWTToken(key.Body, perms.AllPerms) +} + type rpcClientKey struct{} func ParseClientFromCtx(ctx context.Context) (*rpc.Client, error) { From 76ced90ff430cc3bb6d1372de0bb775ab9993421 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 2 Oct 2023 11:47:44 +0300 Subject: [PATCH 241/388] refactor(gateway): remove deprecated endpoints (#2764) --- api/gateway/das.go | 28 ---- api/gateway/endpoints.go | 23 +-- api/gateway/state.go | 209 +++------------------------- api/gateway/state_test.go | 57 -------- nodebuilder/gateway/config.go | 7 +- nodebuilder/gateway/constructors.go | 3 +- nodebuilder/gateway/flags.go | 16 +-- nodebuilder/gateway/module.go | 3 +- nodebuilder/state/cmd/state.go | 44 ------ 9 files changed, 27 insertions(+), 363 deletions(-) delete mode 100644 api/gateway/das.go delete mode 100644 api/gateway/state_test.go diff --git a/api/gateway/das.go b/api/gateway/das.go deleted file mode 100644 index 88dc97927c..0000000000 --- a/api/gateway/das.go +++ /dev/null @@ -1,28 +0,0 @@ -package gateway - -import ( - "encoding/json" - "net/http" -) - -const ( - dasStateEndpoint = "/daser/state" -) - -func (h *Handler) handleDASStateRequest(w http.ResponseWriter, r *http.Request) { - logDeprecation(dasStateEndpoint, "das.SamplingStats") - stats, err := h.das.SamplingStats(r.Context()) - if err != nil { - writeError(w, http.StatusInternalServerError, dasStateEndpoint, err) - return - } - resp, err := json.Marshal(stats) - if err != nil { - writeError(w, http.StatusInternalServerError, dasStateEndpoint, err) - return - } - _, err = w.Write(resp) - if err != nil { - log.Errorw("serving request", "endpoint", dasStateEndpoint, "err", err) - } -} diff --git a/api/gateway/endpoints.go b/api/gateway/endpoints.go index 9600138909..104d01b053 100644 --- a/api/gateway/endpoints.go +++ b/api/gateway/endpoints.go @@ -5,28 +5,7 @@ import ( "net/http" ) -func (h *Handler) RegisterEndpoints(rpc *Server, deprecatedEndpointsEnabled bool) { - if deprecatedEndpointsEnabled { - log.Warn("Deprecated endpoints will be removed from the gateway in the next release. Use the RPC instead.") - // state endpoints - rpc.RegisterHandlerFunc(balanceEndpoint, h.handleBalanceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(submitPFBEndpoint, h.handleSubmitPFB, http.MethodPost) - - // staking queries - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryDelegationEndpoint, addrKey), h.handleQueryDelegation, - http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", queryUnbondingEndpoint, addrKey), h.handleQueryUnbonding, - http.MethodGet) - rpc.RegisterHandlerFunc(queryRedelegationsEndpoint, h.handleQueryRedelegations, - http.MethodPost) - - // DASer endpoints - // only register if DASer service is available - if h.das != nil { - rpc.RegisterHandlerFunc(dasStateEndpoint, h.handleDASStateRequest, http.MethodGet) - } - } - +func (h *Handler) RegisterEndpoints(rpc *Server) { // state endpoints rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", balanceEndpoint, addrKey), h.handleBalanceRequest, http.MethodGet) diff --git a/api/gateway/state.go b/api/gateway/state.go index 69900b0bfc..13cf729cc6 100644 --- a/api/gateway/state.go +++ b/api/gateway/state.go @@ -9,17 +9,12 @@ import ( "github.com/cosmos/cosmos-sdk/types" "github.com/gorilla/mux" - "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/state" ) const ( - balanceEndpoint = "/balance" - submitTxEndpoint = "/submit_tx" - submitPFBEndpoint = "/submit_pfb" - queryDelegationEndpoint = "/query_delegation" - queryUnbondingEndpoint = "/query_unbonding" - queryRedelegationsEndpoint = "/query_redelegations" + balanceEndpoint = "/balance" + submitTxEndpoint = "/submit_tx" ) const addrKey = "address" @@ -34,21 +29,6 @@ type submitTxRequest struct { Tx string `json:"tx"` } -// submitPFBRequest represents a request to submit a PayForBlob -// transaction. -type submitPFBRequest struct { - NamespaceID string `json:"namespace_id"` - Data string `json:"data"` - Fee int64 `json:"fee"` - GasLimit uint64 `json:"gas_limit"` -} - -// queryRedelegationsRequest represents a request to query redelegations -type queryRedelegationsRequest struct { - From string `json:"from"` - To string `json:"to"` -} - func (h *Handler) handleBalanceRequest(w http.ResponseWriter, r *http.Request) { var ( bal *state.Balance @@ -57,24 +37,25 @@ func (h *Handler) handleBalanceRequest(w http.ResponseWriter, r *http.Request) { // read and parse request vars := mux.Vars(r) addrStr, exists := vars[addrKey] - if exists { - // convert address to Address type - var addr state.AccAddress - addr, err = types.AccAddressFromBech32(addrStr) + if !exists { + writeError(w, http.StatusBadRequest, balanceEndpoint, errors.New("balance endpoint requires address")) + return + } + + // convert address to Address type + var addr state.AccAddress + addr, err = types.AccAddressFromBech32(addrStr) + if err != nil { + // first check if it is a validator address and can be converted + valAddr, err := types.ValAddressFromBech32(addrStr) if err != nil { - // first check if it is a validator address and can be converted - valAddr, err := types.ValAddressFromBech32(addrStr) - if err != nil { - writeError(w, http.StatusBadRequest, balanceEndpoint, ErrInvalidAddressFormat) - return - } - addr = valAddr.Bytes() + writeError(w, http.StatusBadRequest, balanceEndpoint, ErrInvalidAddressFormat) + return } - bal, err = h.state.BalanceForAddress(r.Context(), state.Address{Address: addr}) - } else { - logDeprecation(balanceEndpoint, "state.Balance") - bal, err = h.state.Balance(r.Context()) + addr = valAddr.Bytes() } + + bal, err = h.state.BalanceForAddress(r.Context(), state.Address{Address: addr}) if err != nil { writeError(w, http.StatusInternalServerError, balanceEndpoint, err) return @@ -119,157 +100,3 @@ func (h *Handler) handleSubmitTx(w http.ResponseWriter, r *http.Request) { log.Errorw("writing response", "endpoint", submitTxEndpoint, "err", err) } } - -func (h *Handler) handleSubmitPFB(w http.ResponseWriter, r *http.Request) { - logDeprecation(submitPFBEndpoint, "blob.Submit or state.SubmitPayForBlob") - // decode request - var req submitPFBRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) - return - } - namespace, err := hex.DecodeString(req.NamespaceID) - if err != nil { - writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) - return - } - data, err := hex.DecodeString(req.Data) - if err != nil { - writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) - return - } - fee := types.NewInt(req.Fee) - - constructedBlob, err := blob.NewBlobV0(namespace, data) - if err != nil { - writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) - return - } - - // perform request - txResp, err := h.state.SubmitPayForBlob(r.Context(), fee, req.GasLimit, []*blob.Blob{constructedBlob}) - if err != nil { - if txResp == nil { - // no tx data to return - writeError(w, http.StatusBadRequest, submitPFBEndpoint, err) - return - } - // if error returned, change status from 200 to 206 - w.WriteHeader(http.StatusPartialContent) - } - - bs, err := json.Marshal(&txResp) - if err != nil { - writeError(w, http.StatusInternalServerError, submitPFBEndpoint, err) - return - } - - _, err = w.Write(bs) - if err != nil { - log.Errorw("writing response", "endpoint", submitPFBEndpoint, "err", err) - } -} - -func (h *Handler) handleQueryDelegation(w http.ResponseWriter, r *http.Request) { - logDeprecation(queryDelegationEndpoint, "state.QueryDelegation") - // read and parse request - vars := mux.Vars(r) - addrStr, exists := vars[addrKey] - if !exists { - writeError(w, http.StatusBadRequest, queryDelegationEndpoint, ErrMissingAddress) - return - } - - // convert address to Address type - addr, err := types.ValAddressFromBech32(addrStr) - if err != nil { - writeError(w, http.StatusBadRequest, queryDelegationEndpoint, err) - return - } - delegation, err := h.state.QueryDelegation(r.Context(), addr) - if err != nil { - writeError(w, http.StatusInternalServerError, queryDelegationEndpoint, err) - return - } - resp, err := json.Marshal(delegation) - if err != nil { - writeError(w, http.StatusInternalServerError, queryDelegationEndpoint, err) - return - } - _, err = w.Write(resp) - if err != nil { - log.Errorw("writing response", "endpoint", queryDelegationEndpoint, "err", err) - } -} - -func (h *Handler) handleQueryUnbonding(w http.ResponseWriter, r *http.Request) { - logDeprecation(queryUnbondingEndpoint, "state.QueryUnbonding") - // read and parse request - vars := mux.Vars(r) - addrStr, exists := vars[addrKey] - if !exists { - writeError(w, http.StatusBadRequest, queryUnbondingEndpoint, ErrMissingAddress) - return - } - - // convert address to Address type - addr, err := types.ValAddressFromBech32(addrStr) - if err != nil { - writeError(w, http.StatusBadRequest, queryUnbondingEndpoint, err) - return - } - unbonding, err := h.state.QueryUnbonding(r.Context(), addr) - if err != nil { - writeError(w, http.StatusInternalServerError, queryUnbondingEndpoint, err) - return - } - resp, err := json.Marshal(unbonding) - if err != nil { - writeError(w, http.StatusInternalServerError, queryUnbondingEndpoint, err) - return - } - _, err = w.Write(resp) - if err != nil { - log.Errorw("writing response", "endpoint", queryUnbondingEndpoint, "err", err) - } -} - -func (h *Handler) handleQueryRedelegations(w http.ResponseWriter, r *http.Request) { - logDeprecation(queryRedelegationsEndpoint, "state.QueryRedelegations") - var req queryRedelegationsRequest - err := json.NewDecoder(r.Body).Decode(&req) - if err != nil { - writeError(w, http.StatusBadRequest, queryRedelegationsEndpoint, err) - return - } - srcValAddr, err := types.ValAddressFromBech32(req.From) - if err != nil { - writeError(w, http.StatusBadRequest, queryRedelegationsEndpoint, err) - return - } - dstValAddr, err := types.ValAddressFromBech32(req.To) - if err != nil { - writeError(w, http.StatusBadRequest, queryRedelegationsEndpoint, err) - return - } - unbonding, err := h.state.QueryRedelegations(r.Context(), srcValAddr, dstValAddr) - if err != nil { - writeError(w, http.StatusInternalServerError, queryRedelegationsEndpoint, err) - return - } - resp, err := json.Marshal(unbonding) - if err != nil { - writeError(w, http.StatusInternalServerError, queryRedelegationsEndpoint, err) - return - } - _, err = w.Write(resp) - if err != nil { - log.Errorw("writing response", "endpoint", queryRedelegationsEndpoint, "err", err) - } -} - -func logDeprecation(endpoint string, alternative string) { - log.Warn("The " + endpoint + " endpoint is deprecated and will be removed in the next release. Please " + - "use " + alternative + " from the RPC instead.") -} diff --git a/api/gateway/state_test.go b/api/gateway/state_test.go deleted file mode 100644 index aa9196cc8d..0000000000 --- a/api/gateway/state_test.go +++ /dev/null @@ -1,57 +0,0 @@ -package gateway - -import ( - "bytes" - "encoding/hex" - "encoding/json" - "errors" - "net/http" - "net/http/httptest" - "testing" - - "github.com/golang/mock/gomock" - "github.com/stretchr/testify/require" - - stateMock "github.com/celestiaorg/celestia-node/nodebuilder/state/mocks" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/state" -) - -func TestHandleSubmitPFB(t *testing.T) { - ctrl := gomock.NewController(t) - mock := stateMock.NewMockModule(ctrl) - handler := NewHandler(mock, nil, nil, nil) - - t.Run("partial response", func(t *testing.T) { - txResponse := state.TxResponse{ - Height: 1, - TxHash: "hash", - Codespace: "codespace", - Code: 1, - } - // simulate core-app err, since it is not exported - timedErr := errors.New("timed out waiting for tx to be included in a block") - mock.EXPECT().SubmitPayForBlob(gomock.Any(), gomock.Any(), gomock.Any(), gomock.Any()). - Return(&txResponse, timedErr) - - ns, err := share.NewBlobNamespaceV0([]byte("abc")) - require.NoError(t, err) - hexNs := hex.EncodeToString(ns[:]) - - bs, err := json.Marshal(submitPFBRequest{ - NamespaceID: hexNs, - Data: "DEADBEEF", - }) - require.NoError(t, err) - httpreq := httptest.NewRequest("GET", "/", bytes.NewReader(bs)) - respRec := httptest.NewRecorder() - handler.handleSubmitPFB(respRec, httpreq) - - var resp state.TxResponse - err = json.NewDecoder(respRec.Body).Decode(&resp) - require.NoError(t, err) - - require.Equal(t, http.StatusPartialContent, respRec.Code) - require.Equal(t, resp, txResponse) - }) -} diff --git a/nodebuilder/gateway/config.go b/nodebuilder/gateway/config.go index f85f207ceb..903a27489a 100644 --- a/nodebuilder/gateway/config.go +++ b/nodebuilder/gateway/config.go @@ -8,10 +8,9 @@ import ( ) type Config struct { - Address string - Port string - Enabled bool - deprecatedEndpoints bool + Address string + Port string + Enabled bool } func DefaultConfig() Config { diff --git a/nodebuilder/gateway/constructors.go b/nodebuilder/gateway/constructors.go index c771c12023..c28153b0a5 100644 --- a/nodebuilder/gateway/constructors.go +++ b/nodebuilder/gateway/constructors.go @@ -10,7 +10,6 @@ import ( // Handler constructs a new RPC Handler from the given services. func Handler( - cfg *Config, state state.Module, share share.Module, header header.Module, @@ -18,7 +17,7 @@ func Handler( serv *gateway.Server, ) { handler := gateway.NewHandler(state, share, header, daser) - handler.RegisterEndpoints(serv, cfg.deprecatedEndpoints) + handler.RegisterEndpoints(serv) handler.RegisterMiddleware(serv) } diff --git a/nodebuilder/gateway/flags.go b/nodebuilder/gateway/flags.go index 4d72a278e5..cd13e47162 100644 --- a/nodebuilder/gateway/flags.go +++ b/nodebuilder/gateway/flags.go @@ -6,10 +6,9 @@ import ( ) var ( - enabledFlag = "gateway" - addrFlag = "gateway.addr" - portFlag = "gateway.port" - deprecatedEndpoints = "gateway.deprecated-endpoints" + enabledFlag = "gateway" + addrFlag = "gateway.addr" + portFlag = "gateway.port" ) // Flags gives a set of hardcoded node/gateway package flags. @@ -21,11 +20,6 @@ func Flags() *flag.FlagSet { false, "Enables the REST gateway", ) - flags.Bool( - deprecatedEndpoints, - false, - "Enables deprecated endpoints on the gateway. These will be removed in the next release.", - ) flags.String( addrFlag, "", @@ -46,10 +40,6 @@ func ParseFlags(cmd *cobra.Command, cfg *Config) { if cmd.Flags().Changed(enabledFlag) && err == nil { cfg.Enabled = enabled } - deprecatedEndpointsEnabled, err := cmd.Flags().GetBool(deprecatedEndpoints) - if cmd.Flags().Changed(deprecatedEndpoints) && err == nil { - cfg.deprecatedEndpoints = deprecatedEndpointsEnabled - } addr, port := cmd.Flag(addrFlag), cmd.Flag(portFlag) if !cfg.Enabled && (addr.Changed || port.Changed) { log.Warn("custom address or port provided without enabling gateway, setting config values") diff --git a/nodebuilder/gateway/module.go b/nodebuilder/gateway/module.go index b727f4c04d..4cdf325dc0 100644 --- a/nodebuilder/gateway/module.go +++ b/nodebuilder/gateway/module.go @@ -48,13 +48,12 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { "gateway", baseComponents, fx.Invoke(func( - cfg *Config, state stateServ.Module, share shareServ.Module, header headerServ.Module, serv *gateway.Server, ) { - Handler(cfg, state, share, header, nil, serv) + Handler(state, share, header, nil, serv) }), ) default: diff --git a/nodebuilder/state/cmd/state.go b/nodebuilder/state/cmd/state.go index be52b6d26a..d35c4a1b4f 100644 --- a/nodebuilder/state/cmd/state.go +++ b/nodebuilder/state/cmd/state.go @@ -6,10 +6,8 @@ import ( "strconv" "cosmossdk.io/math" - "github.com/cosmos/cosmos-sdk/types" "github.com/spf13/cobra" - "github.com/celestiaorg/celestia-node/blob" cmdnode "github.com/celestiaorg/celestia-node/cmd" "github.com/celestiaorg/celestia-node/state" ) @@ -21,7 +19,6 @@ func init() { balanceForAddressCmd, transferCmd, submitTxCmd, - submitPFBCmd, cancelUnbondingDelegationCmd, beginRedelegateCmd, undelegateCmd, @@ -156,47 +153,6 @@ var submitTxCmd = &cobra.Command{ }, } -var submitPFBCmd = &cobra.Command{ - Use: "submit-pfb [namespace] [data] [fee] [gasLim]", - Short: "Allows to submit PFBs", - Args: cobra.ExactArgs(4), - RunE: func(cmd *cobra.Command, args []string) error { - client, err := cmdnode.ParseClientFromCtx(cmd.Context()) - if err != nil { - return err - } - defer client.Close() - - namespace, err := cmdnode.ParseV0Namespace(args[0]) - if err != nil { - return fmt.Errorf("error parsing a namespace:%v", err) - } - - fee, err := strconv.ParseInt(args[2], 10, 64) - if err != nil { - return fmt.Errorf("error parsing a fee:%v", err) - } - - gasLimit, err := strconv.ParseUint(args[3], 10, 64) - if err != nil { - return fmt.Errorf("error parsing a gasLim:%v", err) - } - - parsedBlob, err := blob.NewBlobV0(namespace, []byte(args[1])) - if err != nil { - return fmt.Errorf("error creating a blob:%v", err) - } - - txResp, err := client.State.SubmitPayForBlob( - cmd.Context(), - types.NewInt(fee), - gasLimit, - []*blob.Blob{parsedBlob}, - ) - return cmdnode.PrintOutput(txResp, err, nil) - }, -} - var cancelUnbondingDelegationCmd = &cobra.Command{ Use: "cancel-unbonding-delegation [address] [amount] [height] [fee] [gasLimit]", Short: "Cancels a user's pending undelegation from a validator.", From 9ef7594c7827cfa53ccabb5ff454fc586eed967c Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 5 Oct 2023 12:27:57 +0300 Subject: [PATCH 242/388] refactoring(rpc): remove authEnvKey (#2810) ## Overview ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- cmd/rpc.go | 13 ++++--------- 1 file changed, 4 insertions(+), 9 deletions(-) diff --git a/cmd/rpc.go b/cmd/rpc.go index 7ea0173795..62e9fe7923 100644 --- a/cmd/rpc.go +++ b/cmd/rpc.go @@ -4,7 +4,6 @@ import ( "context" "errors" "fmt" - "os" "github.com/spf13/cobra" flag "github.com/spf13/pflag" @@ -17,7 +16,6 @@ import ( const ( // defaultRPCAddress is a default address to dial to defaultRPCAddress = "http://localhost:26658" - authEnvKey = "CELESTIA_NODE_AUTH_TOKEN" //nolint:gosec ) var ( @@ -39,7 +37,7 @@ func RPCFlags() *flag.FlagSet { &authTokenFlag, "token", "", - "Authorization token (if not provided, the "+authEnvKey+" environment variable will be used)", + "Authorization token", ) storeFlag := NodeFlags().Lookup(nodeStoreFlag) @@ -48,15 +46,12 @@ func RPCFlags() *flag.FlagSet { } func InitClient(cmd *cobra.Command, _ []string) error { - if authTokenFlag == "" { - authTokenFlag = os.Getenv(authEnvKey) - } - if authTokenFlag == "" { storePath := "" - if cmd.Flag(nodeStoreFlag).Changed { - storePath = cmd.Flag(nodeStoreFlag).Value.String() + if !cmd.Flag(nodeStoreFlag).Changed { + return fmt.Errorf("cant get the access to the auth token: token/node-store flag was not specified") } + storePath = cmd.Flag(nodeStoreFlag).Value.String() token, err := getToken(storePath) if err != nil { return fmt.Errorf("cant get the access to the auth token: %v", err) From e6a69829c1883c32eab96ca8e1dd9e0aa99a4bd6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 13:27:04 +0300 Subject: [PATCH 243/388] chore(deps): Bump the otel group with 2 updates (#2800) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps the otel group with 2 updates: [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp](https://github.com/open-telemetry/opentelemetry-go) and [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp](https://github.com/open-telemetry/opentelemetry-go). Updates `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp` from 0.41.0 to 0.42.0
Release notes

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's releases.

Release v1.19.0/v0.42.0/v0.0.7

This release contains the first stable release of the OpenTelemetry Go metric SDK. Our project stability guarantees now apply to the go.opentelemetry.io/otel/sdk/metric package. See our versioning policy for more information about these stability guarantees.

Added

  • Add the "Roll the dice" getting started application example in go.opentelemetry.io/otel/example/dice. (#4539)
  • The WithWriter and WithPrettyPrint options to go.opentelemetry.io/otel/exporters/stdout/stdoutmetric to set a custom io.Writer, and allow displaying the output in human-readable JSON. (#4507)

Changed

  • Allow '/' characters in metric instrument names. (#4501)
  • The exporter in go.opentelemetry.io/otel/exporters/stdout/stdoutmetric does not prettify its output by default anymore. (#4507)
  • Upgrade gopkg.io/yaml from v2 to v3 in go.opentelemetry.io/otel/schema. (#4535)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the Prometheus metric on every Collect if we know the scope is invalid. (#4499)

Removed

  • Remove "go.opentelemetry.io/otel/bridge/opencensus".NewMetricExporter, which is replaced by NewMetricProducer. (#4566)

Full Changelog: https://github.com/open-telemetry/opentelemetry-go/compare/v1.18.0...v1.19.0

Release v1.19.0-rc.1/v0.42.0-rc.1

This is a release candidate for the v1.19.0/v0.42.0 release. That release is expected to include the v1 release of the OpenTelemetry Go metric SDK and will provide stability guarantees of that SDK. See our versioning policy for more information about these stability guarantees.

Changed

  • Allow '/' characters in metric instrument names. (#4501)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the Prometheus metric on every Collect if we know the scope is invalid. (#4499)

New Contributors

Full Changelog: https://github.com/open-telemetry/opentelemetry-go/compare/v1.18.0...v1.19.0-rc.1

Release v1.18.0/v0.41.0/v0.0.6

This release drops the compatibility guarantee of [Go 1.19].

Added

  • Add WithProducer option in go.opentelemetry.op/otel/exporters/prometheus to restore the ability to register producers on the prometheus exporter's manual reader. (#4473)
  • Add IgnoreValue option in go.opentelemetry.io/otel/sdk/metric/metricdata/metricdatatest to allow ignoring values when comparing metrics. (#4447)

... (truncated)

Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's changelog.

[1.19.0/0.42.0/0.0.7] 2023-09-28

This release contains the first stable release of the OpenTelemetry Go [metric SDK]. Our project stability guarantees now apply to the go.opentelemetry.io/otel/sdk/metric package. See our versioning policy for more information about these stability guarantees.

Added

  • Add the "Roll the dice" getting started application example in go.opentelemetry.io/otel/example/dice. (#4539)
  • The WithWriter and WithPrettyPrint options to go.opentelemetry.io/otel/exporters/stdout/stdoutmetric to set a custom io.Writer, and allow displaying the output in human-readable JSON. (#4507)

Changed

  • Allow '/' characters in metric instrument names. (#4501)
  • The exporter in go.opentelemetry.io/otel/exporters/stdout/stdoutmetric does not prettify its output by default anymore. (#4507)
  • Upgrade gopkg.io/yaml from v2 to v3 in go.opentelemetry.io/otel/schema. (#4535)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the Prometheus metric on every Collect if we know the scope is invalid. (#4499)

Removed

  • Remove "go.opentelemetry.io/otel/bridge/opencensus".NewMetricExporter, which is replaced by NewMetricProducer. (#4566)

[1.19.0-rc.1/0.42.0-rc.1] 2023-09-14

This is a release candidate for the v1.19.0/v0.42.0 release. That release is expected to include the v1 release of the OpenTelemetry Go metric SDK and will provide stability guarantees of that SDK. See our versioning policy for more information about these stability guarantees.

Changed

  • Allow '/' characters in metric instrument names. (#4501)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the prometheus metric on every Collect if we know the scope is invalid. (#4499)
Commits

Updates `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` from 1.18.0 to 1.19.0
Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp's changelog.

[1.19.0/0.42.0/0.0.7] 2023-09-28

This release contains the first stable release of the OpenTelemetry Go [metric SDK]. Our project stability guarantees now apply to the go.opentelemetry.io/otel/sdk/metric package. See our versioning policy for more information about these stability guarantees.

Added

  • Add the "Roll the dice" getting started application example in go.opentelemetry.io/otel/example/dice. (#4539)
  • The WithWriter and WithPrettyPrint options to go.opentelemetry.io/otel/exporters/stdout/stdoutmetric to set a custom io.Writer, and allow displaying the output in human-readable JSON. (#4507)

Changed

  • Allow '/' characters in metric instrument names. (#4501)
  • The exporter in go.opentelemetry.io/otel/exporters/stdout/stdoutmetric does not prettify its output by default anymore. (#4507)
  • Upgrade gopkg.io/yaml from v2 to v3 in go.opentelemetry.io/otel/schema. (#4535)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the Prometheus metric on every Collect if we know the scope is invalid. (#4499)

Removed

  • Remove "go.opentelemetry.io/otel/bridge/opencensus".NewMetricExporter, which is replaced by NewMetricProducer. (#4566)

[1.19.0-rc.1/0.42.0-rc.1] 2023-09-14

This is a release candidate for the v1.19.0/v0.42.0 release. That release is expected to include the v1 release of the OpenTelemetry Go metric SDK and will provide stability guarantees of that SDK. See our versioning policy for more information about these stability guarantees.

Changed

  • Allow '/' characters in metric instrument names. (#4501)

Fixed

  • In go.opentelemetry.op/otel/exporters/prometheus, don't try to create the prometheus metric on every Collect if we know the scope is invalid. (#4499)
Commits

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 8 ++++---- go.sum | 16 ++++++++-------- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 702d964619..13f218440c 100644 --- a/go.mod +++ b/go.mod @@ -56,12 +56,12 @@ require ( github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0 go.opentelemetry.io/otel v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 go.opentelemetry.io/otel/metric v1.19.0 go.opentelemetry.io/otel/sdk v1.19.0 - go.opentelemetry.io/otel/sdk/metric v0.41.0 + go.opentelemetry.io/otel/sdk/metric v1.19.0 go.opentelemetry.io/otel/trace v1.19.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 @@ -316,7 +316,7 @@ require ( github.com/zondax/ledger-go v0.14.1 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0 // indirect + go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.0 // indirect diff --git a/go.sum b/go.sum index 0ee3078273..7c1334a343 100644 --- a/go.sum +++ b/go.sum @@ -2384,14 +2384,14 @@ go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+n go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0 h1:k0k7hFNDd8K4iOMJXj7s8sHaC4mhTlAeppRmZXLgZ6k= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.41.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0 h1:iV3BOgW4fry1Riw9dwypigqlIYWXvSRVT2RJmblzo40= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.41.0/go.mod h1:7PGzqlKrxIRmbj5tlNW0nTkYZ5fHXDgk6Fy8/KjR0CI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0 h1:6pu8ttx76BxHf+xz/H77AUZkPF3cwWzXqAUsXhVKI18= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.18.0/go.mod h1:IOmXxPrxoxFMXdNy7lfDmE8MzE61YPcurbUm0SMjerI= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= @@ -2401,8 +2401,8 @@ go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/sdk/metric v0.41.0 h1:c3sAt9/pQ5fSIUfl0gPtClV3HhE18DCVzByD33R/zsk= -go.opentelemetry.io/otel/sdk/metric v0.41.0/go.mod h1:PmOmSt+iOklKtIg5O4Vz9H/ttcRFSNTgii+E1KGyn1w= +go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= +go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= From 424541113fb68c97850d9e9b4795ce6102cbc2b7 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 13:30:38 +0300 Subject: [PATCH 244/388] chore(deps): Bump github.com/prometheus/client_golang from 1.16.0 to 1.17.0 (#2787) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.16.0 to 1.17.0.
Release notes

Sourced from github.com/prometheus/client_golang's releases.

v1.17.0

What's Changed

  • [CHANGE] Minimum required go version is now 1.19 (we also test client_golang against new 1.21 version). #1325
  • [FEATURE] Add support for Created Timestamps in Counters, Summaries and Historams. #1313
  • [ENHANCEMENT] Enable detection of a native histogram without observations. #1314

New Contributors

... (truncated)

Changelog

Sourced from github.com/prometheus/client_golang's changelog.

1.17.0 / 2023-09-27

  • [CHANGE] Minimum required go version is now 1.19 (we also test client_golang against new 1.21 version). #1325
  • [FEATURE] Add support for Created Timestamps in Counters, Summaries and Historams. #1313
  • [ENHANCEMENT] Enable detection of a native histogram without observations. #1314
Commits
  • fa1408e Merge pull request #1352 from prometheus/arthursens/cut-1.17.0
  • 24a72b8 Add changelog entry for 1.17
  • 1bae6c1 Deprecated comment should begin with "Deprecated:" (#1347)
  • bbab8fe Fix typos in comments, tests, and errors (#1346)
  • df7fa49 Extend Counters, Summaries and Histograms with creation timestamp (#1313)
  • 74cc262 Add go_godebug_non_default_behavior_tlsmaxrsasize_events_total (#1348)
  • d03abf3 Cleanup golangci-lint errcheck (#1339)
  • ca6ba04 Update common Prometheus files (#1338)
  • 51d24f8 Update common Prometheus files (#1332)
  • c17edf0 Merge pull request #1304 from prometheus/dependabot/go_modules/google.golang....
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/prometheus/client_golang&package-manager=go_modules&previous-version=1.16.0&new-version=1.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 13f218440c..ccb71e8fc5 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 - github.com/prometheus/client_golang v1.16.0 + github.com/prometheus/client_golang v1.17.0 github.com/pyroscope-io/client v0.7.2 github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.7.0 @@ -278,7 +278,7 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_model v0.4.0 // indirect + github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect github.com/prometheus/common v0.44.0 // indirect github.com/prometheus/procfs v0.11.1 // indirect github.com/pyroscope-io/godeltaprof v0.1.2 // indirect diff --git a/go.sum b/go.sum index 7c1334a343..07d9f18d02 100644 --- a/go.sum +++ b/go.sum @@ -2032,8 +2032,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -2041,8 +2041,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= +github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= From f6b7b2329115a0d079f47448e3a875453b49ec62 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 10:36:32 +0000 Subject: [PATCH 245/388] chore(deps): Bump github.com/hashicorp/golang-lru/v2 from 2.0.6 to 2.0.7 (#2785) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/hashicorp/golang-lru/v2](https://github.com/hashicorp/golang-lru) from 2.0.6 to 2.0.7.
Release notes

Sourced from github.com/hashicorp/golang-lru/v2's releases.

golang-lru 2.0.7

What's Changed

New Contributors

Full Changelog: https://github.com/hashicorp/golang-lru/compare/v2.0.6...v2.0.7

Commits
  • d851586 add a Resize method to 2Q
  • d46c1d9 expirable LRU: fix so that Get/Peek cannot return an ok and empty value (#156)
  • 56a2dc0 Update arc to base package v2.0.6
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/golang-lru/v2&package-manager=go_modules&previous-version=2.0.6&new-version=2.0.7)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index ccb71e8fc5..e5bec1fcf3 100644 --- a/go.mod +++ b/go.mod @@ -26,7 +26,7 @@ require ( github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.0 github.com/hashicorp/go-retryablehttp v0.7.4 - github.com/hashicorp/golang-lru/v2 v2.0.6 + github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.13.1 github.com/ipfs/go-block-format v0.2.0 diff --git a/go.sum b/go.sum index 07d9f18d02..6d47e155b7 100644 --- a/go.sum +++ b/go.sum @@ -980,8 +980,8 @@ github.com/hashicorp/golang-lru v1.0.2/go.mod h1:iADmTwqILo4mZ8BN3D2Q6+9jd8WM5uG github.com/hashicorp/golang-lru/arc/v2 v2.0.5 h1:l2zaLDubNhW4XO3LnliVj0GXO3+/CGNJAg1dcN2Fpfw= github.com/hashicorp/golang-lru/arc/v2 v2.0.5/go.mod h1:ny6zBSQZi2JxIeYcv7kt2sH2PXJtirBN7RDhRpxPkxU= github.com/hashicorp/golang-lru/v2 v2.0.1/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= -github.com/hashicorp/golang-lru/v2 v2.0.6 h1:3xi/Cafd1NaoEnS/yDssIiuVeDVywU0QdFGl3aQaQHM= -github.com/hashicorp/golang-lru/v2 v2.0.6/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= +github.com/hashicorp/golang-lru/v2 v2.0.7 h1:a+bsQ5rvGLjzHuww6tVxozPZFVghXaHOwFs4luLUK2k= +github.com/hashicorp/golang-lru/v2 v2.0.7/go.mod h1:QeFd9opnmA6QUJc5vARoKUSoFhyfM2/ZepoAG6RGpeM= github.com/hashicorp/hcl v1.0.0 h1:0Anlzjpi4vEasTeNFn2mLJgTSwt0+6sfsiTG8qcWGx4= github.com/hashicorp/hcl v1.0.0/go.mod h1:E5yfLk+7swimpb2L/Alb/PJmXilQ/rhwaUYs4T20WEQ= github.com/hashicorp/logutils v1.0.0/go.mod h1:QIAnNjmIWmVIIkWDTG1z5v++HQmx9WQRO+LraFDTW64= From fc0074676cd8eb7b46091d68db7333bf20c7adc9 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 6 Oct 2023 13:39:23 +0300 Subject: [PATCH 246/388] chore(deps): Bump alpine from 3.18.3 to 3.18.4 (#2794) Bumps alpine from 3.18.3 to 3.18.4. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=alpine&package-manager=docker&previous-version=3.18.3&new-version=3.18.4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bebd61a644..403691d4a7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,7 +15,7 @@ COPY . . RUN make build && make cel-key -FROM --platform=$BUILDPLATFORM docker.io/alpine:3.18.3 +FROM --platform=$BUILDPLATFORM docker.io/alpine:3.18.4 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From 95727d7ce3eedbff6fc5a3540430697af374839c Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 6 Oct 2023 11:43:41 +0100 Subject: [PATCH 247/388] (chore) ensure all dependency group PRs get the default label (#2798) currently only the go.mod dependabot PRs were getting the `kind:deps` label, so others that targer Docker or github actions would fail a label check eg: https://github.com/celestiaorg/celestia-node/pull/2794 this should fix that. minor tweak. --- .github/dependabot.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 4ab88e1d32..e2becce196 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -7,6 +7,8 @@ updates: day: monday time: "11:00" open-pull-requests-limit: 10 + labels: + - kind:deps - package-ecosystem: gomod directory: "/" schedule: @@ -30,3 +32,5 @@ updates: day: monday time: "11:00" open-pull-requests-limit: 10 + labels: + - kind:deps From 49d962e53a772999cf1b365f72f0f73dc40d48f2 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 6 Oct 2023 12:44:08 +0200 Subject: [PATCH 248/388] deps: bump go-header (#2813) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e5bec1fcf3..2bf4f0edf6 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/celestia-app v1.0.0-rc18 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.3.1 + github.com/celestiaorg/go-header v0.3.3 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 6d47e155b7..d0cae595b7 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.3.1 h1:rJuFPVMoI20Du4KHWLz0IMllEIp0XgIWi9lHQLPWmq8= -github.com/celestiaorg/go-header v0.3.1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.3.3 h1:Y04hdJIJfD5hapyqK0ZQMgMTH5PQGV9YpcIf56LGc4E= +github.com/celestiaorg/go-header v0.3.3/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= From 270931bdd329359b89f436fef2b71e951a083afa Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 9 Oct 2023 16:18:30 +0400 Subject: [PATCH 249/388] fix(share/peer-manager): use mutex for peer-manager pool cleanup (#2817) Resolves https://github.com/celestiaorg/celestia-node/issues/2807 --- share/p2p/peers/manager.go | 5 +---- share/p2p/peers/pool.go | 9 +++++++++ 2 files changed, 10 insertions(+), 4 deletions(-) diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index caef242eec..4935db2774 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -7,7 +7,6 @@ import ( "sync" "sync/atomic" "time" - "unsafe" logging "github.com/ipfs/go-log/v2" pubsub "github.com/libp2p/go-libp2p-pubsub" @@ -518,9 +517,7 @@ func (m *Manager) markPoolAsSynced(datahash string) { p := m.getOrCreatePool(datahash) if p.isSynced.CompareAndSwap(false, true) { p.isSynced.Store(true) - old := (*unsafe.Pointer)(unsafe.Pointer(&p.pool)) - // release pointer to old pool to free up memory - atomic.StorePointer(old, unsafe.Pointer(newPool(time.Second))) + p.reset() } } diff --git a/share/p2p/peers/pool.go b/share/p2p/peers/pool.go index c43bbc963b..609d68e0b3 100644 --- a/share/p2p/peers/pool.go +++ b/share/p2p/peers/pool.go @@ -221,3 +221,12 @@ func (p *pool) len() int { defer p.m.RUnlock() return p.activeCount } + +// reset will reset the pool to its initial state. +func (p *pool) reset() { + lock := &p.m + lock.Lock() + defer lock.Lock() + // swap the pool with an empty one + *p = *newPool(time.Second) +} From 1ba92c3d50de87688453ae746960e18462953572 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 17:31:37 +0400 Subject: [PATCH 250/388] chore(deps): Bump go.opentelemetry.io/contrib/instrumentation/runtime from 0.44.0 to 0.45.0 (#2819) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [go.opentelemetry.io/contrib/instrumentation/runtime](https://github.com/open-telemetry/opentelemetry-go-contrib) from 0.44.0 to 0.45.0.
Release notes

Sourced from go.opentelemetry.io/contrib/instrumentation/runtime's releases.

Release v1.20.0/v0.45.0/v0.14.0

Added

  • Set the description for the rpc.server.duration metric in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc. (#4302)
  • Add NewServerHandler and NewClientHandler that return a grpc.StatsHandler used for gRPC instrumentation in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc. (#3002)
  • Add new Prometheus bridge module in go.opentelemetry.io/contrib/bridges/prometheus. (#4227)

Changed

  • Upgrade dependencies of OpenTelemetry Go to use the new v1.19.0/v0.42.0/v0.0.7 release.
  • Use grpc.StatsHandler for gRPC instrumentation in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/example. (#4325)

New Contributors

Full Changelog: https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.19.0...v1.20.0

Release v1.19.0/v0.44.0/v0.13.0

Added

  • Add gcp.gce.instance.name and gcp.gce.instance.hostname resource attributes to go.opentelemetry.io/contrib/detectors/gcp. (#4263)

Changed

  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/ec2 have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/ecs have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/eks have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/detectors/aws/lambda have been upgraded to v1.21.0. (#4265)
  • The semantic conventions used by go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-lambda-go/otellambda have been upgraded to v1.21.0. (#4265)
    • The faas.execution attribute is now faas.invocation_id.
    • The faas.id attribute is now aws.lambda.invoked_arn.
  • The semantic conventions used by go.opentelemetry.io/contrib/instrumentation/github.com/aws/aws-sdk-go-v2/otelaws have been upgraded to v1.21.0. (#4265)
  • The http.request.method attribute will only allow known HTTP methods from the metrics generated by go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#4277)

Removed

  • The high cardinality attributes net.sock.peer.addr, net.sock.peer.port, http.user_agent, enduser.id, and http.client_ip were removed from the metrics generated by go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp. (#4277)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/astaxie/beego/otelbeego module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/go-kit/kit/otelkit module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/Shopify/sarama/otelsarama module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/bradfitz/gomemcache/memcache/otelmemcache module is removed. (#4295)
  • The deprecated go.opentelemetry.io/contrib/instrumentation/github.com/gocql/gocql/otelgocql module is removed. (#4295)

New Contributors

Full Changelog: https://github.com/open-telemetry/opentelemetry-go-contrib/compare/v1.18.0...v1.19.0

Release v1.18.0/v0.43.0/v0.12.0

... (truncated)

Changelog

Sourced from go.opentelemetry.io/contrib/instrumentation/runtime's changelog.

[1.20.0/0.45.0/0.14.0] - 2023-09-28

Added

  • Set the description for the rpc.server.duration metric in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc. (#4302)
  • Add NewServerHandler and NewClientHandler that return a grpc.StatsHandler used for gRPC instrumentation in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc. (#3002)
  • Add new Prometheus bridge module in go.opentelemetry.io/contrib/bridges/prometheus. (#4227)

Changed

  • Upgrade dependencies of OpenTelemetry Go to use the new v1.19.0/v0.42.0/v0.0.7 release.
  • Use grpc.StatsHandler for gRPC instrumentation in go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc/example. (#4325)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.opentelemetry.io/contrib/instrumentation/runtime&package-manager=go_modules&previous-version=0.44.0&new-version=0.45.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2bf4f0edf6..599cef51f6 100644 --- a/go.mod +++ b/go.mod @@ -54,7 +54,7 @@ require ( github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 - go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0 + go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 go.opentelemetry.io/otel v1.19.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 diff --git a/go.sum b/go.sum index d0cae595b7..c907bef452 100644 --- a/go.sum +++ b/go.sum @@ -2376,8 +2376,8 @@ go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= go.opencensus.io v0.24.0 h1:y73uSU6J157QMP2kn2r30vwW1A2W2WFwSCGnAVxeaD0= go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= -go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0 h1:TXu20nL4yYfJlQeqG/D3Ia6b0p2HZmLfJto9hqJTQ/c= -go.opentelemetry.io/contrib/instrumentation/runtime v0.44.0/go.mod h1:tQ5gBnfjndV1su3+DiLuu6rnd9hBBzg4rkRILnjSNFg= +go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 h1:2JydY5UiDpqvj2p7sO9bgHuhTy4hgTZ0ymehdq/Ob0Q= +go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0/go.mod h1:ch3a5QxOqVWxas4CzjCFFOOQe+7HgAXC/N1oVxS9DK4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= From ec4472a50a84fdd33b346e319ee6a91693415c30 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 9 Oct 2023 17:34:07 +0400 Subject: [PATCH 251/388] feat(discovery): discover peers by tag (#2730) Allow to run multiple discovery subcomponents, where each can find new peers based on preset `tag` and notify its subscribers. It could allow any sort of separate discovery subcomponents (versioned, pruned/archived, etc). Essentially it allows discovery to be abstracted away and act as single function component. Involves some minor refactoring here and there just to clean up things. Resolves https://github.com/celestiaorg/celestia-node/issues/2578 --- nodebuilder/share/config.go | 2 +- nodebuilder/share/constructors.go | 17 ++++- nodebuilder/share/module.go | 4 +- nodebuilder/tests/p2p_test.go | 2 - share/availability/full/testing.go | 10 ++- share/getters/shrex_test.go | 9 --- share/p2p/discovery/discovery.go | 64 +++++++--------- share/p2p/discovery/discovery_test.go | 102 ++++++++++++++++++++++---- share/p2p/discovery/metrics.go | 3 +- share/p2p/discovery/options.go | 47 +++++++----- share/p2p/peers/manager.go | 37 ++++------ share/p2p/peers/manager_test.go | 81 ++++++++++---------- 12 files changed, 226 insertions(+), 152 deletions(-) diff --git a/nodebuilder/share/config.go b/nodebuilder/share/config.go index 5d66ea7691..1d984b6dca 100644 --- a/nodebuilder/share/config.go +++ b/nodebuilder/share/config.go @@ -26,7 +26,7 @@ type Config struct { PeerManagerParams peers.Parameters LightAvailability light.Parameters `toml:",omitempty"` - Discovery discovery.Parameters + Discovery *discovery.Parameters } func DefaultConfig(tp node.Type) Config { diff --git a/nodebuilder/share/constructors.go b/nodebuilder/share/constructors.go index 944b7f2d23..aa2ac5bec1 100644 --- a/nodebuilder/share/constructors.go +++ b/nodebuilder/share/constructors.go @@ -17,18 +17,27 @@ import ( "github.com/celestiaorg/celestia-node/share/getters" "github.com/celestiaorg/celestia-node/share/ipld" disc "github.com/celestiaorg/celestia-node/share/p2p/discovery" + "github.com/celestiaorg/celestia-node/share/p2p/peers" ) -func newDiscovery(cfg Config) func(routing.ContentRouting, host.Host) *disc.Discovery { +const ( + // fullNodesTag is the tag used to identify full nodes in the discovery service. + fullNodesTag = "full" +) + +func newDiscovery(cfg *disc.Parameters, +) func(routing.ContentRouting, host.Host, *peers.Manager) (*disc.Discovery, error) { return func( r routing.ContentRouting, h host.Host, - ) *disc.Discovery { + manager *peers.Manager, + ) (*disc.Discovery, error) { return disc.NewDiscovery( + cfg, h, routingdisc.NewRoutingDiscovery(r), - disc.WithPeersLimit(cfg.Discovery.PeersLimit), - disc.WithAdvertiseInterval(cfg.Discovery.AdvertiseInterval), + fullNodesTag, + disc.WithOnPeersUpdate(manager.UpdateFullNodePool), ) } } diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index b2e6eed4e5..9fbd262a32 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -37,7 +37,7 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option fx.Provide(newModule), fx.Invoke(func(disc *disc.Discovery) {}), fx.Provide(fx.Annotate( - newDiscovery(*cfg), + newDiscovery(cfg.Discovery), fx.OnStart(func(ctx context.Context, d *disc.Discovery) error { return d.Start(ctx) }), @@ -147,7 +147,6 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option fx.Provide( func( params peers.Parameters, - discovery *disc.Discovery, host host.Host, connGater *conngater.BasicConnectionGater, shrexSub *shrexsub.PubSub, @@ -158,7 +157,6 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option ) (*peers.Manager, error) { return peers.NewManager( params, - discovery, host, connGater, peers.WithShrexSubPools(shrexSub, headerSub), diff --git a/nodebuilder/tests/p2p_test.go b/nodebuilder/tests/p2p_test.go index d05846f40c..9fe63fb931 100644 --- a/nodebuilder/tests/p2p_test.go +++ b/nodebuilder/tests/p2p_test.go @@ -186,9 +186,7 @@ func TestRestartNodeDiscovery(t *testing.T) { sw.Disconnect(t, nodes[0], nodes[1]) // create and start one more FN with disabled discovery - fullCfg.Share.Discovery.PeersLimit = 0 disabledDiscoveryFN := sw.NewNodeWithConfig(node.Full, fullCfg, nodesConfig) - err = disabledDiscoveryFN.Start(ctx) require.NoError(t, err) // ensure that the FN with disabled discovery is discovered by both of the diff --git a/share/availability/full/testing.go b/share/availability/full/testing.go index a636b26ea6..dd21d398c2 100644 --- a/share/availability/full/testing.go +++ b/share/availability/full/testing.go @@ -41,12 +41,16 @@ func Node(dn *availability_test.TestDagNet) *availability_test.TestNode { } func TestAvailability(t *testing.T, getter share.Getter) *ShareAvailability { - disc := discovery.NewDiscovery( + params := discovery.DefaultParameters() + params.AdvertiseInterval = time.Second + params.PeersLimit = 10 + disc, err := discovery.NewDiscovery( + params, nil, routing.NewRoutingDiscovery(routinghelpers.Null{}), - discovery.WithAdvertiseInterval(time.Second), - discovery.WithPeersLimit(10), + "full", ) + require.NoError(t, err) store, err := eds.NewStore(eds.DefaultParameters(), t.TempDir(), datastore.NewMapDatastore()) require.NoError(t, err) err = store.Start(context.Background()) diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 9b97a4dd5a..85878b204a 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -9,9 +9,7 @@ import ( "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" - routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/host" - routingdisc "github.com/libp2p/go-libp2p/p2p/discovery/routing" "github.com/libp2p/go-libp2p/p2p/net/conngater" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" @@ -25,7 +23,6 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" - "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" @@ -210,18 +207,12 @@ func testManager( return nil, err } - disc := discovery.NewDiscovery(nil, - routingdisc.NewRoutingDiscovery(routinghelpers.Null{}), - discovery.WithPeersLimit(10), - discovery.WithAdvertiseInterval(time.Second), - ) connGater, err := conngater.NewBasicConnectionGater(ds_sync.MutexWrap(datastore.NewMapDatastore())) if err != nil { return nil, err } manager, err := peers.NewManager( peers.DefaultParameters(), - disc, host, connGater, peers.WithShrexSubPools(shrexSub, headerSub), diff --git a/share/p2p/discovery/discovery.go b/share/p2p/discovery/discovery.go index f24df2c88b..0f44d42dbe 100644 --- a/share/p2p/discovery/discovery.go +++ b/share/p2p/discovery/discovery.go @@ -19,9 +19,6 @@ import ( var log = logging.Logger("share/discovery") const ( - // rendezvousPoint is the namespace where peers advertise and discover each other. - rendezvousPoint = "full" - // eventbusBufSize is the size of the buffered channel to handle // events in libp2p. We specify a larger buffer size for the channel // to avoid overflowing and blocking subscription during disconnection bursts. @@ -45,6 +42,8 @@ var discoveryRetryTimeout = retryTimeout // Discovery combines advertise and discover services and allows to store discovered nodes. // TODO: The code here gets horribly hairy, so we should refactor this at some point type Discovery struct { + // Tag is used as rondezvous point for discovery service + tag string set *limitedSet host host.Host disc discovery.Discovery @@ -58,43 +57,50 @@ type Discovery struct { cancel context.CancelFunc - params Parameters + params *Parameters } type OnUpdatedPeers func(peerID peer.ID, isAdded bool) +func (f OnUpdatedPeers) add(next OnUpdatedPeers) OnUpdatedPeers { + return func(peerID peer.ID, isAdded bool) { + f(peerID, isAdded) + next(peerID, isAdded) + } +} + // NewDiscovery constructs a new discovery. func NewDiscovery( + params *Parameters, h host.Host, d discovery.Discovery, + tag string, opts ...Option, -) *Discovery { - params := DefaultParameters() - - for _, opt := range opts { - opt(¶ms) +) (*Discovery, error) { + if err := params.Validate(); err != nil { + return nil, err } + if tag == "" { + return nil, fmt.Errorf("discovery: tag cannot be empty") + } + o := newOptions(opts...) return &Discovery{ + tag: tag, set: newLimitedSet(params.PeersLimit), host: h, disc: d, connector: newBackoffConnector(h, defaultBackoffFactory), - onUpdatedPeers: func(peer.ID, bool) {}, + onUpdatedPeers: o.onUpdatedPeers, params: params, triggerDisc: make(chan struct{}), - } + }, nil } func (d *Discovery) Start(context.Context) error { ctx, cancel := context.WithCancel(context.Background()) d.cancel = cancel - if d.params.PeersLimit == 0 { - log.Warn("peers limit is set to 0. Skipping discovery...") - return nil - } - sub, err := d.host.EventBus().Subscribe(&event.EvtPeerConnectednessChanged{}, eventbus.BufSize(eventbusBufSize)) if err != nil { return fmt.Errorf("subscribing for connection events: %w", err) @@ -111,15 +117,6 @@ func (d *Discovery) Stop(context.Context) error { return nil } -// WithOnPeersUpdate chains OnPeersUpdate callbacks on every update of discovered peers list. -func (d *Discovery) WithOnPeersUpdate(f OnUpdatedPeers) { - prev := d.onUpdatedPeers - d.onUpdatedPeers = func(peerID peer.ID, isAdded bool) { - prev(peerID, isAdded) - f(peerID, isAdded) - } -} - // Peers provides a list of discovered peers in the "full" topic. // If Discovery hasn't found any peers, it blocks until at least one peer is found. func (d *Discovery) Peers(ctx context.Context) ([]peer.ID, error) { @@ -133,7 +130,7 @@ func (d *Discovery) Discard(id peer.ID) bool { return false } - d.host.ConnManager().Unprotect(id, rendezvousPoint) + d.host.ConnManager().Unprotect(id, d.tag) d.connector.Backoff(id) d.set.Remove(id) d.onUpdatedPeers(id, false) @@ -153,21 +150,16 @@ func (d *Discovery) Discard(id peer.ID) bool { // Advertise is a utility function that persistently advertises a service through an Advertiser. // TODO: Start advertising only after the reachability is confirmed by AutoNAT func (d *Discovery) Advertise(ctx context.Context) { - if d.params.AdvertiseInterval == -1 { - log.Warn("AdvertiseInterval is set to -1. Skipping advertising...") - return - } - timer := time.NewTimer(d.params.AdvertiseInterval) defer timer.Stop() for { - _, err := d.disc.Advertise(ctx, rendezvousPoint) + _, err := d.disc.Advertise(ctx, d.tag) d.metrics.observeAdvertise(ctx, err) if err != nil { if ctx.Err() != nil { return } - log.Warnw("error advertising", "rendezvous", rendezvousPoint, "err", err) + log.Warnw("error advertising", "rendezvous", d.tag, "err", err) // we don't want retry indefinitely in busy loop // internal discovery mechanism may need some time before attempts @@ -280,7 +272,7 @@ func (d *Discovery) discover(ctx context.Context) bool { findCancel() }() - peers, err := d.disc.FindPeers(findCtx, rendezvousPoint) + peers, err := d.disc.FindPeers(findCtx, d.tag) if err != nil { log.Error("unable to start discovery", "err", err) return false @@ -371,11 +363,11 @@ func (d *Discovery) handleDiscoveredPeer(ctx context.Context, peer peer.AddrInfo d.metrics.observeHandlePeer(ctx, handlePeerConnected) logger.Debug("added peer to set") - // tag to protect peer from being killed by ConnManager + // Tag to protect peer from being killed by ConnManager // NOTE: This is does not protect from remote killing the connection. // In the future, we should design a protocol that keeps bidirectional agreement on whether // connection should be kept or not, similar to mesh link in GossipSub. - d.host.ConnManager().Protect(peer.ID, rendezvousPoint) + d.host.ConnManager().Protect(peer.ID, d.tag) return true } diff --git a/share/p2p/discovery/discovery_test.go b/share/p2p/discovery/discovery_test.go index 06d88a9079..c02931e1a4 100644 --- a/share/p2p/discovery/discovery_test.go +++ b/share/p2p/discovery/discovery_test.go @@ -17,33 +17,47 @@ import ( "github.com/stretchr/testify/require" ) +const ( + fullNodesTag = "full" +) + func TestDiscovery(t *testing.T) { const nodes = 10 // higher number brings higher coverage discoveryRetryTimeout = time.Millisecond * 100 // defined in discovery.go - ctx, cancel := context.WithTimeout(context.Background(), time.Minute*30) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*20) t.Cleanup(cancel) tn := newTestnet(ctx, t) - peerA := tn.discovery( - WithPeersLimit(nodes), - WithAdvertiseInterval(-1), - ) - type peerUpdate struct { peerID peer.ID isAdded bool } updateCh := make(chan peerUpdate) - peerA.WithOnPeersUpdate(func(peerID peer.ID, isAdded bool) { + submit := func(peerID peer.ID, isAdded bool) { updateCh <- peerUpdate{peerID: peerID, isAdded: isAdded} - }) + } + + host, routingDisc := tn.peer() + params := DefaultParameters() + params.PeersLimit = nodes + + // start discovery listener service for peerA + peerA := tn.startNewDiscovery(params, host, routingDisc, fullNodesTag, + WithOnPeersUpdate(submit), + ) + // start discovery advertisement services for other peers + params.AdvertiseInterval = time.Millisecond * 100 discs := make([]*Discovery, nodes) for i := range discs { - discs[i] = tn.discovery(WithPeersLimit(0), WithAdvertiseInterval(time.Millisecond*100)) + host, routingDisc := tn.peer() + disc, err := NewDiscovery(params, host, routingDisc, fullNodesTag) + require.NoError(t, err) + go disc.Advertise(tn.ctx) + discs[i] = tn.startNewDiscovery(params, host, routingDisc, fullNodesTag) select { case res := <-updateCh: @@ -56,6 +70,7 @@ func TestDiscovery(t *testing.T) { assert.EqualValues(t, nodes, peerA.set.Size()) + // disconnect peerA from all peers and check that notifications are received on updateCh channel for _, disc := range discs { peerID := disc.host.ID() err := peerA.host.Network().ClosePeer(peerID) @@ -73,6 +88,51 @@ func TestDiscovery(t *testing.T) { assert.EqualValues(t, 0, peerA.set.Size()) } +func TestDiscoveryTagged(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + t.Cleanup(cancel) + + tn := newTestnet(ctx, t) + + // launch 2 peers, that advertise with different tags + adv1, routingDisc1 := tn.peer() + adv2, routingDisc2 := tn.peer() + + // sub will discover both peers, but on different tags + sub, routingDisc := tn.peer() + + params := DefaultParameters() + + // create 2 discovery services for sub, each with a different tag + done1 := make(chan struct{}) + tn.startNewDiscovery(params, sub, routingDisc, "tag1", + WithOnPeersUpdate(checkPeer(t, adv1.ID(), done1))) + + done2 := make(chan struct{}) + tn.startNewDiscovery(params, sub, routingDisc, "tag2", + WithOnPeersUpdate(checkPeer(t, adv2.ID(), done2))) + + // run discovery services for advertisers + ds1 := tn.startNewDiscovery(params, adv1, routingDisc1, "tag1") + go ds1.Advertise(tn.ctx) + + ds2 := tn.startNewDiscovery(params, adv2, routingDisc2, "tag2") + go ds2.Advertise(tn.ctx) + + // wait for discovery services to discover each other on different tags + select { + case <-done1: + case <-ctx.Done(): + t.Fatal("did not discover peer in time") + } + + select { + case <-done2: + case <-ctx.Done(): + t.Fatal("did not discover peer in time") + } +} + type testnet struct { ctx context.Context T *testing.T @@ -97,17 +157,21 @@ func newTestnet(ctx context.Context, t *testing.T) *testnet { return &testnet{ctx: ctx, T: t, bootstrapper: *host.InfoFromHost(hst)} } -func (t *testnet) discovery(opts ...Option) *Discovery { - hst, routingDisc := t.peer() - disc := NewDiscovery(hst, routingDisc, opts...) - err := disc.Start(t.ctx) +func (t *testnet) startNewDiscovery( + params *Parameters, + hst host.Host, + routingDisc discovery.Discovery, + tag string, + opts ...Option, +) *Discovery { + disc, err := NewDiscovery(params, hst, routingDisc, tag, opts...) + require.NoError(t.T, err) + err = disc.Start(t.ctx) require.NoError(t.T, err) t.T.Cleanup(func() { err := disc.Stop(t.ctx) require.NoError(t.T, err) }) - - go disc.Advertise(t.ctx) return disc } @@ -134,3 +198,11 @@ func (t *testnet) peer() (host.Host, discovery.Discovery) { return hst, routing.NewRoutingDiscovery(dht) } + +func checkPeer(t *testing.T, expected peer.ID, done chan struct{}) func(peerID peer.ID, isAdded bool) { + return func(peerID peer.ID, isAdded bool) { + defer close(done) + require.Equal(t, expected, peerID) + require.True(t, isAdded) + } +} diff --git a/share/p2p/discovery/metrics.go b/share/p2p/discovery/metrics.go index 99c9bb4548..d0be1c219d 100644 --- a/share/p2p/discovery/metrics.go +++ b/share/p2p/discovery/metrics.go @@ -15,7 +15,6 @@ const ( handlePeerResultKey = "result" handlePeerSkipSelf handlePeerResult = "skip_self" - handlePeerEmptyAddrs handlePeerResult = "skip_empty_addresses" handlePeerEnoughPeers handlePeerResult = "skip_enough_peers" handlePeerBackoff handlePeerResult = "skip_backoff" handlePeerConnected handlePeerResult = "connected" @@ -47,7 +46,7 @@ func (d *Discovery) WithMetrics() error { return fmt.Errorf("discovery: init metrics: %w", err) } d.metrics = metrics - d.WithOnPeersUpdate(metrics.observeOnPeersUpdate) + d.onUpdatedPeers = d.onUpdatedPeers.add(metrics.observeOnPeersUpdate) return nil } diff --git a/share/p2p/discovery/options.go b/share/p2p/discovery/options.go index 9d2735c50d..de4b13a7db 100644 --- a/share/p2p/discovery/options.go +++ b/share/p2p/discovery/options.go @@ -3,6 +3,8 @@ package discovery import ( "fmt" "time" + + "github.com/libp2p/go-libp2p/core/peer" ) // Parameters is the set of Parameters that must be configured for the Discovery module @@ -16,13 +18,19 @@ type Parameters struct { AdvertiseInterval time.Duration } +// options is the set of options that can be configured for the Discovery module +type options struct { + // onUpdatedPeers will be called on peer set changes + onUpdatedPeers OnUpdatedPeers +} + // Option is a function that configures Discovery Parameters -type Option func(*Parameters) +type Option func(*options) // DefaultParameters returns the default Parameters' configuration values // for the Discovery module -func DefaultParameters() Parameters { - return Parameters{ +func DefaultParameters() *Parameters { + return &Parameters{ PeersLimit: 5, AdvertiseInterval: time.Hour, } @@ -30,29 +38,30 @@ func DefaultParameters() Parameters { // Validate validates the values in Parameters func (p *Parameters) Validate() error { - if p.AdvertiseInterval <= 0 { - return fmt.Errorf( - "discovery: invalid option: value AdvertiseInterval %s, %s", - "is 0 or negative.", - "value must be positive", - ) + if p.PeersLimit <= 0 { + return fmt.Errorf("discovery: peers limit cannot be zero or negative") } + if p.AdvertiseInterval <= 0 { + return fmt.Errorf("discovery: advertise interval cannot be zero or negative") + } return nil } -// WithPeersLimit is a functional option that Discovery -// uses to set the PeersLimit configuration param -func WithPeersLimit(peersLimit uint) Option { - return func(p *Parameters) { - p.PeersLimit = peersLimit +// WithOnPeersUpdate chains OnPeersUpdate callbacks on every update of discovered peers list. +func WithOnPeersUpdate(f OnUpdatedPeers) Option { + return func(p *options) { + p.onUpdatedPeers = p.onUpdatedPeers.add(f) } } -// WithAdvertiseInterval is a functional option that Discovery -// uses to set the AdvertiseInterval configuration param -func WithAdvertiseInterval(advInterval time.Duration) Option { - return func(p *Parameters) { - p.AdvertiseInterval = advInterval +func newOptions(opts ...Option) *options { + defaults := &options{ + onUpdatedPeers: func(peer.ID, bool) {}, + } + + for _, opt := range opts { + opt(defaults) } + return defaults } diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index 4935db2774..e39a181150 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -21,7 +21,6 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/p2p/discovery" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -54,7 +53,6 @@ type Manager struct { // header subscription is necessary in order to Validate the inbound eds hash headerSub libhead.Subscriber[*header.ExtendedHeader] shrexSub *shrexsub.PubSub - disc *discovery.Discovery host host.Host connGater *conngater.BasicConnectionGater @@ -98,7 +96,6 @@ type syncPool struct { func NewManager( params Parameters, - discovery *discovery.Discovery, host host.Host, connGater *conngater.BasicConnectionGater, options ...Option, @@ -110,7 +107,6 @@ func NewManager( s := &Manager{ params: params, connGater: connGater, - disc: discovery, host: host, pools: make(map[string]*syncPool), blacklistedHashes: make(map[string]bool), @@ -126,23 +122,6 @@ func NewManager( } s.fullNodes = newPool(s.params.PeerCooldown) - - discovery.WithOnPeersUpdate( - func(peerID peer.ID, isAdded bool) { - if isAdded { - if s.isBlacklistedPeer(peerID) { - log.Debugw("got blacklisted peer from discovery", "peer", peerID.String()) - return - } - s.fullNodes.add(peerID) - log.Debugw("added to full nodes", "peer", peerID) - return - } - - log.Debugw("removing peer from discovered full nodes", "peer", peerID.String()) - s.fullNodes.remove(peerID) - }) - return s, nil } @@ -247,6 +226,22 @@ func (m *Manager) Peer( } } +// UpdateFullNodePool is called by discovery when new full node is discovered or removed +func (m *Manager) UpdateFullNodePool(peerID peer.ID, isAdded bool) { + if isAdded { + if m.isBlacklistedPeer(peerID) { + log.Debugw("got blacklisted peer from discovery", "peer", peerID.String()) + return + } + m.fullNodes.add(peerID) + log.Debugw("added to full nodes", "peer", peerID) + return + } + + log.Debugw("removing peer from discovered full nodes", "peer", peerID.String()) + m.fullNodes.remove(peerID) +} + func (m *Manager) newPeer( ctx context.Context, datahash share.DataHash, diff --git a/share/p2p/peers/manager_test.go b/share/p2p/peers/manager_test.go index 1ad7e65c02..94ec5d5ea2 100644 --- a/share/p2p/peers/manager_test.go +++ b/share/p2p/peers/manager_test.go @@ -10,7 +10,6 @@ import ( "github.com/ipfs/go-datastore/sync" dht "github.com/libp2p/go-libp2p-kad-dht" pubsub "github.com/libp2p/go-libp2p-pubsub" - routinghelpers "github.com/libp2p/go-libp2p-routing-helpers" "github.com/libp2p/go-libp2p/core/host" "github.com/libp2p/go-libp2p/core/peer" routingdisc "github.com/libp2p/go-libp2p/p2p/discovery/routing" @@ -364,13 +363,14 @@ func TestIntegration(t *testing.T) { }) t.Run("get peer from discovery", func(t *testing.T) { + fullNodesTag := "fullNodes" nw, err := mocknet.FullMeshConnected(3) require.NoError(t, err) ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) t.Cleanup(cancel) // set up bootstrapper - bsHost := nw.Hosts()[2] + bsHost := nw.Hosts()[0] bs := host.InfoFromHost(bsHost) opts := []dht.Option{ dht.Mode(dht.ModeAuto), @@ -388,62 +388,74 @@ func TestIntegration(t *testing.T) { require.NoError(t, bsRouter.Bootstrap(ctx)) // set up broadcaster node - bnHost := nw.Hosts()[0] - router1, err := dht.New(ctx, bnHost, opts...) + bnHost := nw.Hosts()[1] + bnRouter, err := dht.New(ctx, bnHost, opts...) require.NoError(t, err) - bnDisc := discovery.NewDiscovery( - nw.Hosts()[0], - routingdisc.NewRoutingDiscovery(router1), - discovery.WithPeersLimit(0), - discovery.WithAdvertiseInterval(time.Second), + + params := discovery.DefaultParameters() + params.AdvertiseInterval = time.Second + + bnDisc, err := discovery.NewDiscovery( + params, + bnHost, + routingdisc.NewRoutingDiscovery(bnRouter), + fullNodesTag, ) + require.NoError(t, err) // set up full node / receiver node - fnHost := nw.Hosts()[0] - router2, err := dht.New(ctx, fnHost, opts...) - require.NoError(t, err) - fnDisc := discovery.NewDiscovery( - nw.Hosts()[1], - routingdisc.NewRoutingDiscovery(router2), - discovery.WithPeersLimit(10), - discovery.WithAdvertiseInterval(time.Second), - ) - err = fnDisc.Start(ctx) + fnHost := nw.Hosts()[2] + fnRouter, err := dht.New(ctx, fnHost, opts...) require.NoError(t, err) - t.Cleanup(func() { - err = fnDisc.Stop(ctx) - require.NoError(t, err) - }) - // hook peer manager to discovery + // init peer manager for full node connGater, err := conngater.NewBasicConnectionGater(sync.MutexWrap(datastore.NewMapDatastore())) require.NoError(t, err) fnPeerManager, err := NewManager( DefaultParameters(), - fnDisc, nil, connGater, ) require.NoError(t, err) waitCh := make(chan struct{}) - fnDisc.WithOnPeersUpdate(func(peerID peer.ID, isAdded bool) { + checkDiscoveredPeer := func(peerID peer.ID, isAdded bool) { defer close(waitCh) - // check that obtained peer id is same as BN - require.Equal(t, nw.Hosts()[0].ID(), peerID) + // check that obtained peer id is BN + require.Equal(t, bnHost.ID(), peerID) + } + + // set up discovery for full node with hook to peer manager and check discovered peer + params = discovery.DefaultParameters() + params.AdvertiseInterval = time.Second + params.PeersLimit = 10 + + fnDisc, err := discovery.NewDiscovery( + params, + fnHost, + routingdisc.NewRoutingDiscovery(fnRouter), + fullNodesTag, + discovery.WithOnPeersUpdate(fnPeerManager.UpdateFullNodePool), + discovery.WithOnPeersUpdate(checkDiscoveredPeer), + ) + require.NoError(t, fnDisc.Start(ctx)) + t.Cleanup(func() { + err = fnDisc.Stop(ctx) + require.NoError(t, err) }) - require.NoError(t, router1.Bootstrap(ctx)) - require.NoError(t, router2.Bootstrap(ctx)) + require.NoError(t, bnRouter.Bootstrap(ctx)) + require.NoError(t, fnRouter.Bootstrap(ctx)) go bnDisc.Advertise(ctx) select { case <-waitCh: - require.Contains(t, fnPeerManager.fullNodes.peersList, fnHost.ID()) + require.Contains(t, fnPeerManager.fullNodes.peersList, bnHost.ID()) case <-ctx.Done(): require.NoError(t, ctx.Err()) } + }) } @@ -457,22 +469,17 @@ func testManager(ctx context.Context, headerSub libhead.Subscriber[*header.Exten return nil, err } - disc := discovery.NewDiscovery(nil, - routingdisc.NewRoutingDiscovery(routinghelpers.Null{}), - discovery.WithPeersLimit(0), - discovery.WithAdvertiseInterval(time.Second), - ) connGater, err := conngater.NewBasicConnectionGater(sync.MutexWrap(datastore.NewMapDatastore())) if err != nil { return nil, err } manager, err := NewManager( DefaultParameters(), - disc, host, connGater, WithShrexSubPools(shrexSub, headerSub), ) + if err != nil { return nil, err } From 5e5f46339cec6c74f82a45a56e086a60208bb93b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 13:37:09 +0000 Subject: [PATCH 252/388] chore(deps): Bump golang.org/x/crypto from 0.13.0 to 0.14.0 (#2821) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.13.0 to 0.14.0.
Commits
  • e3cc52e go.mod: update golang.org/x dependencies
  • 833695f ssh: add server side support for ping@openssh.com protocol extension
  • ec07f4e chacha20: drop Go 1.10 compatibility for arm64
  • b665ba6 all: use crypto/ed25519 instead of golang.org/x/crypto/ed25519
  • a1aeb9b ssh: add test cases for compatibility with old (buggy) clients
  • 28c53ff ssh: add MultiAlgorithmSigner
  • 3f0842a sha3: have ShakeHash extend hash.Hash
  • e90f1e1 cryptobyte: add uint48 methods
  • d359caa ssh: support for marshaling keys using the OpenSSH format
  • c5370d2 ssh: check the declared public key algo against decoded one
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.13.0&new-version=0.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index 599cef51f6..6264c79c56 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.0 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.13.0 + golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 golang.org/x/sync v0.3.0 golang.org/x/text v0.13.0 @@ -324,8 +324,8 @@ require ( golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.14.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect - golang.org/x/sys v0.12.0 // indirect - golang.org/x/term v0.12.0 // indirect + golang.org/x/sys v0.13.0 // indirect + golang.org/x/term v0.13.0 // indirect golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect diff --git a/go.sum b/go.sum index c907bef452..48286df271 100644 --- a/go.sum +++ b/go.sum @@ -2506,8 +2506,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2854,8 +2854,8 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= -golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2864,8 +2864,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.12.0 h1:/ZfYdc3zq+q02Rv9vGqTeSItdzZTSNDmfTi0mBAuidU= -golang.org/x/term v0.12.0/go.mod h1:owVbMEjm3cBLCHdkQu9b1opXd4ETQWc3BhuQGKgXgvU= +golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= +golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From a2cf93d1b9096254d92148ab33e6da9d8b621160 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 9 Oct 2023 17:24:27 +0300 Subject: [PATCH 253/388] fix(docs): fix GetVerifiedRange doc (#2826) --- docs/adr/adr-009-public-api.md | 2 +- nodebuilder/header/header.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/docs/adr/adr-009-public-api.md b/docs/adr/adr-009-public-api.md index 76bb1c5ab9..d2bf6ee817 100644 --- a/docs/adr/adr-009-public-api.md +++ b/docs/adr/adr-009-public-api.md @@ -100,7 +100,7 @@ GetByHeight(ctx context.Context, height uint64) (*header.ExtendedHeader, error) // WaitForHeight blocks until the header at the given height has been processed // by the node's header store or until context deadline is exceeded. WaitForHeight(ctx context.Context, height uint64) (*header.ExtendedHeader, error) -// GetVerifiedRangeByHeight returns the given range [from:to) of ExtendedHeaders +// GetVerifiedRangeByHeight returns the given range (from:to) of ExtendedHeaders // from the node's header store and verifies that the returned headers are // adjacent to each other. GetVerifiedRangeByHeight(ctx context.Context, from, to uint64) ([]*ExtendedHeader, error) diff --git a/nodebuilder/header/header.go b/nodebuilder/header/header.go index d7eb7b5b46..9b9455da26 100644 --- a/nodebuilder/header/header.go +++ b/nodebuilder/header/header.go @@ -19,7 +19,7 @@ type Module interface { // GetByHash returns the header of the given hash from the node's header store. GetByHash(ctx context.Context, hash libhead.Hash) (*header.ExtendedHeader, error) - // GetVerifiedRangeByHeight returns the given range [from:to) of ExtendedHeaders + // GetVerifiedRangeByHeight returns the given range (from:to) of ExtendedHeaders // from the node's header store and verifies that the returned headers are // adjacent to each other. GetVerifiedRangeByHeight( From 88c80e57cd9cd71daf7b28d16b69246a9040e32f Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 9 Oct 2023 18:44:31 +0400 Subject: [PATCH 254/388] chore(deps): Bump golang.org/x/sync from 0.3.0 to 0.4.0 (#2820) Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.3.0 to 0.4.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/sync&package-manager=go_modules&previous-version=0.3.0&new-version=0.4.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 6264c79c56..d39b4696fd 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 - golang.org/x/sync v0.3.0 + golang.org/x/sync v0.4.0 golang.org/x/text v0.13.0 google.golang.org/grpc v1.58.2 google.golang.org/protobuf v1.31.0 diff --git a/go.sum b/go.sum index 48286df271..7a986186d2 100644 --- a/go.sum +++ b/go.sum @@ -2707,8 +2707,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= +golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 321b3aaf39ddd04ecd0e0238a94c2f9d7e5ff064 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 2 Oct 2023 17:10:33 +0300 Subject: [PATCH 255/388] fix(tests)!: fix FraudProofHandling test (#2665) --- nodebuilder/fraud/fraud.go | 14 ++++++------ nodebuilder/fraud/mocks/api.go | 4 ++-- nodebuilder/tests/fraud_test.go | 40 ++++++++++++++++++++++++++------- 3 files changed, 41 insertions(+), 17 deletions(-) diff --git a/nodebuilder/fraud/fraud.go b/nodebuilder/fraud/fraud.go index 45c3863d6f..46af4af67d 100644 --- a/nodebuilder/fraud/fraud.go +++ b/nodebuilder/fraud/fraud.go @@ -18,7 +18,7 @@ var _ Module = (*API)(nil) //go:generate mockgen -destination=mocks/api.go -package=mocks . Module type Module interface { // Subscribe allows to subscribe on a Proof pub sub topic by its type. - Subscribe(context.Context, fraud.ProofType) (<-chan Proof, error) + Subscribe(context.Context, fraud.ProofType) (<-chan *Proof, error) // Get fetches fraud proofs from the disk by its type. Get(context.Context, fraud.ProofType) ([]Proof, error) } @@ -27,12 +27,12 @@ type Module interface { // TODO(@distractedm1nd): These structs need to be autogenerated. type API struct { Internal struct { - Subscribe func(context.Context, fraud.ProofType) (<-chan Proof, error) `perm:"public"` - Get func(context.Context, fraud.ProofType) ([]Proof, error) `perm:"public"` + Subscribe func(context.Context, fraud.ProofType) (<-chan *Proof, error) `perm:"public"` + Get func(context.Context, fraud.ProofType) ([]Proof, error) `perm:"public"` } } -func (api *API) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-chan Proof, error) { +func (api *API) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-chan *Proof, error) { return api.Internal.Subscribe(ctx, proofType) } @@ -49,12 +49,12 @@ type module struct { fraud.Service[*header.ExtendedHeader] } -func (s *module) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-chan Proof, error) { +func (s *module) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-chan *Proof, error) { subscription, err := s.Service.Subscribe(proofType) if err != nil { return nil, err } - proofs := make(chan Proof) + proofs := make(chan *Proof) go func() { defer close(proofs) defer subscription.Cancel() @@ -69,7 +69,7 @@ func (s *module) Subscribe(ctx context.Context, proofType fraud.ProofType) (<-ch select { case <-ctx.Done(): return - case proofs <- Proof{Proof: proof}: + case proofs <- &Proof{Proof: proof}: } } }() diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index 399f8746e1..10111b81a8 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -53,10 +53,10 @@ func (mr *MockModuleMockRecorder) Get(arg0, arg1 interface{}) *gomock.Call { } // Subscribe mocks base method. -func (m *MockModule) Subscribe(arg0 context.Context, arg1 fraud0.ProofType) (<-chan fraud.Proof, error) { +func (m *MockModule) Subscribe(arg0 context.Context, arg1 fraud0.ProofType) (<-chan *fraud.Proof, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Subscribe", arg0, arg1) - ret0, _ := ret[0].(<-chan fraud.Proof) + ret0, _ := ret[0].(<-chan *fraud.Proof) ret1, _ := ret[1].(error) return ret0, ret1 } diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index d708f6d454..61eca88e21 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -5,6 +5,7 @@ import ( "testing" "time" + "github.com/celestiaorg/go-fraud" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" @@ -13,6 +14,7 @@ import ( "github.com/tendermint/tendermint/types" "go.uber.org/fx" + "github.com/celestiaorg/celestia-node/header" headerfraud "github.com/celestiaorg/celestia-node/header/headertest/fraud" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/core" @@ -98,21 +100,45 @@ func TestFraudProofHandling(t *testing.T) { select { case p := <-subscr: require.Equal(t, 10, int(p.Height())) + t.Log("Caught the proof....") subCancel() case <-ctx.Done(): subCancel() t.Fatal("full node did not receive a fraud proof in time") } + getCtx, getCancel := context.WithTimeout(ctx, time.Second) + proofs, err := fullClient.Fraud.Get(getCtx, byzantine.BadEncoding) + getCancel() + + require.NoError(t, err) + require.Len(t, proofs, 1) + require.True(t, proofs[0].Type() == byzantine.BadEncoding) // This is an obscure way to check if the Syncer was stopped. // If we cannot get a height header within a timeframe it means the syncer was stopped // FIXME: Eventually, this should be a check on service registry managing and keeping // lifecycles of each Module. // 6. - syncCtx, syncCancel := context.WithTimeout(context.Background(), blockTime*5) - _, err = fullClient.Header.WaitForHeight(syncCtx, 15) + // random height after befp.height + height := uint64(15) + // initial timeout is set to 5 sec, as we are targeting the height=15, + // blockTime=1 sec, expected befp.height=10 + timeOut := blockTime * 5 + // during befp validation the node can still receive headers and it mostly depends on + // the operating system or hardware(e.g. on macOS tests is working 100% time with a single height=15, + // and on the Linux VM sometimes the last height is 17-18). So, lets give a chance for our befp validator to check + // the fraud proof and stop the syncer. + for height < 20 { + syncCtx, syncCancel := context.WithTimeout(context.Background(), timeOut) + _, err = full.HeaderServ.WaitForHeight(syncCtx, height) + syncCancel() + if err != nil { + break + } + timeOut = blockTime + height++ + } require.ErrorIs(t, err, context.DeadlineExceeded) - syncCancel() // 7. cfg = nodebuilder.DefaultConfig(node.Light) @@ -137,11 +163,9 @@ func TestFraudProofHandling(t *testing.T) { // 9. fN := sw.NewNodeWithStore(node.Full, store) - require.Error(t, fN.Start(ctx)) - fNClient := getAdminClient(ctx, fN, t) - proofs, err := fNClient.Fraud.Get(ctx, byzantine.BadEncoding) - require.NoError(t, err) - require.NotNil(t, proofs) + err = fN.Start(ctx) + var fpExist *fraud.ErrFraudExists[*header.ExtendedHeader] + require.ErrorAs(t, err, &fpExist) sw.StopNode(ctx, bridge) sw.StopNode(ctx, full) From 97474320e53f83c0d8e9fdd78cfa0f8e88435be3 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 4 Oct 2023 13:56:41 +0200 Subject: [PATCH 256/388] chore(share/availability)!: Remove `ProbabilityOfAvailability` endpoint (#2797) Resolves #2769 --------- Co-authored-by: ramin --- api/gateway/availability.go | 14 +++----------- nodebuilder/share/cmd/share.go | 17 ----------------- nodebuilder/share/mocks/api.go | 17 +---------------- nodebuilder/share/share.go | 12 ++---------- nodebuilder/tests/fraud_test.go | 3 ++- share/availability.go | 4 ---- share/availability/full/availability.go | 4 ---- share/availability/light/availability.go | 10 ---------- 8 files changed, 8 insertions(+), 73 deletions(-) diff --git a/api/gateway/availability.go b/api/gateway/availability.go index e5e53e0dba..203fd86d61 100644 --- a/api/gateway/availability.go +++ b/api/gateway/availability.go @@ -15,8 +15,7 @@ const heightAvailabilityEndpoint = "/data_available" // AvailabilityResponse represents the response to a // `/data_available` request. type AvailabilityResponse struct { - Available bool `json:"available"` - Probability string `json:"probability_of_availability"` + Available bool `json:"available"` } func (h *Handler) handleHeightAvailabilityRequest(w http.ResponseWriter, r *http.Request) { @@ -33,16 +32,10 @@ func (h *Handler) handleHeightAvailabilityRequest(w http.ResponseWriter, r *http return } - availResp := &AvailabilityResponse{ - Probability: strconv.FormatFloat( - h.share.ProbabilityOfAvailability(r.Context()), 'g', -1, 64), - } - err = h.share.SharesAvailable(r.Context(), header.DAH) switch err { case nil: - availResp.Available = true - resp, err := json.Marshal(availResp) + resp, err := json.Marshal(&AvailabilityResponse{Available: true}) if err != nil { writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err) return @@ -52,8 +45,7 @@ func (h *Handler) handleHeightAvailabilityRequest(w http.ResponseWriter, r *http log.Errorw("serving request", "endpoint", heightAvailabilityEndpoint, "err", err) } case share.ErrNotAvailable: - availResp.Available = false - resp, err := json.Marshal(availResp) + resp, err := json.Marshal(&AvailabilityResponse{Available: false}) if err != nil { writeError(w, http.StatusInternalServerError, heightAvailabilityEndpoint, err) return diff --git a/nodebuilder/share/cmd/share.go b/nodebuilder/share/cmd/share.go index fbf3e51db4..bc22651238 100644 --- a/nodebuilder/share/cmd/share.go +++ b/nodebuilder/share/cmd/share.go @@ -16,7 +16,6 @@ import ( func init() { Cmd.AddCommand( sharesAvailableCmd, - probabilityOfAvailabilityCmd, getSharesByNamespaceCmd, getShare, getEDS, @@ -73,22 +72,6 @@ var sharesAvailableCmd = &cobra.Command{ }, } -var probabilityOfAvailabilityCmd = &cobra.Command{ - Use: "availability", - Short: "Calculates the probability of the data square being available based on the number of samples collected.", - Args: cobra.NoArgs, - RunE: func(cmd *cobra.Command, args []string) error { - client, err := cmdnode.ParseClientFromCtx(cmd.Context()) - if err != nil { - return err - } - defer client.Close() - - prob := client.Share.ProbabilityOfAvailability(cmd.Context()) - return cmdnode.PrintOutput(prob, nil, nil) - }, -} - var getSharesByNamespaceCmd = &cobra.Command{ Use: "get-by-namespace [dah, namespace]", Short: "Gets all shares from an EDS within the given namespace.", diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 66baa23301..78a124a20d 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" rsmt2d "github.com/celestiaorg/rsmt2d" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -83,20 +82,6 @@ func (mr *MockModuleMockRecorder) GetSharesByNamespace(arg0, arg1, arg2 interfac return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetSharesByNamespace", reflect.TypeOf((*MockModule)(nil).GetSharesByNamespace), arg0, arg1, arg2) } -// ProbabilityOfAvailability mocks base method. -func (m *MockModule) ProbabilityOfAvailability(arg0 context.Context) float64 { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProbabilityOfAvailability", arg0) - ret0, _ := ret[0].(float64) - return ret0 -} - -// ProbabilityOfAvailability indicates an expected call of ProbabilityOfAvailability. -func (mr *MockModuleMockRecorder) ProbabilityOfAvailability(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProbabilityOfAvailability", reflect.TypeOf((*MockModule)(nil).ProbabilityOfAvailability), arg0) -} - // SharesAvailable mocks base method. func (m *MockModule) SharesAvailable(arg0 context.Context, arg1 *da.DataAvailabilityHeader) error { m.ctrl.T.Helper() diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index def3bb4d0f..37d68c1581 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -30,9 +30,6 @@ type Module interface { // SharesAvailable subjectively validates if Shares committed to the given Root are available on // the Network. SharesAvailable(context.Context, *share.Root) error - // ProbabilityOfAvailability calculates the probability of the data square - // being available based on the number of samples collected. - ProbabilityOfAvailability(context.Context) float64 // GetShare gets a Share by coordinates in EDS. GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) // GetEDS gets the full EDS identified by the given root. @@ -46,9 +43,8 @@ type Module interface { // TODO(@distractedm1nd): These structs need to be autogenerated. type API struct { Internal struct { - SharesAvailable func(context.Context, *share.Root) error `perm:"public"` - ProbabilityOfAvailability func(context.Context) float64 `perm:"public"` - GetShare func( + SharesAvailable func(context.Context, *share.Root) error `perm:"public"` + GetShare func( ctx context.Context, dah *share.Root, row, col int, @@ -69,10 +65,6 @@ func (api *API) SharesAvailable(ctx context.Context, root *share.Root) error { return api.Internal.SharesAvailable(ctx, root) } -func (api *API) ProbabilityOfAvailability(ctx context.Context) float64 { - return api.Internal.ProbabilityOfAvailability(ctx) -} - func (api *API) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) { return api.Internal.GetShare(ctx, dah, row, col) } diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index 61eca88e21..1927cdaf42 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -5,7 +5,6 @@ import ( "testing" "time" - "github.com/celestiaorg/go-fraud" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/host" @@ -14,6 +13,8 @@ import ( "github.com/tendermint/tendermint/types" "go.uber.org/fx" + "github.com/celestiaorg/go-fraud" + "github.com/celestiaorg/celestia-node/header" headerfraud "github.com/celestiaorg/celestia-node/header/headertest/fraud" "github.com/celestiaorg/celestia-node/nodebuilder" diff --git a/share/availability.go b/share/availability.go index cd4b3a777f..587469d94b 100644 --- a/share/availability.go +++ b/share/availability.go @@ -30,8 +30,4 @@ type Availability interface { // SharesAvailable subjectively validates if Shares committed to the given Root are available on // the Network. SharesAvailable(context.Context, *Root) error - // ProbabilityOfAvailability calculates the probability of the data square - // being available based on the number of samples collected. - // TODO(@Wondertan): Merge with SharesAvailable method, eventually - ProbabilityOfAvailability(context.Context) float64 } diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index be269077d7..5fecb757fd 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -98,7 +98,3 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro } return nil } - -func (fa *ShareAvailability) ProbabilityOfAvailability(context.Context) float64 { - return 1 -} diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index 0fc073a2df..e7ad900d8d 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -3,7 +3,6 @@ package light import ( "context" "errors" - "math" "sync" "github.com/ipfs/go-datastore" @@ -140,15 +139,6 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Roo return nil } -// ProbabilityOfAvailability calculates the probability that the -// data square is available based on the amount of samples collected -// (params.SampleAmount). -// -// Formula: 1 - (0.75 ** amount of samples) -func (la *ShareAvailability) ProbabilityOfAvailability(context.Context) float64 { - return 1 - math.Pow(0.75, float64(la.params.SampleAmount)) -} - func rootKey(root *share.Root) datastore.Key { return datastore.NewKey(root.String()) } From 6f083b62873f214583b1f3d13ef5dc8ac3fa19e7 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 4 Oct 2023 14:20:54 +0200 Subject: [PATCH 257/388] chore(api | nodebuilder)!: Elevate all api permissions to `read` and disallow `public`, remove `client.NewPublicClient` (#2771) We made a decision synchronously with @adlerjohn and @ramin to require at least `read`-level permissions to be able to access *any* endpoint on the node's API. Therefore, `NewPublicClient` is no longer necessary. --- api/rpc/client/client.go | 5 -- api/rpc_test.go | 116 ++++----------------------------- nodebuilder/fraud/fraud.go | 4 +- nodebuilder/fraud/mocks/api.go | 3 +- nodebuilder/header/header.go | 10 +-- nodebuilder/share/share.go | 8 +-- nodebuilder/state/state.go | 10 +-- 7 files changed, 31 insertions(+), 125 deletions(-) diff --git a/api/rpc/client/client.go b/api/rpc/client/client.go index 7ac8a55b3d..9cd5fe08e3 100644 --- a/api/rpc/client/client.go +++ b/api/rpc/client/client.go @@ -59,11 +59,6 @@ func (c *Client) Close() { c.closer.closeAll() } -// NewPublicClient creates a new Client with one connection per namespace. -func NewPublicClient(ctx context.Context, addr string) (*Client, error) { - return newClient(ctx, addr, nil) -} - // NewClient creates a new Client with one connection per namespace with the // given token as the authorization token. func NewClient(ctx context.Context, addr string, token string) (*Client, error) { diff --git a/api/rpc_test.go b/api/rpc_test.go index 898a307389..9ff35bf1e4 100644 --- a/api/rpc_test.go +++ b/api/rpc_test.go @@ -19,7 +19,6 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/api/rpc/perms" daspkg "github.com/celestiaorg/celestia-node/das" - headerpkg "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/blob" blobMock "github.com/celestiaorg/celestia-node/nodebuilder/blob/mocks" @@ -43,16 +42,24 @@ import ( func TestRPCCallsUnderlyingNode(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - nd, server := setupNodeWithModifiedRPC(t) + + // generate dummy signer and sign admin perms token with it + signer, err := jwt.NewHS256(make([]byte, 32)) + require.NoError(t, err) + + nd, server := setupNodeWithAuthedRPC(t, signer) url := nd.RPCServer.ListenAddr() + + adminToken, err := perms.NewTokenWithPerms(signer, perms.AllPerms) + require.NoError(t, err) + // we need to run this a few times to prevent the race where the server is not yet started var ( rpcClient *client.Client - err error ) for i := 0; i < 3; i++ { time.Sleep(time.Second * 1) - rpcClient, err = client.NewPublicClient(ctx, "http://"+url) + rpcClient, err = client.NewClient(ctx, "http://"+url, string(adminToken)) if err == nil { t.Cleanup(rpcClient.Close) break @@ -154,13 +161,7 @@ func TestAuthedRPC(t *testing.T) { require.NotNil(t, rpcClient) require.NoError(t, err) - // 1. Test method with public permissions - server.Header.EXPECT().NetworkHead(gomock.Any()).Return(new(headerpkg.ExtendedHeader), nil) - got, err := rpcClient.Header.NetworkHead(ctx) - require.NoError(t, err) - require.NotNil(t, got) - - // 2. Test method with read-level permissions + // 1. Test method with read-level permissions expected := daspkg.SamplingStats{ SampledChainHead: 100, CatchupHead: 100, @@ -182,7 +183,7 @@ func TestAuthedRPC(t *testing.T) { require.ErrorContains(t, err, "missing permission") } - // 3. Test method with write-level permissions + // 2. Test method with write-level permissions expectedResp := &state.TxResponse{} if tt.perm > 2 { server.State.EXPECT().SubmitTx(gomock.Any(), gomock.Any()).Return(expectedResp, nil) @@ -195,7 +196,7 @@ func TestAuthedRPC(t *testing.T) { require.ErrorContains(t, err, "missing permission") } - // 4. Test method with admin-level permissions + // 3. Test method with admin-level permissions expectedReachability := network.Reachability(3) if tt.perm > 3 { server.P2P.EXPECT().NATStatus(gomock.Any()).Return(expectedReachability, nil) @@ -213,55 +214,6 @@ func TestAuthedRPC(t *testing.T) { } } -// TestPublicClient tests that the public rpc client can only -// access public methods. -func TestPublicClient(t *testing.T) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - // generate dummy signer and sign admin perms token with it - signer, err := jwt.NewHS256(make([]byte, 32)) - require.NoError(t, err) - - nd, server := setupNodeWithAuthedRPC(t, signer) - url := nd.RPCServer.ListenAddr() - - // we need to run this a few times to prevent the race where the server is not yet started - var rpcClient *client.Client - require.NoError(t, err) - for i := 0; i < 3; i++ { - time.Sleep(time.Second * 1) - rpcClient, err = client.NewPublicClient(ctx, "http://"+url) - if err == nil { - t.Cleanup(rpcClient.Close) - break - } - } - require.NotNil(t, rpcClient) - require.NoError(t, err) - - // 1. Test method with public permissions - server.Header.EXPECT().NetworkHead(gomock.Any()).Return(new(headerpkg.ExtendedHeader), nil) - got, err := rpcClient.Header.NetworkHead(ctx) - require.NoError(t, err) - require.NotNil(t, got) - - // 2. Test method with read-level permissions - _, err = rpcClient.DAS.SamplingStats(ctx) - require.Error(t, err) - require.ErrorContains(t, err, "missing permission") - - // 3. Test method with write-level permissions - _, err = rpcClient.State.SubmitTx(ctx, []byte{}) - require.Error(t, err) - require.ErrorContains(t, err, "missing permission") - - // 4. Test method with admin-level permissions - _, err = rpcClient.P2P.NATStatus(ctx) - require.Error(t, err) - require.ErrorContains(t, err, "missing permission") -} - func TestAllReturnValuesAreMarshalable(t *testing.T) { ra := reflect.TypeOf(new(api)).Elem() for i := 0; i < ra.NumMethod(); i++ { @@ -325,46 +277,6 @@ func implementsMarshaler(t *testing.T, typ reflect.Type) { } -func setupNodeWithModifiedRPC(t *testing.T) (*nodebuilder.Node, *mockAPI) { - ctx, cancel := context.WithCancel(context.Background()) - t.Cleanup(cancel) - - ctrl := gomock.NewController(t) - - mockAPI := &mockAPI{ - stateMock.NewMockModule(ctrl), - shareMock.NewMockModule(ctrl), - fraudMock.NewMockModule(ctrl), - headerMock.NewMockModule(ctrl), - dasMock.NewMockModule(ctrl), - p2pMock.NewMockModule(ctrl), - nodeMock.NewMockModule(ctrl), - blobMock.NewMockModule(ctrl), - } - - // given the behavior of fx.Invoke, this invoke will be called last as it is added at the root - // level module. For further information, check the documentation on fx.Invoke. - invokeRPC := fx.Invoke(func(srv *rpc.Server) { - srv.RegisterService("state", mockAPI.State) - srv.RegisterService("share", mockAPI.Share) - srv.RegisterService("fraud", mockAPI.Fraud) - srv.RegisterService("header", mockAPI.Header) - srv.RegisterService("das", mockAPI.Das) - srv.RegisterService("p2p", mockAPI.P2P) - srv.RegisterService("node", mockAPI.Node) - srv.RegisterService("blob", mockAPI.Blob) - }) - nd := nodebuilder.TestNode(t, node.Full, invokeRPC) - // start node - err := nd.Start(ctx) - require.NoError(t, err) - t.Cleanup(func() { - err = nd.Stop(ctx) - require.NoError(t, err) - }) - return nd, mockAPI -} - // setupNodeWithAuthedRPC sets up a node and overrides its JWT // signer with the given signer. func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, *mockAPI) { diff --git a/nodebuilder/fraud/fraud.go b/nodebuilder/fraud/fraud.go index 46af4af67d..178b0527a1 100644 --- a/nodebuilder/fraud/fraud.go +++ b/nodebuilder/fraud/fraud.go @@ -27,8 +27,8 @@ type Module interface { // TODO(@distractedm1nd): These structs need to be autogenerated. type API struct { Internal struct { - Subscribe func(context.Context, fraud.ProofType) (<-chan *Proof, error) `perm:"public"` - Get func(context.Context, fraud.ProofType) ([]Proof, error) `perm:"public"` + Subscribe func(context.Context, fraud.ProofType) (<-chan *Proof, error) `perm:"read"` + Get func(context.Context, fraud.ProofType) ([]Proof, error) `perm:"read"` } } diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index 10111b81a8..fcc7a58231 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - fraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" fraud0 "github.com/celestiaorg/go-fraud" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/header/header.go b/nodebuilder/header/header.go index 9b9455da26..43b5646a77 100644 --- a/nodebuilder/header/header.go +++ b/nodebuilder/header/header.go @@ -53,18 +53,18 @@ type API struct { GetByHash func( ctx context.Context, hash libhead.Hash, - ) (*header.ExtendedHeader, error) `perm:"public"` + ) (*header.ExtendedHeader, error) `perm:"read"` GetVerifiedRangeByHeight func( context.Context, *header.ExtendedHeader, uint64, - ) ([]*header.ExtendedHeader, error) `perm:"public"` - GetByHeight func(context.Context, uint64) (*header.ExtendedHeader, error) `perm:"public"` + ) ([]*header.ExtendedHeader, error) `perm:"read"` + GetByHeight func(context.Context, uint64) (*header.ExtendedHeader, error) `perm:"read"` WaitForHeight func(context.Context, uint64) (*header.ExtendedHeader, error) `perm:"read"` SyncState func(ctx context.Context) (sync.State, error) `perm:"read"` SyncWait func(ctx context.Context) error `perm:"read"` - NetworkHead func(ctx context.Context) (*header.ExtendedHeader, error) `perm:"public"` - Subscribe func(ctx context.Context) (<-chan *header.ExtendedHeader, error) `perm:"public"` + NetworkHead func(ctx context.Context) (*header.ExtendedHeader, error) `perm:"read"` + Subscribe func(ctx context.Context) (<-chan *header.ExtendedHeader, error) `perm:"read"` } } diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index 37d68c1581..da97f32a33 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -43,21 +43,21 @@ type Module interface { // TODO(@distractedm1nd): These structs need to be autogenerated. type API struct { Internal struct { - SharesAvailable func(context.Context, *share.Root) error `perm:"public"` + SharesAvailable func(context.Context, *share.Root) error `perm:"read"` GetShare func( ctx context.Context, dah *share.Root, row, col int, - ) (share.Share, error) `perm:"public"` + ) (share.Share, error) `perm:"read"` GetEDS func( ctx context.Context, root *share.Root, - ) (*rsmt2d.ExtendedDataSquare, error) `perm:"public"` + ) (*rsmt2d.ExtendedDataSquare, error) `perm:"read"` GetSharesByNamespace func( ctx context.Context, root *share.Root, namespace share.Namespace, - ) (share.NamespacedShares, error) `perm:"public"` + ) (share.NamespacedShares, error) `perm:"read"` } } diff --git a/nodebuilder/state/state.go b/nodebuilder/state/state.go index c66205d594..83408680da 100644 --- a/nodebuilder/state/state.go +++ b/nodebuilder/state/state.go @@ -100,9 +100,9 @@ type Module interface { type API struct { Internal struct { AccountAddress func(ctx context.Context) (state.Address, error) `perm:"read"` - IsStopped func(ctx context.Context) bool `perm:"public"` + IsStopped func(ctx context.Context) bool `perm:"read"` Balance func(ctx context.Context) (*state.Balance, error) `perm:"read"` - BalanceForAddress func(ctx context.Context, addr state.Address) (*state.Balance, error) `perm:"public"` + BalanceForAddress func(ctx context.Context, addr state.Address) (*state.Balance, error) `perm:"read"` Transfer func( ctx context.Context, to state.AccAddress, @@ -150,16 +150,16 @@ type API struct { QueryDelegation func( ctx context.Context, valAddr state.ValAddress, - ) (*types.QueryDelegationResponse, error) `perm:"public"` + ) (*types.QueryDelegationResponse, error) `perm:"read"` QueryUnbonding func( ctx context.Context, valAddr state.ValAddress, - ) (*types.QueryUnbondingDelegationResponse, error) `perm:"public"` + ) (*types.QueryUnbondingDelegationResponse, error) `perm:"read"` QueryRedelegations func( ctx context.Context, srcValAddr, dstValAddr state.ValAddress, - ) (*types.QueryRedelegationsResponse, error) `perm:"public"` + ) (*types.QueryRedelegationsResponse, error) `perm:"read"` } } From d69d3b23683e394c2394b78147a5113610dba298 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 6 Oct 2023 14:32:30 +0200 Subject: [PATCH 258/388] refactor(share)!: replacing share.Root with header.ExtendedHeader in APIS (#2773) --- api/gateway/availability.go | 2 +- api/gateway/share.go | 2 +- blob/service.go | 12 +- das/daser.go | 2 +- das/daser_test.go | 10 +- header/header.go | 17 ++- header/headertest/testing.go | 14 +- nodebuilder/node_test.go | 5 +- nodebuilder/share/cmd/share.go | 35 +++-- nodebuilder/share/module.go | 3 +- nodebuilder/share/share.go | 45 +++--- nodebuilder/tests/blob_test.go | 2 +- nodebuilder/tests/nd_test.go | 13 +- nodebuilder/tests/reconstruct_test.go | 12 +- nodebuilder/tests/sync_test.go | 8 +- share/availability.go | 6 +- share/availability/full/availability.go | 18 +-- share/availability/full/availability_test.go | 15 +- .../availability/full/reconstruction_test.go | 33 +++-- share/availability/light/availability.go | 8 +- share/availability/light/availability_test.go | 50 ++++--- share/availability/light/testing.go | 10 +- share/availability/mocks/availability.go | 19 +-- share/eds/utils.go | 104 ++++++++++++++ share/getter.go | 10 +- share/getters/cascade.go | 19 ++- share/getters/cascade_test.go | 19 +-- share/getters/getter_test.go | 134 +++++++++++------- share/getters/ipld.go | 15 +- share/getters/shrex.go | 30 ++-- share/getters/shrex_test.go | 28 ++-- share/getters/store.go | 53 ++----- share/getters/testing.go | 24 +++- share/getters/utils.go | 66 --------- share/ipld/corrupted_data_test.go | 7 +- share/ipld/utils.go | 18 +++ share/mocks/getter.go | 11 +- share/p2p/shrexnd/exchange_test.go | 34 +---- share/p2p/shrexnd/server.go | 6 +- 39 files changed, 510 insertions(+), 409 deletions(-) create mode 100644 share/ipld/utils.go diff --git a/api/gateway/availability.go b/api/gateway/availability.go index 203fd86d61..70341b34f2 100644 --- a/api/gateway/availability.go +++ b/api/gateway/availability.go @@ -32,7 +32,7 @@ func (h *Handler) handleHeightAvailabilityRequest(w http.ResponseWriter, r *http return } - err = h.share.SharesAvailable(r.Context(), header.DAH) + err = h.share.SharesAvailable(r.Context(), header) switch err { case nil: resp, err := json.Marshal(&AvailabilityResponse{Available: true}) diff --git a/api/gateway/share.go b/api/gateway/share.go index c9dec071f3..36c1f94a0a 100644 --- a/api/gateway/share.go +++ b/api/gateway/share.go @@ -96,7 +96,7 @@ func (h *Handler) getShares(ctx context.Context, height uint64, namespace share. return nil, err } - shares, err := h.share.GetSharesByNamespace(ctx, header.DAH, namespace) + shares, err := h.share.GetSharesByNamespace(ctx, header, namespace) if err != nil { return nil, err } diff --git a/blob/service.go b/blob/service.go index 2130904aa7..cf3daff7de 100644 --- a/blob/service.go +++ b/blob/service.go @@ -126,7 +126,7 @@ func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share. wg.Add(1) go func(i int, namespace share.Namespace) { defer wg.Done() - blobs, err := s.getBlobs(ctx, namespace, header.DAH) + blobs, err := s.getBlobs(ctx, namespace, header) if err != nil { resultErr[i] = fmt.Errorf("getting blobs for namespace(%s): %s", namespace.String(), err) return @@ -204,7 +204,7 @@ func (s *Service) getByCommitment( blobShare *shares.Share ) - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header.DAH, namespace) + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) if err != nil { if errors.Is(err, share.ErrNotFound) { err = ErrBlobNotFound @@ -293,8 +293,12 @@ func (s *Service) getByCommitment( // getBlobs retrieves the DAH and fetches all shares from the requested Namespace and converts // them to Blobs. -func (s *Service) getBlobs(ctx context.Context, namespace share.Namespace, root *share.Root) ([]*Blob, error) { - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, root, namespace) +func (s *Service) getBlobs( + ctx context.Context, + namespace share.Namespace, + header *header.ExtendedHeader, +) ([]*Blob, error) { + namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) if err != nil { return nil, err } diff --git a/das/daser.go b/das/daser.go index 9d3e43a91b..29139d3a5f 100644 --- a/das/daser.go +++ b/das/daser.go @@ -147,7 +147,7 @@ func (d *DASer) Stop(ctx context.Context) error { } func (d *DASer) sample(ctx context.Context, h *header.ExtendedHeader) error { - err := d.da.SharesAvailable(ctx, h.DAH) + err := d.da.SharesAvailable(ctx, h) if err != nil { var byzantineErr *byzantine.ErrByzantine if errors.As(err, &byzantineErr) { diff --git a/das/daser_test.go b/das/daser_test.go index 6a3378cdb5..ca553b0f09 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -214,7 +214,7 @@ func TestDASerSampleTimeout(t *testing.T) { avail := mocks.NewMockAvailability(gomock.NewController(t)) doneCh := make(chan struct{}) avail.EXPECT().SharesAvailable(gomock.Any(), gomock.Any()).DoAndReturn( - func(sampleCtx context.Context, h *share.Root) error { + func(sampleCtx context.Context, h *header.ExtendedHeader) error { select { case <-sampleCtx.Done(): close(doneCh) @@ -291,9 +291,7 @@ func (m *mockGetter) fillSubWithHeaders( for i := startHeight; i < endHeight; i++ { dah := availability_test.RandFillBS(t, 16, bServ) - randHeader := headertest.RandExtendedHeader(t) - randHeader.DataHash = dah.Hash() - randHeader.DAH = dah + randHeader := headertest.RandExtendedHeaderWithRoot(t, dah) randHeader.RawHeader.Height = int64(i + 1) sub.Headers[index] = randHeader @@ -319,9 +317,7 @@ func (m *mockGetter) generateHeaders(t *testing.T, bServ blockservice.BlockServi for i := startHeight; i < endHeight; i++ { dah := availability_test.RandFillBS(t, 16, bServ) - randHeader := headertest.RandExtendedHeader(t) - randHeader.DataHash = dah.Hash() - randHeader.DAH = dah + randHeader := headertest.RandExtendedHeaderWithRoot(t, dah) randHeader.RawHeader.Height = int64(i + 1) m.headers[int64(i+1)] = randHeader diff --git a/header/header.go b/header/header.go index 9778684b7d..01a84a2c0a 100644 --- a/header/header.go +++ b/header/header.go @@ -12,10 +12,9 @@ import ( core "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/rsmt2d" - - "github.com/celestiaorg/celestia-node/share" ) // ConstructFn aliases a function that creates an ExtendedHeader. @@ -36,9 +35,9 @@ type RawHeader = core.Header // block headers and perform Data Availability Sampling. type ExtendedHeader struct { RawHeader `json:"header"` - Commit *core.Commit `json:"commit"` - ValidatorSet *core.ValidatorSet `json:"validator_set"` - DAH *share.Root `json:"dah"` + Commit *core.Commit `json:"commit"` + ValidatorSet *core.ValidatorSet `json:"validator_set"` + DAH *da.DataAvailabilityHeader `json:"dah"` } // MakeExtendedHeader assembles new ExtendedHeader. @@ -49,14 +48,14 @@ func MakeExtendedHeader( eds *rsmt2d.ExtendedDataSquare, ) (*ExtendedHeader, error) { var ( - dah *share.Root + dah da.DataAvailabilityHeader err error ) switch eds { case nil: - dah = share.EmptyRoot() + dah = da.MinDataAvailabilityHeader() default: - dah, err = share.NewRoot(eds) + dah, err = da.NewDataAvailabilityHeader(eds) if err != nil { return nil, err } @@ -64,7 +63,7 @@ func MakeExtendedHeader( eh := &ExtendedHeader{ RawHeader: *h, - DAH: dah, + DAH: &dah, Commit: comm, ValidatorSet: vals, } diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 0b3c6a34dc..9907fd7eb4 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -17,6 +17,7 @@ import ( "github.com/tendermint/tendermint/types" tmtime "github.com/tendermint/tendermint/types/time" + "github.com/celestiaorg/celestia-app/pkg/da" libhead "github.com/celestiaorg/go-header" "github.com/celestiaorg/go-header/headertest" "github.com/celestiaorg/rsmt2d" @@ -202,7 +203,7 @@ func (s *TestSuite) nextProposer() *types.Validator { } // RandExtendedHeader provides an ExtendedHeader fixture. -func RandExtendedHeader(t *testing.T) *header.ExtendedHeader { +func RandExtendedHeader(t testing.TB) *header.ExtendedHeader { dah := share.EmptyRoot() rh := RandRawHeader(t) @@ -224,6 +225,13 @@ func RandExtendedHeader(t *testing.T) *header.ExtendedHeader { } } +func RandExtendedHeaderWithRoot(t testing.TB, dah *da.DataAvailabilityHeader) *header.ExtendedHeader { + h := RandExtendedHeader(t) + h.DataHash = dah.Hash() + h.DAH = dah + return h +} + func RandValidatorSet(numValidators int, votingPower int64) (*types.ValidatorSet, []types.PrivValidator) { var ( valz = make([]*types.Validator, numValidators) @@ -256,7 +264,7 @@ func RandValidator(randPower bool, minPower int64) (*types.Validator, types.Priv } // RandRawHeader provides a RawHeader fixture. -func RandRawHeader(t *testing.T) *header.RawHeader { +func RandRawHeader(t testing.TB) *header.RawHeader { return &header.RawHeader{ Version: version.Consensus{Block: 11, App: 1}, ChainID: "test", @@ -276,7 +284,7 @@ func RandRawHeader(t *testing.T) *header.RawHeader { } // RandBlockID provides a BlockID fixture. -func RandBlockID(*testing.T) types.BlockID { +func RandBlockID(testing.TB) types.BlockID { bid := types.BlockID{ Hash: make([]byte, 32), PartSetHeader: types.PartSetHeader{ diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index 3fc3f4f02a..1d0f3c4fad 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -13,6 +13,7 @@ import ( collectormetricpb "go.opentelemetry.io/proto/otlp/collector/metrics/v1" "google.golang.org/protobuf/proto" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/share" ) @@ -143,7 +144,9 @@ func TestEmptyBlockExists(t *testing.T) { require.NoError(t, err) // ensure an empty block exists in store - err = node.ShareServ.SharesAvailable(ctx, share.EmptyRoot()) + + eh := headertest.RandExtendedHeaderWithRoot(t, share.EmptyRoot()) + err = node.ShareServ.SharesAvailable(ctx, eh) require.NoError(t, err) err = node.Stop(ctx) diff --git a/nodebuilder/share/cmd/share.go b/nodebuilder/share/cmd/share.go index bc22651238..3d1af05289 100644 --- a/nodebuilder/share/cmd/share.go +++ b/nodebuilder/share/cmd/share.go @@ -7,9 +7,8 @@ import ( "github.com/spf13/cobra" - "github.com/celestiaorg/celestia-app/pkg/da" - cmdnode "github.com/celestiaorg/celestia-node/cmd" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" ) @@ -45,13 +44,13 @@ var sharesAvailableCmd = &cobra.Command{ return err } - root := da.MinDataAvailabilityHeader() - err = json.Unmarshal(raw, &root) + var eh *header.ExtendedHeader + err = json.Unmarshal(raw, &eh) if err != nil { return err } - err = client.Share.SharesAvailable(cmd.Context(), &root) + err = client.Share.SharesAvailable(cmd.Context(), eh) formatter := func(data interface{}) interface{} { err, ok := data.(error) available := false @@ -73,7 +72,7 @@ var sharesAvailableCmd = &cobra.Command{ } var getSharesByNamespaceCmd = &cobra.Command{ - Use: "get-by-namespace [dah, namespace]", + Use: "get-by-namespace [extended header, namespace]", Short: "Gets all shares from an EDS within the given namespace.", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { @@ -88,8 +87,8 @@ var getSharesByNamespaceCmd = &cobra.Command{ return err } - root := da.MinDataAvailabilityHeader() - err = json.Unmarshal(raw, &root) + var eh *header.ExtendedHeader + err = json.Unmarshal(raw, &eh) if err != nil { return err } @@ -99,13 +98,13 @@ var getSharesByNamespaceCmd = &cobra.Command{ return err } - shares, err := client.Share.GetSharesByNamespace(cmd.Context(), &root, ns) + shares, err := client.Share.GetSharesByNamespace(cmd.Context(), eh, ns) return cmdnode.PrintOutput(shares, err, nil) }, } var getShare = &cobra.Command{ - Use: "get-share [dah, row, col]", + Use: "get-share [extended header, row, col]", Short: "Gets a Share by coordinates in EDS.", Args: cobra.ExactArgs(3), RunE: func(cmd *cobra.Command, args []string) error { @@ -120,8 +119,8 @@ var getShare = &cobra.Command{ return err } - root := da.MinDataAvailabilityHeader() - err = json.Unmarshal(raw, &root) + var eh *header.ExtendedHeader + err = json.Unmarshal(raw, &eh) if err != nil { return err } @@ -136,7 +135,7 @@ var getShare = &cobra.Command{ return err } - s, err := client.Share.GetShare(cmd.Context(), &root, int(row), int(col)) + s, err := client.Share.GetShare(cmd.Context(), eh, int(row), int(col)) formatter := func(data interface{}) interface{} { sh, ok := data.(share.Share) @@ -159,8 +158,8 @@ var getShare = &cobra.Command{ } var getEDS = &cobra.Command{ - Use: "get-eds [dah]", - Short: "Gets the full EDS identified by the given root", + Use: "get-eds [extended header]", + Short: "Gets the full EDS identified by the given extended header", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) @@ -174,13 +173,13 @@ var getEDS = &cobra.Command{ return err } - root := da.MinDataAvailabilityHeader() - err = json.Unmarshal(raw, &root) + var eh *header.ExtendedHeader + err = json.Unmarshal(raw, &eh) if err != nil { return err } - shares, err := client.Share.GetEDS(cmd.Context(), &root) + shares, err := client.Share.GetEDS(cmd.Context(), eh) return cmdnode.PrintOutput(shares, err, nil) }, } diff --git a/nodebuilder/share/module.go b/nodebuilder/share/module.go index 9fbd262a32..3fa55b2d35 100644 --- a/nodebuilder/share/module.go +++ b/nodebuilder/share/module.go @@ -98,11 +98,10 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option func( host host.Host, store *eds.Store, - getter *getters.StoreGetter, network modp2p.Network, ) (*shrexnd.Server, error) { cfg.ShrExNDParams.WithNetworkID(network.String()) - return shrexnd.NewServer(cfg.ShrExNDParams, host, store, getter) + return shrexnd.NewServer(cfg.ShrExNDParams, host, store) }, fx.OnStart(func(ctx context.Context, server *shrexnd.Server) error { return server.Start(ctx) diff --git a/nodebuilder/share/share.go b/nodebuilder/share/share.go index da97f32a33..a8e1e1c895 100644 --- a/nodebuilder/share/share.go +++ b/nodebuilder/share/share.go @@ -5,6 +5,7 @@ import ( "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" ) @@ -27,58 +28,60 @@ var _ Module = (*API)(nil) // //go:generate mockgen -destination=mocks/api.go -package=mocks . Module type Module interface { - // SharesAvailable subjectively validates if Shares committed to the given Root are available on - // the Network. - SharesAvailable(context.Context, *share.Root) error + // SharesAvailable subjectively validates if Shares committed to the given + // ExtendedHeader are available on the Network. + SharesAvailable(context.Context, *header.ExtendedHeader) error // GetShare gets a Share by coordinates in EDS. - GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) - // GetEDS gets the full EDS identified by the given root. - GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) + GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) + // GetEDS gets the full EDS identified by the given extended header. + GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) // GetSharesByNamespace gets all shares from an EDS within the given namespace. // Shares are returned in a row-by-row order if the namespace spans multiple rows. - GetSharesByNamespace(ctx context.Context, root *share.Root, namespace share.Namespace) (share.NamespacedShares, error) + GetSharesByNamespace( + ctx context.Context, header *header.ExtendedHeader, namespace share.Namespace, + ) (share.NamespacedShares, error) } // API is a wrapper around Module for the RPC. // TODO(@distractedm1nd): These structs need to be autogenerated. type API struct { Internal struct { - SharesAvailable func(context.Context, *share.Root) error `perm:"read"` + SharesAvailable func(context.Context, *header.ExtendedHeader) error `perm:"read"` GetShare func( ctx context.Context, - dah *share.Root, + header *header.ExtendedHeader, row, col int, ) (share.Share, error) `perm:"read"` GetEDS func( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, ) (*rsmt2d.ExtendedDataSquare, error) `perm:"read"` GetSharesByNamespace func( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, namespace share.Namespace, ) (share.NamespacedShares, error) `perm:"read"` } } -func (api *API) SharesAvailable(ctx context.Context, root *share.Root) error { - return api.Internal.SharesAvailable(ctx, root) +func (api *API) SharesAvailable(ctx context.Context, header *header.ExtendedHeader) error { + return api.Internal.SharesAvailable(ctx, header) } -func (api *API) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) { - return api.Internal.GetShare(ctx, dah, row, col) +func (api *API) GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) { + return api.Internal.GetShare(ctx, header, row, col) } -func (api *API) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) { - return api.Internal.GetEDS(ctx, root) +func (api *API) GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { + return api.Internal.GetEDS(ctx, header) } func (api *API) GetSharesByNamespace( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, namespace share.Namespace, ) (share.NamespacedShares, error) { - return api.Internal.GetSharesByNamespace(ctx, root, namespace) + return api.Internal.GetSharesByNamespace(ctx, header, namespace) } type module struct { @@ -86,6 +89,6 @@ type module struct { share.Availability } -func (m module) SharesAvailable(ctx context.Context, root *share.Root) error { - return m.Availability.SharesAvailable(ctx, root) +func (m module) SharesAvailable(ctx context.Context, header *header.ExtendedHeader) error { + return m.Availability.SharesAvailable(ctx, header) } diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 2078fbfa74..ccbc6440cc 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -120,7 +120,7 @@ func TestBlobModule(t *testing.T) { b, err := fullClient.Blob.Get(ctx, height, newBlob.Namespace(), newBlob.Commitment) assert.Nil(t, b) require.Error(t, err) - require.ErrorIs(t, err, blob.ErrBlobNotFound) + require.ErrorContains(t, err, blob.ErrBlobNotFound.Error()) }, }, } diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index 8149ea9fe7..f3338ec294 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -65,9 +65,9 @@ func TestShrexNDFromLights(t *testing.T) { // ensure to fetch random namespace (not the reserved namespace) namespace := h.DAH.RowRoots[1][:share.NamespaceSize] - expected, err := bridgeClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) + expected, err := bridgeClient.Share.GetSharesByNamespace(reqCtx, h, namespace) require.NoError(t, err) - got, err := lightClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) + got, err := lightClient.Share.GetSharesByNamespace(reqCtx, h, namespace) require.NoError(t, err) require.True(t, len(got[0].Shares) > 0) @@ -139,18 +139,18 @@ func TestShrexNDFromLightsWithBadFulls(t *testing.T) { // ensure to fetch random namespace (not the reserved namespace) namespace := h.DAH.RowRoots[1][:share.NamespaceSize] - expected, err := bridgeClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) + expected, err := bridgeClient.Share.GetSharesByNamespace(reqCtx, h, namespace) require.NoError(t, err) require.True(t, len(expected[0].Shares) > 0) // choose a random full to test fN := fulls[len(fulls)/2] fnClient := getAdminClient(ctx, fN, t) - gotFull, err := fnClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) + gotFull, err := fnClient.Share.GetSharesByNamespace(reqCtx, h, namespace) require.NoError(t, err) require.True(t, len(gotFull[0].Shares) > 0) - gotLight, err := lightClient.Share.GetSharesByNamespace(reqCtx, h.DAH, namespace) + gotLight, err := lightClient.Share.GetSharesByNamespace(reqCtx, h, namespace) require.NoError(t, err) require.True(t, len(gotLight[0].Shares) > 0) @@ -176,11 +176,10 @@ func replaceNDServer(cfg *nodebuilder.Config, handler network.StreamHandler) fx. func( host host.Host, store *eds.Store, - getter *getters.StoreGetter, network p2p.Network, ) (*shrexnd.Server, error) { cfg.Share.ShrExNDParams.WithNetworkID(network.String()) - return shrexnd.NewServer(cfg.Share.ShrExNDParams, host, store, getter) + return shrexnd.NewServer(cfg.Share.ShrExNDParams, host, store) }, fx.OnStart(func(ctx context.Context, server *shrexnd.Server) error { // replace handler for server diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index 3c2e8b1f83..89f4a0171a 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -77,7 +77,7 @@ func TestFullReconstructFromBridge(t *testing.T) { return err } - return fullClient.Share.SharesAvailable(bctx, h.DAH) + return fullClient.Share.SharesAvailable(bctx, h) }) } require.NoError(t, <-fillDn) @@ -211,10 +211,10 @@ func TestFullReconstructFromFulls(t *testing.T) { ctxErr, cancelErr := context.WithTimeout(ctx, time.Second*30) errg, errCtx = errgroup.WithContext(ctxErr) errg.Go(func() error { - return fullClient1.Share.SharesAvailable(errCtx, h.DAH) + return fullClient1.Share.SharesAvailable(errCtx, h) }) errg.Go(func() error { - return fullClient1.Share.SharesAvailable(errCtx, h.DAH) + return fullClient1.Share.SharesAvailable(errCtx, h) }) require.Error(t, errg.Wait()) cancelErr() @@ -227,10 +227,10 @@ func TestFullReconstructFromFulls(t *testing.T) { h, err := fullClient1.Header.WaitForHeight(bctx, uint64(i)) require.NoError(t, err) errg.Go(func() error { - return fullClient1.Share.SharesAvailable(bctx, h.DAH) + return fullClient1.Share.SharesAvailable(bctx, h) }) errg.Go(func() error { - return fullClient2.Share.SharesAvailable(bctx, h.DAH) + return fullClient2.Share.SharesAvailable(bctx, h) }) } @@ -344,7 +344,7 @@ func TestFullReconstructFromLights(t *testing.T) { return err } - return fullClient.Share.SharesAvailable(bctx, h.DAH) + return fullClient.Share.SharesAvailable(bctx, h) }) } require.NoError(t, <-fillDn) diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index 0bb0e1c757..8b939749d6 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -75,7 +75,7 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check that the light node has also sampled over the block at height 20 - err = lightClient.Share.SharesAvailable(ctx, h.DAH) + err = lightClient.Share.SharesAvailable(ctx, h) assert.NoError(t, err) // wait until the entire chain (up to network head) has been sampled @@ -95,7 +95,7 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check to ensure the full node can sync the 20th block's data - err = fullClient.Share.SharesAvailable(ctx, h.DAH) + err = fullClient.Share.SharesAvailable(ctx, h) assert.NoError(t, err) // wait for full node to sync up the blocks from genesis -> network head. @@ -165,7 +165,7 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check that the light node has also sampled over the block at height 20 - err = lightClient.Share.SharesAvailable(ctx, h.DAH) + err = lightClient.Share.SharesAvailable(ctx, h) assert.NoError(t, err) // wait until the entire chain (up to network head) has been sampled @@ -185,7 +185,7 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { assert.EqualValues(t, h.Commit.BlockID.Hash, sw.GetCoreBlockHashByHeight(ctx, numBlocks)) // check to ensure the full node can sync the 20th block's data - err = fullClient.Share.SharesAvailable(ctx, h.DAH) + err = fullClient.Share.SharesAvailable(ctx, h) assert.NoError(t, err) // wait for full node to sync up the blocks from genesis -> network head. diff --git a/share/availability.go b/share/availability.go index 587469d94b..f3511da450 100644 --- a/share/availability.go +++ b/share/availability.go @@ -6,6 +6,8 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" ) // ErrNotAvailable is returned whenever DA sampling fails. @@ -26,8 +28,10 @@ func NewRoot(eds *rsmt2d.ExtendedDataSquare) (*Root, error) { } // Availability defines interface for validation of Shares' availability. +// +//go:generate mockgen -destination=availability/mocks/availability.go -package=mocks . Availability type Availability interface { // SharesAvailable subjectively validates if Shares committed to the given Root are available on // the Network. - SharesAvailable(context.Context, *Root) error + SharesAvailable(context.Context, *header.ExtendedHeader) error } diff --git a/share/availability/full/availability.go b/share/availability/full/availability.go index 5fecb757fd..4ea211cb1e 100644 --- a/share/availability/full/availability.go +++ b/share/availability/full/availability.go @@ -8,6 +8,7 @@ import ( "github.com/filecoin-project/dagstore" logging "github.com/ipfs/go-log/v2" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/byzantine" @@ -56,35 +57,36 @@ func (fa *ShareAvailability) Stop(context.Context) error { // SharesAvailable reconstructs the data committed to the given Root by requesting // enough Shares from the network. -func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Root) error { +func (fa *ShareAvailability) SharesAvailable(ctx context.Context, header *header.ExtendedHeader) error { + dah := header.DAH // short-circuit if the given root is minimum DAH of an empty data square, to avoid datastore hit - if share.DataHash(root.Hash()).IsEmptyRoot() { + if share.DataHash(dah.Hash()).IsEmptyRoot() { return nil } // we assume the caller of this method has already performed basic validation on the // given dah/root. If for some reason this has not happened, the node should panic. - if err := root.ValidateBasic(); err != nil { + if err := dah.ValidateBasic(); err != nil { log.Errorw("Availability validation cannot be performed on a malformed DataAvailabilityHeader", "err", err) panic(err) } // a hack to avoid loading the whole EDS in mem if we store it already. - if ok, _ := fa.store.Has(ctx, root.Hash()); ok { + if ok, _ := fa.store.Has(ctx, dah.Hash()); ok { return nil } - adder := ipld.NewProofsAdder(len(root.RowRoots)) + adder := ipld.NewProofsAdder(len(dah.RowRoots)) ctx = ipld.CtxWithProofsAdder(ctx, adder) defer adder.Purge() - eds, err := fa.getter.GetEDS(ctx, root) + eds, err := fa.getter.GetEDS(ctx, header) if err != nil { if errors.Is(err, context.Canceled) { return err } - log.Errorw("availability validation failed", "root", root.String(), "err", err.Error()) + log.Errorw("availability validation failed", "root", dah.String(), "err", err.Error()) var byzantineErr *byzantine.ErrByzantine if errors.Is(err, share.ErrNotFound) || errors.Is(err, context.DeadlineExceeded) && !errors.As(err, &byzantineErr) { return share.ErrNotAvailable @@ -92,7 +94,7 @@ func (fa *ShareAvailability) SharesAvailable(ctx context.Context, root *share.Ro return err } - err = fa.store.Put(ctx, root.Hash(), eds) + err = fa.store.Put(ctx, dah.Hash(), eds) if err != nil && !errors.Is(err, dagstore.ErrShardExists) { return fmt.Errorf("full availability: failed to store eds: %w", err) } diff --git a/share/availability/full/availability_test.go b/share/availability/full/availability_test.go index a769c981c4..8ac0648a87 100644 --- a/share/availability/full/availability_test.go +++ b/share/availability/full/availability_test.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/eds/edstest" @@ -22,10 +23,12 @@ func TestShareAvailableOverMocknet_Full(t *testing.T) { net := availability_test.NewTestDAGNet(ctx, t) _, root := RandNode(net, 32) + + eh := headertest.RandExtendedHeaderWithRoot(t, root) nd := Node(net) net.ConnectAll() - err := nd.SharesAvailable(ctx, root) + err := nd.SharesAvailable(ctx, eh) assert.NoError(t, err) } @@ -35,8 +38,10 @@ func TestSharesAvailable_Full(t *testing.T) { // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it getter, dah := GetterWithRandSquare(t, 16) + + eh := headertest.RandExtendedHeaderWithRoot(t, dah) avail := TestAvailability(t, getter) - err := avail.SharesAvailable(ctx, dah) + err := avail.SharesAvailable(ctx, eh) assert.NoError(t, err) } @@ -46,8 +51,9 @@ func TestSharesAvailable_StoresToEDSStore(t *testing.T) { // RandServiceWithSquare creates a NewShareAvailability inside, so we can test it getter, dah := GetterWithRandSquare(t, 16) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) avail := TestAvailability(t, getter) - err := avail.SharesAvailable(ctx, dah) + err := avail.SharesAvailable(ctx, eh) assert.NoError(t, err) has, err := avail.store.Has(ctx, dah.Hash()) @@ -63,13 +69,14 @@ func TestSharesAvailable_Full_ErrNotAvailable(t *testing.T) { eds := edstest.RandEDS(t, 4) dah, err := da.NewDataAvailabilityHeader(eds) + eh := headertest.RandExtendedHeaderWithRoot(t, &dah) require.NoError(t, err) avail := TestAvailability(t, getter) errors := []error{share.ErrNotFound, context.DeadlineExceeded} for _, getterErr := range errors { getter.EXPECT().GetEDS(gomock.Any(), gomock.Any()).Return(nil, getterErr) - err := avail.SharesAvailable(ctx, &dah) + err := avail.SharesAvailable(ctx, eh) require.ErrorIs(t, err, share.ErrNotAvailable) } } diff --git a/share/availability/full/reconstruction_test.go b/share/availability/full/reconstruction_test.go index f3b6ce91bd..6ac5a3f31e 100644 --- a/share/availability/full/reconstruction_test.go +++ b/share/availability/full/reconstruction_test.go @@ -11,6 +11,7 @@ import ( "github.com/stretchr/testify/require" "golang.org/x/sync/errgroup" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/availability/light" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" @@ -37,7 +38,9 @@ func TestShareAvailable_OneFullNode(t *testing.T) { net := availability_test.NewTestDAGNet(ctx, t) source, root := RandNode(net, origSquareSize) // make a source node, a.k.a bridge - full := Node(net) // make a full availability service which reconstructs data + eh := headertest.RandExtendedHeader(t) + eh.DAH = root + full := Node(net) // make a full availability service which reconstructs data // ensure there is no connection between source and full nodes // so that full reconstructs from the light nodes only @@ -45,14 +48,14 @@ func TestShareAvailable_OneFullNode(t *testing.T) { errg, errCtx := errgroup.WithContext(ctx) errg.Go(func() error { - return full.SharesAvailable(errCtx, root) + return full.SharesAvailable(errCtx, eh) }) lights := make([]*availability_test.TestNode, lightNodes) for i := 0; i < len(lights); i++ { lights[i] = light.Node(net) go func(i int) { - err := lights[i].SharesAvailable(ctx, root) + err := lights[i].SharesAvailable(ctx, eh) if err != nil { t.Log("light errors:", err) } @@ -91,6 +94,8 @@ func TestShareAvailable_ConnectedFullNodes(t *testing.T) { net := availability_test.NewTestDAGNet(ctx, t) source, root := RandNode(net, origSquareSize) + eh := headertest.RandExtendedHeader(t) + eh.DAH = root // create two full nodes and ensure they are disconnected full1 := Node(net) @@ -106,10 +111,10 @@ func TestShareAvailable_ConnectedFullNodes(t *testing.T) { // start reconstruction for fulls errg, errCtx := errgroup.WithContext(ctx) errg.Go(func() error { - return full1.SharesAvailable(errCtx, root) + return full1.SharesAvailable(errCtx, eh) }) errg.Go(func() error { - return full2.SharesAvailable(errCtx, root) + return full2.SharesAvailable(errCtx, eh) }) // create light nodes and start sampling for them immediately @@ -119,7 +124,7 @@ func TestShareAvailable_ConnectedFullNodes(t *testing.T) { for i := 0; i < len(lights1); i++ { lights1[i] = light.Node(net) go func(i int) { - err := lights1[i].SharesAvailable(ctx, root) + err := lights1[i].SharesAvailable(ctx, eh) if err != nil { t.Log("light1 errors:", err) } @@ -127,7 +132,7 @@ func TestShareAvailable_ConnectedFullNodes(t *testing.T) { lights2[i] = light.Node(net) go func(i int) { - err := lights2[i].SharesAvailable(ctx, root) + err := lights2[i].SharesAvailable(ctx, eh) if err != nil { t.Log("light2 errors:", err) } @@ -189,6 +194,8 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { net := availability_test.NewTestDAGNet(ctx, t) source, root := RandNode(net, origSquareSize) + eh := headertest.RandExtendedHeader(t) + eh.DAH = root // create light nodes and start sampling for them immediately lights1, lights2 := make( @@ -201,7 +208,7 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { lights1[i] = light.Node(net) go func(i int) { defer wg.Done() - err := lights1[i].SharesAvailable(ctx, root) + err := lights1[i].SharesAvailable(ctx, eh) if err != nil { t.Log("light1 errors:", err) } @@ -210,7 +217,7 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { lights2[i] = light.Node(net) go func(i int) { defer wg.Done() - err := lights2[i].SharesAvailable(ctx, root) + err := lights2[i].SharesAvailable(ctx, eh) if err != nil { t.Log("light2 errors:", err) } @@ -243,10 +250,10 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { ctxErr, cancelErr := context.WithTimeout(ctx, time.Second*5) errg, errCtx := errgroup.WithContext(ctxErr) errg.Go(func() error { - return full1.SharesAvailable(errCtx, root) + return full1.SharesAvailable(errCtx, eh) }) errg.Go(func() error { - return full2.SharesAvailable(errCtx, root) + return full2.SharesAvailable(errCtx, eh) }) // check that any of the fulls cannot reconstruct on their own @@ -264,10 +271,10 @@ func TestShareAvailable_DisconnectedFullNodes(t *testing.T) { // they both should be able to reconstruct the block errg, bctx := errgroup.WithContext(ctx) errg.Go(func() error { - return full1.SharesAvailable(bctx, root) + return full1.SharesAvailable(bctx, eh) }) errg.Go(func() error { - return full2.SharesAvailable(bctx, root) + return full2.SharesAvailable(bctx, eh) }) require.NoError(t, errg.Wait()) // wait for all routines to finish before exit, in case there are any errors to log diff --git a/share/availability/light/availability.go b/share/availability/light/availability.go index e7ad900d8d..1d35542344 100644 --- a/share/availability/light/availability.go +++ b/share/availability/light/availability.go @@ -11,6 +11,7 @@ import ( ipldFormat "github.com/ipfs/go-ipld-format" logging "github.com/ipfs/go-log/v2" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/getters" ) @@ -58,8 +59,9 @@ func NewShareAvailability( } // SharesAvailable randomly samples `params.SampleAmount` amount of Shares committed to the given -// Root. This way SharesAvailable subjectively verifies that Shares are available. -func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Root) error { +// ExtendedHeader. This way SharesAvailable subjectively verifies that Shares are available. +func (la *ShareAvailability) SharesAvailable(ctx context.Context, header *header.ExtendedHeader) error { + dah := header.DAH // short-circuit if the given root is minimum DAH of an empty data square if share.DataHash(dah.Hash()).IsEmptyRoot() { return nil @@ -97,7 +99,7 @@ func (la *ShareAvailability) SharesAvailable(ctx context.Context, dah *share.Roo for _, s := range samples { go func(s Sample) { log.Debugw("fetching share", "root", dah.String(), "row", s.Row, "col", s.Col) - _, err := la.getter.GetShare(ctx, dah, s.Row, s.Col) + _, err := la.getter.GetShare(ctx, header, s.Row, s.Col) if err != nil { log.Debugw("error fetching share", "root", dah.String(), "row", s.Row, "col", s.Col) } diff --git a/share/availability/light/availability_test.go b/share/availability/light/availability_test.go index 1e6b99cda4..2ace654d50 100644 --- a/share/availability/light/availability_test.go +++ b/share/availability/light/availability_test.go @@ -9,6 +9,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/ipld" @@ -19,7 +20,8 @@ func TestSharesAvailableCaches(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) defer cancel() - getter, dah := GetterWithRandSquare(t, 16) + getter, eh := GetterWithRandSquare(t, 16) + dah := eh.DAH avail := TestAvailability(getter) // cache doesn't have dah yet @@ -27,7 +29,7 @@ func TestSharesAvailableCaches(t *testing.T) { assert.NoError(t, err) assert.False(t, has) - err = avail.SharesAvailable(ctx, dah) + err = avail.SharesAvailable(ctx, eh) assert.NoError(t, err) // is now cached @@ -45,9 +47,10 @@ func TestSharesAvailableHitsCache(t *testing.T) { bServ := ipld.NewMemBlockservice() dah := availability_test.RandFillBS(t, 16, bServ) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) // blockstore doesn't actually have the dah - err := avail.SharesAvailable(ctx, dah) + err := avail.SharesAvailable(ctx, eh) require.Error(t, err) // cache doesn't have dah yet, since it errored @@ -59,7 +62,7 @@ func TestSharesAvailableHitsCache(t *testing.T) { require.NoError(t, err) // should hit cache after putting - err = avail.SharesAvailable(ctx, dah) + err = avail.SharesAvailable(ctx, eh) require.NoError(t, err) } @@ -70,7 +73,8 @@ func TestSharesAvailableEmptyRoot(t *testing.T) { getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) - err := avail.SharesAvailable(ctx, share.EmptyRoot()) + eh := headertest.RandExtendedHeaderWithRoot(t, share.EmptyRoot()) + err := avail.SharesAvailable(ctx, eh) assert.NoError(t, err) } @@ -90,10 +94,11 @@ func TestSharesAvailableFailed(t *testing.T) { bServ := ipld.NewMemBlockservice() dah := availability_test.RandFillBS(t, 16, bServ) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) getter, _ := GetterWithRandSquare(t, 16) avail := TestAvailability(getter) - err := avail.SharesAvailable(ctx, dah) + err := avail.SharesAvailable(ctx, eh) assert.Error(t, err) } @@ -103,10 +108,12 @@ func TestShareAvailableOverMocknet_Light(t *testing.T) { net := availability_test.NewTestDAGNet(ctx, t) _, root := RandNode(net, 16) + eh := headertest.RandExtendedHeader(t) + eh.DAH = root nd := Node(net) net.ConnectAll() - err := nd.SharesAvailable(ctx, root) + err := nd.SharesAvailable(ctx, eh) assert.NoError(t, err) } @@ -115,11 +122,11 @@ func TestGetShare(t *testing.T) { defer cancel() n := 16 - getter, dah := GetterWithRandSquare(t, n) + getter, eh := GetterWithRandSquare(t, n) for i := range make([]bool, n) { for j := range make([]bool, n) { - sh, err := getter.GetShare(ctx, dah, i, j) + sh, err := getter.GetShare(ctx, eh, i, j) assert.NotNil(t, sh) assert.NoError(t, err) } @@ -148,9 +155,11 @@ func TestService_GetSharesByNamespace(t *testing.T) { copy(share.GetNamespace(randShares[idx2]), share.GetNamespace(randShares[idx1])) } root := availability_test.FillBS(t, bServ, randShares) + eh := headertest.RandExtendedHeader(t) + eh.DAH = root randNamespace := share.GetNamespace(randShares[idx1]) - shares, err := getter.GetSharesByNamespace(context.Background(), root, randNamespace) + shares, err := getter.GetSharesByNamespace(context.Background(), eh, randNamespace) require.NoError(t, err) require.NoError(t, shares.Verify(root, randNamespace)) flattened := shares.Flatten() @@ -174,8 +183,10 @@ func TestService_GetSharesByNamespace(t *testing.T) { copy(share.GetNamespace(randShares[i]), lastNID) } root := availability_test.FillBS(t, bServ, randShares) + eh := headertest.RandExtendedHeader(t) + eh.DAH = root - shares, err := getter.GetSharesByNamespace(context.Background(), root, lastNID) + shares, err := getter.GetSharesByNamespace(context.Background(), eh, lastNID) require.NoError(t, err) require.NoError(t, shares.Verify(root, lastNID)) }) @@ -187,21 +198,21 @@ func TestGetShares(t *testing.T) { defer cancel() n := 16 - getter, dah := GetterWithRandSquare(t, n) + getter, eh := GetterWithRandSquare(t, n) - eds, err := getter.GetEDS(ctx, dah) + eds, err := getter.GetEDS(ctx, eh) require.NoError(t, err) gotDAH, err := share.NewRoot(eds) require.NoError(t, err) - require.True(t, dah.Equals(gotDAH)) + require.True(t, eh.DAH.Equals(gotDAH)) } func TestService_GetSharesByNamespaceNotFound(t *testing.T) { - getter, root := GetterWithRandSquare(t, 1) - root.RowRoots = nil + getter, eh := GetterWithRandSquare(t, 1) + eh.DAH.RowRoots = nil - emptyShares, err := getter.GetSharesByNamespace(context.Background(), root, sharetest.RandV0Namespace()) + emptyShares, err := getter.GetSharesByNamespace(context.Background(), eh, sharetest.RandV0Namespace()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) } @@ -218,12 +229,13 @@ func BenchmarkService_GetSharesByNamespace(b *testing.B) { for _, tt := range tests { b.Run(strconv.Itoa(tt.amountShares), func(b *testing.B) { t := &testing.T{} - getter, root := GetterWithRandSquare(t, tt.amountShares) + getter, eh := GetterWithRandSquare(t, tt.amountShares) + root := eh.DAH randNamespace := root.RowRoots[(len(root.RowRoots)-1)/2][:share.NamespaceSize] root.RowRoots[(len(root.RowRoots) / 2)] = root.RowRoots[(len(root.RowRoots)-1)/2] b.ResetTimer() for i := 0; i < b.N; i++ { - _, err := getter.GetSharesByNamespace(context.Background(), root, randNamespace) + _, err := getter.GetSharesByNamespace(context.Background(), eh, randNamespace) require.NoError(t, err) } }) diff --git a/share/availability/light/testing.go b/share/availability/light/testing.go index e68068fc25..9efc9ff14a 100644 --- a/share/availability/light/testing.go +++ b/share/availability/light/testing.go @@ -6,6 +6,8 @@ import ( "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-datastore" + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" "github.com/celestiaorg/celestia-node/share/getters" @@ -14,10 +16,14 @@ import ( // GetterWithRandSquare provides a share.Getter filled with 'n' NMT trees of 'n' random shares, // essentially storing a whole square. -func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *share.Root) { +func GetterWithRandSquare(t *testing.T, n int) (share.Getter, *header.ExtendedHeader) { bServ := ipld.NewMemBlockservice() getter := getters.NewIPLDGetter(bServ) - return getter, availability_test.RandFillBS(t, n, bServ) + root := availability_test.RandFillBS(t, n, bServ) + eh := headertest.RandExtendedHeader(t) + eh.DAH = root + + return getter, eh } // EmptyGetter provides an unfilled share.Getter with corresponding blockservice.BlockService than diff --git a/share/availability/mocks/availability.go b/share/availability/mocks/availability.go index ff4b8e1328..fc68d3d2bc 100644 --- a/share/availability/mocks/availability.go +++ b/share/availability/mocks/availability.go @@ -8,9 +8,8 @@ import ( context "context" reflect "reflect" + header "github.com/celestiaorg/celestia-node/header" gomock "github.com/golang/mock/gomock" - - da "github.com/celestiaorg/celestia-app/pkg/da" ) // MockAvailability is a mock of Availability interface. @@ -36,22 +35,8 @@ func (m *MockAvailability) EXPECT() *MockAvailabilityMockRecorder { return m.recorder } -// ProbabilityOfAvailability mocks base method. -func (m *MockAvailability) ProbabilityOfAvailability(arg0 context.Context) float64 { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "ProbabilityOfAvailability", arg0) - ret0, _ := ret[0].(float64) - return ret0 -} - -// ProbabilityOfAvailability indicates an expected call of ProbabilityOfAvailability. -func (mr *MockAvailabilityMockRecorder) ProbabilityOfAvailability(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ProbabilityOfAvailability", reflect.TypeOf((*MockAvailability)(nil).ProbabilityOfAvailability), arg0) -} - // SharesAvailable mocks base method. -func (m *MockAvailability) SharesAvailable(arg0 context.Context, arg1 *da.DataAvailabilityHeader) error { +func (m *MockAvailability) SharesAvailable(arg0 context.Context, arg1 *header.ExtendedHeader) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SharesAvailable", arg0, arg1) ret0, _ := ret[0].(error) diff --git a/share/eds/utils.go b/share/eds/utils.go index 3417a2aa62..2133565b91 100644 --- a/share/eds/utils.go +++ b/share/eds/utils.go @@ -1,12 +1,21 @@ package eds import ( + "context" + "errors" "fmt" "io" "github.com/filecoin-project/dagstore" + "github.com/ipfs/boxo/blockservice" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/trace" + "golang.org/x/sync/errgroup" + "github.com/celestiaorg/celestia-node/libs/utils" + "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/cache" + "github.com/celestiaorg/celestia-node/share/ipld" ) // readCloser is a helper struct, that combines io.Reader and io.Closer @@ -46,3 +55,98 @@ func closeAndLog(name string, closer io.Closer) { log.Warnw("closing "+name, "err", err) } } + +// RetrieveNamespaceFromStore gets all EDS shares in the given namespace from +// the EDS store through the corresponding CAR-level blockstore. It is extracted +// from the store getter to make it available for reuse in the shrexnd server. +func RetrieveNamespaceFromStore( + ctx context.Context, + store *Store, + dah *share.Root, + namespace share.Namespace, +) (shares share.NamespacedShares, err error) { + if err = namespace.ValidateForData(); err != nil { + return nil, err + } + + bs, err := store.CARBlockstore(ctx, dah.Hash()) + if errors.Is(err, ErrNotFound) { + // convert error to satisfy getter interface contract + err = share.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("failed to retrieve blockstore from eds store: %w", err) + } + defer func() { + if err := bs.Close(); err != nil { + log.Warnw("closing blockstore", "err", err) + } + }() + + // wrap the read-only CAR blockstore in a getter + blockGetter := NewBlockGetter(bs) + shares, err = CollectSharesByNamespace(ctx, blockGetter, dah, namespace) + if errors.Is(err, ipld.ErrNodeNotFound) { + // IPLD node not found after the index pointed to this shard and the CAR + // blockstore has been opened successfully is a strong indicator of + // corruption. We remove the block on bridges and fulls and return + // share.ErrNotFound to ensure the data is retrieved by the next getter. + // Note that this recovery is manual and will only be restored by an RPC + // call to SharesAvailable that fetches the same datahash that was + // removed. + err = store.Remove(ctx, dah.Hash()) + if err != nil { + log.Errorf("failed to remove CAR from store after detected corruption: %w", err) + } + err = share.ErrNotFound + } + if err != nil { + return nil, fmt.Errorf("failed to retrieve shares by namespace from store: %w", err) + } + + return shares, nil +} + +// CollectSharesByNamespace collects NamespaceShares within the given namespace from share.Root. +func CollectSharesByNamespace( + ctx context.Context, + bg blockservice.BlockGetter, + root *share.Root, + namespace share.Namespace, +) (shares share.NamespacedShares, err error) { + ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( + attribute.String("namespace", namespace.String()), + )) + defer func() { + utils.SetStatusAndEnd(span, err) + }() + + rootCIDs := ipld.FilterRootByNamespace(root, namespace) + if len(rootCIDs) == 0 { + return nil, nil + } + + errGroup, ctx := errgroup.WithContext(ctx) + shares = make([]share.NamespacedRow, len(rootCIDs)) + for i, rootCID := range rootCIDs { + // shadow loop variables, to ensure correct values are captured + i, rootCID := i, rootCID + errGroup.Go(func() error { + row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rootCID, namespace, len(root.RowRoots)) + shares[i] = share.NamespacedRow{ + Shares: row, + Proof: proof, + } + if err != nil { + return fmt.Errorf("retrieving shares by namespace %s for row %x: %w", namespace.String(), rootCID, err) + } + return nil + }) + } + + if err := errGroup.Wait(); err != nil { + return nil, err + } + + return shares, nil +} diff --git a/share/getter.go b/share/getter.go index 18d3873de1..3fcc93de33 100644 --- a/share/getter.go +++ b/share/getter.go @@ -8,6 +8,8 @@ import ( "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/header" ) var ( @@ -24,17 +26,17 @@ var ( //go:generate mockgen -destination=mocks/getter.go -package=mocks . Getter type Getter interface { // GetShare gets a Share by coordinates in EDS. - GetShare(ctx context.Context, root *Root, row, col int) (Share, error) + GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (Share, error) - // GetEDS gets the full EDS identified by the given root. - GetEDS(context.Context, *Root) (*rsmt2d.ExtendedDataSquare, error) + // GetEDS gets the full EDS identified by the given extended header. + GetEDS(context.Context, *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) // GetSharesByNamespace gets all shares from an EDS within the given namespace. // Shares are returned in a row-by-row order if the namespace spans multiple rows. // Inclusion of returned data could be verified using Verify method on NamespacedShares. // If no shares are found for target namespace non-inclusion could be also verified by calling // Verify method. - GetSharesByNamespace(context.Context, *Root, Namespace) (NamespacedShares, error) + GetSharesByNamespace(context.Context, *header.ExtendedHeader, Namespace) (NamespacedShares, error) } // NamespacedShares represents all shares with proofs within a specific namespace of an EDS. diff --git a/share/getters/cascade.go b/share/getters/cascade.go index eb3e969c1c..3875127580 100644 --- a/share/getters/cascade.go +++ b/share/getters/cascade.go @@ -9,6 +9,7 @@ import ( "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/byzantine" @@ -32,33 +33,37 @@ func NewCascadeGetter(getters []share.Getter) *CascadeGetter { } // GetShare gets a share from any of registered share.Getters in cascading order. -func (cg *CascadeGetter) GetShare(ctx context.Context, root *share.Root, row, col int) (share.Share, error) { +func (cg *CascadeGetter) GetShare( + ctx context.Context, header *header.ExtendedHeader, row, col int, +) (share.Share, error) { ctx, span := tracer.Start(ctx, "cascade/get-share", trace.WithAttributes( attribute.Int("row", row), attribute.Int("col", col), )) defer span.End() - upperBound := len(root.RowRoots) + upperBound := len(header.DAH.RowRoots) if row >= upperBound || col >= upperBound { err := share.ErrOutOfBounds span.RecordError(err) return nil, err } get := func(ctx context.Context, get share.Getter) (share.Share, error) { - return get.GetShare(ctx, root, row, col) + return get.GetShare(ctx, header, row, col) } return cascadeGetters(ctx, cg.getters, get) } // GetEDS gets a full EDS from any of registered share.Getters in cascading order. -func (cg *CascadeGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) { +func (cg *CascadeGetter) GetEDS( + ctx context.Context, header *header.ExtendedHeader, +) (*rsmt2d.ExtendedDataSquare, error) { ctx, span := tracer.Start(ctx, "cascade/get-eds") defer span.End() get := func(ctx context.Context, get share.Getter) (*rsmt2d.ExtendedDataSquare, error) { - return get.GetEDS(ctx, root) + return get.GetEDS(ctx, header) } return cascadeGetters(ctx, cg.getters, get) @@ -68,7 +73,7 @@ func (cg *CascadeGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d. // order. func (cg *CascadeGetter) GetSharesByNamespace( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, namespace share.Namespace, ) (share.NamespacedShares, error) { ctx, span := tracer.Start(ctx, "cascade/get-shares-by-namespace", trace.WithAttributes( @@ -77,7 +82,7 @@ func (cg *CascadeGetter) GetSharesByNamespace( defer span.End() get := func(ctx context.Context, get share.Getter) (share.NamespacedShares, error) { - return get.GetSharesByNamespace(ctx, root, namespace) + return get.GetSharesByNamespace(ctx, header, namespace) } return cascadeGetters(ctx, cg.getters, get) diff --git a/share/getters/cascade_test.go b/share/getters/cascade_test.go index d955c50682..d2b44883a1 100644 --- a/share/getters/cascade_test.go +++ b/share/getters/cascade_test.go @@ -11,6 +11,7 @@ import ( "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/mocks" ) @@ -20,24 +21,24 @@ func TestCascadeGetter(t *testing.T) { t.Cleanup(cancel) const gettersN = 3 - roots := make([]*share.Root, gettersN) + headers := make([]*header.ExtendedHeader, gettersN) getters := make([]share.Getter, gettersN) - for i := range roots { - getters[i], roots[i] = TestGetter(t) + for i := range headers { + getters[i], headers[i] = TestGetter(t) } getter := NewCascadeGetter(getters) t.Run("GetShare", func(t *testing.T) { - for _, r := range roots { - sh, err := getter.GetShare(ctx, r, 0, 0) + for _, eh := range headers { + sh, err := getter.GetShare(ctx, eh, 0, 0) assert.NoError(t, err) assert.NotEmpty(t, sh) } }) t.Run("GetEDS", func(t *testing.T) { - for _, r := range roots { - sh, err := getter.GetEDS(ctx, r) + for _, eh := range headers { + sh, err := getter.GetEDS(ctx, eh) assert.NoError(t, err) assert.NotEmpty(t, sh) } @@ -54,7 +55,7 @@ func TestCascade(t *testing.T) { successGetter := mocks.NewMockGetter(ctrl) ctxGetter := mocks.NewMockGetter(ctrl) timeoutGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, _ *share.Root) (*rsmt2d.ExtendedDataSquare, error) { + DoAndReturn(func(ctx context.Context, _ *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { return nil, context.DeadlineExceeded }).AnyTimes() immediateFailGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). @@ -62,7 +63,7 @@ func TestCascade(t *testing.T) { successGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). Return(nil, nil).AnyTimes() ctxGetter.EXPECT().GetEDS(gomock.Any(), gomock.Any()). - DoAndReturn(func(ctx context.Context, _ *share.Root) (*rsmt2d.ExtendedDataSquare, error) { + DoAndReturn(func(ctx context.Context, _ *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { return nil, ctx.Err() }).AnyTimes() diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 6c75ddf83f..77c470dae9 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -18,6 +18,8 @@ import ( dsbadger "github.com/celestiaorg/go-ds-badger4" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" @@ -41,92 +43,109 @@ func TestStoreGetter(t *testing.T) { sg := NewStoreGetter(edsStore) t.Run("GetShare", func(t *testing.T) { - randEds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, eh := randomEDS(t) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { - share, err := sg.GetShare(ctx, dah, i, j) + share, err := sg.GetShare(ctx, eh, i, j) require.NoError(t, err) assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } // doesn't panic on indexes too high - _, err := sg.GetShare(ctx, dah, squareSize, squareSize) + _, err := sg.GetShare(ctx, eh, squareSize, squareSize) require.ErrorIs(t, err, share.ErrOutOfBounds) // root not found - _, dah = randomEDS(t) - _, err = sg.GetShare(ctx, dah, 0, 0) + _, eh = randomEDS(t) + _, err = sg.GetShare(ctx, eh, 0, 0) require.ErrorIs(t, err, share.ErrNotFound) }) t.Run("GetEDS", func(t *testing.T) { - randEds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, eh := randomEDS(t) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) - retrievedEDS, err := sg.GetEDS(ctx, dah) + retrievedEDS, err := sg.GetEDS(ctx, eh) require.NoError(t, err) assert.True(t, randEds.Equals(retrievedEDS)) // root not found - root := share.Root{} - _, err = sg.GetEDS(ctx, &root) + emptyRoot := da.MinDataAvailabilityHeader() + eh.DAH = &emptyRoot + _, err = sg.GetEDS(ctx, eh) require.ErrorIs(t, err, share.ErrNotFound) }) t.Run("GetSharesByNamespace", func(t *testing.T) { - randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) - shares, err := sg.GetSharesByNamespace(ctx, dah, namespace) + shares, err := sg.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(dah, namespace)) + require.NoError(t, shares.Verify(eh.DAH, namespace)) assert.Len(t, shares.Flatten(), 2) // namespace not found randNamespace := sharetest.RandV0Namespace() - emptyShares, err := sg.GetSharesByNamespace(ctx, dah, randNamespace) + emptyShares, err := sg.GetSharesByNamespace(ctx, eh, randNamespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) // root not found - root := share.Root{} - _, err = sg.GetSharesByNamespace(ctx, &root, namespace) + emptyRoot := da.MinDataAvailabilityHeader() + eh.DAH = &emptyRoot + _, err = sg.GetSharesByNamespace(ctx, eh, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) t.Run("GetSharesFromNamespace removes corrupted shard", func(t *testing.T) { - randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) // available - shares, err := sg.GetSharesByNamespace(ctx, dah, namespace) + shares, err := sg.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(dah, namespace)) + require.NoError(t, shares.Verify(eh.DAH, namespace)) assert.Len(t, shares.Flatten(), 2) // 'corrupt' existing CAR by overwriting with a random EDS - f, err := os.OpenFile(tmpDir+"/blocks/"+dah.String(), os.O_WRONLY, 0644) + f, err := os.OpenFile(tmpDir+"/blocks/"+eh.DAH.String(), os.O_WRONLY, 0644) require.NoError(t, err) - edsToOverwriteWith, dah := randomEDS(t) + edsToOverwriteWith, eh := randomEDS(t) err = eds.WriteEDS(ctx, edsToOverwriteWith, f) require.NoError(t, err) - shares, err = sg.GetSharesByNamespace(ctx, dah, namespace) + shares, err = sg.GetSharesByNamespace(ctx, eh, namespace) require.ErrorIs(t, err, share.ErrNotFound) require.Nil(t, shares) // corruption detected, shard is removed - has, err := edsStore.Has(ctx, dah.Hash()) - require.False(t, has) - require.NoError(t, err) + // try every 200ms until it passes or the context ends + ticker := time.NewTicker(200 * time.Millisecond) + defer ticker.Stop() + for { + select { + case <-ctx.Done(): + t.Fatal("context ended before successful retrieval") + case <-ticker.C: + has, err := edsStore.Has(ctx, eh.DAH.Hash()) + if err != nil { + t.Fatal(err) + } + if !has { + require.NoError(t, err) + return + } + } + } }) } @@ -150,26 +169,26 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - randEds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, eh := randomEDS(t) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) squareSize := int(randEds.Width()) for i := 0; i < squareSize; i++ { for j := 0; j < squareSize; j++ { - share, err := sg.GetShare(ctx, dah, i, j) + share, err := sg.GetShare(ctx, eh, i, j) require.NoError(t, err) assert.Equal(t, randEds.GetCell(uint(i), uint(j)), share) } } // doesn't panic on indexes too high - _, err := sg.GetShare(ctx, dah, squareSize+1, squareSize+1) + _, err := sg.GetShare(ctx, eh, squareSize+1, squareSize+1) require.ErrorIs(t, err, share.ErrOutOfBounds) // root not found - _, dah = randomEDS(t) - _, err = sg.GetShare(ctx, dah, 0, 0) + _, eh = randomEDS(t) + _, err = sg.GetShare(ctx, eh, 0, 0) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -177,11 +196,11 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - randEds, dah := randomEDS(t) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, eh := randomEDS(t) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) - retrievedEDS, err := sg.GetEDS(ctx, dah) + retrievedEDS, err := sg.GetEDS(ctx, eh) require.NoError(t, err) assert.True(t, randEds.Equals(retrievedEDS)) @@ -196,25 +215,26 @@ func TestIPLDGetter(t *testing.T) { ctx, cancel := context.WithTimeout(ctx, time.Second) t.Cleanup(cancel) - randEds, namespace, dah := randomEDSWithDoubledNamespace(t, 4) - err = edsStore.Put(ctx, dah.Hash(), randEds) + randEds, namespace, eh := randomEDSWithDoubledNamespace(t, 4) + err = edsStore.Put(ctx, eh.DAH.Hash(), randEds) require.NoError(t, err) // first check that shares are returned correctly if they exist - shares, err := sg.GetSharesByNamespace(ctx, dah, namespace) + shares, err := sg.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) - require.NoError(t, shares.Verify(dah, namespace)) + require.NoError(t, shares.Verify(eh.DAH, namespace)) assert.Len(t, shares.Flatten(), 2) // namespace not found randNamespace := sharetest.RandV0Namespace() - emptyShares, err := sg.GetSharesByNamespace(ctx, dah, randNamespace) + emptyShares, err := sg.GetSharesByNamespace(ctx, eh, randNamespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) // nid doesnt exist in root - root := share.Root{} - emptyShares, err = sg.GetSharesByNamespace(ctx, &root, namespace) + emptyRoot := da.MinDataAvailabilityHeader() + eh.DAH = &emptyRoot + emptyShares, err = sg.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) @@ -248,16 +268,19 @@ func BenchmarkIPLDGetterOverBusyCache(b *testing.B) { edsStore := newStore(eds.DefaultParameters()) // generate EDSs and store them - hashes := make([]da.DataAvailabilityHeader, blocks) - for i := range hashes { + headers := make([]*header.ExtendedHeader, blocks) + for i := range headers { eds := edstest.RandEDS(b, size) dah, err := da.NewDataAvailabilityHeader(eds) require.NoError(b, err) err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(b, err) + eh := headertest.RandExtendedHeader(b) + eh.DAH = &dah + // store cids for read loop later - hashes[i] = dah + headers[i] = eh } // restart store to clear cache @@ -277,27 +300,31 @@ func BenchmarkIPLDGetterOverBusyCache(b *testing.B) { b.ResetTimer() g := sync.WaitGroup{} g.Add(blocks) - for _, h := range hashes { + for _, h := range headers { h := h go func() { defer g.Done() - _, err := getter.GetEDS(ctx, &h) + _, err := getter.GetEDS(ctx, h) require.NoError(b, err) }() } g.Wait() } -func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *share.Root) { +func randomEDS(t *testing.T) (*rsmt2d.ExtendedDataSquare, *header.ExtendedHeader) { eds := edstest.RandEDS(t, 4) dah, err := share.NewRoot(eds) require.NoError(t, err) - return eds, dah + eh := headertest.RandExtendedHeaderWithRoot(t, dah) + return eds, eh } // randomEDSWithDoubledNamespace generates a random EDS and ensures that there are two shares in the // middle that share a namespace. -func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedDataSquare, []byte, *share.Root) { +func randomEDSWithDoubledNamespace( + t *testing.T, + size int, +) (*rsmt2d.ExtendedDataSquare, []byte, *header.ExtendedHeader) { n := size * size randShares := sharetest.RandShares(t, n) idx1 := (n - 1) / 2 @@ -321,6 +348,7 @@ func randomEDSWithDoubledNamespace(t *testing.T, size int) (*rsmt2d.ExtendedData require.NoError(t, err, "failure to recompute the extended data square") dah, err := share.NewRoot(eds) require.NoError(t, err) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) - return eds, share.GetNamespace(randShares[idx1]), dah + return eds, share.GetNamespace(randShares[idx1]), eh } diff --git a/share/getters/ipld.go b/share/getters/ipld.go index 4024a67f5b..e9c930248d 100644 --- a/share/getters/ipld.go +++ b/share/getters/ipld.go @@ -13,6 +13,7 @@ import ( "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" @@ -39,7 +40,7 @@ func NewIPLDGetter(bServ blockservice.BlockService) *IPLDGetter { } // GetShare gets a single share at the given EDS coordinates from the bitswap network. -func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) { +func (ig *IPLDGetter) GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) { var err error ctx, span := tracer.Start(ctx, "ipld/get-share", trace.WithAttributes( attribute.Int("row", row), @@ -49,6 +50,7 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in utils.SetStatusAndEnd(span, err) }() + dah := header.DAH upperBound := len(dah.RowRoots) if row >= upperBound || col >= upperBound { err := share.ErrOutOfBounds @@ -71,14 +73,17 @@ func (ig *IPLDGetter) GetShare(ctx context.Context, dah *share.Root, row, col in return s, nil } -func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d.ExtendedDataSquare, err error) { +func (ig *IPLDGetter) GetEDS( + ctx context.Context, + header *header.ExtendedHeader, +) (eds *rsmt2d.ExtendedDataSquare, err error) { ctx, span := tracer.Start(ctx, "ipld/get-eds") defer func() { utils.SetStatusAndEnd(span, err) }() // rtrv.Retrieve calls shares.GetShares until enough shares are retrieved to reconstruct the EDS - eds, err = ig.rtrv.Retrieve(ctx, root) + eds, err = ig.rtrv.Retrieve(ctx, header.DAH) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound @@ -95,7 +100,7 @@ func (ig *IPLDGetter) GetEDS(ctx context.Context, root *share.Root) (eds *rsmt2d func (ig *IPLDGetter) GetSharesByNamespace( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "ipld/get-shares-by-namespace", trace.WithAttributes( @@ -111,7 +116,7 @@ func (ig *IPLDGetter) GetSharesByNamespace( // wrap the blockservice in a session if it has been signaled in the context. blockGetter := getGetter(ctx, ig.bServ) - shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) + shares, err = eds.CollectSharesByNamespace(ctx, blockGetter, header.DAH, namespace) if errors.Is(err, ipld.ErrNodeNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 5a2f696854..1c296a6a33 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -13,8 +13,10 @@ import ( "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" @@ -119,11 +121,11 @@ func (sg *ShrexGetter) Stop(ctx context.Context) error { return sg.peerManager.Stop(ctx) } -func (sg *ShrexGetter) GetShare(context.Context, *share.Root, int, int) (share.Share, error) { +func (sg *ShrexGetter) GetShare(context.Context, *header.ExtendedHeader, int, int) (share.Share, error) { return nil, fmt.Errorf("getter/shrex: GetShare %w", errOperationNotSupported) } -func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) { +func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { var ( attempt int err error @@ -133,6 +135,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex utils.SetStatusAndEnd(span, err) }() + dah := header.DAH for { if ctx.Err() != nil { sg.metrics.recordEDSAttempt(ctx, attempt, false) @@ -140,10 +143,10 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex } attempt++ start := time.Now() - peer, setStatus, getErr := sg.peerManager.Peer(ctx, root.Hash()) + peer, setStatus, getErr := sg.peerManager.Peer(ctx, dah.Hash()) if getErr != nil { log.Debugw("eds: couldn't find peer", - "hash", root.String(), + "hash", dah.String(), "err", getErr, "finished (s)", time.Since(start)) sg.metrics.recordEDSAttempt(ctx, attempt, false) @@ -152,7 +155,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - eds, getErr := sg.edsClient.RequestEDS(reqCtx, root.Hash(), peer) + eds, getErr := sg.edsClient.RequestEDS(reqCtx, dah.Hash(), peer) cancel() switch { case getErr == nil: @@ -175,7 +178,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex err = errors.Join(err, getErr) } log.Debugw("eds: request failed", - "hash", root.String(), + "hash", dah.String(), "peer", peer.String(), "attempt", attempt, "err", getErr, @@ -185,7 +188,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, root *share.Root) (*rsmt2d.Ex func (sg *ShrexGetter) GetSharesByNamespace( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, namespace share.Namespace, ) (share.NamespacedShares, error) { if err := namespace.ValidateForData(); err != nil { @@ -203,7 +206,8 @@ func (sg *ShrexGetter) GetSharesByNamespace( }() // verify that the namespace could exist inside the roots before starting network requests - roots := filterRootsByNamespace(root, namespace) + dah := header.DAH + roots := ipld.FilterRootByNamespace(dah, namespace) if len(roots) == 0 { return nil, nil } @@ -215,10 +219,10 @@ func (sg *ShrexGetter) GetSharesByNamespace( } attempt++ start := time.Now() - peer, setStatus, getErr := sg.peerManager.Peer(ctx, root.Hash()) + peer, setStatus, getErr := sg.peerManager.Peer(ctx, dah.Hash()) if getErr != nil { log.Debugw("nd: couldn't find peer", - "hash", root.String(), + "hash", dah.String(), "namespace", namespace.String(), "err", getErr, "finished (s)", time.Since(start)) @@ -228,12 +232,12 @@ func (sg *ShrexGetter) GetSharesByNamespace( reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - nd, getErr := sg.ndClient.RequestND(reqCtx, root, namespace, peer) + nd, getErr := sg.ndClient.RequestND(reqCtx, dah, namespace, peer) cancel() switch { case getErr == nil: // both inclusion and non-inclusion cases needs verification - if verErr := nd.Verify(root, namespace); verErr != nil { + if verErr := nd.Verify(dah, namespace); verErr != nil { getErr = verErr setStatus(peers.ResultBlacklistPeer) break @@ -257,7 +261,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( err = errors.Join(err, getErr) } log.Debugw("nd: request failed", - "hash", root.String(), + "hash", dah.String(), "namespace", namespace.String(), "peer", peer.String(), "attempt", attempt, diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index 85878b204a..b625bb4c10 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -23,6 +23,7 @@ import ( "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" "github.com/celestiaorg/celestia-node/share/p2p/peers" "github.com/celestiaorg/celestia-node/share/p2p/shrexeds" "github.com/celestiaorg/celestia-node/share/p2p/shrexnd" @@ -62,13 +63,14 @@ func TestShrexGetter(t *testing.T) { // generate test data namespace := sharetest.RandV0Namespace() randEDS, dah := edstest.RandEDSWithNamespace(t, namespace, 64) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - got, err := getter.GetSharesByNamespace(ctx, dah, namespace) + got, err := getter.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) require.NoError(t, got.Verify(dah, namespace)) }) @@ -79,12 +81,13 @@ func TestShrexGetter(t *testing.T) { // generate test data _, dah, namespace := generateTestEDS(t) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - _, err := getter.GetSharesByNamespace(ctx, dah, namespace) + _, err := getter.GetSharesByNamespace(ctx, eh, namespace) require.ErrorIs(t, err, share.ErrNotFound) }) @@ -94,6 +97,7 @@ func TestShrexGetter(t *testing.T) { // generate test data eds, dah, maxNamespace := generateTestEDS(t) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), @@ -103,9 +107,9 @@ func TestShrexGetter(t *testing.T) { nID, err := addToNamespace(maxNamespace, -1) require.NoError(t, err) // check for namespace to be between max and min namespace in root - require.Len(t, filterRootsByNamespace(dah, nID), 1) + require.Len(t, ipld.FilterRootByNamespace(dah, nID), 1) - emptyShares, err := getter.GetSharesByNamespace(ctx, dah, nID) + emptyShares, err := getter.GetSharesByNamespace(ctx, eh, nID) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) @@ -118,6 +122,7 @@ func TestShrexGetter(t *testing.T) { // generate test data eds, dah, maxNamesapce := generateTestEDS(t) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), @@ -127,9 +132,9 @@ func TestShrexGetter(t *testing.T) { namespace, err := addToNamespace(maxNamesapce, 1) require.NoError(t, err) // check for namespace to be not in root - require.Len(t, filterRootsByNamespace(dah, namespace), 0) + require.Len(t, ipld.FilterRootByNamespace(dah, namespace), 0) - emptyShares, err := getter.GetSharesByNamespace(ctx, dah, namespace) + emptyShares, err := getter.GetSharesByNamespace(ctx, eh, namespace) require.NoError(t, err) // no shares should be returned require.Empty(t, emptyShares.Flatten()) @@ -142,13 +147,14 @@ func TestShrexGetter(t *testing.T) { // generate test data randEDS, dah, _ := generateTestEDS(t) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) require.NoError(t, edsStore.Put(ctx, dah.Hash(), randEDS)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - got, err := getter.GetEDS(ctx, dah) + got, err := getter.GetEDS(ctx, eh) require.NoError(t, err) require.Equal(t, randEDS.Flattened(), got.Flattened()) }) @@ -158,13 +164,14 @@ func TestShrexGetter(t *testing.T) { // generate test data _, dah, _ := generateTestEDS(t) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) cancel() - _, err := getter.GetEDS(ctx, dah) + _, err := getter.GetEDS(ctx, eh) require.ErrorIs(t, err, context.Canceled) }) @@ -174,12 +181,13 @@ func TestShrexGetter(t *testing.T) { // generate test data _, dah, _ := generateTestEDS(t) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ DataHash: dah.Hash(), Height: 1, }) - _, err := getter.GetEDS(ctx, dah) + _, err := getter.GetEDS(ctx, eh) require.ErrorIs(t, err, share.ErrNotFound) }) } @@ -226,7 +234,7 @@ func newNDClientServer( params := shrexnd.DefaultParameters() // create server and register handler - server, err := shrexnd.NewServer(params, srvHost, edsStore, NewStoreGetter(edsStore)) + server, err := shrexnd.NewServer(params, srvHost, edsStore) require.NoError(t, err) require.NoError(t, server.Start(ctx)) diff --git a/share/getters/store.go b/share/getters/store.go index ad05cf8f37..d66a057c56 100644 --- a/share/getters/store.go +++ b/share/getters/store.go @@ -10,6 +10,7 @@ import ( "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" @@ -33,7 +34,8 @@ func NewStoreGetter(store *eds.Store) *StoreGetter { // GetShare gets a single share at the given EDS coordinates from the eds.Store through the // corresponding CAR-level blockstore. -func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col int) (share.Share, error) { +func (sg *StoreGetter) GetShare(ctx context.Context, header *header.ExtendedHeader, row, col int) (share.Share, error) { + dah := header.DAH var err error ctx, span := tracer.Start(ctx, "store/get-share", trace.WithAttributes( attribute.Int("row", row), @@ -79,13 +81,15 @@ func (sg *StoreGetter) GetShare(ctx context.Context, dah *share.Root, row, col i } // GetEDS gets the EDS identified by the given root from the EDS store. -func (sg *StoreGetter) GetEDS(ctx context.Context, root *share.Root) (data *rsmt2d.ExtendedDataSquare, err error) { +func (sg *StoreGetter) GetEDS( + ctx context.Context, header *header.ExtendedHeader, +) (data *rsmt2d.ExtendedDataSquare, err error) { ctx, span := tracer.Start(ctx, "store/get-eds") defer func() { utils.SetStatusAndEnd(span, err) }() - data, err = sg.store.Get(ctx, root.Hash()) + data, err = sg.store.Get(ctx, header.DAH.Hash()) if errors.Is(err, eds.ErrNotFound) { // convert error to satisfy getter interface contract err = share.ErrNotFound @@ -100,7 +104,7 @@ func (sg *StoreGetter) GetEDS(ctx context.Context, root *share.Root) (data *rsmt // corresponding CAR-level blockstore. func (sg *StoreGetter) GetSharesByNamespace( ctx context.Context, - root *share.Root, + header *header.ExtendedHeader, namespace share.Namespace, ) (shares share.NamespacedShares, err error) { ctx, span := tracer.Start(ctx, "store/get-shares-by-namespace", trace.WithAttributes( @@ -110,44 +114,9 @@ func (sg *StoreGetter) GetSharesByNamespace( utils.SetStatusAndEnd(span, err) }() - if err = namespace.ValidateForData(); err != nil { - return nil, err - } - - bs, err := sg.store.CARBlockstore(ctx, root.Hash()) - if errors.Is(err, eds.ErrNotFound) { - // convert error to satisfy getter interface contract - err = share.ErrNotFound - } + ns, err := eds.RetrieveNamespaceFromStore(ctx, sg.store, header.DAH, namespace) if err != nil { - return nil, fmt.Errorf("getter/store: failed to retrieve blockstore: %w", err) + return nil, fmt.Errorf("getter/store: %w", err) } - defer func() { - if err := bs.Close(); err != nil { - log.Warnw("closing blockstore", "err", err) - } - }() - - // wrap the read-only CAR blockstore in a getter - blockGetter := eds.NewBlockGetter(bs) - shares, err = collectSharesByNamespace(ctx, blockGetter, root, namespace) - if errors.Is(err, ipld.ErrNodeNotFound) { - // IPLD node not found after the index pointed to this shard and the CAR - // blockstore has been opened successfully is a strong indicator of - // corruption. We remove the block on bridges and fulls and return - // share.ErrNotFound to ensure the data is retrieved by the next getter. - // Note that this recovery is manual and will only be restored by an RPC - // call to SharesAvailable that fetches the same datahash that was - // removed. - err = sg.store.Remove(ctx, root.Hash()) - if err != nil { - log.Errorf("getter/store: failed to remove CAR after detected corruption: %w", err) - } - err = share.ErrNotFound - } - if err != nil { - return nil, fmt.Errorf("getter/store: failed to retrieve shares by namespace: %w", err) - } - - return shares, nil + return ns, nil } diff --git a/share/getters/testing.go b/share/getters/testing.go index aa0cc0e479..fafeb0541c 100644 --- a/share/getters/testing.go +++ b/share/getters/testing.go @@ -10,18 +10,21 @@ import ( "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" + "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/edstest" ) // TestGetter provides a testing SingleEDSGetter and the root of the EDS it holds. -func TestGetter(t *testing.T) (share.Getter, *share.Root) { +func TestGetter(t *testing.T) (share.Getter, *header.ExtendedHeader) { eds := edstest.RandEDS(t, 8) dah, err := share.NewRoot(eds) + eh := headertest.RandExtendedHeaderWithRoot(t, dah) require.NoError(t, err) return &SingleEDSGetter{ EDS: eds, - }, dah + }, eh } // SingleEDSGetter contains a single EDS where data is retrieved from. @@ -31,8 +34,12 @@ type SingleEDSGetter struct { } // GetShare gets a share from a kept EDS if exist and if the correct root is given. -func (seg *SingleEDSGetter) GetShare(_ context.Context, root *share.Root, row, col int) (share.Share, error) { - err := seg.checkRoot(root) +func (seg *SingleEDSGetter) GetShare( + _ context.Context, + header *header.ExtendedHeader, + row, col int, +) (share.Share, error) { + err := seg.checkRoot(header.DAH) if err != nil { return nil, err } @@ -40,8 +47,11 @@ func (seg *SingleEDSGetter) GetShare(_ context.Context, root *share.Root, row, c } // GetEDS returns a kept EDS if the correct root is given. -func (seg *SingleEDSGetter) GetEDS(_ context.Context, root *share.Root) (*rsmt2d.ExtendedDataSquare, error) { - err := seg.checkRoot(root) +func (seg *SingleEDSGetter) GetEDS( + _ context.Context, + header *header.ExtendedHeader, +) (*rsmt2d.ExtendedDataSquare, error) { + err := seg.checkRoot(header.DAH) if err != nil { return nil, err } @@ -49,7 +59,7 @@ func (seg *SingleEDSGetter) GetEDS(_ context.Context, root *share.Root) (*rsmt2d } // GetSharesByNamespace returns NamespacedShares from a kept EDS if the correct root is given. -func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *share.Root, share.Namespace, +func (seg *SingleEDSGetter) GetSharesByNamespace(context.Context, *header.ExtendedHeader, share.Namespace, ) (share.NamespacedShares, error) { panic("SingleEDSGetter: GetSharesByNamespace is not implemented") } diff --git a/share/getters/utils.go b/share/getters/utils.go index e3f24b3857..2260183b4f 100644 --- a/share/getters/utils.go +++ b/share/getters/utils.go @@ -3,20 +3,10 @@ package getters import ( "context" "errors" - "fmt" "time" - "github.com/ipfs/boxo/blockservice" - "github.com/ipfs/go-cid" logging "github.com/ipfs/go-log/v2" "go.opentelemetry.io/otel" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/trace" - "golang.org/x/sync/errgroup" - - "github.com/celestiaorg/celestia-node/libs/utils" - "github.com/celestiaorg/celestia-node/share" - "github.com/celestiaorg/celestia-node/share/ipld" ) var ( @@ -26,62 +16,6 @@ var ( errOperationNotSupported = errors.New("operation is not supported") ) -// filterRootsByNamespace returns the row roots from the given share.Root that contain the passed -// namespace. -func filterRootsByNamespace(root *share.Root, namespace share.Namespace) []cid.Cid { - rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) - for _, row := range root.RowRoots { - if !namespace.IsOutsideRange(row, row) { - rowRootCIDs = append(rowRootCIDs, ipld.MustCidFromNamespacedSha256(row)) - } - } - return rowRootCIDs -} - -// collectSharesByNamespace collects NamespaceShares within the given namespace from share.Root. -func collectSharesByNamespace( - ctx context.Context, - bg blockservice.BlockGetter, - root *share.Root, - namespace share.Namespace, -) (shares share.NamespacedShares, err error) { - ctx, span := tracer.Start(ctx, "collect-shares-by-namespace", trace.WithAttributes( - attribute.String("namespace", namespace.String()), - )) - defer func() { - utils.SetStatusAndEnd(span, err) - }() - - rootCIDs := filterRootsByNamespace(root, namespace) - if len(rootCIDs) == 0 { - return nil, nil - } - - errGroup, ctx := errgroup.WithContext(ctx) - shares = make([]share.NamespacedRow, len(rootCIDs)) - for i, rootCID := range rootCIDs { - // shadow loop variables, to ensure correct values are captured - i, rootCID := i, rootCID - errGroup.Go(func() error { - row, proof, err := ipld.GetSharesByNamespace(ctx, bg, rootCID, namespace, len(root.RowRoots)) - shares[i] = share.NamespacedRow{ - Shares: row, - Proof: proof, - } - if err != nil { - return fmt.Errorf("retrieving shares by namespace %s for row %x: %w", namespace.String(), rootCID, err) - } - return nil - }) - } - - if err := errGroup.Wait(); err != nil { - return nil, err - } - - return shares, nil -} - // ctxWithSplitTimeout will split timeout stored in context by splitFactor and return the result if // it is greater than minTimeout. minTimeout == 0 will be ignored, splitFactor <= 0 will be ignored func ctxWithSplitTimeout( diff --git a/share/ipld/corrupted_data_test.go b/share/ipld/corrupted_data_test.go index 58ddf785fb..d1d6e6b4d5 100644 --- a/share/ipld/corrupted_data_test.go +++ b/share/ipld/corrupted_data_test.go @@ -7,6 +7,7 @@ import ( "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-node/header/headertest" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/availability/full" availability_test "github.com/celestiaorg/celestia-node/share/availability/test" @@ -32,9 +33,11 @@ func TestNamespaceHasher_CorruptedData(t *testing.T) { // before the provider starts attacking, we should be able to retrieve successfully. We pass a size // 16 block, but this is not important to the test and any valid block size behaves the same. root := availability_test.RandFillBS(t, 16, provider.BlockService) + + eh := headertest.RandExtendedHeaderWithRoot(t, root) getCtx, cancelGet := context.WithTimeout(ctx, sharesAvailableTimeout) t.Cleanup(cancelGet) - err := requestor.SharesAvailable(getCtx, root) + err := requestor.SharesAvailable(getCtx, eh) require.NoError(t, err) // clear the storage of the requester so that it must retrieve again, then start attacking @@ -43,6 +46,6 @@ func TestNamespaceHasher_CorruptedData(t *testing.T) { mockBS.Attacking = true getCtx, cancelGet = context.WithTimeout(ctx, sharesAvailableTimeout) t.Cleanup(cancelGet) - err = requestor.SharesAvailable(getCtx, root) + err = requestor.SharesAvailable(getCtx, eh) require.ErrorIs(t, err, share.ErrNotAvailable) } diff --git a/share/ipld/utils.go b/share/ipld/utils.go new file mode 100644 index 0000000000..d3e987e7f3 --- /dev/null +++ b/share/ipld/utils.go @@ -0,0 +1,18 @@ +package ipld + +import ( + "github.com/ipfs/go-cid" + + "github.com/celestiaorg/celestia-node/share" +) + +// FilterRootByNamespace returns the row roots from the given share.Root that contain the namespace. +func FilterRootByNamespace(root *share.Root, namespace share.Namespace) []cid.Cid { + rowRootCIDs := make([]cid.Cid, 0, len(root.RowRoots)) + for _, row := range root.RowRoots { + if !namespace.IsOutsideRange(row, row) { + rowRootCIDs = append(rowRootCIDs, MustCidFromNamespacedSha256(row)) + } + } + return rowRootCIDs +} diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 2a1b84efd5..738e2b246c 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - - da "github.com/celestiaorg/celestia-app/pkg/da" + header "github.com/celestiaorg/celestia-node/header" share "github.com/celestiaorg/celestia-node/share" rsmt2d "github.com/celestiaorg/rsmt2d" + gomock "github.com/golang/mock/gomock" ) // MockGetter is a mock of Getter interface. @@ -39,7 +38,7 @@ func (m *MockGetter) EXPECT() *MockGetterMockRecorder { } // GetEDS mocks base method. -func (m *MockGetter) GetEDS(arg0 context.Context, arg1 *da.DataAvailabilityHeader) (*rsmt2d.ExtendedDataSquare, error) { +func (m *MockGetter) GetEDS(arg0 context.Context, arg1 *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetEDS", arg0, arg1) ret0, _ := ret[0].(*rsmt2d.ExtendedDataSquare) @@ -54,7 +53,7 @@ func (mr *MockGetterMockRecorder) GetEDS(arg0, arg1 interface{}) *gomock.Call { } // GetShare mocks base method. -func (m *MockGetter) GetShare(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2, arg3 int) ([]byte, error) { +func (m *MockGetter) GetShare(arg0 context.Context, arg1 *header.ExtendedHeader, arg2, arg3 int) ([]byte, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetShare", arg0, arg1, arg2, arg3) ret0, _ := ret[0].([]byte) @@ -69,7 +68,7 @@ func (mr *MockGetterMockRecorder) GetShare(arg0, arg1, arg2, arg3 interface{}) * } // GetSharesByNamespace mocks base method. -func (m *MockGetter) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 share.Namespace) (share.NamespacedShares, error) { +func (m *MockGetter) GetSharesByNamespace(arg0 context.Context, arg1 *header.ExtendedHeader, arg2 share.Namespace) (share.NamespacedShares, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSharesByNamespace", arg0, arg1, arg2) ret0, _ := ret[0].(share.NamespacedShares) diff --git a/share/p2p/shrexnd/exchange_test.go b/share/p2p/shrexnd/exchange_test.go index 99a7e2f8fb..cb8bbe9d74 100644 --- a/share/p2p/shrexnd/exchange_test.go +++ b/share/p2p/shrexnd/exchange_test.go @@ -13,8 +13,6 @@ import ( mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" "github.com/stretchr/testify/require" - "github.com/celestiaorg/rsmt2d" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/eds/edstest" @@ -25,7 +23,7 @@ import ( func TestExchange_RequestND_NotFound(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) t.Cleanup(cancel) - edsStore, client, server := makeExchange(t, notFoundGetter{}) + edsStore, client, server := makeExchange(t) require.NoError(t, edsStore.Start(ctx)) require.NoError(t, server.Start(ctx)) @@ -48,8 +46,8 @@ func TestExchange_RequestND_NotFound(t *testing.T) { require.NoError(t, err) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) - randNamespace := dah.RowRoots[(len(dah.RowRoots)-1)/2][:share.NamespaceSize] - emptyShares, err := client.RequestND(ctx, dah, randNamespace, server.host.ID()) + namespace := sharetest.RandV0Namespace() + emptyShares, err := client.RequestND(ctx, dah, namespace, server.host.ID()) require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) }) @@ -62,7 +60,7 @@ func TestExchange_RequestND(t *testing.T) { client, err := NewClient(DefaultParameters(), net.Hosts()[0]) require.NoError(t, err) - server, err := NewServer(DefaultParameters(), net.Hosts()[1], nil, nil) + server, err := NewServer(DefaultParameters(), net.Hosts()[1], nil) require.NoError(t, err) require.NoError(t, server.Start(context.Background())) @@ -103,26 +101,6 @@ func TestExchange_RequestND(t *testing.T) { }) } -type notFoundGetter struct{} - -func (m notFoundGetter) GetShare( - _ context.Context, _ *share.Root, _, _ int, -) (share.Share, error) { - return nil, share.ErrNotFound -} - -func (m notFoundGetter) GetEDS( - _ context.Context, _ *share.Root, -) (*rsmt2d.ExtendedDataSquare, error) { - return nil, share.ErrNotFound -} - -func (m notFoundGetter) GetSharesByNamespace( - _ context.Context, _ *share.Root, _ share.Namespace, -) (share.NamespacedShares, error) { - return nil, nil -} - func newStore(t *testing.T) *eds.Store { t.Helper() @@ -142,14 +120,14 @@ func createMocknet(t *testing.T, amount int) []libhost.Host { return net.Hosts() } -func makeExchange(t *testing.T, getter share.Getter) (*eds.Store, *Client, *Server) { +func makeExchange(t *testing.T) (*eds.Store, *Client, *Server) { t.Helper() store := newStore(t) hosts := createMocknet(t, 2) client, err := NewClient(DefaultParameters(), hosts[0]) require.NoError(t, err) - server, err := NewServer(DefaultParameters(), hosts[1], store, getter) + server, err := NewServer(DefaultParameters(), hosts[1], store) require.NoError(t, err) return store, client, server diff --git a/share/p2p/shrexnd/server.go b/share/p2p/shrexnd/server.go index 153bbbb1ba..33e61ff472 100644 --- a/share/p2p/shrexnd/server.go +++ b/share/p2p/shrexnd/server.go @@ -30,7 +30,6 @@ type Server struct { protocolID protocol.ID handler network.StreamHandler - getter share.Getter store *eds.Store params *Parameters @@ -39,13 +38,12 @@ type Server struct { } // NewServer creates new Server -func NewServer(params *Parameters, host host.Host, store *eds.Store, getter share.Getter) (*Server, error) { +func NewServer(params *Parameters, host host.Host, store *eds.Store) (*Server, error) { if err := params.Validate(); err != nil { return nil, fmt.Errorf("shrex-nd: server creation failed: %w", err) } srv := &Server{ - getter: getter, store: store, host: host, params: params, @@ -183,7 +181,7 @@ func (srv *Server) getNamespaceData(ctx context.Context, return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving DAH: %w", err) } - shares, err := srv.getter.GetSharesByNamespace(ctx, dah, namespace) + shares, err := eds.RetrieveNamespaceFromStore(ctx, srv.store, dah, namespace) if err != nil { return nil, pb.StatusCode_INTERNAL, fmt.Errorf("retrieving shares: %w", err) } From 00b32a808c6d800e4e820ea33a6b6c2811a0ad3a Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 9 Oct 2023 10:56:59 +0200 Subject: [PATCH 259/388] fix!: make API version match node version --- nodebuilder/node/admin.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder/node/admin.go b/nodebuilder/node/admin.go index 03c9cf1b52..8063835f1d 100644 --- a/nodebuilder/node/admin.go +++ b/nodebuilder/node/admin.go @@ -10,7 +10,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/authtoken" ) -const APIVersion = "v0.2.2" +const APIVersion = "v0.11.0" type module struct { tp Type From 3e56a048e5eea627d0df7d028bcc0a171f93c0e2 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Tue, 10 Oct 2023 15:01:34 +0200 Subject: [PATCH 260/388] chore(share/ipld): obliterate expensive ipld getter tracing (#2832) The load these traces can produce is massive. In the case of ODS size 128, and the whole block is sampled for namespace data, it will be 48896 spans (128*2-1)*128*2 - (128 * 128). Besides, there is zero value this tracing brought us so far. --- share/ipld/get.go | 15 --------------- share/ipld/get_shares.go | 6 ------ share/ipld/namespace_data.go | 18 ------------------ share/ipld/nmt.go | 4 +--- 4 files changed, 1 insertion(+), 42 deletions(-) diff --git a/share/ipld/get.go b/share/ipld/get.go index f263877dc0..adf2ffa8c5 100644 --- a/share/ipld/get.go +++ b/share/ipld/get.go @@ -10,8 +10,6 @@ import ( "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" "github.com/celestiaorg/celestia-node/share" ) @@ -100,9 +98,6 @@ func GetLeaves(ctx context.Context, maxShares int, put func(int, ipld.Node), ) { - ctx, span := tracer.Start(ctx, "get-leaves") - defer span.End() - // this buffer ensures writes to 'jobs' are never blocking (bin-tree-feat) jobs := make(chan *job, (maxShares+1)/2) // +1 for the case where 'maxShares' is 1 jobs <- &job{cid: root, ctx: ctx} @@ -120,21 +115,12 @@ func GetLeaves(ctx context.Context, // work over each job concurrently, s.t. shares do not block // processing of each other pool.Submit(func() { - ctx, span := tracer.Start(j.ctx, "process-job") - defer span.End() defer wg.Done() - span.SetAttributes( - attribute.String("cid", j.cid.String()), - attribute.Int("pos", j.sharePos), - ) - nd, err := GetNode(ctx, bGetter, j.cid) if err != nil { // we don't really care about errors here // just fetch as much as possible - span.RecordError(err) - span.SetStatus(codes.Error, err.Error()) return } // check links to know what we should do with the node @@ -142,7 +128,6 @@ func GetLeaves(ctx context.Context, if len(lnks) == 0 { // successfully fetched a share/leaf // ladies and gentlemen, we got em! - span.SetStatus(codes.Ok, "") put(j.sharePos, nd) return } diff --git a/share/ipld/get_shares.go b/share/ipld/get_shares.go index 3fda86941f..98db7012b5 100644 --- a/share/ipld/get_shares.go +++ b/share/ipld/get_shares.go @@ -32,9 +32,6 @@ func GetShare( // Does not return any error, and returns/unblocks only on success // (got all shares) or on context cancellation. func GetShares(ctx context.Context, bg blockservice.BlockGetter, root cid.Cid, shares int, put func(int, share.Share)) { - ctx, span := tracer.Start(ctx, "get-shares") - defer span.End() - putNode := func(i int, leaf format.Node) { put(i, leafToShare(leaf)) } @@ -51,9 +48,6 @@ func GetSharesByNamespace( namespace share.Namespace, maxShares int, ) ([]share.Share, *nmt.Proof, error) { - ctx, span := tracer.Start(ctx, "get-shares-by-namespace") - defer span.End() - data := NewNamespaceData(maxShares, namespace, WithLeaves(), WithProofs()) err := data.CollectLeavesByNamespace(ctx, bGetter, root) if err != nil { diff --git a/share/ipld/namespace_data.go b/share/ipld/namespace_data.go index a3f468ee47..5a6fd2abb4 100644 --- a/share/ipld/namespace_data.go +++ b/share/ipld/namespace_data.go @@ -10,8 +10,6 @@ import ( "github.com/ipfs/boxo/blockservice" "github.com/ipfs/go-cid" ipld "github.com/ipfs/go-ipld-format" - "go.opentelemetry.io/otel/attribute" - "go.opentelemetry.io/otel/codes" "github.com/celestiaorg/nmt" @@ -193,13 +191,6 @@ func (n *NamespaceData) CollectLeavesByNamespace( return err } - ctx, span := tracer.Start(ctx, "get-leaves-by-namespace") - defer span.End() - - span.SetAttributes( - attribute.String("namespace", n.namespace.String()), - ) - // buffer the jobs to avoid blocking, we only need as many // queued as the number of shares in the second-to-last layer jobs := make(chan job, (n.maxShares+1)/2) @@ -227,15 +218,8 @@ func (n *NamespaceData) CollectLeavesByNamespace( return retrievalErr } pool.Submit(func() { - ctx, span := tracer.Start(j.ctx, "process-job") - defer span.End() defer wg.done() - span.SetAttributes( - attribute.String("cid", j.cid.String()), - attribute.Int("pos", j.sharePos), - ) - // if an error is likely to be returned or not depends on // the underlying impl of the blockservice, currently it is not a realistic probability nd, err := GetNode(ctx, bGetter, j.cid) @@ -248,7 +232,6 @@ func (n *NamespaceData) CollectLeavesByNamespace( "pos", j.sharePos, "err", err, ) - span.SetStatus(codes.Error, err.Error()) // we still need to update the bounds n.addLeaf(j.sharePos, nil) return @@ -257,7 +240,6 @@ func (n *NamespaceData) CollectLeavesByNamespace( links := nd.Links() if len(links) == 0 { // successfully fetched a leaf belonging to the namespace - span.SetStatus(codes.Ok, "") // we found a leaf, so we update the bounds n.addLeaf(j.sharePos, nd) return diff --git a/share/ipld/nmt.go b/share/ipld/nmt.go index b43d786a5f..6dba300965 100644 --- a/share/ipld/nmt.go +++ b/share/ipld/nmt.go @@ -15,7 +15,6 @@ import ( logging "github.com/ipfs/go-log/v2" mh "github.com/multiformats/go-multihash" mhcore "github.com/multiformats/go-multihash/core" - "go.opentelemetry.io/otel" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/da" @@ -25,8 +24,7 @@ import ( ) var ( - tracer = otel.Tracer("ipld") - log = logging.Logger("ipld") + log = logging.Logger("ipld") ) const ( From 937eb433d937e4652d54b4356d7fce1aeaf5ded1 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Wed, 11 Oct 2023 18:27:11 +0400 Subject: [PATCH 261/388] fix(share/p2p/peer-manager) fix race for hasPeerCh pointer read (#2835) --- share/p2p/peers/pool.go | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/share/p2p/peers/pool.go b/share/p2p/peers/pool.go index 609d68e0b3..d0cc45ac44 100644 --- a/share/p2p/peers/pool.go +++ b/share/p2p/peers/pool.go @@ -89,8 +89,11 @@ func (p *pool) next(ctx context.Context) <-chan peer.ID { return } + p.m.RLock() + hasPeerCh := p.hasPeerCh + p.m.RUnlock() select { - case <-p.hasPeerCh: + case <-hasPeerCh: case <-ctx.Done(): return } From 127eca71279eb5d3b93ece5f04d997ac6f9a5408 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 11 Oct 2023 11:37:52 -0500 Subject: [PATCH 262/388] chore: bump to the official v1.0.0 tag of celestia-app (#2831) ## Overview bumps to the official tag for celestia-app. This is a non-breaking change. ## Checklist - [x] New and updated code has appropriate documentation - [x] New and updated code has new and/or updated testing - [x] Required CI checks are passing - [x] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- go.mod | 8 ++++---- go.sum | 21 ++++++++++----------- 2 files changed, 14 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index d39b4696fd..1c76383ffa 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0-rc18 + github.com/celestiaorg/celestia-app v1.0.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.3.3 @@ -76,10 +76,10 @@ require ( require ( github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/bits-and-blooms/bitset v1.5.0 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.10.0 // indirect + github.com/consensys/gnark-crypto v0.12.0 // indirect github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect @@ -292,7 +292,7 @@ require ( github.com/regen-network/cosmos-proto v0.3.1 // indirect github.com/rivo/uniseg v0.4.4 // indirect github.com/rs/cors v1.8.2 // indirect - github.com/rs/zerolog v1.29.1 // indirect + github.com/rs/zerolog v1.31.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil v3.21.6+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect diff --git a/go.sum b/go.sum index 7a986186d2..4e31203f91 100644 --- a/go.sum +++ b/go.sum @@ -315,8 +315,8 @@ github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d h1:xDfNPAt8lFiC1U github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d/go.mod h1:6QX/PXZ00z/TKoufEY6K/a0k6AhaJrQKdFe6OfVXsa4= github.com/bgentry/speakeasy v0.1.0 h1:ByYyxL9InA1OWqxJqqp2A5pYHUrCiAL6K3J+LKSsQkY= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= -github.com/bits-and-blooms/bitset v1.5.0 h1:NpE8frKRLGHIcEzkR+gZhiioW1+WbYV6fKwD6ZIpQT8= -github.com/bits-and-blooms/bitset v1.5.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.7.0 h1:YjAGVd3XmtK9ktAbX8Zg2g2PwLIMjGREZJHlV4j7NEo= +github.com/bits-and-blooms/bitset v1.7.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bmizerany/pat v0.0.0-20170815010413-6226ea591a40/go.mod h1:8rLXio+WjiTceGBHIoTvn60HIbs7Hm7bcHjyrSqYB9c= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= @@ -358,8 +358,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0-rc18 h1:d2+KYPr4ri10aOtnXih3hOUd6ap94c/Hv46JFzpWzy0= -github.com/celestiaorg/celestia-app v1.0.0-rc18/go.mod h1:Q375ijriqMv8PjKmMujNs2WzMAsgxjpbeUoQt+AZre0= +github.com/celestiaorg/celestia-app v1.0.0 h1:CJX0y3JE1gEKzb0Hr3/qS0Z/tZgYGJyCHJzc0+SVxYM= +github.com/celestiaorg/celestia-app v1.0.0/go.mod h1:8FN8f/iZC5D6A7QYJhK86nyRKrmQrzXFucV19vLv9Es= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 h1:c4cMVLU2bGTesZW1ZVgeoCB++gOOJTF3OvBsqBvo6n0= @@ -454,8 +454,8 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= -github.com/consensys/gnark-crypto v0.10.0 h1:zRh22SR7o4K35SoNqouS9J/TKHTyU2QWaj5ldehyXtA= -github.com/consensys/gnark-crypto v0.10.0/go.mod h1:Iq/P3HHl0ElSjsg2E1gsMwhAyxnxoKK5nVyZKd+/KhU= +github.com/consensys/gnark-crypto v0.12.0 h1:1OnSpOykNkUIBIBJKdhwy2p0JlW5o+Az02ICzZmvvdg= +github.com/consensys/gnark-crypto v0.12.0/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= @@ -1707,7 +1707,6 @@ github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVc github.com/mattn/go-colorable v0.1.7/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.8/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= github.com/mattn/go-colorable v0.1.9/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= -github.com/mattn/go-colorable v0.1.12/go.mod h1:u5H1YNBxpqRaxsYJYSkiCWKzEfiAb1Gb520KVy5xxl4= github.com/mattn/go-colorable v0.1.13 h1:fFA4WZxdEF4tXPZVKMLwD8oUnCTTo08duU7wxecdEvA= github.com/mattn/go-colorable v0.1.13/go.mod h1:7S9/ev0klgBDR4GtXTXX8a3vIGJpMovkB8vQcUbaXHg= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= @@ -2120,10 +2119,10 @@ github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= -github.com/rs/xid v1.4.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= +github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= -github.com/rs/zerolog v1.29.1 h1:cO+d60CHkknCbvzEWxP0S9K6KqyTjrCNUy1LdQLCGPc= -github.com/rs/zerolog v1.29.1/go.mod h1:Le6ESbR7hc+DP6Lt1THiV8CQSdkkNrd3R0XbEgp3ZBU= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday v1.5.2 h1:HyvC0ARfnZBqnXwABFeSZHpKvJHJJfPz81GNueLj0oo= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -2817,7 +2816,6 @@ golang.org/x/sys v0.0.0-20210819135213-f52c844e1c1c/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20210823070655-63515b42dcdf/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210903071746-97244b99971b/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20210908233432-aa78b53d3365/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.0.0-20210927094055-39ccf1dd6fa6/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211019181941-9d821ace8654/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211124211545-fe61309f8881/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= @@ -2854,6 +2852,7 @@ golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= From 7491d53ee80b14b35ffe86542943406c3fbb55c5 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 12 Oct 2023 15:08:16 +0400 Subject: [PATCH 263/388] fix(share/eds): fix races in cache tests (#2834) Fixes races in eds cache tests: - read / write isClosed flag in Cache mock - write / write for error object from 2 goroutines - read / write of cache object inside edsStore on cache swaps in tests. Resolves https://github.com/celestiaorg/celestia-node/issues/2833 --------- Co-authored-by: Wondertan --- share/eds/blockstore.go | 6 +++--- share/eds/cache/accessor_cache_test.go | 13 ++++++++++++- share/eds/metrics.go | 2 +- share/eds/store.go | 12 ++++++------ share/eds/store_test.go | 22 +++++++++++----------- 5 files changed, 33 insertions(+), 22 deletions(-) diff --git a/share/eds/blockstore.go b/share/eds/blockstore.go index 9cbb3f4e8a..e44601870e 100644 --- a/share/eds/blockstore.go +++ b/share/eds/blockstore.go @@ -6,7 +6,7 @@ import ( "fmt" bstore "github.com/ipfs/boxo/blockstore" - dshelp "github.com/ipfs/boxo/datastore/dshelp" + "github.com/ipfs/boxo/datastore/dshelp" blocks "github.com/ipfs/go-block-format" "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" @@ -154,13 +154,13 @@ func (bs *blockstore) getReadOnlyBlockstore(ctx context.Context, cid cid.Cid) (* // check if either cache contains an accessor shardKey := keys[0] - accessor, err := bs.store.cache.Get(shardKey) + accessor, err := bs.store.cache.Load().Get(shardKey) if err == nil { return blockstoreCloser(accessor) } // load accessor to the blockstore cache and use it as blockstoreCloser - accessor, err = bs.store.cache.Second().GetOrLoad(ctx, shardKey, bs.store.getAccessor) + accessor, err = bs.store.cache.Load().Second().GetOrLoad(ctx, shardKey, bs.store.getAccessor) if err != nil { return nil, fmt.Errorf("failed to get accessor for shard %s: %w", shardKey, err) } diff --git a/share/eds/cache/accessor_cache_test.go b/share/eds/cache/accessor_cache_test.go index 4f12301dee..347b251a88 100644 --- a/share/eds/cache/accessor_cache_test.go +++ b/share/eds/cache/accessor_cache_test.go @@ -5,6 +5,7 @@ import ( "context" "errors" "io" + "sync" "testing" "time" @@ -210,7 +211,8 @@ func TestAccessorCache(t *testing.T) { // initialize close done := make(chan struct{}) go func() { - err = cache.Remove(key) + err := cache.Remove(key) + require.NoError(t, err) close(done) }() @@ -284,16 +286,21 @@ func TestAccessorCache(t *testing.T) { } type mockAccessor struct { + m sync.Mutex data []byte isClosed bool returnedBs int } func (m *mockAccessor) Reader() io.Reader { + m.m.Lock() + defer m.m.Unlock() return bytes.NewBuffer(m.data) } func (m *mockAccessor) Blockstore() (dagstore.ReadBlockstore, error) { + m.m.Lock() + defer m.m.Unlock() if m.returnedBs > 0 { return nil, errors.New("blockstore already returned") } @@ -302,6 +309,8 @@ func (m *mockAccessor) Blockstore() (dagstore.ReadBlockstore, error) { } func (m *mockAccessor) Close() error { + m.m.Lock() + defer m.m.Unlock() if m.isClosed { return errors.New("already closed") } @@ -312,6 +321,8 @@ func (m *mockAccessor) Close() error { func (m *mockAccessor) checkClosed(t *testing.T, expected bool) { // item will be removed in background, so give it some time to settle time.Sleep(time.Millisecond * 100) + m.m.Lock() + defer m.m.Unlock() require.Equal(t, expected, m.isClosed) } diff --git a/share/eds/metrics.go b/share/eds/metrics.go index 8f87643f17..cbebf8321a 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -124,7 +124,7 @@ func (s *Store) WithMetrics() error { return err } - if err = s.cache.EnableMetrics(); err != nil { + if err = s.cache.Load().EnableMetrics(); err != nil { return err } diff --git a/share/eds/store.go b/share/eds/store.go index 9fc90046fc..816065909e 100644 --- a/share/eds/store.go +++ b/share/eds/store.go @@ -49,7 +49,7 @@ type Store struct { mounts *mount.Registry bs *blockstore - cache *cache.DoubleCache + cache atomic.Pointer[cache.DoubleCache] carIdx index.FullIndexRepo invertedIdx *simpleInvertedIndex @@ -129,9 +129,9 @@ func NewStore(params *Parameters, basePath string, ds datastore.Batching) (*Stor gcInterval: params.GCInterval, mounts: r, shardFailures: failureChan, - cache: cache.NewDoubleCache(recentBlocksCache, blockstoreCache), } store.bs = newBlockstore(store, ds) + store.cache.Store(cache.NewDoubleCache(recentBlocksCache, blockstoreCache)) return store, nil } @@ -286,7 +286,7 @@ func (s *Store) put(ctx context.Context, root share.DataHash, square *rsmt2d.Ext go func() { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) defer cancel() - ac, err := s.cache.First().GetOrLoad(ctx, result.Key, s.getAccessor) + ac, err := s.cache.Load().First().GetOrLoad(ctx, result.Key, s.getAccessor) if err != nil { log.Warnw("unable to put accessor to recent blocks accessors cache", "err", err) return @@ -347,7 +347,7 @@ func (s *Store) GetCAR(ctx context.Context, root share.DataHash) (io.ReadCloser, func (s *Store) getCAR(ctx context.Context, root share.DataHash) (io.ReadCloser, error) { key := shard.KeyFromString(root.String()) - accessor, err := s.cache.Get(key) + accessor, err := s.cache.Load().Get(key) if err == nil { return newReadCloser(accessor), nil } @@ -391,7 +391,7 @@ func (s *Store) carBlockstore( root share.DataHash, ) (*BlockstoreCloser, error) { key := shard.KeyFromString(root.String()) - accessor, err := s.cache.Get(key) + accessor, err := s.cache.Load().Get(key) if err == nil { return blockstoreCloser(accessor) } @@ -482,7 +482,7 @@ func (s *Store) Remove(ctx context.Context, root share.DataHash) error { func (s *Store) remove(ctx context.Context, root share.DataHash) (err error) { key := shard.KeyFromString(root.String()) // remove open links to accessor from cache - if err := s.cache.Remove(key); err != nil { + if err := s.cache.Load().Remove(key); err != nil { log.Warnw("remove accessor from cache", "err", err) } ch := make(chan dagstore.ShardResult, 1) diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 7052533555..09357347d0 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -158,7 +158,7 @@ func TestEDSStore(t *testing.T) { time.Sleep(time.Millisecond * 100) // remove non-failed accessor from cache - err = edsStore.cache.Remove(shard.KeyFromString(dah.String())) + err = edsStore.cache.Load().Remove(shard.KeyFromString(dah.String())) assert.NoError(t, err) _, err = edsStore.GetCAR(ctx, dah.Hash()) @@ -205,7 +205,7 @@ func TestEDSStore(t *testing.T) { // check, that the key is in the cache after put shardKey := shard.KeyFromString(dah.String()) - _, err = edsStore.cache.Get(shardKey) + _, err = edsStore.cache.Load().Get(shardKey) assert.NoError(t, err) }) @@ -276,7 +276,7 @@ func TestEDSStore_GC(t *testing.T) { // remove links to the shard from cache time.Sleep(time.Millisecond * 100) key := shard.KeyFromString(share.DataHash(dah.Hash()).String()) - err = edsStore.cache.Remove(key) + err = edsStore.cache.Load().Remove(key) require.NoError(t, err) // doesn't exist yet @@ -305,8 +305,8 @@ func Test_BlockstoreCache(t *testing.T) { require.NoError(t, err) // store eds to the store with noopCache to allow clean cache after put - swap := edsStore.cache - edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) + swap := edsStore.cache.Load() + edsStore.cache.Store(cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{})) eds, dah := randomEDS(t) err = edsStore.Put(ctx, dah.Hash(), eds) require.NoError(t, err) @@ -327,11 +327,11 @@ func Test_BlockstoreCache(t *testing.T) { } // swap back original cache - edsStore.cache = swap + edsStore.cache.Store(swap) // key shouldn't be in cache yet, check for returned errCacheMiss shardKey := shard.KeyFromString(dah.String()) - _, err = edsStore.cache.Get(shardKey) + _, err = edsStore.cache.Load().Get(shardKey) require.Error(t, err) // now get it from blockstore, to trigger storing to cache @@ -339,7 +339,7 @@ func Test_BlockstoreCache(t *testing.T) { require.NoError(t, err) // should be no errCacheMiss anymore - _, err = edsStore.cache.Get(shardKey) + _, err = edsStore.cache.Load().Get(shardKey) require.NoError(t, err) } @@ -362,7 +362,7 @@ func Test_CachedAccessor(t *testing.T) { time.Sleep(time.Millisecond * 100) // accessor should be in cache - _, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) + _, err = edsStore.cache.Load().Get(shard.KeyFromString(dah.String())) require.NoError(t, err) // first read from cached accessor @@ -393,7 +393,7 @@ func Test_NotCachedAccessor(t *testing.T) { err = edsStore.Start(ctx) require.NoError(t, err) // replace cache with noopCache to - edsStore.cache = cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{}) + edsStore.cache.Store(cache.NewDoubleCache(cache.NoopCache{}, cache.NoopCache{})) eds, dah := randomEDS(t) err = edsStore.Put(ctx, dah.Hash(), eds) @@ -403,7 +403,7 @@ func Test_NotCachedAccessor(t *testing.T) { time.Sleep(time.Millisecond * 100) // accessor should not be in cache - _, err = edsStore.cache.Get(shard.KeyFromString(dah.String())) + _, err = edsStore.cache.Load().Get(shard.KeyFromString(dah.String())) require.Error(t, err) // first read from direct accessor (not from cache) From f1a302bba63415a8b228065dbfce5d6ceb37eff6 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Thu, 12 Oct 2023 11:18:10 -0400 Subject: [PATCH 264/388] chore: Update docker-build-publish.yml (#2838) This fixes the bug that rollkit local-celestia-devent was experiencing with the docker images being overwritten. The changes to the reuseable docker action simplify the logic for easier maintenance. The changes to the node CI file are to only run the docker action on a PR or pushes to main. When the workflow was triggering on all push events, the action was running twice on PRs which was contributing to the confusion and causing images to be overwritten. This means if the team wants a docker image built for testing, they must open at least a draft PR. The PR doesn't need to be merged, but the PR trigger is what will built the image. --- .github/workflows/docker-build-publish.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index c2589ff141..72d3f90f95 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -5,7 +5,7 @@ on: merge_group: push: branches: - - "**" + - "main" tags: - "v[0-9]+.[0-9]+.[0-9]+" - "v[0-9]+.[0-9]+.[0-9]+-alpha.[0-9]+" @@ -18,6 +18,6 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.3 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.7 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile From 17619db964bf6d854c38ea3294aef642991f5093 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Fri, 13 Oct 2023 12:11:56 +0200 Subject: [PATCH 265/388] fix(modp2p): temporary disable quic (#2837) Disables QUIC by default yet allows to enable it with ENV var. This is temporary and until libp2p/go-libp2p#2591 is investigated and fixed. The current solution disables QUIC programmatically while we still keep writing in the config QUIC listen addresses by default. This removes the need for users to reinit their configs once we turn QUIC back by default. This works perfectly fine on mocha. --- nodebuilder/p2p/addrs.go | 17 ++++++++++++++--- nodebuilder/p2p/host.go | 20 +++++++++++++++++++- 2 files changed, 33 insertions(+), 4 deletions(-) diff --git a/nodebuilder/p2p/addrs.go b/nodebuilder/p2p/addrs.go index d8f50c8144..528f58d722 100644 --- a/nodebuilder/p2p/addrs.go +++ b/nodebuilder/p2p/addrs.go @@ -2,6 +2,7 @@ package p2p import ( "fmt" + "slices" p2pconfig "github.com/libp2p/go-libp2p/config" hst "github.com/libp2p/go-libp2p/core/host" @@ -11,13 +12,23 @@ import ( // Listen returns invoke function that starts listening for inbound connections with libp2p.Host. func Listen(listen []string) func(h hst.Host) (err error) { return func(h hst.Host) (err error) { - maListen := make([]ma.Multiaddr, len(listen)) - for i, addr := range listen { - maListen[i], err = ma.NewMultiaddr(addr) + maListen := make([]ma.Multiaddr, 0, len(listen)) + for _, addr := range listen { + maddr, err := ma.NewMultiaddr(addr) if err != nil { return fmt.Errorf("failure to parse config.P2P.ListenAddresses: %s", err) } + + // TODO(@WonderTan): Remove this check when QUIC is stable + if slices.ContainsFunc(maddr.Protocols(), func(p ma.Protocol) bool { + return p.Code == ma.P_QUIC_V1 || p.Code == ma.P_WEBTRANSPORT + }) { + continue + } + + maListen = append(maListen, maddr) } + return h.Network().Listen(maListen...) } } diff --git a/nodebuilder/p2p/host.go b/nodebuilder/p2p/host.go index e55cb65d1f..c3943a02fa 100644 --- a/nodebuilder/p2p/host.go +++ b/nodebuilder/p2p/host.go @@ -3,6 +3,7 @@ package p2p import ( "context" "fmt" + "os" "github.com/libp2p/go-libp2p" p2pconfig "github.com/libp2p/go-libp2p/config" @@ -16,12 +17,22 @@ import ( "github.com/libp2p/go-libp2p/core/routing" routedhost "github.com/libp2p/go-libp2p/p2p/host/routed" "github.com/libp2p/go-libp2p/p2p/net/conngater" + quic "github.com/libp2p/go-libp2p/p2p/transport/quic" + "github.com/libp2p/go-libp2p/p2p/transport/tcp" + webtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport" "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "github.com/celestiaorg/celestia-node/nodebuilder/node" ) +var enableQUIC bool + +func init() { + _, ok := os.LookupEnv("CELESTIA_ENABLE_QUIC") + enableQUIC = ok +} + // routedHost constructs a wrapped Host that may fallback to address discovery, // if any top-level operation on the Host is provided with PeerID(Hash(PbK)) only. func routedHost(base HostBase, r routing.PeerRouting) hst.Host { @@ -44,8 +55,15 @@ func host(params hostParams) (HostBase, error) { libp2p.ResourceManager(params.ResourceManager), // to clearly define what defaults we rely upon libp2p.DefaultSecurity, - libp2p.DefaultTransports, libp2p.DefaultMuxers, + libp2p.Transport(tcp.NewTCPTransport), + } + + if enableQUIC { + opts = append(opts, + libp2p.Transport(quic.NewTransport), + libp2p.Transport(webtransport.New), + ) } if params.Registry != nil { From a52e2de4a76de0e25e53b04f82894d0f44888bd1 Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 13 Oct 2023 12:30:26 +0200 Subject: [PATCH 266/388] chore: adding mainnet chain id and setting as default (#2840) :) Also removed blockspace race Part of #2823 --- nodebuilder/p2p/bootstrap.go | 6 +----- nodebuilder/p2p/genesis.go | 8 ++++---- nodebuilder/p2p/network.go | 22 +++++++++++----------- 3 files changed, 16 insertions(+), 20 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index 47b39eed31..e3971753a1 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -37,6 +37,7 @@ func bootstrappersFor(net Network) ([]string, error) { // NOTE: Every time we add a new long-running network, its bootstrap peers have to be added here. var bootstrapList = map[Network][]string{ + Mainnet: {}, Arabica: { "/dns4/da-bridge.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWM3e9MWtyc8GkP8QRt74Riu17QuhGfZMytB2vq5NwkWAu", "/dns4/da-bridge-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWKj8mcdiBGxQRe1jqhaMnh2tGoC3rPDmr5UH2q8H4WA9M", @@ -49,11 +50,6 @@ var bootstrapList = map[Network][]string{ "/dns4/da-full-1-mocha-4.celestia-mocha.com/tcp/2121/p2p/12D3KooWCUHPLqQXZzpTx1x3TAsdn3vYmTNDhzg66yG8hqoxGGN8", "/dns4/da-full-2-mocha-4.celestia-mocha.com/tcp/2121/p2p/12D3KooWR6SHsXPkkvhCRn6vp1RqSefgaT1X1nMNvrVjU2o3GoYy", }, - BlockspaceRace: { - "/dns4/bootstr-incent-3.celestia.tools/tcp/2121/p2p/12D3KooWNzdKcHagtvvr6qtjcPTAdCN6ZBiBLH8FBHbihxqu4GZx", - "/dns4/bootstr-incent-2.celestia.tools/tcp/2121/p2p/12D3KooWNJZyWeCsrKxKrxsNM1RVL2Edp77svvt7Cosa63TggC9m", - "/dns4/bootstr-incent-1.celestia.tools/tcp/2121/p2p/12D3KooWBtxdBzToQwnS4ySGpph9PtGmmjEyATkgX3PfhAo4xmf7", - }, Private: {}, } diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index 5066db5177..c88ed53ef3 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -23,8 +23,8 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ - Arabica: "5904E55478BA4B3002EE885621E007A2A6A2399662841912219AECD5D5CBE393", - Mocha: "B93BBE20A0FBFDF955811B6420F8433904664D45DB4BF51022BE4200C1A1680D", - BlockspaceRace: "1A8491A72F73929680DAA6C93E3B593579261B2E76536BFA4F5B97D6FE76E088", - Private: "", + Mainnet: "", + Arabica: "5904E55478BA4B3002EE885621E007A2A6A2399662841912219AECD5D5CBE393", + Mocha: "B93BBE20A0FBFDF955811B6420F8433904664D45DB4BF51022BE4200C1A1680D", + Private: "", } diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index a6197aa321..a7f9ff7236 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -10,15 +10,15 @@ import ( // NOTE: Every time we add a new long-running network, it has to be added here. const ( // DefaultNetwork is the default network of the current build. - DefaultNetwork = Mocha + DefaultNetwork = Mainnet // Arabica testnet. See: celestiaorg/networks. Arabica Network = "arabica-10" // Mocha testnet. See: celestiaorg/networks. Mocha Network = "mocha-4" - // BlockspaceRace testnet. See: https://docs.celestia.org/nodes/blockspace-race/. - BlockspaceRace Network = "blockspacerace-0" // Private can be used to set up any private network, including local testing setups. Private Network = "private" + // Celestia mainnet. See: celestiaorg/networks. + Mainnet Network = "celestia" // BlockTime is a network block time. // TODO @renaynay @Wondertan (#790) BlockTime = time.Second * 10 @@ -52,20 +52,20 @@ func (n Network) String() string { // networksList is a strict list of all known long-standing networks. var networksList = map[Network]struct{}{ - Arabica: {}, - Mocha: {}, - BlockspaceRace: {}, - Private: {}, + Mainnet: {}, + Arabica: {}, + Mocha: {}, + Private: {}, } // networkAliases is a strict list of all known long-standing networks // mapped from the string representation of their *alias* (rather than // their actual value) to the Network. var networkAliases = map[string]Network{ - "arabica": Arabica, - "mocha": Mocha, - "blockspacerace": BlockspaceRace, - "private": Private, + "mainnet": Mainnet, + "arabica": Arabica, + "mocha": Mocha, + "private": Private, } // listProvidedNetworks provides a string listing all known long-standing networks for things like From 34775bbf11e102f4a2457e09acdfc83c288ded73 Mon Sep 17 00:00:00 2001 From: Matthew Sevey Date: Fri, 13 Oct 2023 11:13:09 -0400 Subject: [PATCH 267/388] bug: fix duplicate docker build stages (#2845) Final bug fix celestiaorg/.github#82 Visual proof of action executing as expected now celestiaorg/celestia-node/actions/runs/6509222904/job/17680437495 Screen Shot 2023-10-13 at 10 13 28 AM --- .github/workflows/docker-build-publish.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/docker-build-publish.yml b/.github/workflows/docker-build-publish.yml index 72d3f90f95..b8322d5e17 100644 --- a/.github/workflows/docker-build-publish.yml +++ b/.github/workflows/docker-build-publish.yml @@ -18,6 +18,6 @@ jobs: permissions: contents: write packages: write - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.7 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_pipeline.yml@v0.2.8 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile From 4f2a44b4573399c61c3ad1ff857df6ebf150ee2a Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 13 Oct 2023 18:22:46 +0300 Subject: [PATCH 268/388] fix(blob/service): fix getByCommitment (#2828) Fixes https://github.com/celestiaorg/celestia-node/issues/2808 Co-authored-by: Hlib Kanunnikov --- blob/blob.go | 36 ++++++++++++ blob/helper.go | 27 +++------ blob/service.go | 130 ++++++++++++++++++++----------------------- blob/service_test.go | 19 +++++++ 4 files changed, 122 insertions(+), 90 deletions(-) diff --git a/blob/blob.go b/blob/blob.go index c5fd04c782..3f7e92dd20 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -3,11 +3,13 @@ package blob import ( "bytes" "encoding/json" + "errors" "fmt" tmproto "github.com/tendermint/tendermint/proto/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/appconsts" + "github.com/celestiaorg/celestia-app/pkg/shares" "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/nmt" @@ -138,3 +140,37 @@ func (b *Blob) UnmarshalJSON(data []byte) error { b.namespace = blob.Namespace return nil } + +// buildBlobsIfExist takes shares and tries building the Blobs from them. +// It will build blobs either until appShares will be empty or the first incomplete blob will appear, so in this +// specific case it will return all built blobs + remaining shares. +func buildBlobsIfExist(appShares []shares.Share) ([]*Blob, []shares.Share, error) { + if len(appShares) == 0 { + return nil, nil, errors.New("empty shares received") + } + blobs := make([]*Blob, 0, len(appShares)) + for { + length, err := appShares[0].SequenceLen() + if err != nil { + return nil, nil, err + } + + amount := shares.SparseSharesNeeded(length) + if amount > len(appShares) { + return blobs, appShares, nil + } + + b, err := parseShares(appShares[:amount]) + if err != nil { + return nil, nil, err + } + + // only 1 blob will be created bc we passed the exact amount of shares + blobs = append(blobs, b[0]) + + if amount == len(appShares) { + return blobs, nil, nil + } + appShares = appShares[amount:] + } +} diff --git a/blob/helper.go b/blob/helper.go index 5627fac998..72a56c7889 100644 --- a/blob/helper.go +++ b/blob/helper.go @@ -17,16 +17,16 @@ func SharesToBlobs(rawShares []share.Share) ([]*Blob, error) { return nil, ErrBlobNotFound } - appShares := make([]shares.Share, 0, len(rawShares)) - for _, shr := range rawShares { - bShare, err := shares.NewShare(shr) - if err != nil { - return nil, err - } - appShares = append(appShares, *bShare) + appShares, err := toAppShares(rawShares...) + if err != nil { + return nil, err } + return parseShares(appShares) +} - shareSequences, err := shares.ParseShares(appShares, true) +// parseShares takes shares and converts them to the []*Blob. +func parseShares(appShrs []shares.Share) ([]*Blob, error) { + shareSequences, err := shares.ParseShares(appShrs, true) if err != nil { return nil, err } @@ -84,14 +84,3 @@ func BlobsToShares(blobs ...*Blob) ([]share.Share, error) { } return shares.ToBytes(rawShares), nil } - -// constructAndVerifyBlob reconstruct a Blob from the passed shares and compares commitments. -func constructAndVerifyBlob(sh []share.Share, commitment Commitment) (*Blob, bool, error) { - blob, err := SharesToBlobs(sh) - if err != nil { - return nil, false, err - } - - equal := blob[0].Commitment.Equal(commitment) - return blob[0], equal, nil -} diff --git a/blob/service.go b/blob/service.go index cf3daff7de..88b6c63d3f 100644 --- a/blob/service.go +++ b/blob/service.go @@ -197,13 +197,6 @@ func (s *Service) getByCommitment( return nil, nil, err } - var ( - rawShares = make([]share.Share, 0) - proofs = make(Proof, 0) - amount int - blobShare *shares.Share - ) - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) if err != nil { if errors.Is(err, share.ErrNotFound) { @@ -211,84 +204,57 @@ func (s *Service) getByCommitment( } return nil, nil, err } - for _, row := range namespacedShares { - if len(row.Shares) == 0 { - break - } - - rawShares = append(rawShares, row.Shares...) - proofs = append(proofs, row.Proof) - // reconstruct the `blobShare` from the first rawShare in range - // in order to get blob's length(first share will contain this info) - if blobShare == nil { - for i, shr := range rawShares { - bShare, err := shares.NewShare(shr) - if err != nil { - return nil, nil, err - } - - // ensure that the first share is not a NamespacePaddingShare - // these shares are used to satisfy the non-interactive default rules - // and are not the part of the blob, so should be removed. - isPadding, err := bShare.IsPadding() - if err != nil { - return nil, nil, err - } - if isPadding { - continue - } - - blobShare = bShare - // save the length. - length, err := blobShare.SequenceLen() - if err != nil { - return nil, nil, err - } - amount = shares.SparseSharesNeeded(length) - rawShares = rawShares[i:] - break - } - } + var ( + rawShares = make([]shares.Share, 0) + proofs = make(Proof, 0) + // spansMultipleRows specifies whether blob is expanded into multiple rows + spansMultipleRows bool + ) - // move to the next row if the blob is incomplete. - if amount > len(rawShares) { - continue + for _, row := range namespacedShares { + appShares, err := toAppShares(row.Shares...) + if err != nil { + return nil, nil, err } + rawShares = append(rawShares, appShares...) + proofs = append(proofs, row.Proof) - blob, same, err := constructAndVerifyBlob(rawShares[:amount], commitment) + var blobs []*Blob + blobs, rawShares, err = buildBlobsIfExist(rawShares) if err != nil { return nil, nil, err } - if same { - return blob, &proofs, nil + for _, b := range blobs { + if b.Commitment.Equal(commitment) { + return b, &proofs, nil + } + // Falling under this flag means that the data from the last row + // was insufficient to create a complete blob. As a result, + // the first blob received spans two rows and includes proofs + // for both of these rows. All other blobs in the result will relate + // to the current row and have a single proof. + if spansMultipleRows { + spansMultipleRows = false + // leave proof only for the current row + proofs = proofs[len(proofs)-1:] + } } - // drop info of the checked blob - rawShares = rawShares[amount:] if len(rawShares) > 0 { - // save proof for the last row in case we have rawShares - proofs = proofs[len(proofs)-1:] - } else { - // otherwise clear proofs - proofs = nil + spansMultipleRows = true + continue } - blobShare = nil + proofs = nil } - if len(rawShares) == 0 { - return nil, nil, ErrBlobNotFound - } - - blob, same, err := constructAndVerifyBlob(rawShares, commitment) - if err != nil { - return nil, nil, err - } - if same { - return blob, &proofs, nil + err = ErrBlobNotFound + if len(rawShares) > 0 { + err = fmt.Errorf("incomplete blob with the "+ + "namespace: %s detected at %d: %w", namespace.String(), height, err) + log.Error(err) } - - return nil, nil, ErrBlobNotFound + return nil, nil, err } // getBlobs retrieves the DAH and fetches all shares from the requested Namespace and converts @@ -304,3 +270,25 @@ func (s *Service) getBlobs( } return SharesToBlobs(namespacedShares.Flatten()) } + +// toAppShares converts node's raw shares to the app shares, skipping padding +func toAppShares(shrs ...share.Share) ([]shares.Share, error) { + appShrs := make([]shares.Share, 0, len(shrs)) + for _, shr := range shrs { + bShare, err := shares.NewShare(shr) + if err != nil { + return nil, err + } + + ok, err := bShare.IsPadding() + if err != nil { + return nil, err + } + if ok { + continue + } + + appShrs = append(appShrs, *bShare) + } + return appShrs, nil +} diff --git a/blob/service_test.go b/blob/service_test.go index 64296b969c..6777084eb4 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -344,6 +344,25 @@ func TestService_GetSingleBlobWithoutPadding(t *testing.T) { assert.Equal(t, newBlob.Commitment, blobs[1].Commitment) } +func TestService_Get(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + sizes := []int{1, 6, 3, 2, 4, 6, 8, 2, 15, 17} + + appBlobs, err := blobtest.GenerateV0Blobs(sizes, true) + require.NoError(t, err) + blobs, err := convertBlobs(appBlobs...) + require.NoError(t, err) + + service := createService(ctx, t, blobs) + for _, blob := range blobs { + b, err := service.Get(ctx, 1, blob.Namespace(), blob.Commitment) + require.NoError(t, err) + assert.Equal(t, b.Commitment, blob.Commitment) + } +} + func TestService_GetAllWithoutPadding(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) From dbf7dbff39bab55359171e9973cd781ad9b9968f Mon Sep 17 00:00:00 2001 From: Ryan Date: Fri, 13 Oct 2023 17:25:17 +0200 Subject: [PATCH 269/388] fix: removing chainid from default store name for mainnet (#2841) Also part of #2823 Stacked on #2840 --- cmd/flags_node.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/cmd/flags_node.go b/cmd/flags_node.go index fe4981b6c6..ef5af26580 100644 --- a/cmd/flags_node.go +++ b/cmd/flags_node.go @@ -87,6 +87,10 @@ func DefaultNodeStorePath(tp string, network string) (string, error) { return "", err } } + if network == p2p.Mainnet.String() { + return fmt.Sprintf("%s/.celestia-%s", home, strings.ToLower(tp)), nil + } + // only include network name in path for testnets and custom networks return fmt.Sprintf( "%s/.celestia-%s-%s", home, From 422dea262a2b32bd1b026d760eaf1c7ed78e141b Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 13 Oct 2023 18:27:25 +0200 Subject: [PATCH 270/388] deps!: Bumps go-header (#2844) Bumps go-header - includes breaks to Getter interface + metrics instantiation for header pkg Co-authored-by: Hlib Kanunnikov --- core/exchange.go | 45 +++++++++++++++--------------- core/exchange_test.go | 23 +++++++++++++-- core/listener_test.go | 14 +++++++--- das/daser_test.go | 6 +--- docs/adr/adr-009-public-api.md | 4 +-- go.mod | 2 +- go.sum | 4 +-- nodebuilder/header/config.go | 3 ++ nodebuilder/header/constructors.go | 27 ++++++++++++++---- nodebuilder/header/header.go | 10 +++---- nodebuilder/header/mocks/api.go | 15 +++++----- nodebuilder/header/module.go | 18 +++++++++--- nodebuilder/header/opts.go | 28 ------------------- nodebuilder/header/service.go | 4 +-- nodebuilder/settings.go | 7 +++-- 15 files changed, 117 insertions(+), 93 deletions(-) delete mode 100644 nodebuilder/header/opts.go diff --git a/core/exchange.go b/core/exchange.go index f8e1606a3e..06f648edad 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -42,7 +42,29 @@ func (ce *Exchange) GetByHeight(ctx context.Context, height uint64) (*header.Ext return ce.getExtendedHeaderByHeight(ctx, &intHeight) } -func (ce *Exchange) GetRangeByHeight(ctx context.Context, from, amount uint64) ([]*header.ExtendedHeader, error) { +func (ce *Exchange) GetRangeByHeight( + ctx context.Context, + from *header.ExtendedHeader, + to uint64, +) ([]*header.ExtendedHeader, error) { + amount := to - (from.Height() + 1) + headers, err := ce.getRangeByHeight(ctx, from.Height()+1, amount) + if err != nil { + return nil, err + } + + for _, h := range headers { + err := from.Verify(h) + if err != nil { + return nil, fmt.Errorf("verifying next header against last verified height: %d: %w", + from.Height(), err) + } + from = h + } + return headers, nil +} + +func (ce *Exchange) getRangeByHeight(ctx context.Context, from, amount uint64) ([]*header.ExtendedHeader, error) { if amount == 0 { return nil, nil } @@ -73,27 +95,6 @@ func (ce *Exchange) GetRangeByHeight(ctx context.Context, from, amount uint64) ( return headers, nil } -func (ce *Exchange) GetVerifiedRange( - ctx context.Context, - from *header.ExtendedHeader, - amount uint64, -) ([]*header.ExtendedHeader, error) { - headers, err := ce.GetRangeByHeight(ctx, from.Height()+1, amount) - if err != nil { - return nil, err - } - - for _, h := range headers { - err := from.Verify(h) - if err != nil { - return nil, fmt.Errorf("verifying next header against last verified height: %d: %w", - from.Height(), err) - } - from = h - } - return headers, nil -} - func (ce *Exchange) Get(ctx context.Context, hash libhead.Hash) (*header.ExtendedHeader, error) { log.Debugw("requesting header", "hash", hash.String()) block, err := ce.fetcher.GetBlockByHash(ctx, hash) diff --git a/core/exchange_test.go b/core/exchange_test.go index 8aa62593c8..853b5a8dc6 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -17,6 +17,9 @@ import ( ) func TestCoreExchange_RequestHeaders(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + t.Cleanup(cancel) + fetcher, _ := createCoreFetcher(t, DefaultTestConfig()) // generate 10 blocks @@ -25,10 +28,26 @@ func TestCoreExchange_RequestHeaders(t *testing.T) { store := createStore(t) ce := NewExchange(fetcher, store, header.MakeExtendedHeader) - headers, err := ce.GetRangeByHeight(context.Background(), 1, 10) + + // initialize store with genesis block + genHeight := int64(1) + genBlock, err := fetcher.GetBlock(ctx, &genHeight) + require.NoError(t, err) + genHeader, err := ce.Get(ctx, genBlock.Header.Hash().Bytes()) + require.NoError(t, err) + + to := uint64(10) + expectedFirstHeightInRange := genHeader.Height() + 1 + expectedLastHeightInRange := to - 1 + expectedLenHeaders := to - expectedFirstHeightInRange + + // request headers from height 1 to 10 [2:10) + headers, err := ce.GetRangeByHeight(context.Background(), genHeader, to) require.NoError(t, err) - assert.Equal(t, 10, len(headers)) + assert.Len(t, headers, int(expectedLenHeaders)) + assert.Equal(t, expectedFirstHeightInRange, headers[0].Height()) + assert.Equal(t, expectedLastHeightInRange, headers[len(headers)-1].Height()) } func createCoreFetcher(t *testing.T, cfg *testnode.Config) (*BlockFetcher, testnode.Context) { diff --git a/core/listener_test.go b/core/listener_test.go index 90bdefb72f..eab634a330 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -31,8 +31,13 @@ func TestListener(t *testing.T) { // create mocknet with two pubsub endpoints ps0, ps1 := createMocknetWithTwoPubsubEndpoints(ctx, t) - subscriber := p2p.NewSubscriber[*header.ExtendedHeader](ps1, header.MsgID, networkID) - err := subscriber.SetVerifier(func(context.Context, *header.ExtendedHeader) error { + subscriber, err := p2p.NewSubscriber[*header.ExtendedHeader]( + ps1, + header.MsgID, + p2p.WithSubscriberNetworkID(networkID), + ) + require.NoError(t, err) + err = subscriber.SetVerifier(func(context.Context, *header.ExtendedHeader) error { return nil }) require.NoError(t, err) @@ -162,8 +167,9 @@ func createListener( edsSub *shrexsub.PubSub, store *eds.Store, ) *Listener { - p2pSub := p2p.NewSubscriber[*header.ExtendedHeader](ps, header.MsgID, networkID) - err := p2pSub.Start(ctx) + p2pSub, err := p2p.NewSubscriber[*header.ExtendedHeader](ps, header.MsgID, p2p.WithSubscriberNetworkID(networkID)) + require.NoError(t, err) + err = p2pSub.Start(ctx) require.NoError(t, err) t.Cleanup(func() { require.NoError(t, p2pSub.Stop(ctx)) diff --git a/das/daser_test.go b/das/daser_test.go index ca553b0f09..1ec160e224 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -384,11 +384,7 @@ func (m getterStub) GetByHeight(_ context.Context, height uint64) (*header.Exten DAH: &share.Root{RowRoots: make([][]byte, 0)}}, nil } -func (m getterStub) GetRangeByHeight(context.Context, uint64, uint64) ([]*header.ExtendedHeader, error) { - return nil, nil -} - -func (m getterStub) GetVerifiedRange( +func (m getterStub) GetRangeByHeight( context.Context, *header.ExtendedHeader, uint64, diff --git a/docs/adr/adr-009-public-api.md b/docs/adr/adr-009-public-api.md index d2bf6ee817..be9e8827eb 100644 --- a/docs/adr/adr-009-public-api.md +++ b/docs/adr/adr-009-public-api.md @@ -100,10 +100,10 @@ GetByHeight(ctx context.Context, height uint64) (*header.ExtendedHeader, error) // WaitForHeight blocks until the header at the given height has been processed // by the node's header store or until context deadline is exceeded. WaitForHeight(ctx context.Context, height uint64) (*header.ExtendedHeader, error) -// GetVerifiedRangeByHeight returns the given range (from:to) of ExtendedHeaders +// GetRangeByHeight returns the given range (from:to) of ExtendedHeaders // from the node's header store and verifies that the returned headers are // adjacent to each other. -GetVerifiedRangeByHeight(ctx context.Context, from, to uint64) ([]*ExtendedHeader, error) +GetRangeByHeight(ctx context.Context, from, to uint64) ([]*ExtendedHeader, error) // Subscribe creates long-living Subscription for newly validated // ExtendedHeaders. Multiple Subscriptions can be created. Subscribe(context.Context) (<-chan *header.ExtendedHeader, error) diff --git a/go.mod b/go.mod index 1c76383ffa..dfae47fb29 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/celestia-app v1.0.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.3.3 + github.com/celestiaorg/go-header v0.4.0 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 4e31203f91..0c46be987a 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.3.3 h1:Y04hdJIJfD5hapyqK0ZQMgMTH5PQGV9YpcIf56LGc4E= -github.com/celestiaorg/go-header v0.3.3/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.4.0 h1:Ine/xpvFx8o9p6fXW+h2RSPp68rn7VUxTkW1okJxcEY= +github.com/celestiaorg/go-header v0.4.0/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= diff --git a/nodebuilder/header/config.go b/nodebuilder/header/config.go index 238fb39afd..d7265373ce 100644 --- a/nodebuilder/header/config.go +++ b/nodebuilder/header/config.go @@ -16,6 +16,9 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/p2p" ) +// MetricsEnabled will be set during runtime if metrics are enabled on the node. +var MetricsEnabled = false + // Config contains configuration parameters for header retrieval and management. type Config struct { // TrustedHash is the Block/Header hash that Nodes use as starting point for header synchronization. diff --git a/nodebuilder/header/constructors.go b/nodebuilder/header/constructors.go index be4bcbd427..15d2da09b1 100644 --- a/nodebuilder/header/constructors.go +++ b/nodebuilder/header/constructors.go @@ -40,12 +40,18 @@ func newP2PExchange[H libhead.Header[H]]( ids[index] = peer.ID host.Peerstore().AddAddrs(peer.ID, peer.Addrs, peerstore.PermanentAddrTTL) } - exchange, err := p2p.NewExchange[H](host, ids, conngater, + + opts := []p2p.Option[p2p.ClientParameters]{ p2p.WithParams(cfg.Client), p2p.WithNetworkID[p2p.ClientParameters](network.String()), p2p.WithChainID(network.String()), p2p.WithPeerIDStore[p2p.ClientParameters](pidstore), - ) + } + if MetricsEnabled { + opts = append(opts, p2p.WithMetrics[p2p.ClientParameters]()) + } + + exchange, err := p2p.NewExchange[H](host, ids, conngater, opts...) if err != nil { return nil, err } @@ -68,10 +74,12 @@ func newSyncer[H libhead.Header[H]]( sub libhead.Subscriber[H], cfg Config, ) (*sync.Syncer[H], *modfraud.ServiceBreaker[*sync.Syncer[H], H], error) { - syncer, err := sync.NewSyncer[H](ex, store, sub, - sync.WithParams(cfg.Syncer), - sync.WithBlockTime(modp2p.BlockTime), - ) + opts := []sync.Option{sync.WithParams(cfg.Syncer), sync.WithBlockTime(modp2p.BlockTime)} + if MetricsEnabled { + opts = append(opts, sync.WithMetrics()) + } + + syncer, err := sync.NewSyncer[H](ex, store, sub, opts...) if err != nil { return nil, nil, err } @@ -96,6 +104,13 @@ func newInitStore[H libhead.Header[H]]( return nil, err } + if MetricsEnabled { + err = libhead.WithMetrics[H](s) + if err != nil { + return nil, err + } + } + trustedHash, err := cfg.trustedHash(net) if err != nil { return nil, err diff --git a/nodebuilder/header/header.go b/nodebuilder/header/header.go index 43b5646a77..f807796eb6 100644 --- a/nodebuilder/header/header.go +++ b/nodebuilder/header/header.go @@ -19,10 +19,10 @@ type Module interface { // GetByHash returns the header of the given hash from the node's header store. GetByHash(ctx context.Context, hash libhead.Hash) (*header.ExtendedHeader, error) - // GetVerifiedRangeByHeight returns the given range (from:to) of ExtendedHeaders + // GetRangeByHeight returns the given range (from:to) of ExtendedHeaders // from the node's header store and verifies that the returned headers are // adjacent to each other. - GetVerifiedRangeByHeight( + GetRangeByHeight( ctx context.Context, from *header.ExtendedHeader, to uint64, @@ -54,7 +54,7 @@ type API struct { ctx context.Context, hash libhead.Hash, ) (*header.ExtendedHeader, error) `perm:"read"` - GetVerifiedRangeByHeight func( + GetRangeByHeight func( context.Context, *header.ExtendedHeader, uint64, @@ -72,12 +72,12 @@ func (api *API) GetByHash(ctx context.Context, hash libhead.Hash) (*header.Exten return api.Internal.GetByHash(ctx, hash) } -func (api *API) GetVerifiedRangeByHeight( +func (api *API) GetRangeByHeight( ctx context.Context, from *header.ExtendedHeader, to uint64, ) ([]*header.ExtendedHeader, error) { - return api.Internal.GetVerifiedRangeByHeight(ctx, from, to) + return api.Internal.GetRangeByHeight(ctx, from, to) } func (api *API) GetByHeight(ctx context.Context, u uint64) (*header.ExtendedHeader, error) { diff --git a/nodebuilder/header/mocks/api.go b/nodebuilder/header/mocks/api.go index 7d6661ff5d..b0d2b961d9 100644 --- a/nodebuilder/header/mocks/api.go +++ b/nodebuilder/header/mocks/api.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - header "github.com/celestiaorg/celestia-node/header" header0 "github.com/celestiaorg/go-header" sync "github.com/celestiaorg/go-header/sync" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -68,19 +67,19 @@ func (mr *MockModuleMockRecorder) GetByHeight(arg0, arg1 interface{}) *gomock.Ca return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetByHeight", reflect.TypeOf((*MockModule)(nil).GetByHeight), arg0, arg1) } -// GetVerifiedRangeByHeight mocks base method. -func (m *MockModule) GetVerifiedRangeByHeight(arg0 context.Context, arg1 *header.ExtendedHeader, arg2 uint64) ([]*header.ExtendedHeader, error) { +// GetRangeByHeight mocks base method. +func (m *MockModule) GetRangeByHeight(arg0 context.Context, arg1 *header.ExtendedHeader, arg2 uint64) ([]*header.ExtendedHeader, error) { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "GetVerifiedRangeByHeight", arg0, arg1, arg2) + ret := m.ctrl.Call(m, "GetRangeByHeight", arg0, arg1, arg2) ret0, _ := ret[0].([]*header.ExtendedHeader) ret1, _ := ret[1].(error) return ret0, ret1 } -// GetVerifiedRangeByHeight indicates an expected call of GetVerifiedRangeByHeight. -func (mr *MockModuleMockRecorder) GetVerifiedRangeByHeight(arg0, arg1, arg2 interface{}) *gomock.Call { +// GetRangeByHeight indicates an expected call of GetRangeByHeight. +func (mr *MockModuleMockRecorder) GetRangeByHeight(arg0, arg1, arg2 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetVerifiedRangeByHeight", reflect.TypeOf((*MockModule)(nil).GetVerifiedRangeByHeight), arg0, arg1, arg2) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRangeByHeight", reflect.TypeOf((*MockModule)(nil).GetRangeByHeight), arg0, arg1, arg2) } // LocalHead mocks base method. diff --git a/nodebuilder/header/module.go b/nodebuilder/header/module.go index 557cbcfab7..4be25f7125 100644 --- a/nodebuilder/header/module.go +++ b/nodebuilder/header/module.go @@ -50,8 +50,12 @@ func ConstructModule[H libhead.Header[H]](tp node.Type, cfg *Config) fx.Option { }), )), fx.Provide(fx.Annotate( - func(ps *pubsub.PubSub, network modp2p.Network) *p2p.Subscriber[H] { - return p2p.NewSubscriber[H](ps, header.MsgID, network.String()) + func(ps *pubsub.PubSub, network modp2p.Network) (*p2p.Subscriber[H], error) { + opts := []p2p.SubscriberOption{p2p.WithSubscriberNetworkID(network.String())} + if MetricsEnabled { + opts = append(opts, p2p.WithSubscriberMetrics()) + } + return p2p.NewSubscriber[H](ps, header.MsgID, opts...) }, fx.OnStart(func(ctx context.Context, sub *p2p.Subscriber[H]) error { return sub.Start(ctx) @@ -62,14 +66,20 @@ func ConstructModule[H libhead.Header[H]](tp node.Type, cfg *Config) fx.Option { )), fx.Provide(fx.Annotate( func( + cfg Config, host host.Host, store libhead.Store[H], network modp2p.Network, ) (*p2p.ExchangeServer[H], error) { - return p2p.NewExchangeServer[H](host, store, + opts := []p2p.Option[p2p.ServerParameters]{ p2p.WithParams(cfg.Server), p2p.WithNetworkID[p2p.ServerParameters](network.String()), - ) + } + if MetricsEnabled { + opts = append(opts, p2p.WithMetrics[p2p.ServerParameters]()) + } + + return p2p.NewExchangeServer[H](host, store, opts...) }, fx.OnStart(func(ctx context.Context, server *p2p.ExchangeServer[H]) error { return server.Start(ctx) diff --git a/nodebuilder/header/opts.go b/nodebuilder/header/opts.go deleted file mode 100644 index decedcad85..0000000000 --- a/nodebuilder/header/opts.go +++ /dev/null @@ -1,28 +0,0 @@ -package header - -import ( - libhead "github.com/celestiaorg/go-header" - "github.com/celestiaorg/go-header/p2p" - "github.com/celestiaorg/go-header/sync" - - "github.com/celestiaorg/celestia-node/header" -) - -// WithMetrics provides sets `MetricsEnabled` to true on ClientParameters for the header exchange -func WithMetrics( - store libhead.Store[*header.ExtendedHeader], - ex libhead.Exchange[*header.ExtendedHeader], - sync *sync.Syncer[*header.ExtendedHeader], -) error { - if p2pex, ok := ex.(*p2p.Exchange[*header.ExtendedHeader]); ok { - if err := p2pex.InitMetrics(); err != nil { - return err - } - } - - if err := sync.InitMetrics(); err != nil { - return err - } - - return libhead.WithMetrics[*header.ExtendedHeader](store) -} diff --git a/nodebuilder/header/service.go b/nodebuilder/header/service.go index 2b208cb88d..072ef070c6 100644 --- a/nodebuilder/header/service.go +++ b/nodebuilder/header/service.go @@ -52,12 +52,12 @@ func (s *Service) GetByHash(ctx context.Context, hash libhead.Hash) (*header.Ext return s.store.Get(ctx, hash) } -func (s *Service) GetVerifiedRangeByHeight( +func (s *Service) GetRangeByHeight( ctx context.Context, from *header.ExtendedHeader, to uint64, ) ([]*header.ExtendedHeader, error) { - return s.store.GetVerifiedRange(ctx, from, to) + return s.store.GetRangeByHeight(ctx, from, to) } func (s *Service) GetByHeight(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index d56125209c..9767081971 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -24,7 +24,7 @@ import ( "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/nodebuilder/das" - modheader "github.com/celestiaorg/celestia-node/nodebuilder/header" + modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/share" @@ -72,6 +72,10 @@ func WithPyroscope(endpoint string, nodeType node.Type) fx.Option { // WithMetrics enables metrics exporting for the node. func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Option { + // TODO @renaynay: this will be refactored when there is more granular + // control over which module to enable metrics for + modhead.MetricsEnabled = true + baseComponents := fx.Options( fx.Supply(metricOpts), fx.Invoke(initializeMetrics), @@ -83,7 +87,6 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti }), fx.Invoke(fraud.WithMetrics[*header.ExtendedHeader]), fx.Invoke(node.WithMetrics), - fx.Invoke(modheader.WithMetrics), fx.Invoke(share.WithDiscoveryMetrics), ) From 9f2b74accfed10c447b972f8b86d71666df53f8f Mon Sep 17 00:00:00 2001 From: Borys Semerenko Date: Mon, 16 Oct 2023 13:06:59 +0300 Subject: [PATCH 271/388] misc(blob): Add debug logs for the GetAll method (#2749) Co-authored-by: Ryan Co-authored-by: ramin --- blob/service.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/blob/service.go b/blob/service.go index 88b6c63d3f..49bcee6af3 100644 --- a/blob/service.go +++ b/blob/service.go @@ -121,6 +121,10 @@ func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share. resultErr = make([]error, len(namespaces)) ) + for _, ns := range namespaces { + log.Debugw("performing GetAll request", "namespace", ns.String(), "height", height) + } + wg := sync.WaitGroup{} for i, namespace := range namespaces { wg.Add(1) @@ -131,6 +135,8 @@ func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share. resultErr[i] = fmt.Errorf("getting blobs for namespace(%s): %s", namespace.String(), err) return } + + log.Debugw("receiving blobs", "height", height, "total", len(blobs)) resultBlobs[i] = blobs }(i, namespace) } From b54f2b72c2d3d6b6dd251f446c6ab3259e0c68e3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 12:52:31 +0000 Subject: [PATCH 272/388] chore(deps): Bump tspascoal/get-user-teams-membership from 2 to 3 (#2827) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [tspascoal/get-user-teams-membership](https://github.com/tspascoal/get-user-teams-membership) from 2 to 3.
Release notes

Sourced from tspascoal/get-user-teams-membership's releases.

3.0.0

What's Changed

Full Changelog: https://github.com/tspascoal/get-user-teams-membership/compare/v2...v3.0.0

v2.1.0

What's Changed

Features

Dependency Updates

New Contributors

Full Changelog: https://github.com/tspascoal/get-user-teams-membership/compare/v2...v2.1.0

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=tspascoal/get-user-teams-membership&package-manager=github_actions&previous-version=2&new-version=3)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/issue-label-automation.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/issue-label-automation.yml b/.github/workflows/issue-label-automation.yml index 965dbe8e9f..0d2d37ca49 100644 --- a/.github/workflows/issue-label-automation.yml +++ b/.github/workflows/issue-label-automation.yml @@ -16,7 +16,7 @@ jobs: steps: - name: Check for External Contributor - uses: tspascoal/get-user-teams-membership@v2 + uses: tspascoal/get-user-teams-membership@v3 id: teamCheck with: username: ${{ github.actor }} From d16c35526fde51efa6e7bf5076fef4f8902c62ce Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 16 Oct 2023 13:24:46 +0000 Subject: [PATCH 273/388] chore(deps): Bump github.com/celestiaorg/celestia-app from 1.0.0 to 1.1.0 (#2850) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/celestiaorg/celestia-app](https://github.com/celestiaorg/celestia-app) from 1.0.0 to 1.1.0.
Release notes

Sourced from github.com/celestiaorg/celestia-app's releases.

v1.1.0

Upgrade Notice

celestia-app v1.1.0 is compatible with celestia-app 1.0.0. Consensus nodes that are already running v1.0.0 don't need to upgrade to v1.1.0 with urgency. This release contains a fix so that Ledger hardware wallets with firmware 2.2.2 can sign transactions.

What's Changed

Full Changelog: https://github.com/celestiaorg/celestia-app/compare/v1.0.0...v1.1.0

Commits
  • 2b8cc9e fix: restore ability to sign transaction with ledger firmware 2.2.2 (backport...
  • dafc6d5 chore: decrease SlashFractionDoubleSign to 2% (backport #2630) (#2666)
  • f2e5bce chore: increase MinDeposit from 1K to 10K TIA (backport #2631) (#2660)
  • 21c45aa chore: remove unnecessary legacy handler from qgb module (backport #2635) (#2...
  • a861911 ci: generate pre-built binaries for darwin and arm64 (backport #2654) (#2662)
  • 01dad84 fix: LatestAttestationNonce constant name typo (backport #2646) (#2653)
  • 3fa42b5 fix: specs for MaxDepositPeriod (backport #2626) (#2649)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/celestiaorg/celestia-app&package-manager=go_modules&previous-version=1.0.0&new-version=1.1.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 30 ++++++++++++++--------------- go.sum | 60 +++++++++++++++++++++++++++++----------------------------- 2 files changed, 45 insertions(+), 45 deletions(-) diff --git a/go.mod b/go.mod index dfae47fb29..fcd3c27a0d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.0.0 + github.com/celestiaorg/celestia-app v1.1.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.4.0 @@ -67,10 +67,10 @@ require ( go.uber.org/fx v1.20.0 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 + golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.4.0 golang.org/x/text v0.13.0 - google.golang.org/grpc v1.58.2 + google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 ) @@ -88,10 +88,10 @@ require ( ) require ( - cloud.google.com/go v0.110.6 // indirect + cloud.google.com/go v0.110.8 // indirect cloud.google.com/go/compute v1.23.0 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.1 // indirect + cloud.google.com/go/iam v1.1.2 // indirect cloud.google.com/go/storage v1.30.1 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -126,7 +126,7 @@ require ( github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.6 // indirect github.com/cosmos/ibc-go/v6 v6.2.0 // indirect - github.com/cosmos/ledger-cosmos-go v0.12.2 // indirect + github.com/cosmos/ledger-cosmos-go v0.13.1 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect @@ -178,8 +178,8 @@ require ( github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.1 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.3 // indirect - github.com/googleapis/gax-go/v2 v2.11.0 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect + github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect @@ -313,7 +313,7 @@ require ( github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/zondax/hid v0.9.1 // indirect - github.com/zondax/ledger-go v0.14.1 // indirect + github.com/zondax/ledger-go v0.14.2 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect @@ -322,18 +322,18 @@ require ( go.uber.org/dig v1.17.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect - golang.org/x/net v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.10.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect - golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 // indirect + golang.org/x/tools v0.13.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect - google.golang.org/api v0.126.0 // indirect + google.golang.org/api v0.128.0 // indirect google.golang.org/appengine v1.6.7 // indirect - google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 // indirect + google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 0c46be987a..1db8ac6d65 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w9 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.110.6 h1:8uYAkj3YHTP/1iwReuHPxLSbdcyc+dSBbzFMrVwDR6Q= -cloud.google.com/go v0.110.6/go.mod h1:+EYjdK8e5RME/VY/qLCAtuyALQ9q67dvuum8i+H5xsI= +cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= +cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -116,8 +116,8 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v1.1.1 h1:lW7fzj15aVIXYHREOqjRBV9PsH0Z6u8Y46a1YGvQP4Y= -cloud.google.com/go/iam v1.1.1/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= +cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -358,8 +358,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.0.0 h1:CJX0y3JE1gEKzb0Hr3/qS0Z/tZgYGJyCHJzc0+SVxYM= -github.com/celestiaorg/celestia-app v1.0.0/go.mod h1:8FN8f/iZC5D6A7QYJhK86nyRKrmQrzXFucV19vLv9Es= +github.com/celestiaorg/celestia-app v1.1.0 h1:8TM/8xnZ+xf7NfEjYZ9XBEGuVG1QT8hPD3uh05RYEsE= +github.com/celestiaorg/celestia-app v1.1.0/go.mod h1:1uAaesXbMkV6EwfBfULXBKYhjHP8DGSVpA3BfKM4QUw= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 h1:c4cMVLU2bGTesZW1ZVgeoCB++gOOJTF3OvBsqBvo6n0= @@ -494,8 +494,8 @@ github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU= github.com/cosmos/iavl v0.19.6/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= github.com/cosmos/ibc-go/v6 v6.2.0 h1:HKS5WNxQrlmjowHb73J9LqlNJfvTnvkbhXZ9QzNTU7Q= github.com/cosmos/ibc-go/v6 v6.2.0/go.mod h1:+S3sxcNwOhgraYDJAhIFDg5ipXHaUnJrg7tOQqGyWlc= -github.com/cosmos/ledger-cosmos-go v0.12.2 h1:/XYaBlE2BJxtvpkHiBm97gFGSGmYGKunKyF3nNqAXZA= -github.com/cosmos/ledger-cosmos-go v0.12.2/go.mod h1:ZcqYgnfNJ6lAXe4HPtWgarNEY+B74i+2/8MhZw4ziiI= +github.com/cosmos/ledger-cosmos-go v0.13.1 h1:12ac9+GwBb9BjP7X5ygpFk09Itwzjzfmg6A2CWFjoVs= +github.com/cosmos/ledger-cosmos-go v0.13.1/go.mod h1:5tv2RVJEd2+Y38TIQN4CRjJeQGyqOEiKJDfqhk5UjqE= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -877,8 +877,8 @@ github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.3 h1:yk9/cqRKtT9wXZSsRH9aurXEpJX+U6FLtpYTdC3R06k= -github.com/googleapis/enterprise-certificate-proxy v0.2.3/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= +github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -890,8 +890,8 @@ github.com/googleapis/gax-go/v2 v2.3.0/go.mod h1:b8LNqSzNabLiUpXKkY7HAR5jr6bIT99 github.com/googleapis/gax-go/v2 v2.4.0/go.mod h1:XOTVJ59hdnfJLIP/dh8n5CGryZR2LxK9wbMD5+iXC6c= github.com/googleapis/gax-go/v2 v2.5.1/go.mod h1:h6B0KMMFNtI2ddbGJn3T3ZbwkeT6yqEF02fYlzkUCyo= github.com/googleapis/gax-go/v2 v2.6.0/go.mod h1:1mjbznJAPHFpesgE5ucqfYEscaz5kMdcIDwU/6+DDoY= -github.com/googleapis/gax-go/v2 v2.11.0 h1:9V9PWXEsWnPpQhu/PeQIkS4eGzMlTLGgt80cUUI8Ki4= -github.com/googleapis/gax-go/v2 v2.11.0/go.mod h1:DxmR61SGKkGLa2xigwuZIQpkCI2S5iydzRfb3peWZJI= +github.com/googleapis/gax-go/v2 v2.12.0 h1:A+gCJKdRfqXkr+BIRGtZLibNXf0m1f9E4HG56etFpas= +github.com/googleapis/gax-go/v2 v2.12.0/go.mod h1:y+aIqrI5eb1YGMVJfuV3185Ts/D7qKpsEkdD5+I6QGU= github.com/googleapis/go-type-adapters v1.0.0/go.mod h1:zHW75FOG2aur7gAO2B+MLby+cLsWGBF62rFAi7WjWO4= github.com/googleapis/google-cloud-go-testing v0.0.0-20200911160855-bcd43fbb19e8/go.mod h1:dvDLG8qkwmyD9a/MJJN3XJcT3xFxOKAvTZGvuZmac9g= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= @@ -2351,8 +2351,8 @@ github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1 github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= -github.com/zondax/ledger-go v0.14.1 h1:Pip65OOl4iJ84WTpA4BKChvOufMhhbxED3BaihoZN4c= -github.com/zondax/ledger-go v0.14.1/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= +github.com/zondax/ledger-go v0.14.2 h1:NDaba434N7JUVKg4P4nFv6TOjSkUosYb9sdiw3c61Zk= +github.com/zondax/ledger-go v0.14.2/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= @@ -2529,8 +2529,8 @@ golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63 h1:m64FZMko/V45gv0bNmrNYoDEq8U5YUhetc9cBWKS1TQ= -golang.org/x/exp v0.0.0-20230817173708-d852ddb80c63/go.mod h1:0v4NqG35kSWCMzLaMeX+IQrlSnVE/bqGSyC2cz/9Le8= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= +golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2659,8 +2659,8 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.14.0 h1:BONx9s002vGdD9umnlX1Po8vOZmrgH34qlHcD1MfK14= -golang.org/x/net v0.14.0/go.mod h1:PpSgVXXLK0OxS0F31C1/tv6XNguvCrnXIDrFMspZIUI= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2965,8 +2965,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846 h1:Vve/L0v7CXXuxUmaMGIEK/dEeq7uiqb5qBgQrZzIE7E= -golang.org/x/tools v0.12.1-0.20230815132531-74c255bcf846/go.mod h1:Sc0INKfu04TlqNoRA1hgpFZbhYXHPr4V5DzpSBTPqQM= +golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= +golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= @@ -3036,8 +3036,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.126.0 h1:q4GJq+cAdMAC7XP7njvQ4tvohGLiSlytuL4BQxbIZ+o= -google.golang.org/api v0.126.0/go.mod h1:mBwVAtz+87bEN6CbA1GtZPDOqY2R5ONPqJeIlvyo4Aw= +google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= +google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -3161,12 +3161,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5 h1:L6iMMGrtzgHsWofoFcihmDEMYeDR9KN/ThbPWGrh++g= -google.golang.org/genproto v0.0.0-20230803162519-f966b187b2e5/go.mod h1:oH/ZOT02u4kWEp7oYBGYFFkCdKS/uYR9Z7+0/xuuFp8= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e h1:z3vDksarJxsAKM5dmEGv0GHwE2hKJ096wZra71Vs4sw= -google.golang.org/genproto/googleapis/api v0.0.0-20230726155614-23370e0ffb3e/go.mod h1:rsr7RhLuwsDKL7RmgDDCUc6yaGr1iqceVb5Wv6f6YvQ= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878 h1:lv6/DhyiFFGsmzxbsUUTOkN29II+zeWHxvT8Lpdxsv0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230815205213-6bfd019c3878/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= +google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= +google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 h1:U7+wNaVuSTaUqNvK2+osJ9ejEZxbjHHk8F2b6Hpx0AE= +google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -3212,8 +3212,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.58.2 h1:SXUpjxeVF3FKrTYQI4f4KvbGD5u2xccdYdurwowix5I= -google.golang.org/grpc v1.58.2/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 0b854485d4ee7d84ffdaf8c2e8bd1fcedf940680 Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 16 Oct 2023 15:46:01 +0200 Subject: [PATCH 274/388] fix(share/getters): short-circuit on empty root in ShrexGetter (#2846) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Should fix issue where GetEDS hangs on empty data root To test run: `celestia share get-eds “$(celestia header get-by-height 500 --node.store ~/.celestia-light-mocha-4 | jq .result)” --node.store ~/.celestia-light-mocha-4` with and without the PR --- share/getters/shrex.go | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 1c296a6a33..9576da252b 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -136,6 +136,10 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader }() dah := header.DAH + // short circuit if the data root is empty + if dah.Equals(share.EmptyRoot()) { + return share.EmptyExtendedDataSquare(), nil + } for { if ctx.Err() != nil { sg.metrics.recordEDSAttempt(ctx, attempt, false) From a7a17e4bb3209456867b099a9ca061e513b283e4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Tomasz=20Zdyba=C5=82?= Date: Tue, 17 Oct 2023 15:23:03 +0200 Subject: [PATCH 275/388] fix(blob): properly wrap error in GetAll method (#2853) Error returned by GetAll is not formatted correctly. It's especially important, because users of the method will most probably try to detect ErrBlobNotFound (to differentiate from other errors). --- blob/service.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/blob/service.go b/blob/service.go index 49bcee6af3..67fcae349a 100644 --- a/blob/service.go +++ b/blob/service.go @@ -132,7 +132,7 @@ func (s *Service) GetAll(ctx context.Context, height uint64, namespaces []share. defer wg.Done() blobs, err := s.getBlobs(ctx, namespace, header) if err != nil { - resultErr[i] = fmt.Errorf("getting blobs for namespace(%s): %s", namespace.String(), err) + resultErr[i] = fmt.Errorf("getting blobs for namespace(%s): %w", namespace.String(), err) return } From dd53c3b318be3d9c51f9acd62db75c2a2bc288cf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Wed, 18 Oct 2023 16:09:44 +0000 Subject: [PATCH 276/388] chore(deps): Bump celestiaorg/.github from 0.2.3 to 0.2.8 (#2851) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Hlib Kanunnikov --- .github/workflows/ci_release.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 271cab472f..6d641edc73 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -25,7 +25,7 @@ on: jobs: # Dockerfile Linting hadolint: - uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.3 # yamllint disable-line rule:line-length + uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.8 # yamllint disable-line rule:line-length with: dockerfile: Dockerfile @@ -33,7 +33,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.3 + - uses: celestiaorg/.github/.github/actions/yamllint@v0.2.8 markdown-lint: name: Markdown Lint @@ -68,7 +68,7 @@ jobs: uses: mathieudutour/github-tag-action@v6.1 - name: Version Release - uses: celestiaorg/.github/.github/actions/version-release@v0.2.3 + uses: celestiaorg/.github/.github/actions/version-release@v0.2.8 with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.version }} From 40eea9084d7230538a9af33991af796a057a308d Mon Sep 17 00:00:00 2001 From: Yiannis Marangos Date: Wed, 18 Oct 2023 19:15:25 +0300 Subject: [PATCH 277/388] fix(p2p): Start listening to QUIC when CELESTIA_ENABLE_QUIC is set (#2857) --- nodebuilder/p2p/addrs.go | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/nodebuilder/p2p/addrs.go b/nodebuilder/p2p/addrs.go index 528f58d722..27fbd244a3 100644 --- a/nodebuilder/p2p/addrs.go +++ b/nodebuilder/p2p/addrs.go @@ -19,11 +19,13 @@ func Listen(listen []string) func(h hst.Host) (err error) { return fmt.Errorf("failure to parse config.P2P.ListenAddresses: %s", err) } - // TODO(@WonderTan): Remove this check when QUIC is stable - if slices.ContainsFunc(maddr.Protocols(), func(p ma.Protocol) bool { - return p.Code == ma.P_QUIC_V1 || p.Code == ma.P_WEBTRANSPORT - }) { - continue + if !enableQUIC { + // TODO(@WonderTan): Remove this check when QUIC is stable + if slices.ContainsFunc(maddr.Protocols(), func(p ma.Protocol) bool { + return p.Code == ma.P_QUIC_V1 || p.Code == ma.P_WEBTRANSPORT + }) { + continue + } } maListen = append(maListen, maddr) From 2c5eef712ded6f5b383990d55df73da5d143aada Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 18 Oct 2023 18:16:14 +0200 Subject: [PATCH 278/388] fix(core): Increase listener timeout to 3x block time (#2852) Block time can vary quite a bit so do not trigger re-subscription unless there's a significant lag (3x blocktime). This issue was found by bridge node runners on mocha-4 where block time sometimes exceeded 20s --- core/listener.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/listener.go b/core/listener.go index 1c79fbbe71..de0d84ed35 100644 --- a/core/listener.go +++ b/core/listener.go @@ -57,7 +57,7 @@ func NewListener( hashBroadcaster: hashBroadcaster, construct: construct, store: store, - listenerTimeout: 2 * blocktime, + listenerTimeout: 5 * blocktime, } } From db16322eef01538ef40a103d657dbeafaa27bd8f Mon Sep 17 00:00:00 2001 From: Nguyen Nhu Viet Date: Thu, 19 Oct 2023 16:35:47 +0200 Subject: [PATCH 279/388] chore: fix docker builds for arm64 (#2862) ## Overview Now, this is finally fixed Ref: https://github.com/rollkit/local-celestia-devnet/pull/63 Closes https://github.com/celestiaorg/devops/issues/527 ## Checklist - [ ] New and updated code has appropriate documentation - [ ] New and updated code has new and/or updated testing - [ ] Required CI checks are passing - [ ] Visual proof for any user facing features like CLI or documentation updates - [ ] Linked issues closed with keywords --- Dockerfile | 19 +++++++++++++++---- 1 file changed, 15 insertions(+), 4 deletions(-) diff --git a/Dockerfile b/Dockerfile index 403691d4a7..a3ac41f7aa 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,7 +1,15 @@ FROM --platform=$BUILDPLATFORM docker.io/golang:1.21-alpine3.18 as builder +ARG TARGETPLATFORM +ARG BUILDPLATFORM +ARG TARGETOS +ARG TARGETARCH + +ENV CGO_ENABLED=0 +ENV GO111MODULE=on + # hadolint ignore=DL3018 -RUN apk update && apk add --no-cache \ +RUN uname -a && apk update && apk add --no-cache \ bash \ gcc \ git \ @@ -13,9 +21,11 @@ COPY go.mod go.sum ./ RUN go mod download COPY . . -RUN make build && make cel-key +RUN uname -a &&\ + CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ + make build && make cel-key -FROM --platform=$BUILDPLATFORM docker.io/alpine:3.18.4 +FROM docker.io/alpine:3.18.4 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 @@ -28,7 +38,8 @@ ENV NODE_TYPE bridge ENV P2P_NETWORK mocha # hadolint ignore=DL3018 -RUN apk update && apk add --no-cache \ +RUN uname -a &&\ + apk update && apk add --no-cache \ bash \ curl \ jq \ From 39b5c77faaaef8fd77baf0291e5330ad821bc350 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 23 Oct 2023 08:05:09 -0400 Subject: [PATCH 280/388] override organization pull_request_template.md (#2805) As discussed, overrides the organization wide `.github/pull_request_template.md` with the checklist everyone ignores, instead favoring just a comment when opening the PR and a request to add a reference to an issue for a PR check we'll add shortly, opening a PR should just look like this now: pr-message --- .github/pull_request_template.md | 14 ++++++++++++++ 1 file changed, 14 insertions(+) create mode 100644 .github/pull_request_template.md diff --git a/.github/pull_request_template.md b/.github/pull_request_template.md new file mode 100644 index 0000000000..61c286f936 --- /dev/null +++ b/.github/pull_request_template.md @@ -0,0 +1,14 @@ + From 1a5d63cee99d562d2fd50717205928fa5a8f75b1 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 23 Oct 2023 09:24:12 -0400 Subject: [PATCH 281/388] (ci) matrix tests for different OS (#2855) Refs https://github.com/celestiaorg/celestia-node/issues/1584 Lets run our test suite on multi configurations as they sometimes pass one not other (as reported by @vgonkivs) - Task: run tests on > 1 OS (previously ubuntu-latest only) now `ubuntu-latest` and `macos-latest` - currently ONLY for the unit tests (do we want to change this for all the tests?) - also makes a small unrequested tweak to make the execution in go-ci run AFTER the mod and lint checks, so it'll pause if those two fail and fail the entire pipeline faster. Save some wasted compute time, which will be at least x2 worse once we matrix these example run: https://github.com/celestiaorg/celestia-node/actions/runs/6559066381/job/17813974060 --- .github/workflows/go-ci.yml | 12 +++++++++++- blob/service_test.go | 2 ++ 2 files changed, 13 insertions(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 288a9cfb2f..c12fd409d3 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -44,8 +44,14 @@ jobs: run: git diff --exit-code test_coverage: + needs: [lint, go_mod_tidy_check] name: Unit Tests Coverage - runs-on: ubuntu-latest + strategy: + matrix: + os: [ubuntu-latest, macos-latest] + runs-on: ${{ matrix.os }} + env: + OS: ${{ matrix.os }} steps: - uses: actions/checkout@v4 @@ -61,10 +67,13 @@ jobs: - name: upload coverage uses: codecov/codecov-action@v3.1.4 with: + env_vars: OS token: ${{ secrets.CODECOV_TOKEN }} file: ./coverage.txt + name: coverage-${{ matrix.os }} unit_race_test: + needs: [lint, go_mod_tidy_check] name: Run Unit Tests with Race Detector runs-on: ubuntu-latest @@ -80,6 +89,7 @@ jobs: run: make test-unit-race integration_test: + needs: [lint, go_mod_tidy_check] name: Run Integration Tests runs-on: ubuntu-latest diff --git a/blob/service_test.go b/blob/service_test.go index 6777084eb4..152058bed0 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -1,3 +1,5 @@ +//go:build linux + package blob import ( From e8eaf4a9401a0c1207396b9e942712c807ff2dbc Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 23 Oct 2023 09:35:27 -0400 Subject: [PATCH 282/388] update golangci-lint (#2868) - updates `golangci-lint` to 1.55 - removes "double cache" when setting up golang, thus removing linter noise (`Cannot open file...file exists`) so easier to see actual lint errors closes https://github.com/celestiaorg/celestia-node/issues/2590 --- .github/workflows/go-ci.yml | 4 +++- share/availability/test/corrupt_data.go | 4 ++-- share/eds/cache/metrics.go | 8 +++++--- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index c12fd409d3..9d80281c6c 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -25,7 +25,9 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3.7.0 with: - version: v1.54.2 + version: v1.55 + skip-pkg-cache: true + skip-build-cache: true go_mod_tidy_check: name: Go Mod Tidy Check diff --git a/share/availability/test/corrupt_data.go b/share/availability/test/corrupt_data.go index f0bd8fbbc5..1ff553f8b3 100644 --- a/share/availability/test/corrupt_data.go +++ b/share/availability/test/corrupt_data.go @@ -63,7 +63,7 @@ func (fb FraudulentBlockstore) Has(context.Context, cid.Cid) (bool, error) { func (fb FraudulentBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Block, error) { key := cid.String() if fb.Attacking { - key = "corrupt" + key + key = "corrupt_get" + key } data, err := fb.Datastore.Get(ctx, ds.NewKey(key)) @@ -76,7 +76,7 @@ func (fb FraudulentBlockstore) Get(ctx context.Context, cid cid.Cid) (blocks.Blo func (fb FraudulentBlockstore) GetSize(ctx context.Context, cid cid.Cid) (int, error) { key := cid.String() if fb.Attacking { - key = "corrupt" + key + key = "corrupt_size" + key } return fb.Datastore.GetSize(ctx, ds.NewKey(key)) diff --git a/share/eds/cache/metrics.go b/share/eds/cache/metrics.go index 24056aa553..b2e3bec8d8 100644 --- a/share/eds/cache/metrics.go +++ b/share/eds/cache/metrics.go @@ -18,19 +18,21 @@ type metrics struct { } func newMetrics(bc *AccessorCache) (*metrics, error) { - evictedCounter, err := meter.Int64Counter("eds_blockstore_cache_"+bc.name+"_evicted_counter", + metricsPrefix := "eds_blockstore_cache_" + bc.name + + evictedCounter, err := meter.Int64Counter(metricsPrefix+"_evicted_counter", metric.WithDescription("eds blockstore cache evicted event counter")) if err != nil { return nil, err } - getCounter, err := meter.Int64Counter("eds_blockstore_cache_"+bc.name+"_get_counter", + getCounter, err := meter.Int64Counter(metricsPrefix+"_get_counter", metric.WithDescription("eds blockstore cache evicted event counter")) if err != nil { return nil, err } - cacheSize, err := meter.Int64ObservableGauge("eds_blockstore_cache_"+bc.name+"_size", + cacheSize, err := meter.Int64ObservableGauge(metricsPrefix+"_size", metric.WithDescription("total amount of items in blockstore cache"), ) if err != nil { From 7ac72876afa0b6ead00651c00c7fec9d7142e1c9 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 24 Oct 2023 12:49:44 +0300 Subject: [PATCH 283/388] fix(docgen): fix das naming for the openrpc spec (#2870) --- nodebuilder/default_services.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder/default_services.go b/nodebuilder/default_services.go index 0373e44244..430f6ba66b 100644 --- a/nodebuilder/default_services.go +++ b/nodebuilder/default_services.go @@ -18,7 +18,7 @@ var PackageToAPI = map[string]interface{}{ "state": &state.API{}, "share": &share.API{}, "header": &header.API{}, - "daser": &das.API{}, + "das": &das.API{}, "p2p": &p2p.API{}, "blob": &blob.API{}, "node": &node.API{}, From 43e6bbf592f47786bf0b8523e39aa256fe368105 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 24 Oct 2023 14:08:35 +0300 Subject: [PATCH 284/388] chore(nodebuilder/das): change daser module name to das (#2871) --- nodebuilder/das/module.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/nodebuilder/das/module.go b/nodebuilder/das/module.go index d9f7e700e2..0545fffc27 100644 --- a/nodebuilder/das/module.go +++ b/nodebuilder/das/module.go @@ -38,7 +38,7 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { switch tp { case node.Light, node.Full: return fx.Module( - "daser", + "das", baseComponents, fx.Provide(fx.Annotate( newDASer, @@ -56,7 +56,7 @@ func ConstructModule(tp node.Type, cfg *Config) fx.Option { ) case node.Bridge: return fx.Module( - "daser", + "das", baseComponents, fx.Provide(newDaserStub), ) From 565691f713f5e7abc233b9ada95f53a2ec34f216 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Wed, 25 Oct 2023 23:06:50 +0300 Subject: [PATCH 285/388] tests(share/byzantine): extend befp tests (#2864) --- share/eds/byzantine/bad_encoding.go | 48 +++++-- share/eds/byzantine/bad_encoding_test.go | 153 +++++++++++++++++++++-- 2 files changed, 181 insertions(+), 20 deletions(-) diff --git a/share/eds/byzantine/bad_encoding.go b/share/eds/byzantine/bad_encoding.go index e3a862e38a..fbb6b592ea 100644 --- a/share/eds/byzantine/bad_encoding.go +++ b/share/eds/byzantine/bad_encoding.go @@ -102,15 +102,31 @@ func (p *BadEncodingProof) UnmarshalBinary(data []byte) error { return nil } +var ( + errHeightMismatch = errors.New("height reported in proof does not match with the header's height") + errIncorrectIndex = errors.New("row/col index is more then the roots amount") + errIncorrectAmountOfShares = errors.New("incorrect amount of shares") + errIncorrectShare = errors.New("incorrect share received") + errNMTTreeRootsMatch = errors.New("recomputed root matches the DAH root") +) + +var ( + invalidProofPrefix = fmt.Sprintf("invalid %s proof", BadEncoding) +) + // Validate ensures that fraud proof is correct. // Validate checks that provided Merkle Proofs correspond to the shares, // rebuilds bad row or col from received shares, computes Merkle Root // and compares it with block's Merkle Root. func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { if hdr.Height() != p.BlockHeight { - return fmt.Errorf("incorrect block height during BEFP validation: expected %d, got %d", - p.BlockHeight, hdr.Height(), + log.Debugf("%s: %s. expected block's height: %d, got: %d", + invalidProofPrefix, + errHeightMismatch, + hdr.Height(), + p.BlockHeight, ) + return errHeightMismatch } if len(hdr.DAH.RowRoots) != len(hdr.DAH.ColumnRoots) { @@ -132,9 +148,10 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { } if int(p.Index) >= len(merkleRoots) { - return fmt.Errorf("invalid %s proof: index out of bounds (%d >= %d)", - BadEncoding, int(p.Index), len(merkleRoots), + log.Debugf("%s:%s (%d >= %d)", + invalidProofPrefix, errIncorrectIndex, int(p.Index), len(merkleRoots), ) + return errIncorrectIndex } if len(p.Shares) != len(merkleRoots) { @@ -142,9 +159,10 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { // column, it should exactly match the number of row roots. In this // context, the number of row roots is the width of the extended data // square. - return fmt.Errorf("invalid %s proof: incorrect number of shares %d != %d", - BadEncoding, len(p.Shares), len(merkleRoots), + log.Infof("%s: %s (%d >= %d)", + invalidProofPrefix, errIncorrectAmountOfShares, int(p.Index), len(merkleRoots), ) + return errIncorrectAmountOfShares } odsWidth := uint64(len(merkleRoots) / 2) @@ -160,7 +178,9 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { } if amount < odsWidth { - return errors.New("fraud: invalid proof: not enough shares provided to reconstruct row/col") + log.Debugf("%s: %s. not enough shares provided to reconstruct row/col", + invalidProofPrefix, errIncorrectAmountOfShares) + return errIncorrectAmountOfShares } // verify that Merkle proofs correspond to particular shares. @@ -171,7 +191,8 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { } // validate inclusion of the share into one of the DAHeader roots if ok := shr.Validate(ipld.MustCidFromNamespacedSha256(merkleRoots[index])); !ok { - return fmt.Errorf("invalid %s proof: incorrect share received at index %d", BadEncoding, index) + log.Debugf("%s: %s at index %d", invalidProofPrefix, errIncorrectShare, index) + return errIncorrectShare } // NMTree commits the additional namespace while rsmt2d does not know about, so we trim it // this is ugliness from NMTWrapper that we have to embrace ¯\_(ツ)_/¯ @@ -184,7 +205,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { // the row/col can't be reconstructed, or the building of NMTree fails. rebuiltShares, err := codec.Decode(shares) if err != nil { - log.Infow("failed to decode shares at height", + log.Debugw("failed to decode shares at height", "height", hdr.Height(), "err", err, ) return nil @@ -192,7 +213,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { rebuiltExtendedShares, err := codec.Encode(rebuiltShares[0:odsWidth]) if err != nil { - log.Infow("failed to encode shares at height", + log.Debugw("failed to encode shares at height", "height", hdr.Height(), "err", err, ) return nil @@ -203,7 +224,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { for _, share := range rebuiltShares { err = tree.Push(share) if err != nil { - log.Infow("failed to build a tree from the reconstructed shares at height", + log.Debugw("failed to build a tree from the reconstructed shares at height", "height", hdr.Height(), "err", err, ) return nil @@ -212,7 +233,7 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { expectedRoot, err := tree.Root() if err != nil { - log.Infow("failed to build a tree root at height", + log.Debugw("failed to build a tree root at height", "height", hdr.Height(), "err", err, ) return nil @@ -226,7 +247,8 @@ func (p *BadEncodingProof) Validate(hdr *header.ExtendedHeader) error { // comparing rebuilt Merkle Root of bad row/col with respective Merkle Root of row/col from block. if bytes.Equal(expectedRoot, root) { - return fmt.Errorf("invalid %s proof: recomputed Merkle root matches the DAH's row/column root", BadEncoding) + log.Debugf("invalid %s proof:%s", BadEncoding, errNMTTreeRootsMatch) + return errNMTTreeRootsMatch } return nil } diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index 3e245c1ab3..e7032107ca 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -5,11 +5,12 @@ import ( "testing" "time" - "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" core "github.com/tendermint/tendermint/types" "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/test/util/malicious" + "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" @@ -19,8 +20,8 @@ import ( "github.com/celestiaorg/celestia-node/share/sharetest" ) -func TestBadEncodingFraudProof(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*15) +func TestBEFP_Validate(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) defer t.Cleanup(cancel) bServ := ipld.NewMemBlockservice() @@ -37,10 +38,115 @@ func TestBadEncodingFraudProof(t *testing.T) { errByz := NewErrByzantine(ctx, bServ, &dah, errRsmt2d) befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) - err = befp.Validate(&header.ExtendedHeader{ - DAH: &dah, - }) - assert.NoError(t, err) + + var test = []struct { + name string + prepareFn func() error + expectedResult func(error) + }{ + { + name: "valid BEFP", + prepareFn: func() error { + return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + }, + expectedResult: func(err error) { + require.NoError(t, err) + }, + }, + { + name: "invalid BEFP for valid header", + prepareFn: func() error { + validSquare := edstest.RandEDS(t, 2) + validDah, err := da.NewDataAvailabilityHeader(validSquare) + require.NoError(t, err) + err = ipld.ImportEDS(ctx, validSquare, bServ) + require.NoError(t, err) + validShares := validSquare.Flattened() + errInvalidByz := NewErrByzantine(ctx, bServ, &validDah, + &rsmt2d.ErrByzantineData{ + Axis: rsmt2d.Row, + Index: 0, + Shares: validShares[0:4], + }, + ) + invalidBefp := CreateBadEncodingProof([]byte("hash"), 0, errInvalidByz) + return invalidBefp.Validate(&header.ExtendedHeader{DAH: &validDah}) + }, + expectedResult: func(err error) { + require.ErrorIs(t, err, errNMTTreeRootsMatch) + }, + }, + { + name: "incorrect share with Proof", + prepareFn: func() error { + befp, ok := befp.(*BadEncodingProof) + require.True(t, ok) + befp.Shares[0].Share = befp.Shares[1].Share + return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + }, + expectedResult: func(err error) { + require.ErrorIs(t, err, errIncorrectShare) + }, + }, + { + name: "invalid amount of shares", + prepareFn: func() error { + befp, ok := befp.(*BadEncodingProof) + require.True(t, ok) + befp.Shares = befp.Shares[0 : len(befp.Shares)/2] + return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + }, + expectedResult: func(err error) { + require.ErrorIs(t, err, errIncorrectAmountOfShares) + }, + }, + { + name: "not enough shares to recompute the root", + prepareFn: func() error { + befp, ok := befp.(*BadEncodingProof) + require.True(t, ok) + befp.Shares[0] = nil + return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + }, + expectedResult: func(err error) { + require.ErrorIs(t, err, errIncorrectAmountOfShares) + }, + }, + { + name: "index out of bounds", + prepareFn: func() error { + befp, ok := befp.(*BadEncodingProof) + require.True(t, ok) + befpCopy := *befp + befpCopy.Index = 100 + return befpCopy.Validate(&header.ExtendedHeader{DAH: &dah}) + }, + expectedResult: func(err error) { + require.ErrorIs(t, err, errIncorrectIndex) + }, + }, + { + name: "heights mismatch", + prepareFn: func() error { + return befp.Validate(&header.ExtendedHeader{ + RawHeader: core.Header{ + Height: 42, + }, + DAH: &dah, + }) + }, + expectedResult: func(err error) { + require.ErrorIs(t, err, errHeightMismatch) + }, + }, + } + + for _, tt := range test { + t.Run(tt.name, func(t *testing.T) { + err = tt.prepareFn() + tt.expectedResult(err) + }) + } } // TestIncorrectBadEncodingFraudProof asserts that BEFP is not generated for the correct data @@ -90,3 +196,36 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { err = proof.Validate(h) require.Error(t, err) } + +func TestBEFP_ValidateOutOfOrderShares(t *testing.T) { + // skipping it for now because `malicious` package has a small issue: Constructor does not apply + // passed options, so it's not possible to store shares and thus get proofs for them. + // should be ok once app team will fix it. + t.Skip() + eds := edstest.RandEDS(t, 16) + shares := eds.Flattened() + shares[0], shares[1] = shares[1], shares[0] // corrupting eds + bServ := ipld.NewMemBlockservice() + batchAddr := ipld.NewNmtNodeAdder(context.Background(), bServ, ipld.MaxSizeBatchOption(16*2)) + eds, err := rsmt2d.ImportExtendedDataSquare(shares, + share.DefaultRSMT2DCodec(), + malicious.NewConstructor(16, nmt.NodeVisitor(batchAddr.Visit)), + ) + require.NoError(t, err, "failure to recompute the extended data square") + + err = batchAddr.Commit() + require.NoError(t, err) + + dah, err := da.NewDataAvailabilityHeader(eds) + require.NoError(t, err) + + var errRsmt2d *rsmt2d.ErrByzantineData + err = eds.Repair(dah.RowRoots, dah.ColumnRoots) + require.ErrorAs(t, err, &errRsmt2d) + + errByz := NewErrByzantine(context.Background(), bServ, &dah, errRsmt2d) + + befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) + err = befp.Validate(&header.ExtendedHeader{DAH: &dah}) + require.Error(t, err) +} From 7d545457e2d533863450baba2066101bb5ec8e7d Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 25 Oct 2023 22:49:31 -0400 Subject: [PATCH 286/388] signed binaries for goreleaser (#2869) mimics the signed binaries by goreleaser as implemented in `celestia-app` so node can also generate signed binaries @MSevey confirmed that we have an org level secret for `GPG_SIGNING_KEY` and `GPG_PASSPHRASE` so should require no repo setup implementation for app: https://github.com/celestiaorg/celestia-app/commit/934fdeda062229f5c150e9dce74f05b0557ea123 Will follow up with some work to ensure this workflow actually gets triggered (currently the steps before do not pass and generate binaries) closes: https://github.com/celestiaorg/celestia-node/issues/2679 --- .github/workflows/ci_release.yml | 7 +++++++ .goreleaser.yaml | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 6d641edc73..f795ca4a86 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -87,6 +87,12 @@ jobs: - uses: actions/setup-go@v4 with: go-version: 1.21 + - name: Import GPG key + id: import_gpg + uses: crazy-max/ghaction-import-gpg@v4 + with: + gpg_private_key: ${{ secrets.GPG_SIGNING_KEY }} + passphrase: ${{ secrets.GPG_PASSPHRASE }} # Generate the binaries and release - uses: goreleaser/goreleaser-action@v5 with: @@ -95,6 +101,7 @@ jobs: args: release --clean env: GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }} + GPG_FINGERPRINT: ${{ steps.import_gpg.outputs.fingerprint }} # TODO: permission issue, but not worth fixing as this should be refactored # into the celestiaorg/.github repo, at which point any permission issues will diff --git a/.goreleaser.yaml b/.goreleaser.yaml index 70336dabc0..b229b4c348 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -45,6 +45,18 @@ archives: {{- if .Arm }}v{{ .Arm }}{{ end }} checksum: name_template: "checksums.txt" +signs: + - artifacts: checksum + args: + [ + "--batch", + "-u", + "{{ .Env.GPG_FINGERPRINT }}", + "--output", + "${signature}", + "--detach-sign", + "${artifact}", + ] snapshot: name_template: "{{ incpatch .Version }}-next" changelog: From aa6e0d8e2bc171f7937d90036154e31ed6c88898 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 26 Oct 2023 15:55:25 +0200 Subject: [PATCH 287/388] chore: bump go-header (#2877) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index fcd3c27a0d..5a48d30251 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/celestia-app v1.1.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.4.0 + github.com/celestiaorg/go-header v0.4.1 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 1db8ac6d65..922f91385f 100644 --- a/go.sum +++ b/go.sum @@ -370,8 +370,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.4.0 h1:Ine/xpvFx8o9p6fXW+h2RSPp68rn7VUxTkW1okJxcEY= -github.com/celestiaorg/go-header v0.4.0/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.4.1 h1:bjbUcKDnhrJJ9EoE7vtPpgleNLVjc2S+cB4/qe8nQmo= +github.com/celestiaorg/go-header v0.4.1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= From 6a1e8044c892fa55a923256a8b25eb773460fb0c Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Thu, 26 Oct 2023 22:45:12 +0800 Subject: [PATCH 288/388] fix(core/listener): make listener to retry subscribing on errors (#2876) Resolves https://github.com/celestiaorg/celestia-node/issues/2875 --- core/listener.go | 37 ++++++++++++++++++++++++++++--------- 1 file changed, 28 insertions(+), 9 deletions(-) diff --git a/core/listener.go b/core/listener.go index de0d84ed35..1754cb62b3 100644 --- a/core/listener.go +++ b/core/listener.go @@ -20,7 +20,10 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) -var tracer = otel.Tracer("core/listener") +var ( + tracer = otel.Tracer("core/listener") + retrySubscriptionDelay = 5 * time.Second +) // Listener is responsible for listening to Core for // new block events and converting new Core blocks into @@ -94,18 +97,34 @@ func (cl *Listener) runSubscriber(ctx context.Context, sub <-chan types.EventDat // listener stopped because external context was canceled return } - log.Warnw("listener: subscriber error, resubscribing...", "err", err) - err = cl.fetcher.UnsubscribeNewBlockEvent(ctx) - if err != nil { - log.Errorw("listener: unsubscribe error", "err", err) + log.Warnw("listener: subscriber error, resubscribing...", "err", err) + sub = cl.resubscribe(ctx) + if sub == nil { return } + } +} - sub, err = cl.fetcher.SubscribeNewBlockEvent(ctx) - if err != nil { - log.Errorw("listener: resubscribe error", "err", err) - return +func (cl *Listener) resubscribe(ctx context.Context) <-chan types.EventDataSignedBlock { + err := cl.fetcher.UnsubscribeNewBlockEvent(ctx) + if err != nil { + log.Warnw("listener: unsubscribe", "err", err) + } + + ticker := time.NewTicker(retrySubscriptionDelay) + defer ticker.Stop() + for { + sub, err := cl.fetcher.SubscribeNewBlockEvent(ctx) + if err == nil { + return sub + } + log.Errorw("listener: resubscribe", "err", err) + + select { + case <-ctx.Done(): + return nil + case <-ticker.C: } } } From 8e2aaa40003d5639d131a863ae4507436e93d5be Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 27 Oct 2023 15:51:25 +0200 Subject: [PATCH 289/388] chore(deps): Bumps app to v1.3.0 (#2881) --- core/listener_test.go | 5 +++++ go.mod | 10 +++++----- go.sum | 20 ++++++++++---------- 3 files changed, 20 insertions(+), 15 deletions(-) diff --git a/core/listener_test.go b/core/listener_test.go index eab634a330..9537860d78 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -169,8 +169,13 @@ func createListener( ) *Listener { p2pSub, err := p2p.NewSubscriber[*header.ExtendedHeader](ps, header.MsgID, p2p.WithSubscriberNetworkID(networkID)) require.NoError(t, err) + err = p2pSub.Start(ctx) require.NoError(t, err) + err = p2pSub.SetVerifier(func(ctx context.Context, msg *header.ExtendedHeader) error { + return nil + }) + require.NoError(t, err) t.Cleanup(func() { require.NoError(t, p2pSub.Stop(ctx)) }) diff --git a/go.mod b/go.mod index 5a48d30251..432ba0710d 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.1.0 + github.com/celestiaorg/celestia-app v1.3.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.4.1 @@ -79,7 +79,7 @@ require ( github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2 // indirect github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.0 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect @@ -126,7 +126,7 @@ require ( github.com/cosmos/gorocksdb v1.2.0 // indirect github.com/cosmos/iavl v0.19.6 // indirect github.com/cosmos/ibc-go/v6 v6.2.0 // indirect - github.com/cosmos/ledger-cosmos-go v0.13.1 // indirect + github.com/cosmos/ledger-cosmos-go v0.13.2 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect @@ -312,8 +312,8 @@ require ( github.com/whyrusleeping/cbor v0.0.0-20171005072247-63513f603b11 // indirect github.com/whyrusleeping/cbor-gen v0.0.0-20230818171029-f91ae536ca25 // indirect github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect - github.com/zondax/hid v0.9.1 // indirect - github.com/zondax/ledger-go v0.14.2 // indirect + github.com/zondax/hid v0.9.2 // indirect + github.com/zondax/ledger-go v0.14.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect diff --git a/go.sum b/go.sum index 922f91385f..4d555d3915 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.1.0 h1:8TM/8xnZ+xf7NfEjYZ9XBEGuVG1QT8hPD3uh05RYEsE= -github.com/celestiaorg/celestia-app v1.1.0/go.mod h1:1uAaesXbMkV6EwfBfULXBKYhjHP8DGSVpA3BfKM4QUw= +github.com/celestiaorg/celestia-app v1.3.0 h1:vW1zMc1tQ216utzDOYSC3wGO023sKVt8+zujVXnXlOc= +github.com/celestiaorg/celestia-app v1.3.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 h1:c4cMVLU2bGTesZW1ZVgeoCB++gOOJTF3OvBsqBvo6n0= @@ -454,8 +454,8 @@ github.com/consensys/bavard v0.1.13 h1:oLhMLOFGTLdlda/kma4VOJazblc7IM5y5QPd2A/Yj github.com/consensys/bavard v0.1.13/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/consensys/gnark-crypto v0.5.3/go.mod h1:hOdPlWQV1gDLp7faZVeg8Y0iEPFaOUnCc4XeCCk96p0= -github.com/consensys/gnark-crypto v0.12.0 h1:1OnSpOykNkUIBIBJKdhwy2p0JlW5o+Az02ICzZmvvdg= -github.com/consensys/gnark-crypto v0.12.0/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= +github.com/consensys/gnark-crypto v0.12.1 h1:lHH39WuuFgVHONRl3J0LRBtuYdQTumFSDtJF7HpyG8M= +github.com/consensys/gnark-crypto v0.12.1/go.mod h1:v2Gy7L/4ZRosZ7Ivs+9SfUDr0f5UlG+EM5t7MPHiLuY= github.com/containerd/cgroups v0.0.0-20201119153540-4cbc285b3327/go.mod h1:ZJeTFisyysqgcCdecO57Dj79RfL0LNeGiFUqLYQRYLE= github.com/containerd/cgroups v1.0.3/go.mod h1:/ofk34relqNjSGyqPrmEULrO4Sc8LJhvJmWbUCUKqj8= github.com/containerd/cgroups v1.0.4/go.mod h1:nLNQtsF7Sl2HxNebu77i1R0oDlhiTG+kO4JTrUzo6IA= @@ -494,8 +494,8 @@ github.com/cosmos/iavl v0.19.6 h1:XY78yEeNPrEYyNCKlqr9chrwoeSDJ0bV2VjocTk//OU= github.com/cosmos/iavl v0.19.6/go.mod h1:X9PKD3J0iFxdmgNLa7b2LYWdsGd90ToV5cAONApkEPw= github.com/cosmos/ibc-go/v6 v6.2.0 h1:HKS5WNxQrlmjowHb73J9LqlNJfvTnvkbhXZ9QzNTU7Q= github.com/cosmos/ibc-go/v6 v6.2.0/go.mod h1:+S3sxcNwOhgraYDJAhIFDg5ipXHaUnJrg7tOQqGyWlc= -github.com/cosmos/ledger-cosmos-go v0.13.1 h1:12ac9+GwBb9BjP7X5ygpFk09Itwzjzfmg6A2CWFjoVs= -github.com/cosmos/ledger-cosmos-go v0.13.1/go.mod h1:5tv2RVJEd2+Y38TIQN4CRjJeQGyqOEiKJDfqhk5UjqE= +github.com/cosmos/ledger-cosmos-go v0.13.2 h1:aY0KZSmUwNKbBm9OvbIjvf7Ozz2YzzpAbgvN2C8x2T0= +github.com/cosmos/ledger-cosmos-go v0.13.2/go.mod h1:HENcEP+VtahZFw38HZ3+LS3Iv5XV6svsnkk9vdJtLr8= github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF22jk= github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= @@ -2349,10 +2349,10 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= -github.com/zondax/hid v0.9.1 h1:gQe66rtmyZ8VeGFcOpbuH3r7erYtNEAezCAYu8LdkJo= -github.com/zondax/hid v0.9.1/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= -github.com/zondax/ledger-go v0.14.2 h1:NDaba434N7JUVKg4P4nFv6TOjSkUosYb9sdiw3c61Zk= -github.com/zondax/ledger-go v0.14.2/go.mod h1:fZ3Dqg6qcdXWSOJFKMG8GCTnD7slO/RL2feOQv8K320= +github.com/zondax/hid v0.9.2 h1:WCJFnEDMiqGF64nlZz28E9qLVZ0KSJ7xpc5DLEyma2U= +github.com/zondax/hid v0.9.2/go.mod h1:l5wttcP0jwtdLjqjMMWFVEE7d1zO0jvSPA9OPZxWpEM= +github.com/zondax/ledger-go v0.14.3 h1:wEpJt2CEcBJ428md/5MgSLsXLBos98sBOyxNmCjfUCw= +github.com/zondax/ledger-go v0.14.3/go.mod h1:IKKaoxupuB43g4NxeQmbLXv7T9AlQyie1UpHb342ycI= gitlab.com/NebulousLabs/errors v0.0.0-20171229012116-7ead97ef90b8/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975 h1:L/ENs/Ar1bFzUeKx6m3XjlmBgIUlykX9dzvp5k9NGxc= gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ0dpQyWwlENaeZVBiQRjhMEZvk6VTXquzl3FOFP8= From 1552e9e2c837ebabb49591202dd785e089f19728 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 30 Oct 2023 09:34:28 -0400 Subject: [PATCH 290/388] (ci) update github actions ci to use a single GO_VERSION variable (#2806) This sets a single variable (`GO_VERSION`) in the main `ci.yml` workflow, which will be the single place we set the current go version. We then run a `setup` job that sets an required output for subsequent jobs to use, adding: ``` with: go-version: ${{ needs.setup.outputs.go-version }} ``` Where we had previously either: 1) duplicated the GO_VERSION env var in that workflow 2) passed in `1.21` or whatever go version explicitly When the rpc openrpc.json workflow check changes are merged, this will also prevent another duplication of the go version appearing. refs #2698 --- .github/workflows/ci_release.yml | 20 ++++++++++++++++++-- .github/workflows/go-ci.yml | 18 ++++++++++-------- 2 files changed, 28 insertions(+), 10 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index f795ca4a86..71baf03050 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -9,7 +9,6 @@ on: - "v*" pull_request: workflow_dispatch: - # Inputs the workflow accepts. inputs: version: # Friendly description to be shown in the UI instead of 'name' @@ -23,6 +22,20 @@ on: - major jobs: + # set up go version for use through pipelines, setting + # variable one time and setting outputs to access passing it + # to other jobs + setup: + runs-on: ubuntu-latest + env: + # upgrade go version throughout pipeline here + GO_VERSION: "1.21" + outputs: + go-version: ${{ steps.set-vars.outputs.go-version }} + steps: + - id: set-vars + run: echo "go-version=${{env.GO_VERSION}}" >> "$GITHUB_OUTPUT" + # Dockerfile Linting hadolint: uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.8 # yamllint disable-line rule:line-length @@ -48,7 +61,10 @@ jobs: markdownlint --config .markdownlint.yaml '**/*.md' go-ci: + needs: setup uses: ./.github/workflows/go-ci.yml + with: + go-version: ${{ needs.setup.outputs.go-version }} # If this was a workflow dispatch event, we need to generate and push a tag # for goreleaser to grab @@ -86,7 +102,7 @@ jobs: - run: git fetch --force --tags - uses: actions/setup-go@v4 with: - go-version: 1.21 + go-version: ${{ needs.setup.outputs.go-version }} - name: Import GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@v4 diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 9d80281c6c..39405704eb 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -2,9 +2,11 @@ name: Go CI on: workflow_call: - -env: - GO_VERSION: '1.21' + inputs: + go-version: + description: 'Go version' + required: true + type: string concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} @@ -20,7 +22,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ inputs.go-version }} - name: golangci-lint uses: golangci/golangci-lint-action@v3.7.0 @@ -38,7 +40,7 @@ jobs: - uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ inputs.go-version }} - run: go mod tidy @@ -61,7 +63,7 @@ jobs: - name: set up go uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ inputs.go-version }} - name: run unit tests run: make test-unit @@ -85,7 +87,7 @@ jobs: - name: set up go uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ inputs.go-version }} - name: execute test run run: make test-unit-race @@ -101,7 +103,7 @@ jobs: - name: set up go uses: actions/setup-go@v4 with: - go-version: ${{ env.GO_VERSION }} + go-version: ${{ inputs.go-version }} - name: Swamp Tests run: make test-swamp From f6a649e59c20c64b798a5d2314c3179033c3cc33 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Mon, 30 Oct 2023 14:56:33 +0100 Subject: [PATCH 291/388] chore(nodebuilder): bind OTLP error logging (#2861) A minor slight improvement. It was easy to miss errors in the log wall with the default logger. Here, I use our logging system. I tested it and it looks good --- nodebuilder/settings.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 9767081971..29019c7e77 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -5,6 +5,7 @@ import ( "fmt" "time" + logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" "github.com/pyroscope-io/client/pyroscope" otelpyroscope "github.com/pyroscope-io/otel-profiling-go" @@ -208,5 +209,14 @@ func initializeMetrics( }, }) otel.SetMeterProvider(provider) + otel.SetErrorHandler(&loggingErrorHandler{}) return nil } + +var metricsLogger = logging.Logger("otlp") + +type loggingErrorHandler struct{} + +func (loggingErrorHandler) Handle(err error) { + metricsLogger.Error(err) +} From 62e1769a6c012d4d1aa2fd5ef654b3ddcaa95899 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 30 Oct 2023 13:03:16 -0400 Subject: [PATCH 292/388] (ci) mark flakey test suites as "continue-on-error: true" (#2874) in the interest of having a full pipeline for release 0.12.0, have marked the swamp and unit w/ race tests to `continue-on-error: true` so though the job will still fail in ci, the pipeline and dependent steps after those tests (namely version_bump + goreleaser) will still proceed as though they were successful, and we can get signed binaries with the tagged release --- .github/workflows/ci_release.yml | 18 ++++++++++++++---- .github/workflows/go-ci.yml | 3 +++ 2 files changed, 17 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 71baf03050..2a45e8337c 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -32,10 +32,17 @@ jobs: GO_VERSION: "1.21" outputs: go-version: ${{ steps.set-vars.outputs.go-version }} + branch: ${{ steps.trim_ref.outputs.branch }} steps: - - id: set-vars + - name: Set go version + id: set-vars run: echo "go-version=${{env.GO_VERSION}}" >> "$GITHUB_OUTPUT" + - name: Trim branch name + id: trim_ref + run: | + echo "branch=$(${${{ github.ref }}:11})" >> $GITHUB_OUTPUT + # Dockerfile Linting hadolint: uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.8 # yamllint disable-line rule:line-length @@ -74,6 +81,7 @@ jobs: permissions: "write-all" steps: - uses: actions/checkout@v4 + - name: Bump version and push tag # Placing the if condition here is a workaround for needing to block # on this step during workflow dispatch events but the step not @@ -82,12 +90,10 @@ jobs: # in goreleaser not running either. if: ${{ github.event_name == 'workflow_dispatch' }} uses: mathieudutour/github-tag-action@v6.1 - - - name: Version Release - uses: celestiaorg/.github/.github/actions/version-release@v0.2.8 with: github_token: ${{ secrets.GITHUB_TOKEN }} default_bump: ${{ inputs.version }} + release_branches: ${{ needs.setup.outputs.branch }} # Generate the release with goreleaser to include pre-built binaries goreleaser: @@ -99,16 +105,20 @@ jobs: permissions: "write-all" steps: - uses: actions/checkout@v4 + - run: git fetch --force --tags + - uses: actions/setup-go@v4 with: go-version: ${{ needs.setup.outputs.go-version }} + - name: Import GPG key id: import_gpg uses: crazy-max/ghaction-import-gpg@v4 with: gpg_private_key: ${{ secrets.GPG_SIGNING_KEY }} passphrase: ${{ secrets.GPG_PASSPHRASE }} + # Generate the binaries and release - uses: goreleaser/goreleaser-action@v5 with: diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 39405704eb..9aeef0d4da 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -91,6 +91,7 @@ jobs: - name: execute test run run: make test-unit-race + continue-on-error: true integration_test: needs: [lint, go_mod_tidy_check] @@ -107,6 +108,8 @@ jobs: - name: Swamp Tests run: make test-swamp + continue-on-error: true - name: Swamp Tests with Race Detector run: make test-swamp-race + continue-on-error: true From 20664789d8d5119ec197de304cb9ee103f80c4c1 Mon Sep 17 00:00:00 2001 From: Bao Pham <145053932+bao1029p@users.noreply.github.com> Date: Tue, 31 Oct 2023 01:48:48 +0700 Subject: [PATCH 293/388] refactor(nodebuilder/share/cmd)!: Take block height instead of `ExtendedHeader` as argument for share cmds (#2872) Co-authored-by: Bao Co-authored-by: heh im a handsome boi Replace extended headers with block height as an argument in shares cmd **Issues close** Close [#2848 ](https://github.com/celestiaorg/celestia-node/issues/2848) **Changes made** - Replace all uses of `extended header` as an argument with block height in `nodebuilder/share/cmd/share.go.` - Add logic to fetch header first with `block height` for --- nodebuilder/share/cmd/share.go | 54 +++++++++++++--------------------- 1 file changed, 21 insertions(+), 33 deletions(-) diff --git a/nodebuilder/share/cmd/share.go b/nodebuilder/share/cmd/share.go index 3d1af05289..b890f2d4c0 100644 --- a/nodebuilder/share/cmd/share.go +++ b/nodebuilder/share/cmd/share.go @@ -1,12 +1,14 @@ package cmd import ( + "context" "encoding/hex" - "encoding/json" + "fmt" "strconv" "github.com/spf13/cobra" + rpc "github.com/celestiaorg/celestia-node/api/rpc/client" cmdnode "github.com/celestiaorg/celestia-node/cmd" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" @@ -39,13 +41,8 @@ var sharesAvailableCmd = &cobra.Command{ } defer client.Close() - raw, err := parseJSON(args[0]) - if err != nil { - return err - } + eh, err := getExtendedHeaderFromCmdArg(cmd.Context(), client, args[0]) - var eh *header.ExtendedHeader - err = json.Unmarshal(raw, &eh) if err != nil { return err } @@ -72,7 +69,7 @@ var sharesAvailableCmd = &cobra.Command{ } var getSharesByNamespaceCmd = &cobra.Command{ - Use: "get-by-namespace [extended header, namespace]", + Use: "get-by-namespace (height | hash) namespace", Short: "Gets all shares from an EDS within the given namespace.", Args: cobra.ExactArgs(2), RunE: func(cmd *cobra.Command, args []string) error { @@ -82,13 +79,8 @@ var getSharesByNamespaceCmd = &cobra.Command{ } defer client.Close() - raw, err := parseJSON(args[0]) - if err != nil { - return err - } + eh, err := getExtendedHeaderFromCmdArg(cmd.Context(), client, args[0]) - var eh *header.ExtendedHeader - err = json.Unmarshal(raw, &eh) if err != nil { return err } @@ -104,7 +96,7 @@ var getSharesByNamespaceCmd = &cobra.Command{ } var getShare = &cobra.Command{ - Use: "get-share [extended header, row, col]", + Use: "get-share (height | hash) row col", Short: "Gets a Share by coordinates in EDS.", Args: cobra.ExactArgs(3), RunE: func(cmd *cobra.Command, args []string) error { @@ -114,13 +106,8 @@ var getShare = &cobra.Command{ } defer client.Close() - raw, err := parseJSON(args[0]) - if err != nil { - return err - } + eh, err := getExtendedHeaderFromCmdArg(cmd.Context(), client, args[0]) - var eh *header.ExtendedHeader - err = json.Unmarshal(raw, &eh) if err != nil { return err } @@ -158,8 +145,8 @@ var getShare = &cobra.Command{ } var getEDS = &cobra.Command{ - Use: "get-eds [extended header]", - Short: "Gets the full EDS identified by the given extended header", + Use: "get-eds (height | hash)", + Short: "Gets the full EDS identified by the given block height", Args: cobra.ExactArgs(1), RunE: func(cmd *cobra.Command, args []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) @@ -168,13 +155,8 @@ var getEDS = &cobra.Command{ } defer client.Close() - raw, err := parseJSON(args[0]) - if err != nil { - return err - } + eh, err := getExtendedHeaderFromCmdArg(cmd.Context(), client, args[0]) - var eh *header.ExtendedHeader - err = json.Unmarshal(raw, &eh) if err != nil { return err } @@ -184,8 +166,14 @@ var getEDS = &cobra.Command{ }, } -func parseJSON(param string) (json.RawMessage, error) { - var raw json.RawMessage - err := json.Unmarshal([]byte(param), &raw) - return raw, err +func getExtendedHeaderFromCmdArg(ctx context.Context, client *rpc.Client, arg string) (*header.ExtendedHeader, error) { + hash, err := hex.DecodeString(arg) + if err == nil { + return client.Header.GetByHash(ctx, hash) + } + height, err := strconv.ParseUint(arg, 10, 64) + if err != nil { + return nil, fmt.Errorf("can't parse the height/hash argument: %w", err) + } + return client.Header.GetByHeight(ctx, height) } From 08f394d694a8bd2555af971783a39796af28b5a1 Mon Sep 17 00:00:00 2001 From: smuu <18609909+smuu@users.noreply.github.com> Date: Tue, 31 Oct 2023 13:39:01 +0100 Subject: [PATCH 294/388] feat: celestia mainnet bootstrappers (#2888) --------- Signed-off-by: Smuu <18609909+Smuu@users.noreply.github.com> --- nodebuilder/p2p/bootstrap.go | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index e3971753a1..3e9da1d77d 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -37,7 +37,17 @@ func bootstrappersFor(net Network) ([]string, error) { // NOTE: Every time we add a new long-running network, its bootstrap peers have to be added here. var bootstrapList = map[Network][]string{ - Mainnet: {}, + Mainnet: { + "/dns4/da-bridge-1.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWSqZaLcn5Guypo2mrHr297YPJnV8KMEMXNjs3qAS8msw8", + "/dns4/da-bridge-2.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWQpuTFELgsUypqp9N4a1rKBccmrmQVY8Em9yhqppTJcXf", + "/dns4/da-bridge-3.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWSGa4huD6ts816navn7KFYiStBiy5LrBQH1HuEahk4TzQ", + "/dns4/da-bridge-4.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWHBXCmXaUNat6ooynXG837JXPsZpSTeSzZx6DpgNatMmR", + "/dns4/da-bridge-5.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWDGTBK1a2Ru1qmnnRwP6Dmc44Zpsxi3xbgFk7ATEPfmEU", + "/dns4/da-bridge-6.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWLTUFyf3QEGqYkHWQS2yCtuUcL78vnKBdXU5gABM1YDeH", + "/dns4/da-full-1.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWKZCMcwGCYbL18iuw3YVpAZoyb1VBGbx9Kapsjw3soZgr", + "/dns4/da-full-2.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWE3fmRtHgfk9DCuQFfY3H3JYEnTU3xZozv1Xmo8KWrWbK", + "/dns4/da-full-3.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWK6Ftsd4XsWCsQZgZPNhTrE5urwmkoo5P61tGvnKmNVyv", + }, Arabica: { "/dns4/da-bridge.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWM3e9MWtyc8GkP8QRt74Riu17QuhGfZMytB2vq5NwkWAu", "/dns4/da-bridge-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWKj8mcdiBGxQRe1jqhaMnh2tGoC3rPDmr5UH2q8H4WA9M", From 7d9c59639c737409e6aaaf2bfc3ce1c5fe3d7bd2 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 31 Oct 2023 15:05:06 +0100 Subject: [PATCH 295/388] feat(nodebuilder/p2p): Add mainnet genesis hash (#2889) ![IMG_A965E35247A9-1](https://github.com/celestiaorg/celestia-node/assets/41963722/f2ff92df-567f-44fe-9378-0179aa14df6f) --- nodebuilder/p2p/genesis.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index c88ed53ef3..dcc19dfa49 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -23,7 +23,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ - Mainnet: "", + Mainnet: "6BE39EFD10BA412A9DB5288488303F5DD32CF386707A5BEF33617F4C43301872", Arabica: "5904E55478BA4B3002EE885621E007A2A6A2399662841912219AECD5D5CBE393", Mocha: "B93BBE20A0FBFDF955811B6420F8433904664D45DB4BF51022BE4200C1A1680D", Private: "", From ddd3d6a9d0fea7119b7f1effb3a8c8dd0570d2d7 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 2 Nov 2023 18:46:32 +0100 Subject: [PATCH 296/388] chore(cmd): add a newline after printing token so it doesn't run into shell prompt (#2887) minor annoyance where running `auth admin` always has the token output run right into the next shell prompt. --- cmd/auth.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/auth.go b/cmd/auth.go index 3006526b15..a637373242 100644 --- a/cmd/auth.go +++ b/cmd/auth.go @@ -55,7 +55,7 @@ func AuthCmd(fsets ...*flag.FlagSet) *cobra.Command { if err != nil { return err } - fmt.Printf("%s", token) + fmt.Printf("%s\n", token) return nil }, } From 33f5cd306502ffa39d941e042e624d80c786c89b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 3 Nov 2023 12:20:06 +0000 Subject: [PATCH 297/388] chore(deps): Bump actions/setup-node from 3 to 4 (#2886) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-node](https://github.com/actions/setup-node) from 3 to 4.
Release notes

Sourced from actions/setup-node's releases.

v4.0.0

What's Changed

In scope of this release we changed version of node runtime for action from node16 to node20 and updated dependencies in actions/setup-node#866

Besides, release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-node/compare/v3...v4.0.0

v3.8.2

What's Changed

Full Changelog: https://github.com/actions/setup-node/compare/v3...v3.8.2

v3.8.1

What's Changed

In scope of this release, the filter was removed within the cache-save step by @​dmitry-shibanov in actions/setup-node#831. It is filtered and checked in the toolkit/cache library.

Full Changelog: https://github.com/actions/setup-node/compare/v3...v3.8.1

v3.8.0

What's Changed

Bug fixes:

Feature implementations:

Documentation changes:

Update dependencies:

... (truncated)

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-node&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 2a45e8337c..0e7cb044fa 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -60,7 +60,7 @@ jobs: runs-on: ubuntu-latest steps: - uses: actions/checkout@v4 - - uses: actions/setup-node@v3 + - uses: actions/setup-node@v4 with: node-version: 18 - run: | From 38ef88387688f62a02e03220e9e1f5ec1b7fc3a6 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 9 Nov 2023 14:38:55 +0200 Subject: [PATCH 298/388] chore: increase lint timeout (#2917) Increasing timeout helps to avoid the `timeout exceeded` error. --- .github/workflows/go-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 9aeef0d4da..f2767618b4 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -27,6 +27,7 @@ jobs: - name: golangci-lint uses: golangci/golangci-lint-action@v3.7.0 with: + args: --timeout 10m version: v1.55 skip-pkg-cache: true skip-build-cache: true From 806274b036d5f1a128ffb6cc570c2ec18ecbc761 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Thu, 9 Nov 2023 14:51:50 +0200 Subject: [PATCH 299/388] fix(test/daser): stop the eds store before finishing the test (#2918) --- share/availability/full/testing.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/share/availability/full/testing.go b/share/availability/full/testing.go index dd21d398c2..46e97581f2 100644 --- a/share/availability/full/testing.go +++ b/share/availability/full/testing.go @@ -55,5 +55,10 @@ func TestAvailability(t *testing.T, getter share.Getter) *ShareAvailability { require.NoError(t, err) err = store.Start(context.Background()) require.NoError(t, err) + + t.Cleanup(func() { + err = store.Stop(context.Background()) + require.NoError(t, err) + }) return NewShareAvailability(store, getter, disc) } From f96a46197b078b5bb06d13f202a8110cd877e4b8 Mon Sep 17 00:00:00 2001 From: Josh Stein <46639943+jcstein@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:56:22 +0100 Subject: [PATCH 300/388] chore: update link to node rpc docs (#2916) --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 89cb67dc15..3a552495ad 100644 --- a/README.md +++ b/README.md @@ -55,7 +55,7 @@ For more information on setting up a node and the hardware requirements needed, ## API docs -The celestia-node public API is documented [here](https://docs.celestia.org/category/node-api/). +The celestia-node public API is documented [here](https://node-rpc-docs.celestia.org/). ## Node types From 0c78183aa533edbb3924aba697aadeacf11d149a Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:37:59 +0000 Subject: [PATCH 301/388] chore(deps): Bump crazy-max/ghaction-import-gpg from 4 to 6 (#2885) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [crazy-max/ghaction-import-gpg](https://github.com/crazy-max/ghaction-import-gpg) from 4 to 6.
Release notes

Sourced from crazy-max/ghaction-import-gpg's releases.

v6.0.0

Full Changelog: https://github.com/crazy-max/ghaction-import-gpg/compare/v5.4.0...v6.0.0

v5.4.0

Full Changelog: https://github.com/crazy-max/ghaction-import-gpg/compare/v5.3.0...v5.4.0

v5.3.0

Full Changelog: https://github.com/crazy-max/ghaction-import-gpg/compare/v5.2.0...v5.3.0

v5.2.0

  • Remove setOutput workaround by @​crazy-max (#152)
  • Bump @​actions/core from 1.9.0 to 1.10.0 (#147 #151)
  • Bump openpgp from 5.3.1 to 5.5.0 (#149)

Full Changelog: https://github.com/crazy-max/ghaction-import-gpg/compare/v5.1.0...v5.2.0

v5.1.0

  • Bump openpgp from 5.2.1 to 5.3.1 (#145)
  • Bump @​actions/core from 1.6.0 to 1.9.0 (#143)

v5.0.0

  • Node 16 as default runtime (#136)
    • This requires a minimum Actions Runner version of v2.285.0, which is by default available in GHES 3.4 or later.

v4.4.0

  • Use eslint prettier plugin (#132)
  • Bump openpgp from 5.2.0 to 5.2.1 (#131)
  • Bump minimist from 1.2.5 to 1.2.6 (#134)
  • Bump @​actions/exec from 1.1.0 to 1.1.1 (#133)

v4.3.0

  • Update dev dependencies and workflow (#130)
  • Fix error deleting keys with short key id (#129)
  • Bump openpgp from 5.1.0 to 5.2.0 (#128)

... (truncated)

Commits
  • 82a020f Merge pull request #182 from crazy-max/dependabot/github_actions/actions/chec...
  • 7ad3b9b Merge pull request #183 from crazy-max/update-node20
  • 40ca14f chore: node 20 as default runtime
  • 418bb95 chore: update generated content
  • ecf7766 chore: update dev dependencies
  • d3591c7 chore: update to node 20
  • 255def4 chore: update yarn to 3.6.3
  • 3a06279 codecov: update config
  • c0d4620 codecov: update config
  • f30daee Bump actions/checkout from 3 to 4
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=crazy-max/ghaction-import-gpg&package-manager=github_actions&previous-version=4&new-version=6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan --- .github/workflows/ci_release.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 0e7cb044fa..e4dfdf1be4 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -114,7 +114,7 @@ jobs: - name: Import GPG key id: import_gpg - uses: crazy-max/ghaction-import-gpg@v4 + uses: crazy-max/ghaction-import-gpg@v6 with: gpg_private_key: ${{ secrets.GPG_SIGNING_KEY }} passphrase: ${{ secrets.GPG_PASSPHRASE }} From 62733627e0b25492999c34a556984b2bf91a2c93 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 14:50:46 +0000 Subject: [PATCH 302/388] chore(deps): Bump golang.org/x/sync from 0.4.0 to 0.5.0 (#2900) Bumps [golang.org/x/sync](https://github.com/golang/sync) from 0.4.0 to 0.5.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/sync&package-manager=go_modules&previous-version=0.4.0&new-version=0.5.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 432ba0710d..1174b79ade 100644 --- a/go.mod +++ b/go.mod @@ -68,7 +68,7 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - golang.org/x/sync v0.4.0 + golang.org/x/sync v0.5.0 golang.org/x/text v0.13.0 google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 diff --git a/go.sum b/go.sum index 4d555d3915..ffe39ebc70 100644 --- a/go.sum +++ b/go.sum @@ -2706,8 +2706,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.4.0 h1:zxkM55ReGkDlKSM+Fu41A+zmbZuaPVbGMzvvdUPznYQ= -golang.org/x/sync v0.4.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= +golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= +golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 541cdaf87296d6778ae2f8d76aefc04f3cee5887 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 9 Nov 2023 15:18:16 +0000 Subject: [PATCH 303/388] chore(deps): Bump github.com/gorilla/mux from 1.8.0 to 1.8.1 (#2899) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/gorilla/mux](https://github.com/gorilla/mux) from 1.8.0 to 1.8.1.
Release notes

Sourced from github.com/gorilla/mux's releases.

Release v1.8.1

What's Changed

New Contributors

Full Changelog: https://github.com/gorilla/mux/compare/v1.8.0...v1.8.1

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/gorilla/mux&package-manager=go_modules&previous-version=1.8.0&new-version=1.8.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 3 ++- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/go.mod b/go.mod index 1174b79ade..db0e3d2ddb 100644 --- a/go.mod +++ b/go.mod @@ -24,7 +24,7 @@ require ( github.com/gammazero/workerpool v1.1.3 github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 - github.com/gorilla/mux v1.8.0 + github.com/gorilla/mux v1.8.1 github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 diff --git a/go.sum b/go.sum index ffe39ebc70..f27ce66336 100644 --- a/go.sum +++ b/go.sum @@ -902,8 +902,9 @@ github.com/gorilla/handlers v1.5.1 h1:9lRY6j8DEeeBT10CvO9hGW0gmky0BprnvDI5vfhUHH github.com/gorilla/handlers v1.5.1/go.mod h1:t8XrUpc4KVXb7HGyJ4/cEnwQiaxrX/hz1Zv/4g96P1Q= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= -github.com/gorilla/mux v1.8.0 h1:i40aqfkR1h2SlN9hojwV5ZA91wcXFOvkdNIeFDP5koI= github.com/gorilla/mux v1.8.0/go.mod h1:DVbg23sWSpFRCP0SfiEN6jmj59UnW/n46BH5rLB71So= +github.com/gorilla/mux v1.8.1 h1:TuBL49tXwgrFYWhqrNgrUNEY92u81SPhu7sTdzQEiWY= +github.com/gorilla/mux v1.8.1/go.mod h1:AKf9I4AEqPTmMytcMc0KkNouC66V3BtZ4qD5fmWSiMQ= github.com/gorilla/websocket v0.0.0-20170926233335-4201258b820c/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= From 5756ad10a44482c0858bff737f92782a7f1c187d Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 10 Nov 2023 09:46:41 +0000 Subject: [PATCH 304/388] chore(ci): disable fail-fast on matrix tests (#2919) Disable "fail-fast" so if the `ubuntu-latest` matrix unit test fails then it wont cancel the run of `macos-latest`, this will help with easing ability to merge as we've reduced the requirement for both to merge --- .github/workflows/go-ci.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index f2767618b4..30e7a1294d 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -52,6 +52,7 @@ jobs: needs: [lint, go_mod_tidy_check] name: Unit Tests Coverage strategy: + fail-fast: false matrix: os: [ubuntu-latest, macos-latest] runs-on: ${{ matrix.os }} From 36cdfb35a4d0ac4fb0c86161894bd2cc67818e41 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:25:53 +0000 Subject: [PATCH 305/388] chore(deps): Bump golang.org/x/text from 0.13.0 to 0.14.0 (#2901) Bumps [golang.org/x/text](https://github.com/golang/text) from 0.13.0 to 0.14.0.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/text&package-manager=go_modules&previous-version=0.13.0&new-version=0.14.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db0e3d2ddb..d6266d0314 100644 --- a/go.mod +++ b/go.mod @@ -69,7 +69,7 @@ require ( golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.5.0 - golang.org/x/text v0.13.0 + golang.org/x/text v0.14.0 google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 ) diff --git a/go.sum b/go.sum index f27ce66336..6e59b38907 100644 --- a/go.sum +++ b/go.sum @@ -2880,8 +2880,8 @@ golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.5.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.6.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= -golang.org/x/text v0.13.0 h1:ablQoSUd0tRdKxZewP80B+BaqeKJuVhuRxj/dkrun3k= -golang.org/x/text v0.13.0/go.mod h1:TvPlkZtksWOMsz7fbANvkp4WM8x/WCo/om8BMLbz+aE= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= From 1ae18c9bae15cb390255644ff62e261e713fa0ac Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 10:38:19 +0000 Subject: [PATCH 306/388] chore(deps): Bump github.com/spf13/cobra from 1.7.0 to 1.8.0 (#2902) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/spf13/cobra](https://github.com/spf13/cobra) from 1.7.0 to 1.8.0.
Release notes

Sourced from github.com/spf13/cobra's releases.

v1.8.0

✨ Features

🐛 Bug fixes

🔧 Maintenance

🧪 Testing & CI/CD

✏️ Documentation

Thank you everyone who contributed to this release and all your hard work! Cobra and this community would never be possible without all of you!!!! 🐍

Full Changelog: https://github.com/spf13/cobra/compare/v1.7.0...v1.8.0

Commits
  • a0a6ae0 Improve API to get flag completion function (#2063)
  • 890302a Support usage as plugin for tools like kubectl (#2018)
  • 48cea5c build(deps): bump actions/checkout from 3 to 4 (#2028)
  • 22953d8 Replace all non-alphanumerics in active help env var program prefix (#1940)
  • 00b68a1 Add tests for flag completion registration (#2053)
  • b711e87 Don't complete --help flag when flag parsing disabled (#2061)
  • 8b1eba4 Fix linter errors (#2052)
  • 4cafa37 Allow running persistent run hooks of all parents (#2044)
  • 5c962a2 build(deps): bump github.com/cpuguy83/go-md2man/v2 from 2.0.2 to 2.0.3 (#2047)
  • efe8fa3 build(deps): bump actions/setup-go from 3 to 4 (#1934)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/spf13/cobra&package-manager=go_modules&previous-version=1.7.0&new-version=1.8.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 8 ++++---- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/go.mod b/go.mod index d6266d0314..a3d1a8e694 100644 --- a/go.mod +++ b/go.mod @@ -50,7 +50,7 @@ require ( github.com/prometheus/client_golang v1.17.0 github.com/pyroscope-io/client v0.7.2 github.com/pyroscope-io/otel-profiling-go v0.4.0 - github.com/spf13/cobra v1.7.0 + github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 diff --git a/go.sum b/go.sum index 6e59b38907..a7ffc8dbff 100644 --- a/go.sum +++ b/go.sum @@ -500,8 +500,8 @@ github.com/cpuguy83/go-md2man v1.0.10 h1:BSKMNlYxDvnunlTymqtgONjNnaRV1sTpcovwwjF github.com/cpuguy83/go-md2man v1.0.10/go.mod h1:SmD6nW6nTyfqj6ABTjUi3V3JVMnlJmwcJI5acqYI6dE= github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= -github.com/cpuguy83/go-md2man/v2 v2.0.2 h1:p1EgwI/C7NhT0JmVkwCD2ZBK8j4aeHQX2pMHHBfMQ6w= -github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= +github.com/cpuguy83/go-md2man/v2 v2.0.3 h1:qMCsGGgs+MAzDFyp9LpAe1Lqy/fY/qCovCm0qnXZOBM= +github.com/cpuguy83/go-md2man/v2 v2.0.3/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3 h1:HVTnpeuvF6Owjd5mniCL8DEXo7uYXdQEmOP4FJbV5tg= github.com/crackcomm/go-gitignore v0.0.0-20170627025303-887ab5e44cc3/go.mod h1:p1d6YEZWvFzEh4KLyvBcVSnrfNDDvK2zfK/4x2v/4pE= github.com/crate-crypto/go-kzg-4844 v0.3.0 h1:UBlWE0CgyFqqzTI+IFyCzA7A3Zw4iip6uzRv5NIXG0A= @@ -2199,8 +2199,8 @@ github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU= -github.com/spf13/cobra v1.7.0 h1:hyqWnYt1ZQShIddO5kBpj3vu05/++x6tJ6dg8EC572I= -github.com/spf13/cobra v1.7.0/go.mod h1:uLxZILRyS/50WlhOIKD7W6V5bgeIt+4sICxh6uRMrb0= +github.com/spf13/cobra v1.8.0 h1:7aJaZx1B85qltLMc546zn58BxxfZdR/W22ej9CFoEf0= +github.com/spf13/cobra v1.8.0/go.mod h1:WXLWApfZ71AjXPya3WOlMsY9yMs7YeiHhFVlvLyhcho= github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo= github.com/spf13/jwalterweatherman v1.1.0 h1:ue6voC5bR5F8YxI5S67j9i582FU4Qvo2bmqnqMYADFk= github.com/spf13/jwalterweatherman v1.1.0/go.mod h1:aNWZUN0dPAAO/Ljvb5BEdw96iTZ0EXowPYD95IqWIGo= From 1823aeedf9645eb2196827e1099088e0f720d053 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 10 Nov 2023 12:53:52 +0200 Subject: [PATCH 307/388] fix(blob/service): handle absence proof (#2911) Fixes #2860 --- blob/service.go | 7 +++++++ blob/service_test.go | 2 -- share/eds/utils.go | 2 +- share/getters/shrex.go | 2 +- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/blob/service.go b/blob/service.go index 67fcae349a..79e7dd7937 100644 --- a/blob/service.go +++ b/blob/service.go @@ -219,6 +219,13 @@ func (s *Service) getByCommitment( ) for _, row := range namespacedShares { + if len(row.Shares) == 0 { + // the above condition means that we've faced with an Absence Proof. + // This Proof proves that the namespace was not found in the DAH, so + // we can return `ErrBlobNotFound`. + return nil, nil, ErrBlobNotFound + } + appShares, err := toAppShares(row.Shares...) if err != nil { return nil, nil, err diff --git a/blob/service_test.go b/blob/service_test.go index 152058bed0..6777084eb4 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -1,5 +1,3 @@ -//go:build linux - package blob import ( diff --git a/share/eds/utils.go b/share/eds/utils.go index 2133565b91..b897dd14b5 100644 --- a/share/eds/utils.go +++ b/share/eds/utils.go @@ -123,7 +123,7 @@ func CollectSharesByNamespace( rootCIDs := ipld.FilterRootByNamespace(root, namespace) if len(rootCIDs) == 0 { - return nil, nil + return []share.NamespacedRow{}, nil } errGroup, ctx := errgroup.WithContext(ctx) diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 9576da252b..0586826e22 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -213,7 +213,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( dah := header.DAH roots := ipld.FilterRootByNamespace(dah, namespace) if len(roots) == 0 { - return nil, nil + return []share.NamespacedRow{}, nil } for { From 65f0f53dcd7bf25c009e19026aaf775204ac4fad Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 11:03:54 +0000 Subject: [PATCH 308/388] chore(deps): Bump go.uber.org/fx from 1.20.0 to 1.20.1 (#2879) Bumps [go.uber.org/fx](https://github.com/uber-go/fx) from 1.20.0 to 1.20.1.
Release notes

Sourced from go.uber.org/fx's releases.

v1.20.1

Added

  • Provided, Decorated, Supplied, and Replaced events now include a trace of module locations through which the option was given to the App.
  • wasi support.
Changelog

Sourced from go.uber.org/fx's changelog.

1.20.1 - 2023-10-17

Added

  • Provided, Decorated, Supplied, and Replaced events now include a trace of module locations through which the option was given to the App.
  • wasi support.
Commits
  • 9636854 Prepare release v1.20.1 (#1126)
  • 1a5cc04 Include Module Traces in Events (#1122)
  • bbb3783 chore(deps): Bump golang.org/x/net from 0.16.0 to 0.17.0 in /tools (#1124)
  • 4b3089c chore(deps): Bump golang.org/x/tools from 0.13.0 to 0.14.0 in /tools (#1123)
  • f594306 adding support for wasi (#1121)
  • 580de58 chore(deps): bump actions/checkout from 3 to 4 (#1115)
  • e03bc58 chore(deps): bump honnef.co/go/tools from 0.4.5 to 0.4.6 in /tools (#1119)
  • fe6a7db chore(deps): bump golang.org/x/tools from 0.12.0 to 0.13.0 in /tools (#1118)
  • 00970bd chore(deps): bump honnef.co/go/tools from 0.4.3 to 0.4.5 in /tools (#1112)
  • 391edcd Use Go 1.21 (#1111)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=go.uber.org/fx&package-manager=go_modules&previous-version=1.20.0&new-version=1.20.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index a3d1a8e694..77ec4d5bfa 100644 --- a/go.mod +++ b/go.mod @@ -64,7 +64,7 @@ require ( go.opentelemetry.io/otel/sdk/metric v1.19.0 go.opentelemetry.io/otel/trace v1.19.0 go.opentelemetry.io/proto/otlp v1.0.0 - go.uber.org/fx v1.20.0 + go.uber.org/fx v1.20.1 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 golang.org/x/exp v0.0.0-20230905200255-921286631fa9 diff --git a/go.sum b/go.sum index a7ffc8dbff..ba656c61ff 100644 --- a/go.sum +++ b/go.sum @@ -2425,8 +2425,8 @@ go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= -go.uber.org/fx v1.20.0 h1:ZMC/pnRvhsthOZh9MZjMq5U8Or3mA9zBSPaLnzs3ihQ= -go.uber.org/fx v1.20.0/go.mod h1:qCUj0btiR3/JnanEr1TYEePfSw6o/4qYJscgvzQ5Ub0= +go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk= +go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg= go.uber.org/goleak v1.0.0/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.10/go.mod h1:8a7PlsEVH3e/a/GLqe5IIrQx6GzcnRmZEufDUTk4A7A= go.uber.org/goleak v1.1.11-0.20210813005559-691160354723/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= From 70d7a3992bc6f0473a1ffff8b91302971c49ae03 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 10 Nov 2023 12:34:19 +0000 Subject: [PATCH 309/388] (ci) enable optional verbose logging on unit tests and upload artifacts (#2865) - Optionally enabled verbose logging for unit tests - adds ability to verbose log to a file, while sending "normal" quiet output to stdout - updates CI to upload the test.log artifact and store for 5 days for later debugging - local workflow and test runs will stay the same - only enabled for `make test-unit` for now as a proposal for how we achieve this, can expand to rest if we like it @renaynay and @vgonkivs If everyone likes, i can expand to all test cases fixes https://github.com/celestiaorg/celestia-node/issues/1211 --- .github/workflows/ci_release.yml | 10 ++++++++++ .github/workflows/go-ci.yml | 28 +++++++++++++++++++++++++++- Makefile | 9 ++++++++- 3 files changed, 45 insertions(+), 2 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index e4dfdf1be4..25970fdb11 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -33,6 +33,7 @@ jobs: outputs: go-version: ${{ steps.set-vars.outputs.go-version }} branch: ${{ steps.trim_ref.outputs.branch }} + debug: ${{ steps.debug.outputs.debug }} steps: - name: Set go version id: set-vars @@ -43,6 +44,15 @@ jobs: run: | echo "branch=$(${${{ github.ref }}:11})" >> $GITHUB_OUTPUT + - name: Set debug output + id: debug + run: | + if [[ "${{ runner.debug }}" == "true" ]]; then + echo "debug=true" >> $GITHUB_ENV + else + echo "debug=false" >> $GITHUB_ENV + fi + # Dockerfile Linting hadolint: uses: celestiaorg/.github/.github/workflows/reusable_dockerfile_lint.yml@v0.2.8 # yamllint disable-line rule:line-length diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 30e7a1294d..d27e08e516 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -13,7 +13,22 @@ concurrency: cancel-in-progress: true jobs: + setup: + runs-on: ubuntu-latest + outputs: + debug: ${{ steps.debug.outputs.debug }} + steps: + - name: Set debug output + id: debug + run: | + if [[ "${{ runner.debug }}" == "true" ]]; then + echo "debug=true" >> $GITHUB_ENV + else + echo "debug=false" >> $GITHUB_ENV + fi + lint: + needs: [setup] name: Lint runs-on: ubuntu-latest @@ -33,6 +48,7 @@ jobs: skip-build-cache: true go_mod_tidy_check: + needs: [setup] name: Go Mod Tidy Check runs-on: ubuntu-latest @@ -68,7 +84,17 @@ jobs: go-version: ${{ inputs.go-version }} - name: run unit tests - run: make test-unit + run: make test-unit ENABLE_VERBOSE=${{ needs.setup.outputs.debug }} + + - name: Upload unit test output + uses: actions/upload-artifact@v3 + if: always() && needs.setup.outputs.debug == 'true' + with: + name: unit-test-output-${{ matrix.os }} + path: | + debug.log + coverage.txt + retention-days: 5 - name: upload coverage uses: codecov/codecov-action@v3.1.4 diff --git a/Makefile b/Makefile index d90075431a..1133db0533 100644 --- a/Makefile +++ b/Makefile @@ -6,6 +6,13 @@ LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versionin ifeq (${PREFIX},) PREFIX := /usr/local endif +ifeq ($(ENABLE_VERBOSE),true) + LOG_AND_FILTER = | tee debug.log + VERBOSE = -v +else + VERBOSE = + LOG_AND_FILTER = +endif ## help: Get more info on make commands. help: Makefile @echo " Choose a command run in "$(PROJECTNAME)":" @@ -99,7 +106,7 @@ lint: lint-imports ## test-unit: Running unit tests test-unit: @echo "--> Running unit tests" - @go test -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` + @go test $(VERBOSE) -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER) .PHONY: test-unit ## test-unit-race: Running unit tests with data race detector From 48352ebfb033ff10cf76b7f85e9fad4addbdbd91 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 10 Nov 2023 12:48:34 +0000 Subject: [PATCH 310/388] chore(ci): remove 'continue-on-error' configuration from flakey tests (#2913) Added these in the run up to cutting 0.12.0 so hopefully the tag/release pipeline would generate signed binaries and not halt on any of these three flakey test runs. Setting these back to fail explicitly so we can see them go red and feel bad, and now start to fix them --- .github/workflows/go-ci.yml | 3 --- 1 file changed, 3 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index d27e08e516..056e4816bf 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -119,7 +119,6 @@ jobs: - name: execute test run run: make test-unit-race - continue-on-error: true integration_test: needs: [lint, go_mod_tidy_check] @@ -136,8 +135,6 @@ jobs: - name: Swamp Tests run: make test-swamp - continue-on-error: true - name: Swamp Tests with Race Detector run: make test-swamp-race - continue-on-error: true From db4490c530f8c73e24366f1f563bbd89b7ef1edc Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 13:07:33 +0000 Subject: [PATCH 311/388] chore(deps): Bump github.com/multiformats/go-multiaddr from 0.11.0 to 0.12.0 (#2847) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/multiformats/go-multiaddr](https://github.com/multiformats/go-multiaddr) from 0.11.0 to 0.12.0.
Release notes

Sourced from github.com/multiformats/go-multiaddr's releases.

v0.12.0

What's Changed

Full Changelog: https://github.com/multiformats/go-multiaddr/compare/v0.11.0...v0.12.0

Commits
  • 5dd793c release v0.12.0 (#223)
  • f9a66bc net: consider /dns/localhost as private address (#221)
  • a124954 net: consider dns addresses as public (#220)
  • dfafc59 chore: bump go.mod to Go 1.20 and run go fix (#217)
  • 92a246d Merge pull request #216 from multiformats/uci/copy-templates
  • db92edd chore: add or force update .github/workflows/tagpush.yml
  • 438fa23 chore: add or force update .github/workflows/release-check.yml
  • 841bae1 chore: add or force update .github/workflows/releaser.yml
  • 9bfab28 chore: add or force update .github/workflows/go-check.yml
  • aeaad91 chore: add or force update .github/workflows/go-test.yml
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/multiformats/go-multiaddr&package-manager=go_modules&previous-version=0.11.0&new-version=0.12.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 77ec4d5bfa..febfb03a1b 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.11.0 + github.com/multiformats/go-multiaddr v0.12.0 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 diff --git a/go.sum b/go.sum index ba656c61ff..5f13e39e08 100644 --- a/go.sum +++ b/go.sum @@ -1827,8 +1827,8 @@ github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= -github.com/multiformats/go-multiaddr v0.11.0 h1:XqGyJ8ufbCE0HmTDwx2kPdsrQ36AGPZNZX6s6xfJH10= -github.com/multiformats/go-multiaddr v0.11.0/go.mod h1:gWUm0QLR4thQ6+ZF6SXUw8YjtwQSPapICM+NmCkxHSM= +github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE= +github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= From 0847919876fe6211b523c79eeb4ac07b7583321d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 13:34:13 +0000 Subject: [PATCH 312/388] chore(deps): Bump google.golang.org/grpc from 1.58.3 to 1.59.0 (#2866) [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.58.3&new-version=1.59.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
--------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index febfb03a1b..113c9a02ae 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( golang.org/x/exp v0.0.0-20230905200255-921286631fa9 golang.org/x/sync v0.5.0 golang.org/x/text v0.14.0 - google.golang.org/grpc v1.58.3 + google.golang.org/grpc v1.59.0 google.golang.org/protobuf v1.31.0 ) @@ -166,7 +166,7 @@ require ( github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/gateway v1.1.0 // indirect - github.com/golang/glog v1.1.0 // indirect + github.com/golang/glog v1.1.2 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect @@ -323,7 +323,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.12.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.10.0 // indirect + golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect golang.org/x/tools v0.13.0 // indirect diff --git a/go.sum b/go.sum index 5f13e39e08..1de7057386 100644 --- a/go.sum +++ b/go.sum @@ -756,8 +756,8 @@ github.com/golang-jwt/jwt/v4 v4.3.0/go.mod h1:/xlHOz8bRuivTWchD4jCa+NbatV+wEUSzw github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k= github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI= github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q= -github.com/golang/glog v1.1.0 h1:/d3pCKDPWNnvIWe0vVUpNP32qc8U3PDVxySP/y360qE= -github.com/golang/glog v1.1.0/go.mod h1:pfYeQZ3JWZoXTV5sFc986z3HTpwQs9At6P4ImfuP3NQ= +github.com/golang/glog v1.1.2 h1:DVjP2PbBOzHyzA+dn3WhHIq4NdVu3Q+pvivFICf/7fo= +github.com/golang/glog v1.1.2/go.mod h1:zR+okUeTbrL6EL3xHUDxZuEtGv04p5shwip1+mL/rLQ= github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= github.com/golang/groupcache v0.0.0-20191027212112-611e8accdfc9/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc= @@ -2689,8 +2689,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.10.0 h1:zHCpF2Khkwy4mMB4bv0U37YtJdTGW8jI0glAApi0Kh8= -golang.org/x/oauth2 v0.10.0/go.mod h1:kTpgurOux7LqtuxjuyZa4Gj2gdezIt/jQtGnNFfypQI= +golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= +golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -3213,8 +3213,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= -google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= +google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 3fde8c9acb90bb9d4d1abbdc2c5917d7f3693388 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 10 Nov 2023 13:51:59 +0000 Subject: [PATCH 313/388] chore(deps): Bump github.com/ipfs/boxo from 0.13.1 to 0.15.0 (#2903) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/boxo](https://github.com/ipfs/boxo) from 0.13.1 to 0.15.0.
Release notes

Sourced from github.com/ipfs/boxo's releases.

v0.15.0

Changed

What's Changed

Full Changelog: https://github.com/ipfs/boxo/compare/v0.14.0...v0.15.0

v0.14.0

Added

  • boxo/gateway:
    • A new WithResolver(...) option can be used with NewBlocksBackend(...) allowing the user to pass their custom Resolver implementation.
    • The gateway now sets a Cache-Control header for requests under the /ipns/ namespace if the TTL for the corresponding IPNS Records or DNSLink entities is known.
  • boxo/bitswap/client:
    • A new WithoutDuplicatedBlockStats() option can be used with bitswap.New and bsclient.New. This disable accounting for duplicated blocks, which requires a blockstore.Has() lookup for every received block and thus, can impact performance.
  • ✨ Migrated repositories into Boxo

Changed

  • boxo/gateway
    • 🛠 The IPFSBackend interface was updated to make the responses of the Head method more explicit. It now returns a HeadResponse instead of a files.Node.
  • boxo/routing/http/client.Client is now exported. This means you can now pass it around functions, or add it to a struct if you want.
  • 🛠 The path package has been massively refactored. With this refactor, we have condensed the different path-related and/or Kubo-specific packages under a single generic one. Therefore, there are many breaking changes. Please consult the documentation for more details on how to use the new package.
    • Note: content paths created with boxo/path are automatically normalized:
      • Replace multiple slashes with a single slash.
      • Eliminate each . path name element (the current directory).
      • Eliminate each inner .. path name element (the parent directory) along with the non-.. element that precedes it.
      • Eliminate .. elements that begin a rooted path: that is, replace "/.." by "/" at the beginning of a path.
  • 🛠 The signature of CoreAPI.ResolvePath in coreiface has changed to now return the remainder segments as a second return value, matching the signature of resolver.ResolveToLastNode.
  • 🛠 routing/http/client.FindPeers now returns iter.ResultIter[types.PeerRecord] instead of iter.ResultIter[types.Record]. The specification indicates that records for this method will always be Peer Records.
  • 🛠 The namesys package has been refactored. The following are the largest modifications:
    • The options in coreiface/options/namesys have been moved to namesys and their names have been made more consistent.
    • Many of the exported structs and functions have been renamed in order to be consistent with the remaining packages.
    • namesys.Resolver.Resolve now returns a TTL, in addition to the resolved path. If the TTL is unknown, 0 is returned. IPNSResolver is able to resolve a TTL, while DNSResolver is not.
    • namesys/resolver.ResolveIPNS has been moved to namesys.ResolveIPNS and now returns a TTL in addition to the resolved path.
  • ✨ boxo/ipns record defaults follow recommendations from IPNS Record Specification:
    • DefaultRecordTTL is now set to 1h
    • DefaultRecordLifetime follows the increased expiration window of Amino DHT (go-libp2p-kad-dht#793) and is set to 48h
  • 🛠 The gateway's IPFSBackend.ResolveMutable is now expected to return a TTL in addition to the resolved path. If the TTL is unknown, 0 should be returned.

Removed

... (truncated)

Changelog

Sourced from github.com/ipfs/boxo's changelog.

[v0.15.0]

Changed

[v0.14.0]

Added

  • boxo/gateway:
    • A new WithResolver(...) option can be used with NewBlocksBackend(...) allowing the user to pass their custom Resolver implementation.
    • The gateway now sets a Cache-Control header for requests under the /ipns/ namespace if the TTL for the corresponding IPNS Records or DNSLink entities is known.
  • boxo/bitswap/client:
    • A new WithoutDuplicatedBlockStats() option can be used with bitswap.New and bsclient.New. This disable accounting for duplicated blocks, which requires a blockstore.Has() lookup for every received block and thus, can impact performance.
  • ✨ Migrated repositories into Boxo

Changed

  • boxo/gateway
    • 🛠 The IPFSBackend interface was updated to make the responses of the Head method more explicit. It now returns a HeadResponse instead of a files.Node.
  • boxo/routing/http/client.Client is now exported. This means you can now pass it around functions, or add it to a struct if you want.
  • 🛠 The path package has been massively refactored. With this refactor, we have condensed the different path-related and/or Kubo-specific packages under a single generic one. Therefore, there are many breaking changes. Please consult the documentation for more details on how to use the new package.
    • Note: content paths created with boxo/path are automatically normalized:
      • Replace multiple slashes with a single slash.
      • Eliminate each . path name element (the current directory).
      • Eliminate each inner .. path name element (the parent directory) along with the non-.. element that precedes it.
      • Eliminate .. elements that begin a rooted path: that is, replace "/.." by "/" at the beginning of a path.
  • 🛠 The signature of CoreAPI.ResolvePath in coreiface has changed to now return the remainder segments as a second return value, matching the signature of resolver.ResolveToLastNode.
  • 🛠 routing/http/client.FindPeers now returns iter.ResultIter[types.PeerRecord] instead of iter.ResultIter[types.Record]. The specification indicates that records for this method will always be Peer Records.
  • 🛠 The namesys package has been refactored. The following are the largest modifications:
    • The options in coreiface/options/namesys have been moved to namesys and their names have been made more consistent.
    • Many of the exported structs and functions have been renamed in order to be consistent with the remaining packages.
    • namesys.Resolver.Resolve now returns a TTL, in addition to the resolved path. If the TTL is unknown, 0 is returned. IPNSResolver is able to resolve a TTL, while DNSResolver is not.
    • namesys/resolver.ResolveIPNS has been moved to namesys.ResolveIPNS and now returns a TTL

... (truncated)

Commits
  • 521f8d4 Merge pull request #505 from ipfs/release-v0.15.0
  • 5af54b2 chore: release 0.15.0
  • 75ca06e docs: release notes for 0.15.0
  • 3982cdc chore: bump to go-libp2p@0.32.0
  • 9c22812 Merge pull request #501 from ipfs/release
  • 07c5719 Merge pull request #500 from ipfs/release-v0.14.0
  • 50f0e2c chore: bump version to 0.14.0
  • 66c76e5 docs: update changelog for 0.14.0
  • 08b11e5 Merge pull request #474 from ipfs/chore/migrate-bootstrap
  • 45426db Update bootstrap/bootstrap.go
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/boxo&package-manager=go_modules&previous-version=0.13.1&new-version=0.15.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 33 ++++++++++++++-------------- go.sum | 68 +++++++++++++++++++++++++++++++--------------------------- 2 files changed, 53 insertions(+), 48 deletions(-) diff --git a/go.mod b/go.mod index 113c9a02ae..04dd804cc4 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.4 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.13.1 + github.com/ipfs/boxo v0.15.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 @@ -36,9 +36,9 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car v0.6.2 - github.com/libp2p/go-libp2p v0.31.0 + github.com/libp2p/go-libp2p v0.32.0 github.com/libp2p/go-libp2p-kad-dht v0.25.1 - github.com/libp2p/go-libp2p-pubsub v0.9.3 + github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/mitchellh/go-homedir v1.1.0 @@ -67,7 +67,7 @@ require ( go.uber.org/fx v1.20.1 go.uber.org/zap v1.26.0 golang.org/x/crypto v0.14.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.5.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.59.0 @@ -84,6 +84,7 @@ require ( github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/supranational/blst v0.3.11 // indirect + go.uber.org/mock v0.3.0 // indirect rsc.io/tmplfunc v0.0.3 // indirect ) @@ -175,7 +176,7 @@ require ( github.com/google/go-cmp v0.5.9 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect - github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f // indirect + github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect github.com/google/s2a-go v0.1.4 // indirect github.com/google/uuid v1.3.1 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect @@ -229,7 +230,7 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.16.7 // indirect + github.com/klauspost/compress v1.17.2 // indirect github.com/klauspost/cpuid/v2 v2.2.5 // indirect github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect @@ -249,9 +250,9 @@ require ( github.com/manifoldco/promptui v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect - github.com/miekg/dns v1.1.55 // indirect + github.com/miekg/dns v1.1.56 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect github.com/mimoo/StrobeGo v0.0.0-20210601165009-122bf33a46e0 // indirect @@ -265,9 +266,9 @@ require ( github.com/multiformats/go-multiaddr-fmt v0.1.0 // indirect github.com/multiformats/go-multibase v0.2.0 // indirect github.com/multiformats/go-multicodec v0.9.0 // indirect - github.com/multiformats/go-multistream v0.4.1 // indirect + github.com/multiformats/go-multistream v0.5.0 // indirect github.com/multiformats/go-varint v0.0.7 // indirect - github.com/onsi/ginkgo/v2 v2.11.0 // indirect + github.com/onsi/ginkgo/v2 v2.13.0 // indirect github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect @@ -283,9 +284,9 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect - github.com/quic-go/qtls-go1-20 v0.3.3 // indirect - github.com/quic-go/quic-go v0.38.1 // indirect - github.com/quic-go/webtransport-go v0.5.3 // indirect + github.com/quic-go/qtls-go1-20 v0.3.4 // indirect + github.com/quic-go/quic-go v0.39.3 // indirect + github.com/quic-go/webtransport-go v0.6.0 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect @@ -319,14 +320,14 @@ require ( go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect - go.uber.org/dig v1.17.0 // indirect + go.uber.org/dig v1.17.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.12.0 // indirect + golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect golang.org/x/sys v0.13.0 // indirect golang.org/x/term v0.13.0 // indirect - golang.org/x/tools v0.13.0 // indirect + golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.128.0 // indirect diff --git a/go.sum b/go.sum index 1de7057386..0458580b74 100644 --- a/go.sum +++ b/go.sum @@ -861,8 +861,8 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S37WGHujM7tX//fmj1uioxKzKxz3lo4HJo= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f h1:pDhu5sgp8yJlEF/g6osliIIpF9K4F5jvkULXa4daRDQ= -github.com/google/pprof v0.0.0-20230821062121-407c9e7a662f/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= +github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= +github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= @@ -1038,8 +1038,8 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.13.1 h1:nQ5oQzcMZR3oL41REJDcTbrvDvuZh3J9ckc9+ILeRQI= -github.com/ipfs/boxo v0.13.1/go.mod h1:btrtHy0lmO1ODMECbbEY1pxNtrLilvKSYLoGQt1yYCk= +github.com/ipfs/boxo v0.15.0 h1:BriLydj2nlK1nKeJQHxcKSuG5ZXcoutzhBklOtxC5pk= +github.com/ipfs/boxo v0.15.0/go.mod h1:X5ulcbR5Nh7sm3Db8+08AApUo6FsGC5mb23QDKAoB/M= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= @@ -1332,8 +1332,8 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.16.7 h1:2mk3MPGNzKyxErAw8YaohYh69+pa4sIQSC0fPGCFR9I= -github.com/klauspost/compress v1.16.7/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= +github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1414,8 +1414,8 @@ github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBx github.com/libp2p/go-libp2p v0.23.4/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= github.com/libp2p/go-libp2p v0.25.0/go.mod h1:vXHmFpcfl+xIGN4qW58Bw3a0/SKGAesr5/T4IuJHE3o= github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= -github.com/libp2p/go-libp2p v0.31.0 h1:LFShhP8F6xthWiBBq3euxbKjZsoRajVEyBS9snfHxYg= -github.com/libp2p/go-libp2p v0.31.0/go.mod h1:W/FEK1c/t04PbRH3fA9i5oucu5YcgrG0JVoBWT1B7Eg= +github.com/libp2p/go-libp2p v0.32.0 h1:86I4B7nBUPIyTgw3+5Ibq6K7DdKRCuZw8URCfPc1hQM= +github.com/libp2p/go-libp2p v0.32.0/go.mod h1:hXXC3kXPlBZ1eu8Q2hptGrMB4mZ3048JUoS4EKaHW5c= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= @@ -1506,8 +1506,8 @@ github.com/libp2p/go-libp2p-peerstore v0.2.7/go.mod h1:ss/TWTgHZTMpsU/oKVVPQCGuD github.com/libp2p/go-libp2p-peerstore v0.4.0/go.mod h1:rDJUFyzEWPpXpEwywkcTYYzDHlwza8riYMaUzaN6hX0= github.com/libp2p/go-libp2p-peerstore v0.6.0/go.mod h1:DGEmKdXrcYpK9Jha3sS7MhqYdInxJy84bIPtSu65bKc= github.com/libp2p/go-libp2p-pnet v0.2.0/go.mod h1:Qqvq6JH/oMZGwqs3N1Fqhv8NVhrdYcO0BW4wssv21LA= -github.com/libp2p/go-libp2p-pubsub v0.9.3 h1:ihcz9oIBMaCK9kcx+yHWm3mLAFBMAUsM4ux42aikDxo= -github.com/libp2p/go-libp2p-pubsub v0.9.3/go.mod h1:RYA7aM9jIic5VV47WXu4GkcRxRhrdElWf8xtyli+Dzc= +github.com/libp2p/go-libp2p-pubsub v0.10.0 h1:wS0S5FlISavMaAbxyQn3dxMOe2eegMfswM471RuHJwA= +github.com/libp2p/go-libp2p-pubsub v0.10.0/go.mod h1:1OxbaT/pFRO5h+Dpze8hdHQ63R0ke55XTs6b6NwLLkw= github.com/libp2p/go-libp2p-quic-transport v0.10.0/go.mod h1:RfJbZ8IqXIhxBRm5hqUEJqjiiY8xmEuq3HUDS993MkA= github.com/libp2p/go-libp2p-quic-transport v0.13.0/go.mod h1:39/ZWJ1TW/jx1iFkKzzUg00W6tDJh73FC0xYudjr7Hc= github.com/libp2p/go-libp2p-quic-transport v0.16.0/go.mod h1:1BXjVMzr+w7EkPfiHkKnwsWjPjtfaNT0q8RS3tGDvEQ= @@ -1720,8 +1720,9 @@ github.com/mattn/go-isatty v0.0.13/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Ky github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= github.com/mattn/go-isatty v0.0.17/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/mattn/go-pointer v0.0.1/go.mod h1:2zXcozF6qYGgmsG+SeTZz3oAbFLdD3OWqnUbNvJZAlc= github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= github.com/mattn/go-runewidth v0.0.3/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -1744,8 +1745,8 @@ github.com/miekg/dns v1.1.41/go.mod h1:p6aan82bvRIyn+zDIv9xYNUpwa73JcSh9BKwknJys github.com/miekg/dns v1.1.43/go.mod h1:+evo5L0630/F6ca/Z9+GAqzhjGyn8/c+TBaOyfEl0V4= github.com/miekg/dns v1.1.48/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= github.com/miekg/dns v1.1.50/go.mod h1:e3IlAVfNqAllflbibAZEWOXOQ+Ynzk/dDozDxY7XnME= -github.com/miekg/dns v1.1.55 h1:GoQ4hpsj0nFLYe+bWiCToyrBEJXkQfOOIvFGFy0lEgo= -github.com/miekg/dns v1.1.55/go.mod h1:uInx36IzPl7FYnDcMeVWxj9byh7DutNykX4G9Sj60FY= +github.com/miekg/dns v1.1.56 h1:5imZaSeoRNvpM9SzWNhEcP9QliKiz20/dA2QabIGVnE= +github.com/miekg/dns v1.1.56/go.mod h1:cRm6Oo2C8TY9ZS/TqsSrseAcncm74lfK5G+ikN2SWWY= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c h1:bzE/A84HN25pxAuk9Eej1Kz9OUelF97nAc82bDquQI8= github.com/mikioh/tcp v0.0.0-20190314235350-803a9b46060c/go.mod h1:0SQS9kMwD2VsyFEB++InYyBJroV/FRmBgcydeSUcJms= github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b h1:z78hV3sbSMAUoyUMM0I83AUIT6Hu17AWfgjzIbtrYFc= @@ -1882,8 +1883,9 @@ github.com/multiformats/go-multistream v0.3.0/go.mod h1:ODRoqamLUsETKS9BNcII4gcR github.com/multiformats/go-multistream v0.3.1/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-multistream v0.3.3/go.mod h1:ODRoqamLUsETKS9BNcII4gcRsJBU5VAwRIv7O39cEXg= github.com/multiformats/go-multistream v0.4.0/go.mod h1:BS6ZSYcA4NwYEaIMeCtpJydp2Dc+fNRA6uJMSu/m8+4= -github.com/multiformats/go-multistream v0.4.1 h1:rFy0Iiyn3YT0asivDUIR05leAdwZq3de4741sbiSdfo= github.com/multiformats/go-multistream v0.4.1/go.mod h1:Mz5eykRVAjJWckE2U78c6xqdtyNUEhKSM0Lwar2p77Q= +github.com/multiformats/go-multistream v0.5.0 h1:5htLSLl7lvJk3xx3qT/8Zm9J4K8vEOf/QGkvOGQAyiE= +github.com/multiformats/go-multistream v0.5.0/go.mod h1:n6tMZiwiP2wUsR8DgfDWw1dydlEqV3l6N3/GBsX6ILA= github.com/multiformats/go-varint v0.0.1/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.2/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= github.com/multiformats/go-varint v0.0.5/go.mod h1:3Ls8CIEsrijN6+B7PbrXRPxHRPuXSrVKRY101jdMZYE= @@ -1934,8 +1936,8 @@ github.com/onsi/ginkgo/v2 v2.2.0/go.mod h1:MEH45j8TBi6u9BMogfbp0stKC5cdGjumZj5Y7 github.com/onsi/ginkgo/v2 v2.3.0/go.mod h1:Eew0uilEqZmIEZr8JrvYlvOM7Rr6xzTmMV8AyFNU9d0= github.com/onsi/ginkgo/v2 v2.4.0/go.mod h1:iHkDK1fKGcBoEHT5W7YBq4RFWaQulw+caOMkAt4OrFo= github.com/onsi/ginkgo/v2 v2.5.1/go.mod h1:63DOGlLAH8+REH8jUGdL3YpCpu7JODesutUjdENfUAc= -github.com/onsi/ginkgo/v2 v2.11.0 h1:WgqUCUt/lT6yXoQ8Wef0fsNn5cAuMK7+KT9UFRz2tcU= -github.com/onsi/ginkgo/v2 v2.11.0/go.mod h1:ZhrRA5XmEE3x3rhlzamx/JJvujdZoJ2uvgI7kR0iZvM= +github.com/onsi/ginkgo/v2 v2.13.0 h1:0jY9lJquiL8fcf3M4LAXN5aMlS/b2BV86HFFPCPMgE4= +github.com/onsi/ginkgo/v2 v2.13.0/go.mod h1:TE309ZR8s5FsKKpuB1YAQYBzCaAfUgatB/xlT/ETL/o= github.com/onsi/gomega v1.4.1/go.mod h1:C1qb7wdrVGGVU+Z6iS04AVkA3Q65CEZX59MT0QO5uiA= github.com/onsi/gomega v1.4.3/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= @@ -1949,8 +1951,8 @@ github.com/onsi/gomega v1.20.1/go.mod h1:DtrZpjmvpn2mPm4YWQa0/ALMDj9v4YxLgojwPeR github.com/onsi/gomega v1.21.1/go.mod h1:iYAIXgPSaDHak0LCMA+AWBpIKBr8WZicMxnE8luStNc= github.com/onsi/gomega v1.22.1/go.mod h1:x6n7VNe4hw0vkyYUM4mjIXx3JbLiPaBPNgB7PRQ1tuM= github.com/onsi/gomega v1.24.0/go.mod h1:Z/NWtiqwBrwUt4/2loMmHL63EDLnYHmVbuBpDr2vQAg= -github.com/onsi/gomega v1.27.8 h1:gegWiwZjBsf2DgiSbf5hpokZ98JVDMcWkUiigk6/KXc= -github.com/onsi/gomega v1.27.8/go.mod h1:2J8vzI/s+2shY9XHRApDkdgPo1TKT7P2u6fXeJKFnNQ= +github.com/onsi/gomega v1.27.10 h1:naR28SdDFlqrG6kScpT8VWpu1xWY5nJRCF3XaYyBjhI= +github.com/onsi/gomega v1.27.10/go.mod h1:RsS8tutOdbdgzbPtzzATp12yT7kM5I5aElG3evPbQ0M= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 h1:CznVS40zms0Dj5he4ERo+fRPtO0qxUk8lA8Xu3ddet0= github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333/go.mod h1:Ag6rSXkHIckQmjFBCweJEEt1mrTPBv8b9W4aU/NQWfI= @@ -2084,14 +2086,14 @@ github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1 github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= github.com/quic-go/qtls-go1-19 v0.2.0/go.mod h1:ySOI96ew8lnoKPtSqx2BlI5wCpUVPT05RMAlajtnyOI= github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8uT7EiStoU1SM= -github.com/quic-go/qtls-go1-20 v0.3.3 h1:17/glZSLI9P9fDAeyCHBFSWSqJcwx1byhLwP5eUIDCM= -github.com/quic-go/qtls-go1-20 v0.3.3/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= +github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= +github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= -github.com/quic-go/quic-go v0.38.1 h1:M36YWA5dEhEeT+slOu/SwMEucbYd0YFidxG3KlGPZaE= -github.com/quic-go/quic-go v0.38.1/go.mod h1:ijnZM7JsFIkp4cRyjxJNIzdSfCLmUMg9wdyhGmg+SN4= +github.com/quic-go/quic-go v0.39.3 h1:o3YB6t2SR+HU/pgwF29kJ6g4jJIJEwEZ8CKia1h1TKg= +github.com/quic-go/quic-go v0.39.3/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= github.com/quic-go/webtransport-go v0.5.1/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= -github.com/quic-go/webtransport-go v0.5.3 h1:5XMlzemqB4qmOlgIus5zB45AcZ2kCgCy2EptUrfOPWU= -github.com/quic-go/webtransport-go v0.5.3/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= +github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY= +github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= github.com/rakyll/statik v0.1.7 h1:OF3QCZUuyPxuGEP7B4ypUa7sB/iHtqOTDYZXGM8KOdQ= github.com/rakyll/statik v0.1.7/go.mod h1:AlZONWzMtEnMs7W4e/1LURLiI49pIMmp6V9Unghqrcc= github.com/raulk/clock v1.1.0/go.mod h1:3MpVxdZ/ODBQDxbN+kzshf5OSZwPjtMDx6BBXBmOeY0= @@ -2422,8 +2424,8 @@ go.uber.org/atomic v1.10.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/atomic v1.11.0 h1:ZvwS0R+56ePWxUNi+Atn9dWONBPp/AUETXlHW0DxSjE= go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0= go.uber.org/dig v1.15.0/go.mod h1:pKHs0wMynzL6brANhB2hLMro+zalv1osARTviTcqHLM= -go.uber.org/dig v1.17.0 h1:5Chju+tUvcC+N7N6EV08BJz41UZuO3BmHcN4A287ZLI= -go.uber.org/dig v1.17.0/go.mod h1:rTxpf7l5I0eBTlE6/9RL+lDybC7WFwY2QH55ZSjy1mU= +go.uber.org/dig v1.17.1 h1:Tga8Lz8PcYNsWsyHMZ1Vm0OQOUaJNDyvPImgbAu9YSc= +go.uber.org/dig v1.17.1/go.mod h1:Us0rSJiThwCv2GteUN0Q7OKvU7n5J4dxZ9JKUXozFdE= go.uber.org/fx v1.18.2/go.mod h1:g0V1KMQ66zIRk8bLu3Ea5Jt2w/cHlOIp4wdRsgh0JaY= go.uber.org/fx v1.20.1 h1:zVwVQGS8zYvhh9Xxcu4w1M6ESyeMzebzj2NbSayZ4Mk= go.uber.org/fx v1.20.1/go.mod h1:iSYNbHf2y55acNCwCXKx7LbWb5WG1Bnue5RDXz1OREg= @@ -2434,6 +2436,8 @@ go.uber.org/goleak v1.1.11/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ go.uber.org/goleak v1.1.12/go.mod h1:cwTWslyiVhfpKIDGSZEM2HlOvcqm+tG4zioyIeLoqMQ= go.uber.org/goleak v1.2.0 h1:xqgm/S+aQvhWFTtR0XK3Jvg7z8kGV8P4X14IzwN3Eqk= go.uber.org/goleak v1.2.0/go.mod h1:XJYK+MuIchqpmGmUSAzotztawfKvYLUIgg7guXrwVUo= +go.uber.org/mock v0.3.0 h1:3mUxI1No2/60yUYax92Pt8eNOEecx2D3lcXZh2NEZJo= +go.uber.org/mock v0.3.0/go.mod h1:a6FSlNadKUHUa9IP5Vyt1zh4fC7uAwxMutEAscFbkZc= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/multierr v1.3.0/go.mod h1:VgVr7evmIr6uPjLBxg28wmKNXyqE9akIJ5XnfpiKl+4= go.uber.org/multierr v1.5.0/go.mod h1:FeouvMocqHpRaaGuG9EjoKcStLC43Zu/fmqdUMPcKYU= @@ -2530,8 +2534,8 @@ golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2567,8 +2571,8 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.12.0 h1:rmsUpXtvNzj340zd98LZ4KntptpfRHwpFOHG188oHXc= -golang.org/x/mod v0.12.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= +golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= +golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2966,8 +2970,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.13.0 h1:Iey4qkscZuv0VvIt8E0neZjtPVQFSc870HQ448QgEmQ= -golang.org/x/tools v0.13.0/go.mod h1:HvlwmtVNQAhOuCjW7xxvovg8wbNq7LwfXh/k7wXUl58= +golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= +golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From a4ef6d135605c700928700662103332615c293b3 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Mon, 20 Nov 2023 11:09:54 +0100 Subject: [PATCH 314/388] feat(core): Enable metrics for core package (#2863) While we do plan to remove the `bridge` node type eventually, it's useful to have some visibility into the core package in the short term. --- core/exchange.go | 28 ++++++++++- core/exchange_metrics.go | 49 +++++++++++++++++++ core/exchange_test.go | 3 +- core/listener.go | 29 ++++++++++-- core/listener_metrics.go | 81 ++++++++++++++++++++++++++++++++ core/listener_test.go | 4 +- core/option.go | 15 ++++++ nodebuilder/core/config.go | 2 + nodebuilder/core/module.go | 24 ++++++++-- nodebuilder/settings.go | 2 + nodebuilder/tests/swamp/swamp.go | 3 +- 11 files changed, 229 insertions(+), 11 deletions(-) create mode 100644 core/exchange_metrics.go create mode 100644 core/listener_metrics.go create mode 100644 core/option.go diff --git a/core/exchange.go b/core/exchange.go index 06f648edad..79f3d6337a 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -22,18 +22,38 @@ type Exchange struct { fetcher *BlockFetcher store *eds.Store construct header.ConstructFn + + metrics *exchangeMetrics } func NewExchange( fetcher *BlockFetcher, store *eds.Store, construct header.ConstructFn, -) *Exchange { + opts ...Option, +) (*Exchange, error) { + p := new(params) + for _, opt := range opts { + opt(p) + } + + var ( + metrics *exchangeMetrics + err error + ) + if p.metrics { + metrics, err = newExchangeMetrics() + if err != nil { + return nil, err + } + } + return &Exchange{ fetcher: fetcher, store: store, construct: construct, - } + metrics: metrics, + }, nil } func (ce *Exchange) GetByHeight(ctx context.Context, height uint64) (*header.ExtendedHeader, error) { @@ -47,12 +67,16 @@ func (ce *Exchange) GetRangeByHeight( from *header.ExtendedHeader, to uint64, ) ([]*header.ExtendedHeader, error) { + start := time.Now() + amount := to - (from.Height() + 1) headers, err := ce.getRangeByHeight(ctx, from.Height()+1, amount) if err != nil { return nil, err } + ce.metrics.requestDurationPerHeader(ctx, time.Since(start), amount) + for _, h := range headers { err := from.Verify(h) if err != nil { diff --git a/core/exchange_metrics.go b/core/exchange_metrics.go new file mode 100644 index 0000000000..0b454a6e4d --- /dev/null +++ b/core/exchange_metrics.go @@ -0,0 +1,49 @@ +package core + +import ( + "context" + "time" + + "go.opentelemetry.io/otel/metric" +) + +type exchangeMetrics struct { + getByHeightDuration metric.Float64Histogram +} + +func newExchangeMetrics() (*exchangeMetrics, error) { + m := new(exchangeMetrics) + + var err error + m.getByHeightDuration, err = meter.Float64Histogram( + "core_ex_get_by_height_request_time", + metric.WithDescription("core exchange client getByHeight request time in seconds (per single height)"), + ) + if err != nil { + return nil, err + } + + return m, nil +} + +func (m *exchangeMetrics) observe(ctx context.Context, observeFn func(ctx context.Context)) { + if m == nil { + return + } + + if ctx.Err() != nil { + ctx = context.Background() + } + + observeFn(ctx) +} + +func (m *exchangeMetrics) requestDurationPerHeader(ctx context.Context, duration time.Duration, amount uint64) { + m.observe(ctx, func(ctx context.Context) { + if amount == 0 { + return + } + durationPerHeader := duration.Seconds() / float64(amount) + m.getByHeightDuration.Record(ctx, durationPerHeader) + }) +} diff --git a/core/exchange_test.go b/core/exchange_test.go index 853b5a8dc6..c43084c57d 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -27,7 +27,8 @@ func TestCoreExchange_RequestHeaders(t *testing.T) { store := createStore(t) - ce := NewExchange(fetcher, store, header.MakeExtendedHeader) + ce, err := NewExchange(fetcher, store, header.MakeExtendedHeader) + require.NoError(t, err) // initialize store with genesis block genHeight := int64(1) diff --git a/core/listener.go b/core/listener.go index 1754cb62b3..8447733506 100644 --- a/core/listener.go +++ b/core/listener.go @@ -43,6 +43,8 @@ type Listener struct { listenerTimeout time.Duration + metrics *listenerMetrics + cancel context.CancelFunc } @@ -53,7 +55,24 @@ func NewListener( construct header.ConstructFn, store *eds.Store, blocktime time.Duration, -) *Listener { + opts ...Option, +) (*Listener, error) { + p := new(params) + for _, opt := range opts { + opt(p) + } + + var ( + metrics *listenerMetrics + err error + ) + if p.metrics { + metrics, err = newListenerMetrics() + if err != nil { + return nil, err + } + } + return &Listener{ fetcher: fetcher, headerBroadcaster: bcast, @@ -61,7 +80,8 @@ func NewListener( construct: construct, store: store, listenerTimeout: 5 * blocktime, - } + metrics: metrics, + }, nil } // Start kicks off the Listener listener loop. @@ -85,7 +105,7 @@ func (cl *Listener) Start(context.Context) error { func (cl *Listener) Stop(context.Context) error { cl.cancel() cl.cancel = nil - return nil + return cl.metrics.Close() } // runSubscriber runs a subscriber to receive event data of new signed blocks. It will attempt to @@ -144,6 +164,7 @@ func (cl *Listener) listen(ctx context.Context, sub <-chan types.EventDataSigned } log.Debugw("listener: new block from core", "height", b.Header.Height) + err := cl.handleNewSignedBlock(ctx, b) if err != nil { log.Errorw("listener: handling new block msg", @@ -157,6 +178,7 @@ func (cl *Listener) listen(ctx context.Context, sub <-chan types.EventDataSigned } timeout.Reset(cl.listenerTimeout) case <-timeout.C: + cl.metrics.subscriptionStuck(ctx) return errors.New("underlying subscription is stuck") case <-ctx.Done(): return ctx.Err() @@ -178,6 +200,7 @@ func (cl *Listener) handleNewSignedBlock(ctx context.Context, b types.EventDataS if err != nil { return fmt.Errorf("extending block data: %w", err) } + // generate extended header eh, err := cl.construct(&b.Header, &b.Commit, &b.ValidatorSet, eds) if err != nil { diff --git a/core/listener_metrics.go b/core/listener_metrics.go new file mode 100644 index 0000000000..a325149b5d --- /dev/null +++ b/core/listener_metrics.go @@ -0,0 +1,81 @@ +package core + +import ( + "context" + "time" + + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/metric" +) + +var meter = otel.Meter("core") + +type listenerMetrics struct { + lastTimeSubscriptionStuck time.Time + lastTimeSubscriptionStuckInst metric.Int64ObservableGauge + lastTimeSubscriptionStuckReg metric.Registration + + subscriptionStuckInst metric.Int64Counter +} + +func newListenerMetrics() (*listenerMetrics, error) { + m := new(listenerMetrics) + + var err error + m.subscriptionStuckInst, err = meter.Int64Counter( + "core_listener_subscription_stuck_count", + metric.WithDescription("number of times core listener block subscription has been stuck/retried"), + ) + if err != nil { + return nil, err + } + + m.lastTimeSubscriptionStuckInst, err = meter.Int64ObservableGauge( + "core_listener_last_time_subscription_stuck_timestamp", + metric.WithDescription("last time the listener subscription was stuck"), + ) + if err != nil { + return nil, err + } + m.lastTimeSubscriptionStuckReg, err = meter.RegisterCallback( + m.observeLastTimeStuckCallback, + m.lastTimeSubscriptionStuckInst, + ) + if err != nil { + return nil, err + } + + return m, nil +} + +func (m *listenerMetrics) observe(ctx context.Context, observeFn func(ctx context.Context)) { + if m == nil { + return + } + + if ctx.Err() != nil { + ctx = context.Background() + } + + observeFn(ctx) +} + +func (m *listenerMetrics) subscriptionStuck(ctx context.Context) { + m.observe(ctx, func(ctx context.Context) { + m.subscriptionStuckInst.Add(ctx, 1) + m.lastTimeSubscriptionStuck = time.Now() + }) +} + +func (m *listenerMetrics) observeLastTimeStuckCallback(_ context.Context, obs metric.Observer) error { + obs.ObserveInt64(m.lastTimeSubscriptionStuckInst, m.lastTimeSubscriptionStuck.Unix()) + return nil +} + +func (m *listenerMetrics) Close() error { + if m == nil { + return nil + } + + return m.lastTimeSubscriptionStuckReg.Unregister() +} diff --git a/core/listener_test.go b/core/listener_test.go index 9537860d78..bf84c07b41 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -180,7 +180,9 @@ func createListener( require.NoError(t, p2pSub.Stop(ctx)) }) - return NewListener(p2pSub, fetcher, edsSub.Broadcast, header.MakeExtendedHeader, store, nodep2p.BlockTime) + listener, err := NewListener(p2pSub, fetcher, edsSub.Broadcast, header.MakeExtendedHeader, store, nodep2p.BlockTime) + require.NoError(t, err) + return listener } func createEdsPubSub(ctx context.Context, t *testing.T) *shrexsub.PubSub { diff --git a/core/option.go b/core/option.go new file mode 100644 index 0000000000..6e06fade48 --- /dev/null +++ b/core/option.go @@ -0,0 +1,15 @@ +package core + +type Option func(*params) + +type params struct { + metrics bool +} + +// WithMetrics is a functional option that enables metrics +// inside the core package. +func WithMetrics() Option { + return func(p *params) { + p.metrics = true + } +} diff --git a/nodebuilder/core/config.go b/nodebuilder/core/config.go index 4affcd3087..bb5eea5b83 100644 --- a/nodebuilder/core/config.go +++ b/nodebuilder/core/config.go @@ -7,6 +7,8 @@ import ( "github.com/celestiaorg/celestia-node/libs/utils" ) +var MetricsEnabled bool + // Config combines all configuration fields for managing the relationship with a Core node. type Config struct { IP string diff --git a/nodebuilder/core/module.go b/nodebuilder/core/module.go index 02863eae7e..fec7c14b1b 100644 --- a/nodebuilder/core/module.go +++ b/nodebuilder/core/module.go @@ -35,7 +35,20 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option return fx.Module("core", baseComponents, fx.Provide(core.NewBlockFetcher), - fxutil.ProvideAs(core.NewExchange, new(libhead.Exchange[*header.ExtendedHeader])), + fxutil.ProvideAs( + func( + fetcher *core.BlockFetcher, + store *eds.Store, + construct header.ConstructFn, + ) (*core.Exchange, error) { + var opts []core.Option + if MetricsEnabled { + opts = append(opts, core.WithMetrics()) + } + + return core.NewExchange(fetcher, store, construct, opts...) + }, + new(libhead.Exchange[*header.ExtendedHeader])), fx.Invoke(fx.Annotate( func( bcast libhead.Broadcaster[*header.ExtendedHeader], @@ -43,8 +56,13 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option pubsub *shrexsub.PubSub, construct header.ConstructFn, store *eds.Store, - ) *core.Listener { - return core.NewListener(bcast, fetcher, pubsub.Broadcast, construct, store, p2p.BlockTime) + ) (*core.Listener, error) { + var opts []core.Option + if MetricsEnabled { + opts = append(opts, core.WithMetrics()) + } + + return core.NewListener(bcast, fetcher, pubsub.Broadcast, construct, store, p2p.BlockTime, opts...) }, fx.OnStart(func(ctx context.Context, listener *core.Listener) error { return listener.Start(ctx) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 29019c7e77..2d924af000 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -24,6 +24,7 @@ import ( "github.com/celestiaorg/go-fraud" "github.com/celestiaorg/celestia-node/header" + modcore "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/das" modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" @@ -76,6 +77,7 @@ func WithMetrics(metricOpts []otlpmetrichttp.Option, nodeType node.Type) fx.Opti // TODO @renaynay: this will be refactored when there is more granular // control over which module to enable metrics for modhead.MetricsEnabled = true + modcore.MetricsEnabled = true baseComponents := fx.Options( fx.Supply(metricOpts), diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 55426cf70c..e3ac3ad4f2 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -176,11 +176,12 @@ func (s *Swamp) setupGenesis() { store, err := eds.NewStore(eds.DefaultParameters(), s.t.TempDir(), ds) require.NoError(s.t, err) - ex := core.NewExchange( + ex, err := core.NewExchange( core.NewBlockFetcher(s.ClientContext.Client), store, header.MakeExtendedHeader, ) + require.NoError(s.t, err) h, err := ex.GetByHeight(ctx, 1) require.NoError(s.t, err) From d1883d6ddb9f34b2a8aa37d0a4d74f11721152f5 Mon Sep 17 00:00:00 2001 From: Evan Forbes <42654277+evan-forbes@users.noreply.github.com> Date: Wed, 22 Nov 2023 19:11:10 -0600 Subject: [PATCH 315/388] chore: bump celestia-app to v1.4.0 (#2948) This PR bumps to v1.4.0 of celestia-app, which should be a non-breaking version and has been tested by syncing from scratch on mainnet. We still need to test on robusta before moving onto arabica-11 and mocha-4. --- go.mod | 4 ++-- go.sum | 8 ++++---- state/core_access_test.go | 2 +- 3 files changed, 7 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 04dd804cc4..c648c01889 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 - github.com/celestiaorg/celestia-app v1.3.0 + github.com/celestiaorg/celestia-app v1.4.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.4.1 @@ -344,7 +344,7 @@ require ( ) replace ( - github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 + github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app diff --git a/go.sum b/go.sum index 0458580b74..d45b9f0c06 100644 --- a/go.sum +++ b/go.sum @@ -358,12 +358,12 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.3.0 h1:vW1zMc1tQ216utzDOYSC3wGO023sKVt8+zujVXnXlOc= -github.com/celestiaorg/celestia-app v1.3.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= +github.com/celestiaorg/celestia-app v1.4.0 h1:hTId3xL8GssN5sHSHPP7svHi/iWp+XVxphStiR7ADiY= +github.com/celestiaorg/celestia-app v1.4.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= -github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14 h1:c4cMVLU2bGTesZW1ZVgeoCB++gOOJTF3OvBsqBvo6n0= -github.com/celestiaorg/cosmos-sdk v1.18.1-sdk-v0.46.14/go.mod h1:D5y5Exw0bJkcDv9fvYDiZfZrDV1b6+xsFyiungxrCsU= +github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 h1:+Te28r5Zp4Vp69f82kcON9/BIF8f1BNXb0go2+achuc= +github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14/go.mod h1:Og5KKgoBEiQlI6u56lDLG191pfknIdXctFn3COWLQP8= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= diff --git a/state/core_access_test.go b/state/core_access_test.go index 69e9f251c0..68d05a955a 100644 --- a/state/core_access_test.go +++ b/state/core_access_test.go @@ -27,7 +27,7 @@ func TestSubmitPayForBlob(t *testing.T) { tmCfg.Consensus.TimeoutCommit = time.Millisecond * 1 appConf := testnode.DefaultAppConfig() appConf.API.Enable = true - appConf.MinGasPrices = fmt.Sprintf("0.1%s", app.BondDenom) + appConf.MinGasPrices = fmt.Sprintf("0.002%s", app.BondDenom) config := testnode.DefaultConfig().WithTendermintConfig(tmCfg).WithAppConfig(appConf).WithAccounts(accounts) cctx, rpcAddr, grpcAddr := testnode.NewNetwork(t, config) From dd3b8e1411dee3acaad1b4dd21833893f5daedba Mon Sep 17 00:00:00 2001 From: Javed Khan Date: Thu, 23 Nov 2023 08:52:36 -0800 Subject: [PATCH 316/388] feat(cmd): pass Start options; add WithFlagSet option (#2950) Fixes #2936 This PR updates the `Start` command to accept a variadic number of `funcs` which can be used to customise the command. This allows the caller to hook into and modify the command, for example to add a `PreRun` hook, as required for `celestia-da`. With this PR in place, the `celestia-da` requirement, which needs to add some required flags and add a `PreRun` hook can be satisfied as: ```go bridgeCmd := bridge.NewCommand(WithPreRun()) lightCmd := light.NewCommand(WithPreRun()) fullCmd := full.NewCommand(WithPreRun()) ``` Ideally this can be applied for other commands `Init`, `Auth` as well, but it's not required for the current requirement. --- cmd/celestia/bridge.go | 47 ---------------------- cmd/celestia/full.go | 51 ------------------------ cmd/celestia/light.go | 51 ------------------------ cmd/celestia/main.go | 19 +++++++++ cmd/node.go | 88 ++++++++++++++++++++++++++++++++++++++++++ cmd/start.go | 8 ++-- cmd/util.go | 10 +++++ 7 files changed, 121 insertions(+), 153 deletions(-) delete mode 100644 cmd/celestia/bridge.go delete mode 100644 cmd/celestia/full.go delete mode 100644 cmd/celestia/light.go create mode 100644 cmd/node.go diff --git a/cmd/celestia/bridge.go b/cmd/celestia/bridge.go deleted file mode 100644 index c0e2ab0d1a..0000000000 --- a/cmd/celestia/bridge.go +++ /dev/null @@ -1,47 +0,0 @@ -package main - -import ( - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - cmdnode "github.com/celestiaorg/celestia-node/cmd" - "github.com/celestiaorg/celestia-node/nodebuilder/core" - "github.com/celestiaorg/celestia-node/nodebuilder/gateway" - "github.com/celestiaorg/celestia-node/nodebuilder/node" - "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/nodebuilder/rpc" - "github.com/celestiaorg/celestia-node/nodebuilder/state" -) - -// NOTE: We should always ensure that the added Flags below are parsed somewhere, like in the -// PersistentPreRun func on parent command. - -func init() { - flags := []*pflag.FlagSet{ - cmdnode.NodeFlags(), - p2p.Flags(), - core.Flags(), - cmdnode.MiscFlags(), - rpc.Flags(), - gateway.Flags(), - state.Flags(), - } - - bridgeCmd.AddCommand( - cmdnode.Init(flags...), - cmdnode.Start(flags...), - cmdnode.AuthCmd(flags...), - cmdnode.ResetStore(flags...), - cmdnode.RemoveConfigCmd(flags...), - cmdnode.UpdateConfigCmd(flags...), - ) -} - -var bridgeCmd = &cobra.Command{ - Use: "bridge [subcommand]", - Args: cobra.NoArgs, - Short: "Manage your Bridge node", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return cmdnode.PersistentPreRunEnv(cmd, node.Bridge, args) - }, -} diff --git a/cmd/celestia/full.go b/cmd/celestia/full.go deleted file mode 100644 index 8baff1080e..0000000000 --- a/cmd/celestia/full.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - cmdnode "github.com/celestiaorg/celestia-node/cmd" - "github.com/celestiaorg/celestia-node/nodebuilder/core" - "github.com/celestiaorg/celestia-node/nodebuilder/gateway" - "github.com/celestiaorg/celestia-node/nodebuilder/header" - "github.com/celestiaorg/celestia-node/nodebuilder/node" - "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/nodebuilder/rpc" - "github.com/celestiaorg/celestia-node/nodebuilder/state" -) - -// NOTE: We should always ensure that the added Flags below are parsed somewhere, like in the -// PersistentPreRun func on parent command. - -func init() { - flags := []*pflag.FlagSet{ - cmdnode.NodeFlags(), - p2p.Flags(), - header.Flags(), - cmdnode.MiscFlags(), - // NOTE: for now, state-related queries can only be accessed - // over an RPC connection with a celestia-core node. - core.Flags(), - rpc.Flags(), - gateway.Flags(), - state.Flags(), - } - - fullCmd.AddCommand( - cmdnode.Init(flags...), - cmdnode.Start(flags...), - cmdnode.AuthCmd(flags...), - cmdnode.ResetStore(flags...), - cmdnode.RemoveConfigCmd(flags...), - cmdnode.UpdateConfigCmd(flags...), - ) -} - -var fullCmd = &cobra.Command{ - Use: "full [subcommand]", - Args: cobra.NoArgs, - Short: "Manage your Full node", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return cmdnode.PersistentPreRunEnv(cmd, node.Full, args) - }, -} diff --git a/cmd/celestia/light.go b/cmd/celestia/light.go deleted file mode 100644 index 553660c5d3..0000000000 --- a/cmd/celestia/light.go +++ /dev/null @@ -1,51 +0,0 @@ -package main - -import ( - "github.com/spf13/cobra" - "github.com/spf13/pflag" - - cmdnode "github.com/celestiaorg/celestia-node/cmd" - "github.com/celestiaorg/celestia-node/nodebuilder/core" - "github.com/celestiaorg/celestia-node/nodebuilder/gateway" - "github.com/celestiaorg/celestia-node/nodebuilder/header" - "github.com/celestiaorg/celestia-node/nodebuilder/node" - "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/nodebuilder/rpc" - "github.com/celestiaorg/celestia-node/nodebuilder/state" -) - -// NOTE: We should always ensure that the added Flags below are parsed somewhere, like in the -// PersistentPreRun func on parent command. - -func init() { - flags := []*pflag.FlagSet{ - cmdnode.NodeFlags(), - p2p.Flags(), - header.Flags(), - cmdnode.MiscFlags(), - // NOTE: for now, state-related queries can only be accessed - // over an RPC connection with a celestia-core node. - core.Flags(), - rpc.Flags(), - gateway.Flags(), - state.Flags(), - } - - lightCmd.AddCommand( - cmdnode.Init(flags...), - cmdnode.Start(flags...), - cmdnode.AuthCmd(flags...), - cmdnode.ResetStore(flags...), - cmdnode.RemoveConfigCmd(flags...), - cmdnode.UpdateConfigCmd(flags...), - ) -} - -var lightCmd = &cobra.Command{ - Use: "light [subcommand]", - Args: cobra.NoArgs, - Short: "Manage your Light node", - PersistentPreRunE: func(cmd *cobra.Command, args []string) error { - return cmdnode.PersistentPreRunEnv(cmd, node.Light, args) - }, -} diff --git a/cmd/celestia/main.go b/cmd/celestia/main.go index 24c9423100..76287d998f 100644 --- a/cmd/celestia/main.go +++ b/cmd/celestia/main.go @@ -5,9 +5,28 @@ import ( "os" "github.com/spf13/cobra" + "github.com/spf13/pflag" + + cmdnode "github.com/celestiaorg/celestia-node/cmd" ) +func WithSubcommands() func(*cobra.Command, []*pflag.FlagSet) { + return func(c *cobra.Command, flags []*pflag.FlagSet) { + c.AddCommand( + cmdnode.Init(flags...), + cmdnode.Start(cmdnode.WithFlagSet(flags)), + cmdnode.AuthCmd(flags...), + cmdnode.ResetStore(flags...), + cmdnode.RemoveConfigCmd(flags...), + cmdnode.UpdateConfigCmd(flags...), + ) + } +} + func init() { + bridgeCmd := cmdnode.NewBridge(WithSubcommands()) + lightCmd := cmdnode.NewLight(WithSubcommands()) + fullCmd := cmdnode.NewFull(WithSubcommands()) rootCmd.AddCommand( bridgeCmd, lightCmd, diff --git a/cmd/node.go b/cmd/node.go new file mode 100644 index 0000000000..3cdcb29c05 --- /dev/null +++ b/cmd/node.go @@ -0,0 +1,88 @@ +package cmd + +import ( + "github.com/celestiaorg/celestia-node/nodebuilder/core" + "github.com/celestiaorg/celestia-node/nodebuilder/gateway" + "github.com/celestiaorg/celestia-node/nodebuilder/header" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/rpc" + "github.com/celestiaorg/celestia-node/nodebuilder/state" + + "github.com/spf13/cobra" + "github.com/spf13/pflag" +) + +func NewBridge(options ...func(*cobra.Command, []*pflag.FlagSet)) *cobra.Command { + flags := []*pflag.FlagSet{ + NodeFlags(), + p2p.Flags(), + MiscFlags(), + core.Flags(), + rpc.Flags(), + gateway.Flags(), + state.Flags(), + } + cmd := &cobra.Command{ + Use: "bridge [subcommand]", + Args: cobra.NoArgs, + Short: "Manage your Bridge node", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return PersistentPreRunEnv(cmd, node.Bridge, args) + }, + } + for _, option := range options { + option(cmd, flags) + } + return cmd +} + +func NewLight(options ...func(*cobra.Command, []*pflag.FlagSet)) *cobra.Command { + flags := []*pflag.FlagSet{ + NodeFlags(), + p2p.Flags(), + header.Flags(), + MiscFlags(), + core.Flags(), + rpc.Flags(), + gateway.Flags(), + state.Flags(), + } + cmd := &cobra.Command{ + Use: "light [subcommand]", + Args: cobra.NoArgs, + Short: "Manage your Light node", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return PersistentPreRunEnv(cmd, node.Light, args) + }, + } + for _, option := range options { + option(cmd, flags) + } + return cmd +} + +func NewFull(options ...func(*cobra.Command, []*pflag.FlagSet)) *cobra.Command { + flags := []*pflag.FlagSet{ + NodeFlags(), + p2p.Flags(), + header.Flags(), + MiscFlags(), + core.Flags(), + rpc.Flags(), + gateway.Flags(), + state.Flags(), + } + cmd := &cobra.Command{ + Use: "full [subcommand]", + Args: cobra.NoArgs, + Short: "Manage your Full node", + PersistentPreRunE: func(cmd *cobra.Command, args []string) error { + return PersistentPreRunEnv(cmd, node.Full, args) + }, + } + for _, option := range options { + option(cmd, flags) + } + return cmd +} diff --git a/cmd/start.go b/cmd/start.go index d46b553e20..281dfcc0e4 100644 --- a/cmd/start.go +++ b/cmd/start.go @@ -9,7 +9,6 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/spf13/cobra" - flag "github.com/spf13/pflag" "github.com/celestiaorg/celestia-app/app" "github.com/celestiaorg/celestia-app/app/encoding" @@ -18,7 +17,7 @@ import ( ) // Start constructs a CLI command to start Celestia Node daemon of any type with the given flags. -func Start(fsets ...*flag.FlagSet) *cobra.Command { +func Start(options ...func(*cobra.Command)) *cobra.Command { cmd := &cobra.Command{ Use: "start", Short: `Starts Node daemon. First stopping signal gracefully stops the Node and second terminates it. @@ -72,8 +71,9 @@ Options passed on start override configuration options only on start and are not return nd.Stop(ctx) }, } - for _, set := range fsets { - cmd.Flags().AddFlagSet(set) + // Apply each passed option to the command + for _, option := range options { + option(cmd) } return cmd } diff --git a/cmd/util.go b/cmd/util.go index 625685fe0b..020699449a 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -9,6 +9,7 @@ import ( "strings" "github.com/spf13/cobra" + flag "github.com/spf13/pflag" "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/gateway" @@ -125,3 +126,12 @@ func PersistentPreRunEnv(cmd *cobra.Command, nodeType node.Type, _ []string) err cmd.SetContext(ctx) return nil } + +// WithFlagSet adds the given flagset to the command. +func WithFlagSet(fset []*flag.FlagSet) func(*cobra.Command) { + return func(c *cobra.Command) { + for _, set := range fset { + c.Flags().AddFlagSet(set) + } + } +} From 7fe691e58381e3ea8870322fd7f83d7b6241f1d5 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 29 Nov 2023 17:15:54 +0100 Subject: [PATCH 317/388] chore(node): tweak pyroscope options (#2974) * Increases UploadRate to 15 secs. * Send goroutines profiles additionally Extracted from #2960 --- nodebuilder/settings.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 2d924af000..298976fda4 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -52,6 +52,7 @@ func WithPyroscope(endpoint string, nodeType node.Type) fx.Option { return fx.Options( fx.Invoke(func(peerID peer.ID) error { _, err := pyroscope.Start(pyroscope.Config{ + UploadRate: 15 * time.Second, ApplicationName: "celestia.da-node", ServerAddress: endpoint, Tags: map[string]string{ @@ -65,6 +66,7 @@ func WithPyroscope(endpoint string, nodeType node.Type) fx.Option { pyroscope.ProfileAllocSpace, pyroscope.ProfileInuseObjects, pyroscope.ProfileInuseSpace, + pyroscope.ProfileGoroutines, }, }) return err From 24b333dff31d958fcd73f4967501ec08255b7af3 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 29 Nov 2023 18:20:19 +0100 Subject: [PATCH 318/388] chore(modheader): lower store cache size for LN (#2975) Part of an effort to decrease the memory footprint for LNs. Headers are pretty beefy. Extracted from #2960 --- nodebuilder/header/config.go | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/nodebuilder/header/config.go b/nodebuilder/header/config.go index d7265373ce..3fb7162ae0 100644 --- a/nodebuilder/header/config.go +++ b/nodebuilder/header/config.go @@ -43,13 +43,18 @@ func DefaultConfig(tp node.Type) Config { Store: store.DefaultParameters(), Syncer: sync.DefaultParameters(), Server: p2p_exchange.DefaultServerParameters(), + Client: p2p_exchange.DefaultClientParameters(), } switch tp { case node.Bridge: return cfg - case node.Light, node.Full: - cfg.Client = p2p_exchange.DefaultClientParameters() + case node.Full: + return cfg + case node.Light: + cfg.Store.StoreCacheSize = 512 + cfg.Store.IndexCacheSize = 2048 + cfg.Store.WriteBatchSize = 512 return cfg default: panic("header: invalid node type") From 72ace5dedf83672460672bd23b190a6f10c89bf2 Mon Sep 17 00:00:00 2001 From: ramin Date: Thu, 30 Nov 2023 11:24:06 +0000 Subject: [PATCH 319/388] misc(config): add @ramin to codeowners (#2973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit 👀 por favor --- .github/CODEOWNERS | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index e7e610edf8..f9ac165e7a 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -7,6 +7,6 @@ # global owners are only requested if there isn't a more specific # codeowner specified below. For this reason, the global codeowners # are often repeated in package-level definitions. -* @renaynay @Wondertan @vgonkivs @distractedm1nd @walldiss +* @renaynay @Wondertan @vgonkivs @distractedm1nd @walldiss @ramin docs/adr @adlerjohn @liamsi From fa6a1378a39522a46e3a6b93a96443b9263ebd9b Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 30 Nov 2023 17:22:35 +0100 Subject: [PATCH 320/388] fix(node/store): substantial memory usage improvements (#2960) * Fixes memory leak #2905 * Reduces memory consumption (from up-to 3gib to up-to 300mib) (TODO: proofs in pyroscope) * Improves cache hit rate on Badger * The warning log with Badger's low cache hit rate has disappeared. NOTE: More details in code comments --- Makefile | 33 +++++++++++ blob/blob.go | 4 +- cmd/node.go | 6 +- go.mod | 6 +- go.sum | 4 +- nodebuilder/fraud/mocks/api.go | 3 +- nodebuilder/header/mocks/api.go | 3 +- nodebuilder/share/mocks/api.go | 3 +- nodebuilder/store.go | 73 ++++++++++++++++++++++-- nodebuilder/tests/fraud_test.go | 6 +- share/availability/mocks/availability.go | 3 +- share/eds/inverted_index.go | 18 +++++- share/mocks/getter.go | 3 +- 13 files changed, 141 insertions(+), 24 deletions(-) diff --git a/Makefile b/Makefile index 1133db0533..9e1635308b 100644 --- a/Makefile +++ b/Makefile @@ -31,6 +31,12 @@ build: @go build -o build/ ${LDFLAGS} ./cmd/celestia .PHONY: build +## build-jemalloc: Build celestia-node binary with jemalloc allocator for BadgerDB instead of Go's native one +build-jemalloc: jemalloc + @echo "--> Building Celestia with jemalloc" + @go build -o build/ ${LDFLAGS} -tags jemalloc ./cmd/celestia +.PHONY: build-jemalloc + ## clean: Clean up celestia-node binary. clean: @echo "--> Cleaning up ./build" @@ -213,3 +219,30 @@ goreleaser-build: goreleaser-release: goreleaser release --clean --fail-fast --skip-publish .PHONY: goreleaser-release + +# Copied from https://github.com/dgraph-io/badger/blob/main/Makefile + +USER_ID = $(shell id -u) +HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc") +JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" + +## jemalloc installs jemalloc allocator +jemalloc: + @if [ -z "$(HAS_JEMALLOC)" ] ; then \ + mkdir -p /tmp/jemalloc-temp && cd /tmp/jemalloc-temp ; \ + echo "Downloading jemalloc..." ; \ + curl -s -L ${JEMALLOC_URL} -o jemalloc.tar.bz2 ; \ + tar xjf ./jemalloc.tar.bz2 ; \ + cd jemalloc-5.2.1 ; \ + ./configure --with-jemalloc-prefix='je_' --with-malloc-conf='background_thread:true,metadata_thp:auto'; \ + make ; \ + if [ "$(USER_ID)" -eq "0" ]; then \ + make install ; \ + else \ + echo "==== Need sudo access to install jemalloc" ; \ + sudo make install ; \ + fi ; \ + cd /tmp ; \ + rm -rf /tmp/jemalloc-temp ; \ + fi +.PHONY: jemalloc diff --git a/blob/blob.go b/blob/blob.go index 3f7e92dd20..945ffdf587 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -142,8 +142,8 @@ func (b *Blob) UnmarshalJSON(data []byte) error { } // buildBlobsIfExist takes shares and tries building the Blobs from them. -// It will build blobs either until appShares will be empty or the first incomplete blob will appear, so in this -// specific case it will return all built blobs + remaining shares. +// It will build blobs either until appShares will be empty or the first incomplete blob will +// appear, so in this specific case it will return all built blobs + remaining shares. func buildBlobsIfExist(appShares []shares.Share) ([]*Blob, []shares.Share, error) { if len(appShares) == 0 { return nil, nil, errors.New("empty shares received") diff --git a/cmd/node.go b/cmd/node.go index 3cdcb29c05..51ac4a6d2e 100644 --- a/cmd/node.go +++ b/cmd/node.go @@ -1,6 +1,9 @@ package cmd import ( + "github.com/spf13/cobra" + "github.com/spf13/pflag" + "github.com/celestiaorg/celestia-node/nodebuilder/core" "github.com/celestiaorg/celestia-node/nodebuilder/gateway" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -8,9 +11,6 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder/p2p" "github.com/celestiaorg/celestia-node/nodebuilder/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/state" - - "github.com/spf13/cobra" - "github.com/spf13/pflag" ) func NewBridge(options ...func(*cobra.Command, []*pflag.FlagSet)) *cobra.Command { diff --git a/go.mod b/go.mod index c648c01889..b4441c6f6b 100644 --- a/go.mod +++ b/go.mod @@ -82,6 +82,7 @@ require ( github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/ethereum/c-kzg-4844 v0.3.1 // indirect + github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/supranational/blst v0.3.11 // indirect go.uber.org/mock v0.3.0 // indirect @@ -138,7 +139,7 @@ require ( github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect - github.com/dgraph-io/badger/v4 v4.1.0 // indirect + github.com/dgraph-io/badger/v4 v4.2.1-0.20231013074411-fb1b00959581 github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -197,7 +198,6 @@ require ( github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect - github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/holiman/uint256 v1.2.3 // indirect @@ -351,3 +351,5 @@ replace ( github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 ) + +replace github.com/dgraph-io/badger/v4 => github.com/celestiaorg/badger/v4 v4.0.0-20231125230536-2b9e13346f75 diff --git a/go.sum b/go.sum index d45b9f0c06..962e46067b 100644 --- a/go.sum +++ b/go.sum @@ -358,6 +358,8 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= +github.com/celestiaorg/badger/v4 v4.0.0-20231125230536-2b9e13346f75 h1:dfArr6rKfcB46+/sR/ZZuiBbcB1ZQcpZQT4E4HHcIuE= +github.com/celestiaorg/badger/v4 v4.0.0-20231125230536-2b9e13346f75/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/celestiaorg/celestia-app v1.4.0 h1:hTId3xL8GssN5sHSHPP7svHi/iWp+XVxphStiR7ADiY= github.com/celestiaorg/celestia-app v1.4.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= @@ -549,8 +551,6 @@ github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= -github.com/dgraph-io/badger/v4 v4.1.0 h1:E38jc0f+RATYrycSUf9LMv/t47XAy+3CApyYSq4APOQ= -github.com/dgraph-io/badger/v4 v4.1.0/go.mod h1:P50u28d39ibBRmIJuQC/NSdBOg46HnHw7al2SW5QRHg= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index fcc7a58231..10111b81a8 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -8,9 +8,10 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + fraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" fraud0 "github.com/celestiaorg/go-fraud" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/header/mocks/api.go b/nodebuilder/header/mocks/api.go index b0d2b961d9..9b15f2242e 100644 --- a/nodebuilder/header/mocks/api.go +++ b/nodebuilder/header/mocks/api.go @@ -8,10 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + header "github.com/celestiaorg/celestia-node/header" header0 "github.com/celestiaorg/go-header" sync "github.com/celestiaorg/go-header/sync" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 78a124a20d..87f98e4f73 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -8,10 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + da "github.com/celestiaorg/celestia-app/pkg/da" share "github.com/celestiaorg/celestia-node/share" rsmt2d "github.com/celestiaorg/rsmt2d" - gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/store.go b/nodebuilder/store.go index 6d313893b1..f2238c8c3f 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -4,10 +4,12 @@ import ( "errors" "fmt" "path/filepath" + "runtime" "sync" "time" "github.com/cosmos/cosmos-sdk/crypto/keyring" + "github.com/dgraph-io/badger/v4/options" "github.com/ipfs/go-datastore" "github.com/mitchellh/go-homedir" @@ -15,6 +17,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/keystore" + "github.com/celestiaorg/celestia-node/share" ) var ( @@ -118,10 +121,8 @@ func (f *fsStore) Datastore() (datastore.Batching, error) { return f.data, nil } - opts := dsbadger.DefaultOptions // this should be copied - opts.GcInterval = time.Minute * 10 - - ds, err := dsbadger.NewDatastore(dataPath(f.path), &opts) + cfg := constraintBadgerConfig() + ds, err := dsbadger.NewDatastore(dataPath(f.path), cfg) if err != nil { return nil, fmt.Errorf("node: can't open Badger Datastore: %w", err) } @@ -182,3 +183,67 @@ func indexPath(base string) string { func dataPath(base string) string { return filepath.Join(base, "data") } + +// constraintBadgerConfig returns BadgerDB configuration optimized for low memory usage and more frequent +// compaction which prevents memory spikes. +// This is particularly important for LNs with restricted memory resources. +// +// With the following configuration, a LN uses up to 300iB of RAM during initial sync/sampling +// and up to 200MiB during normal operation. (on 4 core CPU, 8GiB RAM droplet) +// +// With the following configuration and "-tags=jemalloc", a LN uses no more than 180MiB during initial +// sync/sampling and up to 100MiB during normal operation. (same hardware spec) +// NOTE: To enable jemalloc, build celestia-node with "-tags=jemalloc" flag, which configures Badger to +// use jemalloc instead of Go's default allocator. +// +// TODO(@Wondertan): Consider alternative less constraint configuration for FN/BN +// TODO(@Wondertan): Consider dynamic memory allocation based on available RAM +func constraintBadgerConfig() *dsbadger.Options { + opts := dsbadger.DefaultOptions // this must be copied + // ValueLog: + // 2mib default => share.Size - makes sure headers and samples are stored in value log + // This *tremendously* reduces the amount of memory used by the node, up to 10 times less during + // compaction + opts.ValueThreshold = share.Size + // make sure we don't have any limits for stored headers + opts.ValueLogMaxEntries = 100000000 + // run value log GC more often to spread the work over time + opts.GcInterval = time.Minute * 1 + // default 0.5 => 0.125 - makes sure value log GC is more aggressive on reclaiming disk space + opts.GcDiscardRatio = 0.125 + + // badger stores checksum for every value, but doesn't verify it by default + // enabling this option may allow us to see detect corrupted data + opts.ChecksumVerificationMode = options.OnBlockRead + opts.VerifyValueChecksum = true + // default 64mib => 0 - disable block cache + // most of our component maintain their own caches, so this is not needed + opts.BlockCacheSize = 0 + // not much gain as it compresses the LSM only as well compression requires block cache + opts.Compression = options.None + + // MemTables: + // default 64mib => 16mib - decreases memory usage and makes compaction more often + opts.MemTableSize = 16 << 20 + // default 5 => 3 + opts.NumMemtables = 3 + // default 5 => 3 + opts.NumLevelZeroTables = 3 + // default 15 => 5 - this prevents memory growth on CPU constraint systems by blocking all writers + opts.NumLevelZeroTablesStall = 5 + + // Compaction: + // Dynamic compactor allocation + compactors := runtime.NumCPU() / 2 + if compactors < 2 { + compactors = 2 // can't be less than 2 + } + if compactors > opts.MaxLevels { // ensure there is no more compactors than db table levels + compactors = opts.MaxLevels + } + opts.NumCompactors = compactors + // makes sure badger is always compacted on shutdown + opts.CompactL0OnClose = true + + return &opts +} diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index 1927cdaf42..cb07dbb73a 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -126,9 +126,9 @@ func TestFraudProofHandling(t *testing.T) { // blockTime=1 sec, expected befp.height=10 timeOut := blockTime * 5 // during befp validation the node can still receive headers and it mostly depends on - // the operating system or hardware(e.g. on macOS tests is working 100% time with a single height=15, - // and on the Linux VM sometimes the last height is 17-18). So, lets give a chance for our befp validator to check - // the fraud proof and stop the syncer. + // the operating system or hardware(e.g. on macOS tests is working 100% time with a single + // height=15, and on the Linux VM sometimes the last height is 17-18). So, lets give a chance for + // our befp validator to check the fraud proof and stop the syncer. for height < 20 { syncCtx, syncCancel := context.WithTimeout(context.Background(), timeOut) _, err = full.HeaderServ.WaitForHeight(syncCtx, height) diff --git a/share/availability/mocks/availability.go b/share/availability/mocks/availability.go index fc68d3d2bc..bb1c01bf85 100644 --- a/share/availability/mocks/availability.go +++ b/share/availability/mocks/availability.go @@ -8,8 +8,9 @@ import ( context "context" reflect "reflect" - header "github.com/celestiaorg/celestia-node/header" gomock "github.com/golang/mock/gomock" + + header "github.com/celestiaorg/celestia-node/header" ) // MockAvailability is a mock of Availability interface. diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index 8b9dcb5d95..dc33c70447 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -4,7 +4,9 @@ import ( "context" "errors" "fmt" + "runtime" + "github.com/dgraph-io/badger/v4/options" "github.com/filecoin-project/dagstore/index" "github.com/filecoin-project/dagstore/shard" ds "github.com/ipfs/go-datastore" @@ -29,14 +31,24 @@ type simpleInvertedIndex struct { // don't care which shard is used to serve a cid. func newSimpleInvertedIndex(storePath string) (*simpleInvertedIndex, error) { opts := dsbadger.DefaultOptions // this should be copied - // turn off value log GC + // turn off value log GC as we don't use value log opts.GcInterval = 0 - // 20 compactors show to have no hangups on put operation up to 40k blocks with eds size 128. - opts.NumCompactors = 20 // use minimum amount of NumLevelZeroTables to trigger L0 compaction faster opts.NumLevelZeroTables = 1 // MaxLevels = 8 will allow the db to grow to ~11.1 TiB opts.MaxLevels = 8 + // inverted index stores unique hash keys, so we don't need to detect conflicts + opts.DetectConflicts = false + // we don't need compression for inverted index as it just hashes + opts.Compression = options.None + compactors := runtime.NumCPU() + if compactors < 2 { + compactors = 2 + } + if compactors > opts.MaxLevels { // ensure there is no more compactors than db table levels + compactors = opts.MaxLevels + } + opts.NumCompactors = compactors ds, err := dsbadger.NewDatastore(storePath+invertedIndexPath, &opts) if err != nil { diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 738e2b246c..2adfa50cfe 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -8,10 +8,11 @@ import ( context "context" reflect "reflect" + gomock "github.com/golang/mock/gomock" + header "github.com/celestiaorg/celestia-node/header" share "github.com/celestiaorg/celestia-node/share" rsmt2d "github.com/celestiaorg/rsmt2d" - gomock "github.com/golang/mock/gomock" ) // MockGetter is a mock of Getter interface. From 31f0e25a89e229620d14b15f9a22f4b8ec2a20f8 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 30 Nov 2023 17:53:16 +0100 Subject: [PATCH 321/388] refactor(modp2p): decouple Bitswap (#2962) Decouples Bitswap into Client and Server. This is necessary to provide new custom options to the Client. In particular, this brings client.WithoutDuplicatedBlockStats() option to the table as nice little optimization for LNs. It disables one metrics we don't care much about atm, but avoids expensive Has call to DB as profiles show. --- nodebuilder/p2p/bitswap.go | 54 +++++++++++++++++++++++++++++--------- 1 file changed, 41 insertions(+), 13 deletions(-) diff --git a/nodebuilder/p2p/bitswap.go b/nodebuilder/p2p/bitswap.go index 19f98609fc..014435071a 100644 --- a/nodebuilder/p2p/bitswap.go +++ b/nodebuilder/p2p/bitswap.go @@ -2,10 +2,12 @@ package p2p import ( "context" + "errors" "fmt" - "github.com/ipfs/boxo/bitswap" + "github.com/ipfs/boxo/bitswap/client" "github.com/ipfs/boxo/bitswap/network" + "github.com/ipfs/boxo/bitswap/server" "github.com/ipfs/boxo/blockstore" "github.com/ipfs/boxo/exchange" "github.com/ipfs/go-datastore" @@ -28,17 +30,38 @@ const ( // dataExchange provides a constructor for IPFS block's DataExchange over BitSwap. func dataExchange(params bitSwapParams) exchange.Interface { - prefix := protocol.ID(fmt.Sprintf("/celestia/%s", params.Net)) - return bitswap.New( + prefix := protocolID(params.Net) + net := network.NewFromIpfsHost(params.Host, &routinghelpers.Null{}, network.Prefix(prefix)) + srvr := server.New( params.Ctx, - network.NewFromIpfsHost(params.Host, &routinghelpers.Null{}, network.Prefix(prefix)), + net, params.Bs, - bitswap.ProvideEnabled(false), - // NOTE: These below ar required for our protocol to work reliably. - // See https://github.com/celestiaorg/celestia-node/issues/732 - bitswap.SetSendDontHaves(false), - bitswap.SetSimulateDontHavesOnTimeout(false), + server.ProvideEnabled(false), // we don't provide blocks over DHT + // NOTE: These below are required for our protocol to work reliably. + // // See https://github.com/celestiaorg/celestia-node/issues/732 + server.SetSendDontHaves(false), ) + + clnt := client.New( + params.Ctx, + net, + params.Bs, + client.WithBlockReceivedNotifier(srvr), + client.SetSimulateDontHavesOnTimeout(false), + client.WithoutDuplicatedBlockStats(), + ) + net.Start(srvr, clnt) // starting with hook does not work + + params.Lifecycle.Append(fx.Hook{ + OnStop: func(ctx context.Context) (err error) { + err = errors.Join(err, clnt.Close()) + err = errors.Join(err, srvr.Close()) + net.Stop() + return err + }, + }) + + return clnt } func blockstoreFromDatastore(ctx context.Context, ds datastore.Batching) (blockstore.Blockstore, error) { @@ -66,8 +89,13 @@ func blockstoreFromEDSStore(ctx context.Context, store *eds.Store) (blockstore.B type bitSwapParams struct { fx.In - Ctx context.Context - Net Network - Host hst.Host - Bs blockstore.Blockstore + Lifecycle fx.Lifecycle + Ctx context.Context + Net Network + Host hst.Host + Bs blockstore.Blockstore +} + +func protocolID(network Network) protocol.ID { + return protocol.ID(fmt.Sprintf("/celestia/%s", network)) } From f2d62f8c15a71fcfb658da5d00e96aea6d859c2f Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 4 Dec 2023 13:15:12 +0200 Subject: [PATCH 322/388] test(blob): add submitting equal blobs test case (#2914) --- nodebuilder/tests/blob_test.go | 72 +++++++++++++++++++++++++++++++++- 1 file changed, 71 insertions(+), 1 deletion(-) diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index ccbc6440cc..627dd850d0 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -73,6 +73,8 @@ func TestBlobModule(t *testing.T) { { name: "Get", doFn: func(t *testing.T) { + // https://github.com/celestiaorg/celestia-node/issues/2915 + time.Sleep(time.Second) blob1, err := fullClient.Blob.Get(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) require.Equal(t, blobs[0], blob1) @@ -81,6 +83,8 @@ func TestBlobModule(t *testing.T) { { name: "GetAll", doFn: func(t *testing.T) { + // https://github.com/celestiaorg/celestia-node/issues/2915 + time.Sleep(time.Second) newBlobs, err := fullClient.Blob.GetAll(ctx, height, []share.Namespace{blobs[0].Namespace()}) require.NoError(t, err) require.Len(t, newBlobs, len(appBlobs0)) @@ -91,6 +95,8 @@ func TestBlobModule(t *testing.T) { { name: "Included", doFn: func(t *testing.T) { + // https://github.com/celestiaorg/celestia-node/issues/2915 + time.Sleep(time.Second) proof, err := fullClient.Blob.GetProof(ctx, height, blobs[0].Namespace(), blobs[0].Commitment) require.NoError(t, err) @@ -123,12 +129,76 @@ func TestBlobModule(t *testing.T) { require.ErrorContains(t, err, blob.ErrBlobNotFound.Error()) }, }, + { + name: "Submit equal blobs", + doFn: func(t *testing.T) { + appBlob, err := blobtest.GenerateV0Blobs([]int{8, 4}, true) + require.NoError(t, err) + b, err := blob.NewBlob( + appBlob[0].ShareVersion, + append([]byte{appBlob[0].NamespaceVersion}, appBlob[0].NamespaceID...), + appBlob[0].Data, + ) + require.NoError(t, err) + + height, err := fullClient.Blob.Submit(ctx, []*blob.Blob{b, b}, nil) + require.NoError(t, err) + + _, err = fullClient.Header.WaitForHeight(ctx, height) + require.NoError(t, err) + + b0, err := fullClient.Blob.Get(ctx, height, b.Namespace(), b.Commitment) + require.NoError(t, err) + require.Equal(t, b, b0) + + // give some time to store the data, + // otherwise the test will hang on the IPLD level. + // https://github.com/celestiaorg/celestia-node/issues/2915 + time.Sleep(time.Second) + + proof, err := fullClient.Blob.GetProof(ctx, height, b.Namespace(), b.Commitment) + require.NoError(t, err) + + included, err := fullClient.Blob.Included(ctx, height, b.Namespace(), proof, b.Commitment) + require.NoError(t, err) + require.True(t, included) + }, + }, + { + // This test allows to check that the blob won't be + // deduplicated if it will be sent multiple times in + // different pfbs. + name: "Submit the same blob in different pfb", + doFn: func(t *testing.T) { + h, err := fullClient.Blob.Submit(ctx, []*blob.Blob{blobs[0]}, nil) + require.NoError(t, err) + + _, err = fullClient.Header.WaitForHeight(ctx, h) + require.NoError(t, err) + + b0, err := fullClient.Blob.Get(ctx, h, blobs[0].Namespace(), blobs[0].Commitment) + require.NoError(t, err) + require.Equal(t, blobs[0], b0) + + // give some time to store the data, + // otherwise the test will hang on the IPLD level. + // https://github.com/celestiaorg/celestia-node/issues/2915 + time.Sleep(time.Second) + + proof, err := fullClient.Blob.GetProof(ctx, h, blobs[0].Namespace(), blobs[0].Commitment) + require.NoError(t, err) + + included, err := fullClient.Blob.Included(ctx, h, blobs[0].Namespace(), proof, blobs[0].Commitment) + require.NoError(t, err) + require.True(t, included) + + }, + }, } for _, tt := range test { tt := tt t.Run(tt.name, func(t *testing.T) { - t.Parallel() tt.doFn(t) }) } From d84bc9eb283b9848658619934176e1af27b1439b Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 4 Dec 2023 15:14:28 +0200 Subject: [PATCH 323/388] chore(deps): Bump alpine from 3.18.4 to 3.18.5 (#2987) Bumps alpine from 3.18.4 to 3.18.5. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=alpine&package-manager=docker&previous-version=3.18.4&new-version=3.18.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index a3ac41f7aa..bc01f24149 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN uname -a &&\ CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ make build && make cel-key -FROM docker.io/alpine:3.18.4 +FROM docker.io/alpine:3.18.5 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From b7b6350f8921d4454b8a38c8485d74f908fb6a61 Mon Sep 17 00:00:00 2001 From: Rootul P Date: Tue, 5 Dec 2023 06:59:50 -0500 Subject: [PATCH 324/388] ci: configure prerelease-suffix (#2990) --- .goreleaser.yaml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.goreleaser.yaml b/.goreleaser.yaml index b229b4c348..9a3991ecae 100644 --- a/.goreleaser.yaml +++ b/.goreleaser.yaml @@ -65,3 +65,5 @@ changelog: exclude: - "^docs:" - "^test:" +git: + prerelease_suffix: "-" From dc1bd000272e25b880f312cc5f4532727824f339 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 5 Dec 2023 17:56:20 +0100 Subject: [PATCH 325/388] pruner: Implement skeleton for pruner package (#2972) This PR implements a skeleton for the pruner package and integrates the pruner's AvailabilityWindow determination into the DASer sampling routine. Namings are up for discussion. --------- Co-authored-by: New Author Name --- das/daser.go | 15 +++++++++++ das/daser_test.go | 3 ++- das/options.go | 15 ++++++++++- libs/utils/fs.go | 4 ++- libs/utils/square.go | 4 ++- logs/logs.go | 4 ++- nodebuilder/das/constructors.go | 5 ++++ nodebuilder/module.go | 2 ++ nodebuilder/prune/module.go | 47 +++++++++++++++++++++++++++++++++ nodebuilder/state/config.go | 4 ++- pruner/archival/pruner.go | 20 ++++++++++++++ pruner/archival/window.go | 5 ++++ pruner/light/pruner.go | 17 ++++++++++++ pruner/light/window.go | 9 +++++++ pruner/pruner.go | 13 +++++++++ pruner/service.go | 25 ++++++++++++++++++ pruner/window.go | 7 +++++ 17 files changed, 193 insertions(+), 6 deletions(-) create mode 100644 nodebuilder/prune/module.go create mode 100644 pruner/archival/pruner.go create mode 100644 pruner/archival/window.go create mode 100644 pruner/light/pruner.go create mode 100644 pruner/light/window.go create mode 100644 pruner/pruner.go create mode 100644 pruner/service.go create mode 100644 pruner/window.go diff --git a/das/daser.go b/das/daser.go index 29139d3a5f..40eee3d316 100644 --- a/das/daser.go +++ b/das/daser.go @@ -5,6 +5,7 @@ import ( "errors" "fmt" "sync/atomic" + "time" "github.com/ipfs/go-datastore" logging "github.com/ipfs/go-log/v2" @@ -147,6 +148,12 @@ func (d *DASer) Stop(ctx context.Context) error { } func (d *DASer) sample(ctx context.Context, h *header.ExtendedHeader) error { + // short-circuit if pruning is enabled and the header is outside the + // availability window + if !d.isWithinSamplingWindow(h) { + return nil + } + err := d.da.SharesAvailable(ctx, h) if err != nil { var byzantineErr *byzantine.ErrByzantine @@ -162,6 +169,14 @@ func (d *DASer) sample(ctx context.Context, h *header.ExtendedHeader) error { return nil } +func (d *DASer) isWithinSamplingWindow(eh *header.ExtendedHeader) bool { + // if sampling window is not set, then all headers are within the window + if d.params.SamplingWindow == 0 { + return true + } + return time.Since(eh.Time()) <= d.params.SamplingWindow +} + // SamplingStats returns the current statistics over the DA sampling process. func (d *DASer) SamplingStats(ctx context.Context) (SamplingStats, error) { return d.sampler.stats(ctx) diff --git a/das/daser_test.go b/das/daser_test.go index 1ec160e224..fd1eb39f7d 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -230,7 +230,8 @@ func TestDASerSampleTimeout(t *testing.T) { fserv := &fraudtest.DummyService[*header.ExtendedHeader]{} // create and start DASer - daser, err := NewDASer(avail, sub, getter, ds, fserv, newBroadcastMock(1), WithSampleTimeout(1)) + daser, err := NewDASer(avail, sub, getter, ds, fserv, newBroadcastMock(1), + WithSampleTimeout(1)) require.NoError(t, err) require.NoError(t, daser.Start(ctx)) diff --git a/das/options.go b/das/options.go index 6af8a02174..c2f2b2fedb 100644 --- a/das/options.go +++ b/das/options.go @@ -40,6 +40,11 @@ type Parameters struct { // divided between parallel workers. SampleTimeout should be adjusted proportionally to // ConcurrencyLimit. SampleTimeout time.Duration + + // SamplingWindow determines the time window that headers should fall into + // in order to be sampled. If set to 0, the sampling window will include + // all headers. + SamplingWindow time.Duration } // DefaultParameters returns the default configuration values for the daser parameters @@ -148,10 +153,18 @@ func WithSampleFrom(sampleFrom uint64) Option { } } -// WithSampleFrom is a functional option to configure the daser's `SampleTimeout` parameter +// WithSampleTimeout is a functional option to configure the daser's `SampleTimeout` parameter // Refer to WithSamplingRange documentation to see an example of how to use this func WithSampleTimeout(sampleTimeout time.Duration) Option { return func(d *DASer) { d.params.SampleTimeout = sampleTimeout } } + +// WithSamplingWindow is a functional option to configure the DASer's +// `SamplingWindow` parameter. +func WithSamplingWindow(samplingWindow time.Duration) Option { + return func(d *DASer) { + d.params.SamplingWindow = samplingWindow + } +} diff --git a/libs/utils/fs.go b/libs/utils/fs.go index d67e9a1eaa..4ad8b6443e 100644 --- a/libs/utils/fs.go +++ b/libs/utils/fs.go @@ -1,6 +1,8 @@ package utils -import "os" +import ( + "os" +) // Exists checks whether file or directory exists under the given 'path' on the system. func Exists(path string) bool { diff --git a/libs/utils/square.go b/libs/utils/square.go index ce2663fd81..68d7fc5ce7 100644 --- a/libs/utils/square.go +++ b/libs/utils/square.go @@ -1,6 +1,8 @@ package utils -import "math" +import ( + "math" +) // SquareSize returns the size of the square based on the given amount of shares. func SquareSize(lenShares int) uint64 { diff --git a/logs/logs.go b/logs/logs.go index 23d0683996..5cb9ed16c6 100644 --- a/logs/logs.go +++ b/logs/logs.go @@ -1,6 +1,8 @@ package logs -import logging "github.com/ipfs/go-log/v2" +import ( + logging "github.com/ipfs/go-log/v2" +) func SetAllLoggers(level logging.LogLevel) { logging.SetAllLoggers(level) diff --git a/nodebuilder/das/constructors.go b/nodebuilder/das/constructors.go index 7c6b5bed4f..973aca5679 100644 --- a/nodebuilder/das/constructors.go +++ b/nodebuilder/das/constructors.go @@ -3,6 +3,7 @@ package das import ( "context" "fmt" + "time" "github.com/ipfs/go-datastore" @@ -12,6 +13,7 @@ import ( "github.com/celestiaorg/celestia-node/das" "github.com/celestiaorg/celestia-node/header" modfraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" + "github.com/celestiaorg/celestia-node/pruner" "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds/byzantine" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" @@ -44,8 +46,11 @@ func newDASer( batching datastore.Batching, fraudServ fraud.Service[*header.ExtendedHeader], bFn shrexsub.BroadcastFn, + availWindow pruner.AvailabilityWindow, options ...das.Option, ) (*das.DASer, *modfraud.ServiceBreaker[*das.DASer, *header.ExtendedHeader], error) { + options = append(options, das.WithSamplingWindow(time.Duration(availWindow))) + ds, err := das.NewDASer(da, hsub, store, batching, fraudServ, bFn, options...) if err != nil { return nil, nil, err diff --git a/nodebuilder/module.go b/nodebuilder/module.go index 3068113102..26dc1d9e33 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -15,6 +15,7 @@ import ( modhead "github.com/celestiaorg/celestia-node/nodebuilder/header" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + "github.com/celestiaorg/celestia-node/nodebuilder/prune" "github.com/celestiaorg/celestia-node/nodebuilder/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/share" "github.com/celestiaorg/celestia-node/nodebuilder/state" @@ -56,6 +57,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store fraud.ConstructModule(tp), blob.ConstructModule(), node.ConstructModule(tp), + prune.ConstructModule(tp), ) return fx.Module( diff --git a/nodebuilder/prune/module.go b/nodebuilder/prune/module.go new file mode 100644 index 0000000000..330ef21cdc --- /dev/null +++ b/nodebuilder/prune/module.go @@ -0,0 +1,47 @@ +package prune + +import ( + "context" + + "go.uber.org/fx" + + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/pruner" + "github.com/celestiaorg/celestia-node/pruner/archival" + "github.com/celestiaorg/celestia-node/pruner/light" +) + +func ConstructModule(tp node.Type) fx.Option { + baseComponents := fx.Options( + fx.Provide(fx.Annotate( + pruner.NewService, + fx.OnStart(func(ctx context.Context, p *pruner.Service) error { + return p.Start(ctx) + }), + fx.OnStop(func(ctx context.Context, p *pruner.Service) error { + return p.Stop(ctx) + }), + )), + ) + + switch tp { + case node.Full, node.Bridge: + return fx.Module("prune", + baseComponents, + fx.Provide(func() pruner.Pruner { + return archival.NewPruner() + }), + fx.Supply(archival.Window), + ) + case node.Light: + return fx.Module("prune", + baseComponents, + fx.Provide(func() pruner.Pruner { + return light.NewPruner() + }), + fx.Supply(archival.Window), // TODO @renaynay: turn this into light.Window in following PR + ) + default: + panic("unknown node type") + } +} diff --git a/nodebuilder/state/config.go b/nodebuilder/state/config.go index e6db813e06..f42e646b76 100644 --- a/nodebuilder/state/config.go +++ b/nodebuilder/state/config.go @@ -1,6 +1,8 @@ package state -import "github.com/cosmos/cosmos-sdk/crypto/keyring" +import ( + "github.com/cosmos/cosmos-sdk/crypto/keyring" +) var defaultKeyringBackend = keyring.BackendTest diff --git a/pruner/archival/pruner.go b/pruner/archival/pruner.go new file mode 100644 index 0000000000..7b1cb935f3 --- /dev/null +++ b/pruner/archival/pruner.go @@ -0,0 +1,20 @@ +package archival + +import ( + "context" + + "github.com/celestiaorg/celestia-node/header" +) + +// Pruner is a noop implementation of the pruner.Factory interface +// that allows archival nodes to sync and retain historical data +// that is out of the availability window. +type Pruner struct{} + +func NewPruner() *Pruner { + return &Pruner{} +} + +func (p *Pruner) Prune(context.Context, ...*header.ExtendedHeader) error { + return nil +} diff --git a/pruner/archival/window.go b/pruner/archival/window.go new file mode 100644 index 0000000000..b89a779816 --- /dev/null +++ b/pruner/archival/window.go @@ -0,0 +1,5 @@ +package archival + +import "github.com/celestiaorg/celestia-node/pruner" + +const Window = pruner.AvailabilityWindow(0) diff --git a/pruner/light/pruner.go b/pruner/light/pruner.go new file mode 100644 index 0000000000..513bfa2b66 --- /dev/null +++ b/pruner/light/pruner.go @@ -0,0 +1,17 @@ +package light + +import ( + "context" + + "github.com/celestiaorg/celestia-node/header" +) + +type Pruner struct{} + +func NewPruner() *Pruner { + return &Pruner{} +} + +func (p *Pruner) Prune(context.Context, ...*header.ExtendedHeader) error { + return nil +} diff --git a/pruner/light/window.go b/pruner/light/window.go new file mode 100644 index 0000000000..53bfe4a163 --- /dev/null +++ b/pruner/light/window.go @@ -0,0 +1,9 @@ +package light + +import ( + "time" + + "github.com/celestiaorg/celestia-node/pruner" +) + +const Window = pruner.AvailabilityWindow(time.Second * 86400 * 30) diff --git a/pruner/pruner.go b/pruner/pruner.go new file mode 100644 index 0000000000..fae60e483c --- /dev/null +++ b/pruner/pruner.go @@ -0,0 +1,13 @@ +package pruner + +import ( + "context" + + "github.com/celestiaorg/celestia-node/header" +) + +// Pruner contains methods necessary to prune data +// from the node's datastore. +type Pruner interface { + Prune(context.Context, ...*header.ExtendedHeader) error +} diff --git a/pruner/service.go b/pruner/service.go new file mode 100644 index 0000000000..f67265977a --- /dev/null +++ b/pruner/service.go @@ -0,0 +1,25 @@ +package pruner + +import ( + "context" +) + +// Service handles the pruning routine for the node using the +// prune Pruner. +type Service struct { + pruner Pruner +} + +func NewService(p Pruner) *Service { + return &Service{ + pruner: p, + } +} + +func (s *Service) Start(context.Context) error { + return nil +} + +func (s *Service) Stop(context.Context) error { + return nil +} diff --git a/pruner/window.go b/pruner/window.go new file mode 100644 index 0000000000..0a86c535ce --- /dev/null +++ b/pruner/window.go @@ -0,0 +1,7 @@ +package pruner + +import ( + "time" +) + +type AvailabilityWindow time.Duration From b709b4eb85803fa86310d4f235ce1988efbf3c19 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 8 Dec 2023 13:20:34 +0200 Subject: [PATCH 326/388] fix(cmd): explicitly set a string representation of the error (#2997) --- cmd/util.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/cmd/util.go b/cmd/util.go index 020699449a..08fa02155b 100644 --- a/cmd/util.go +++ b/cmd/util.go @@ -24,7 +24,7 @@ import ( func PrintOutput(data interface{}, err error, formatData func(interface{}) interface{}) error { switch { case err != nil: - data = err + data = err.Error() case formatData != nil: data = formatData(data) } From 327291292a87b4f7fadfe27b271e28d0f84dd59e Mon Sep 17 00:00:00 2001 From: Bork Bork <107079055+BorkBorked@users.noreply.github.com> Date: Tue, 12 Dec 2023 22:26:15 +0100 Subject: [PATCH 327/388] share: typos fix (#2993) --- share/getters/getter_test.go | 2 +- share/p2p/params.go | 2 +- share/p2p/shrexsub/pubsub_test.go | 2 +- 3 files changed, 3 insertions(+), 3 deletions(-) diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 77c470dae9..8264f6e822 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -231,7 +231,7 @@ func TestIPLDGetter(t *testing.T) { require.NoError(t, err) require.Empty(t, emptyShares.Flatten()) - // nid doesnt exist in root + // nid doesn't exist in root emptyRoot := da.MinDataAvailabilityHeader() eh.DAH = &emptyRoot emptyShares, err = sg.GetSharesByNamespace(ctx, eh, namespace) diff --git a/share/p2p/params.go b/share/p2p/params.go index f11179293d..6636e38fc5 100644 --- a/share/p2p/params.go +++ b/share/p2p/params.go @@ -45,7 +45,7 @@ func (p *Parameters) Validate() error { return fmt.Errorf("invalid write timeout: %v, %s", p.ServerWriteTimeout, errSuffix) } if p.HandleRequestTimeout <= 0 { - return fmt.Errorf("invalid hadnle request timeout: %v, %s", p.HandleRequestTimeout, errSuffix) + return fmt.Errorf("invalid handle request timeout: %v, %s", p.HandleRequestTimeout, errSuffix) } if p.ConcurrencyLimit <= 0 { return fmt.Errorf("invalid concurrency limit: %s", errSuffix) diff --git a/share/p2p/shrexsub/pubsub_test.go b/share/p2p/shrexsub/pubsub_test.go index dcf85515e8..85b16c055b 100644 --- a/share/p2p/shrexsub/pubsub_test.go +++ b/share/p2p/shrexsub/pubsub_test.go @@ -45,7 +45,7 @@ func TestPubSub(t *testing.T) { errExpected bool }{ { - name: "vaild height, valid hash", + name: "valid height, valid hash", notif: Notification{ Height: 1, DataHash: rand.Bytes(32), From 31e06606820bddf40111b8daeae7acaa189ca761 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 13 Dec 2023 16:21:21 +0100 Subject: [PATCH 328/388] Revert "fix(modp2p): temporary disable quic (#2837)" (#2963) Reverts disabled quic. We weren't able to reproduce [the issue](https://github.com/libp2p/go-libp2p/issues/2591) since we first observed it, as well as quic released changes to address this. Even if the issue is there, I think it's more or less safe as we always have TCP as fallback --- nodebuilder/p2p/addrs.go | 19 +++---------------- nodebuilder/p2p/host.go | 20 +------------------- 2 files changed, 4 insertions(+), 35 deletions(-) diff --git a/nodebuilder/p2p/addrs.go b/nodebuilder/p2p/addrs.go index 27fbd244a3..d8f50c8144 100644 --- a/nodebuilder/p2p/addrs.go +++ b/nodebuilder/p2p/addrs.go @@ -2,7 +2,6 @@ package p2p import ( "fmt" - "slices" p2pconfig "github.com/libp2p/go-libp2p/config" hst "github.com/libp2p/go-libp2p/core/host" @@ -12,25 +11,13 @@ import ( // Listen returns invoke function that starts listening for inbound connections with libp2p.Host. func Listen(listen []string) func(h hst.Host) (err error) { return func(h hst.Host) (err error) { - maListen := make([]ma.Multiaddr, 0, len(listen)) - for _, addr := range listen { - maddr, err := ma.NewMultiaddr(addr) + maListen := make([]ma.Multiaddr, len(listen)) + for i, addr := range listen { + maListen[i], err = ma.NewMultiaddr(addr) if err != nil { return fmt.Errorf("failure to parse config.P2P.ListenAddresses: %s", err) } - - if !enableQUIC { - // TODO(@WonderTan): Remove this check when QUIC is stable - if slices.ContainsFunc(maddr.Protocols(), func(p ma.Protocol) bool { - return p.Code == ma.P_QUIC_V1 || p.Code == ma.P_WEBTRANSPORT - }) { - continue - } - } - - maListen = append(maListen, maddr) } - return h.Network().Listen(maListen...) } } diff --git a/nodebuilder/p2p/host.go b/nodebuilder/p2p/host.go index c3943a02fa..e55cb65d1f 100644 --- a/nodebuilder/p2p/host.go +++ b/nodebuilder/p2p/host.go @@ -3,7 +3,6 @@ package p2p import ( "context" "fmt" - "os" "github.com/libp2p/go-libp2p" p2pconfig "github.com/libp2p/go-libp2p/config" @@ -17,22 +16,12 @@ import ( "github.com/libp2p/go-libp2p/core/routing" routedhost "github.com/libp2p/go-libp2p/p2p/host/routed" "github.com/libp2p/go-libp2p/p2p/net/conngater" - quic "github.com/libp2p/go-libp2p/p2p/transport/quic" - "github.com/libp2p/go-libp2p/p2p/transport/tcp" - webtransport "github.com/libp2p/go-libp2p/p2p/transport/webtransport" "github.com/prometheus/client_golang/prometheus" "go.uber.org/fx" "github.com/celestiaorg/celestia-node/nodebuilder/node" ) -var enableQUIC bool - -func init() { - _, ok := os.LookupEnv("CELESTIA_ENABLE_QUIC") - enableQUIC = ok -} - // routedHost constructs a wrapped Host that may fallback to address discovery, // if any top-level operation on the Host is provided with PeerID(Hash(PbK)) only. func routedHost(base HostBase, r routing.PeerRouting) hst.Host { @@ -55,15 +44,8 @@ func host(params hostParams) (HostBase, error) { libp2p.ResourceManager(params.ResourceManager), // to clearly define what defaults we rely upon libp2p.DefaultSecurity, + libp2p.DefaultTransports, libp2p.DefaultMuxers, - libp2p.Transport(tcp.NewTCPTransport), - } - - if enableQUIC { - opts = append(opts, - libp2p.Transport(quic.NewTransport), - libp2p.Transport(webtransport.New), - ) } if params.Registry != nil { From 1403f1d7faa8a5420af7a5bffde138e839405a88 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 10:59:39 +0000 Subject: [PATCH 329/388] chore(deps): Bump alpine from 3.18.5 to 3.19.0 (#3002) Bumps alpine from 3.18.5 to 3.19.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=alpine&package-manager=docker&previous-version=3.18.5&new-version=3.19.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index bc01f24149..e15d7d4f1c 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN uname -a &&\ CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ make build && make cel-key -FROM docker.io/alpine:3.18.5 +FROM docker.io/alpine:3.19.0 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From 791f53f236c6263c078c5a080787cc8a549d3670 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 11:17:42 +0000 Subject: [PATCH 330/388] chore(deps): Bump actions/stale from 8 to 9 (#3001) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/stale](https://github.com/actions/stale) from 8 to 9.
Release notes

Sourced from actions/stale's releases.

v9.0.0

Breaking Changes

  1. Action is now stateful: If the action ends because of operations-per-run then the next run will start from the first unprocessed issue skipping the issues processed during the previous run(s). The state is reset when all the issues are processed. This should be considered for scheduling workflow runs.
  2. Version 9 of this action updated the runtime to Node.js 20. All scripts are now run with Node.js 20 instead of Node.js 16 and are affected by any breaking changes between Node.js 16 and 20.

What Else Changed

  1. Performance optimization that removes unnecessary API calls by @​dsame #1033 fixes #792
  2. Logs displaying current github API rate limit by @​dsame #1032 addresses #1029

For more information, please read the action documentation and its section about statefulness

New Contributors

Full Changelog: https://github.com/actions/stale/compare/v8...v9.0.0

Changelog

Sourced from actions/stale's changelog.

Changelog

[7.0.0]

:warning: Breaking change :warning:

[6.0.1]

Update @​actions/core to v1.10.0 (#839)

[6.0.0]

:warning: Breaking change :warning:

Issues/PRs default close-issue-reason is now not_planned(#789)

[5.1.0]

Don't process stale issues right after they're marked stale [Add close-issue-reason option]#764#772 Various dependabot/dependency updates

4.1.0 (2021-07-14)

Features

4.0.0 (2021-07-14)

Features

Bug Fixes

  • dry-run: forbid mutations in dry-run (#500) (f1017f3), closes #499
  • logs: coloured logs (#465) (5fbbfba)
  • operations: fail fast the current batch to respect the operations limit (#474) (5f6f311), closes #466
  • label comparison: make label comparison case insensitive #517, closes #516
  • filtering comments by actor could have strange behavior: "stale" comments are now detected based on if the message is the stale message not who made the comment(#519), fixes #441, #509, #518

Breaking Changes

... (truncated)

Commits
  • 28ca103 Upgrade Node to v20 (#1110)
  • b69b346 build(deps-dev): bump @​types/node from 18.16.18 to 20.5.1 (#1079)
  • 88a6f4f build(deps-dev): bump typescript from 5.1.3 to 5.2.2 (#1083)
  • 796531a Merge pull request #1080 from akv-platform/fix-delete-cache
  • 8986f62 Don not try to delete cache if it does not exists
  • cab99b3 fix typo proceeded/processed
  • 184e7af Merge pull request #1064 from actions/dependabot/npm_and_yarn/typescript-esli...
  • 523885c chore: update eslint-plugin, parser and eslint-plugin-jest
  • 2487a1d build(deps-dev): bump @​typescript-eslint/eslint-plugin
  • 60c722e Merge pull request #1063 from actions/dependabot/npm_and_yarn/jest-29.6.2
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/stale&package-manager=github_actions&previous-version=8&new-version=9)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/stale.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/stale.yml b/.github/workflows/stale.yml index d1088e5f33..595effed1f 100644 --- a/.github/workflows/stale.yml +++ b/.github/workflows/stale.yml @@ -7,7 +7,7 @@ jobs: stale: runs-on: ubuntu-latest steps: - - uses: actions/stale@v8 + - uses: actions/stale@v9 with: repo-token: ${{ secrets.GITHUB_TOKEN }} days-before-stale: 60 From cde1065deb8d1407b0cf855aab3d573bd87d5c7d Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Fri, 15 Dec 2023 13:22:04 +0000 Subject: [PATCH 331/388] chore(deps): Bump actions/setup-go from 4 to 5 (#3000) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/setup-go](https://github.com/actions/setup-go) from 4 to 5.
Release notes

Sourced from actions/setup-go's releases.

v5.0.0

What's Changed

In scope of this release, we change Nodejs runtime from node16 to node20 (actions/setup-go#421). Moreover, we update some dependencies to the latest versions (actions/setup-go#445).

Besides, this release contains such changes as:

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v5.0.0

v4.1.0

What's Changed

In scope of this release, slow installation on Windows was fixed by @​dsame in actions/setup-go#393 and OS version was added to primaryKey for Ubuntu runners to avoid conflicts (actions/setup-go#383)

This release also includes the following changes:

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.1.0

v4.0.1

What's Changed

New Contributors

Full Changelog: https://github.com/actions/setup-go/compare/v4...v4.0.1

Commits
  • 0c52d54 Update dependencies for node20 (#445)
  • bfd2fb3 Merge pull request #421 from chenrui333/node20-runtime
  • 3d65fa5 feat: bump to use actions/checkout@v4
  • 8a505c9 feat: bump to use node20 runtime
  • 883490d Merge pull request #417 from artemgavrilov/main
  • d45ebba Rephrase sentence
  • 317c661 Replace wildcards term with globs.
  • f90673a Merge pull request #1 from artemgavrilov/caching-docs-improvement
  • 8018234 Improve documentation regarding dependencies cachin
  • d085b4f Merge pull request #411 from galargh/fix/windows-hostedtoolcache
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/setup-go&package-manager=github_actions&previous-version=4&new-version=5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/ci_release.yml | 2 +- .github/workflows/go-ci.yml | 10 +++++----- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/ci_release.yml b/.github/workflows/ci_release.yml index 25970fdb11..ad15a083bc 100644 --- a/.github/workflows/ci_release.yml +++ b/.github/workflows/ci_release.yml @@ -118,7 +118,7 @@ jobs: - run: git fetch --force --tags - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ needs.setup.outputs.go-version }} diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 056e4816bf..c14692cd58 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -35,7 +35,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ inputs.go-version }} @@ -55,7 +55,7 @@ jobs: steps: - uses: actions/checkout@v4 - - uses: actions/setup-go@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ inputs.go-version }} @@ -79,7 +79,7 @@ jobs: - uses: actions/checkout@v4 - name: set up go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ inputs.go-version }} @@ -113,7 +113,7 @@ jobs: - uses: actions/checkout@v4 - name: set up go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ inputs.go-version }} @@ -129,7 +129,7 @@ jobs: - uses: actions/checkout@v4 - name: set up go - uses: actions/setup-go@v4 + uses: actions/setup-go@v5 with: go-version: ${{ inputs.go-version }} From fae3024a92396fa4217aa21b46fedb2505cb65b4 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 18 Dec 2023 10:39:12 +0000 Subject: [PATCH 332/388] feat(rpc): default binding to localhost vs 0.0.0.0 open to world (#2955) fixes #2824 which requests we bind to localhost by default (previously was world accessible) - sets defaults to bind to localhost properly (previously flags SAID the defaults were localhost but were actually 0.0.0.0) - removes an unused `DefaultConfig` defined in `api/gateway/package` - moves defaults (bind address and port) to vars so we can use them to set in flag usage messages so they don't drift out of sync with reality - adds tests for the defaults - adds test cases for flag definitions - adds test cases for flag parsing Other skips suggestion of binding vhosts, we don't do any kind of middleware checking in rpc (i believe in geth this would be a way to bind to 0.0.0.0 then limit to allowed remote IPs making requests). I'd suggest this is a more advanced configuration use case and belongs outside of the node itself to be managed by the infra operator where it'd be handled by a ingress gateway or routing rule to the machine/instance/orchestrator where the node software is running --- api/gateway/config.go | 8 --- nodebuilder/config_test.go | 4 +- nodebuilder/gateway/config.go | 4 +- nodebuilder/gateway/config_test.go | 18 ++++++ nodebuilder/gateway/defaults.go | 6 ++ nodebuilder/gateway/defaults_test.go | 12 ++++ nodebuilder/gateway/flags.go | 6 +- nodebuilder/gateway/flags_test.go | 95 ++++++++++++++++++++++++++++ nodebuilder/rpc/config.go | 4 +- nodebuilder/rpc/config_test.go | 59 +++++++++++++++++ nodebuilder/rpc/defaults.go | 6 ++ nodebuilder/rpc/defaults_test.go | 12 ++++ nodebuilder/rpc/flags.go | 6 +- nodebuilder/rpc/flags_test.go | 95 ++++++++++++++++++++++++++++ 14 files changed, 317 insertions(+), 18 deletions(-) create mode 100644 nodebuilder/gateway/config_test.go create mode 100644 nodebuilder/gateway/defaults.go create mode 100644 nodebuilder/gateway/defaults_test.go create mode 100644 nodebuilder/gateway/flags_test.go create mode 100644 nodebuilder/rpc/config_test.go create mode 100644 nodebuilder/rpc/defaults.go create mode 100644 nodebuilder/rpc/defaults_test.go create mode 100644 nodebuilder/rpc/flags_test.go diff --git a/api/gateway/config.go b/api/gateway/config.go index f7d8bb44b1..0485da486e 100644 --- a/api/gateway/config.go +++ b/api/gateway/config.go @@ -11,14 +11,6 @@ type Config struct { Port string } -func DefaultConfig() Config { - return Config{ - Address: "0.0.0.0", - // do NOT expose the same port as celestia-core by default so that both can run on the same machine - Port: "26658", - } -} - func (cfg *Config) Validate() error { if ip := net.ParseIP(cfg.Address); ip == nil { return fmt.Errorf("service/gateway: invalid listen address format: %s", cfg.Address) diff --git a/nodebuilder/config_test.go b/nodebuilder/config_test.go index db9af3a64d..e7b64b0aed 100644 --- a/nodebuilder/config_test.go +++ b/nodebuilder/config_test.go @@ -69,7 +69,7 @@ var outdatedConfig = ` KeyringBackend = "test" [P2P] - ListenAddresses = ["/ip4/0.0.0.0/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1", "/ip4/0.0.0.0/tcp/2121", + ListenAddresses = ["/ip4/0.0.0.0/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1", "/ip4/0.0.0.0/tcp/2121", "/ip6/::/tcp/2121"] AnnounceAddresses = [] NoAnnounceAddresses = ["/ip4/0.0.0.0/udp/2121/quic-v1", "/ip4/127.0.0.1/udp/2121/quic-v1", "/ip6/::/udp/2121/quic-v1", @@ -91,7 +91,7 @@ var outdatedConfig = ` [Gateway] Address = "0.0.0.0" Port = "26659" - Enabled = true + Enabled = true [Share] PeersLimit = 5 diff --git a/nodebuilder/gateway/config.go b/nodebuilder/gateway/config.go index 903a27489a..c49a4749a3 100644 --- a/nodebuilder/gateway/config.go +++ b/nodebuilder/gateway/config.go @@ -15,9 +15,9 @@ type Config struct { func DefaultConfig() Config { return Config{ - Address: "0.0.0.0", + Address: defaultBindAddress, // do NOT expose the same port as celestia-core by default so that both can run on the same machine - Port: "26659", + Port: defaultPort, Enabled: false, } } diff --git a/nodebuilder/gateway/config_test.go b/nodebuilder/gateway/config_test.go new file mode 100644 index 0000000000..9ef3f1e310 --- /dev/null +++ b/nodebuilder/gateway/config_test.go @@ -0,0 +1,18 @@ +package gateway + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestDefaultConfig tests that the default gateway config is correct. +func TestDefaultConfig(t *testing.T) { + expected := Config{ + Address: defaultBindAddress, + Port: defaultPort, + Enabled: false, + } + + assert.Equal(t, expected, DefaultConfig()) +} diff --git a/nodebuilder/gateway/defaults.go b/nodebuilder/gateway/defaults.go new file mode 100644 index 0000000000..e6c48d5d4e --- /dev/null +++ b/nodebuilder/gateway/defaults.go @@ -0,0 +1,6 @@ +package gateway + +const ( + defaultBindAddress = "localhost" + defaultPort = "26659" +) diff --git a/nodebuilder/gateway/defaults_test.go b/nodebuilder/gateway/defaults_test.go new file mode 100644 index 0000000000..c504f8cca4 --- /dev/null +++ b/nodebuilder/gateway/defaults_test.go @@ -0,0 +1,12 @@ +package gateway + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestServerDefaultConstants(t *testing.T) { + assert.Equal(t, "localhost", defaultBindAddress) + assert.Equal(t, "26659", defaultPort) +} diff --git a/nodebuilder/gateway/flags.go b/nodebuilder/gateway/flags.go index cd13e47162..6da4a66f03 100644 --- a/nodebuilder/gateway/flags.go +++ b/nodebuilder/gateway/flags.go @@ -1,6 +1,8 @@ package gateway import ( + "fmt" + "github.com/spf13/cobra" flag "github.com/spf13/pflag" ) @@ -23,12 +25,12 @@ func Flags() *flag.FlagSet { flags.String( addrFlag, "", - "Set a custom gateway listen address (default: localhost)", + fmt.Sprintf("Set a custom gateway listen address (default: %s)", defaultBindAddress), ) flags.String( portFlag, "", - "Set a custom gateway port (default: 26659)", + fmt.Sprintf("Set a custom gateway port (default: %s)", defaultPort), ) return flags diff --git a/nodebuilder/gateway/flags_test.go b/nodebuilder/gateway/flags_test.go new file mode 100644 index 0000000000..5f55ac77f2 --- /dev/null +++ b/nodebuilder/gateway/flags_test.go @@ -0,0 +1,95 @@ +package gateway + +import ( + "fmt" + "strconv" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFlags(t *testing.T) { + flags := Flags() + + enabled := flags.Lookup(enabledFlag) + require.NotNil(t, enabled) + assert.Equal(t, "false", enabled.Value.String()) + assert.Equal(t, "Enables the REST gateway", enabled.Usage) + + addr := flags.Lookup(addrFlag) + require.NotNil(t, addr) + assert.Equal(t, "", addr.Value.String()) + assert.Equal(t, fmt.Sprintf("Set a custom gateway listen address (default: %s)", defaultBindAddress), addr.Usage) + + port := flags.Lookup(portFlag) + require.NotNil(t, port) + assert.Equal(t, "", port.Value.String()) + assert.Equal(t, fmt.Sprintf("Set a custom gateway port (default: %s)", defaultPort), port.Usage) +} + +func TestParseFlags(t *testing.T) { + tests := []struct { + name string + enabledFlag bool + addrFlag string + portFlag string + expectedCfg *Config + }{ + { + name: "Enabled flag is true", + enabledFlag: true, + addrFlag: "127.0.0.1", + portFlag: "8080", + expectedCfg: &Config{ + Enabled: true, + Address: "127.0.0.1", + Port: "8080", + }, + }, + { + name: "Enabled flag is false", + enabledFlag: false, + addrFlag: "127.0.0.1", + portFlag: "8080", + expectedCfg: &Config{ + Enabled: false, + Address: "127.0.0.1", + Port: "8080", + }, + }, + { + name: "Enabled flag is false and address/port flags are not changed", + enabledFlag: false, + addrFlag: "", + portFlag: "", + expectedCfg: &Config{ + Enabled: false, + Address: "", + Port: "", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := &cobra.Command{} + cfg := &Config{} + + cmd.Flags().AddFlagSet(Flags()) + + err := cmd.Flags().Set(enabledFlag, strconv.FormatBool(test.enabledFlag)) + assert.NoError(t, err) + err = cmd.Flags().Set(addrFlag, test.addrFlag) + assert.NoError(t, err) + err = cmd.Flags().Set(portFlag, test.portFlag) + assert.NoError(t, err) + + ParseFlags(cmd, cfg) + assert.Equal(t, test.expectedCfg.Enabled, cfg.Enabled) + assert.Equal(t, test.expectedCfg.Address, cfg.Address) + assert.Equal(t, test.expectedCfg.Port, cfg.Port) + }) + } +} diff --git a/nodebuilder/rpc/config.go b/nodebuilder/rpc/config.go index 306dd562e3..a270768646 100644 --- a/nodebuilder/rpc/config.go +++ b/nodebuilder/rpc/config.go @@ -14,9 +14,9 @@ type Config struct { func DefaultConfig() Config { return Config{ - Address: "0.0.0.0", + Address: defaultBindAddress, // do NOT expose the same port as celestia-core by default so that both can run on the same machine - Port: "26658", + Port: defaultPort, } } diff --git a/nodebuilder/rpc/config_test.go b/nodebuilder/rpc/config_test.go new file mode 100644 index 0000000000..1c78a1a19f --- /dev/null +++ b/nodebuilder/rpc/config_test.go @@ -0,0 +1,59 @@ +package rpc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +// TestDefaultConfig tests that the default gateway config is correct. +func TestDefaultConfig(t *testing.T) { + expected := Config{ + Address: defaultBindAddress, + Port: defaultPort, + } + + assert.Equal(t, expected, DefaultConfig()) +} + +func TestConfigValidate(t *testing.T) { + tests := []struct { + name string + cfg Config + err bool + }{ + { + name: "valid config", + cfg: Config{ + Address: "127.0.0.1", + Port: "8080", + }, + err: false, + }, + { + name: "invalid address", + cfg: Config{ + Address: "invalid", + Port: "8080", + }, + err: true, + }, + { + name: "invalid port", + cfg: Config{ + Address: "127.0.0.1", + Port: "invalid", + }, + err: true, + }, + } + + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + err := tt.cfg.Validate() + if (err != nil) != tt.err { + t.Errorf("Config.Validate() error = %v, err %v", err, tt.err) + } + }) + } +} diff --git a/nodebuilder/rpc/defaults.go b/nodebuilder/rpc/defaults.go new file mode 100644 index 0000000000..55e51a7c9b --- /dev/null +++ b/nodebuilder/rpc/defaults.go @@ -0,0 +1,6 @@ +package rpc + +const ( + defaultBindAddress = "localhost" + defaultPort = "26658" +) diff --git a/nodebuilder/rpc/defaults_test.go b/nodebuilder/rpc/defaults_test.go new file mode 100644 index 0000000000..74d9c98cfc --- /dev/null +++ b/nodebuilder/rpc/defaults_test.go @@ -0,0 +1,12 @@ +package rpc + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestServerDefaultConstants(t *testing.T) { + assert.Equal(t, "localhost", defaultBindAddress) + assert.Equal(t, "26658", defaultPort) +} diff --git a/nodebuilder/rpc/flags.go b/nodebuilder/rpc/flags.go index 167dbc803a..b7bad333df 100644 --- a/nodebuilder/rpc/flags.go +++ b/nodebuilder/rpc/flags.go @@ -1,6 +1,8 @@ package rpc import ( + "fmt" + "github.com/spf13/cobra" flag "github.com/spf13/pflag" ) @@ -17,12 +19,12 @@ func Flags() *flag.FlagSet { flags.String( addrFlag, "", - "Set a custom RPC listen address (default: localhost)", + fmt.Sprintf("Set a custom RPC listen address (default: %s)", defaultBindAddress), ) flags.String( portFlag, "", - "Set a custom RPC port (default: 26658)", + fmt.Sprintf("Set a custom RPC port (default: %s)", defaultPort), ) return flags diff --git a/nodebuilder/rpc/flags_test.go b/nodebuilder/rpc/flags_test.go new file mode 100644 index 0000000000..1370995833 --- /dev/null +++ b/nodebuilder/rpc/flags_test.go @@ -0,0 +1,95 @@ +package rpc + +import ( + "fmt" + "testing" + + "github.com/spf13/cobra" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestFlags(t *testing.T) { + flags := Flags() + + // Test addrFlag + addr := flags.Lookup(addrFlag) + require.NotNil(t, addr) + assert.Equal(t, "", addr.Value.String()) + assert.Equal(t, fmt.Sprintf("Set a custom RPC listen address (default: %s)", defaultBindAddress), addr.Usage) + + // Test portFlag + port := flags.Lookup(portFlag) + require.NotNil(t, port) + assert.Equal(t, "", port.Value.String()) + assert.Equal(t, fmt.Sprintf("Set a custom RPC port (default: %s)", defaultPort), port.Usage) +} + +// TestParseFlags tests the ParseFlags function in rpc/flags.go +func TestParseFlags(t *testing.T) { + tests := []struct { + name string + addrFlag string + portFlag string + expected *Config + }{ + { + name: "addrFlag is set", + addrFlag: "127.0.0.1:8080", + portFlag: "", + expected: &Config{ + Address: "127.0.0.1:8080", + Port: "", + }, + }, + { + name: "portFlag is set", + addrFlag: "", + portFlag: "9090", + expected: &Config{ + Address: "", + Port: "9090", + }, + }, + { + name: "both addrFlag and portFlag are set", + addrFlag: "192.168.0.1:1234", + portFlag: "5678", + expected: &Config{ + Address: "192.168.0.1:1234", + Port: "5678", + }, + }, + { + name: "neither addrFlag nor portFlag are set", + addrFlag: "", + portFlag: "", + expected: &Config{ + Address: "", + Port: "", + }, + }, + } + + for _, test := range tests { + t.Run(test.name, func(t *testing.T) { + cmd := &cobra.Command{} + cfg := &Config{} + + cmd.Flags().AddFlagSet(Flags()) + + err := cmd.Flags().Set(addrFlag, test.addrFlag) + if err != nil { + t.Errorf(err.Error()) + } + err = cmd.Flags().Set(portFlag, test.portFlag) + if err != nil { + t.Errorf(err.Error()) + } + + ParseFlags(cmd, cfg) + assert.Equal(t, test.expected.Address, cfg.Address) + assert.Equal(t, test.expected.Port, cfg.Port) + }) + } +} From 1d228f0cd1a88490871fefe9496fcefcd40a8f4a Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 18 Dec 2023 11:40:07 +0000 Subject: [PATCH 333/388] ignore otel patch updates (#2988) opening this as a possible solution and to discuss if we want to make the matching more greedy, but this implements a suggestion by @walldiss last week to have dependabot be less aggressive with suggesting EVERY patch release for all dependencies, only limiting to major and minor updates as suggestion. Arguably, we could expand the wildcard to more than `*otel*` and apply to everything, but maybe we just let it run on otel for a bit and see what the consequences are before considering applying it more broadly. Also, i think we should review the security alert configuration side of dependabot as a companion to this to ensure we DO get security alerts if a CVE or issue is patched. But that will be separate --- .github/dependabot.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index e2becce196..edcd86e11d 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -21,6 +21,9 @@ updates: - renaynay labels: - kind:deps + ignore: + - dependency-name: "*otel*" + update-types: ["version-update:semver-patch"] groups: otel: patterns: From e38a7bfbd8e5e589044083be4f334ce140fa2735 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Mon, 18 Dec 2023 14:04:41 +0200 Subject: [PATCH 334/388] fix(share/byzantine): fix proof collection (#2957) This PR fixes an issue in the proof collection method. Previously, we were sending `len(dah.RowRoots)/2` requests in order to get proofs but there is a use case when the process can get stuck for a particular share(out-of-order case). The new approach sends n requests, where n is the number of non-empty shares received from rsmt2d library, and then waits until `len(dah.RowRoots)/2` will be received. I've also improved error handling as previously we were panicking in case any error appeared. The new approach will rely only on `context.DeadlineExceeded ` which basically means that for some reason we can't get proofs, so the data is not available. --------- Co-authored-by: ramin --- share/eds/byzantine/bad_encoding_test.go | 111 ++++++++++++++++++++--- share/eds/byzantine/byzantine.go | 46 +++++----- 2 files changed, 119 insertions(+), 38 deletions(-) diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index e7032107ca..adb23d3549 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -2,9 +2,15 @@ package byzantine import ( "context" + "crypto/sha256" + "hash" "testing" "time" + "github.com/ipfs/boxo/blockservice" + blocks "github.com/ipfs/go-block-format" + "github.com/ipfs/go-cid" + mhcore "github.com/multiformats/go-multihash/core" "github.com/stretchr/testify/require" core "github.com/tendermint/tendermint/types" @@ -35,10 +41,11 @@ func TestBEFP_Validate(t *testing.T) { err = square.Repair(dah.RowRoots, dah.ColumnRoots) require.ErrorAs(t, err, &errRsmt2d) - errByz := NewErrByzantine(ctx, bServ, &dah, errRsmt2d) + byzantine := NewErrByzantine(ctx, bServ, &dah, errRsmt2d) + var errByz *ErrByzantine + require.ErrorAs(t, byzantine, &errByz) befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) - var test = []struct { name string prepareFn func() error @@ -69,7 +76,9 @@ func TestBEFP_Validate(t *testing.T) { Shares: validShares[0:4], }, ) - invalidBefp := CreateBadEncodingProof([]byte("hash"), 0, errInvalidByz) + var errInvalid *ErrByzantine + require.ErrorAs(t, errInvalidByz, &errInvalid) + invalidBefp := CreateBadEncodingProof([]byte("hash"), 0, errInvalid) return invalidBefp.Validate(&header.ExtendedHeader{DAH: &validDah}) }, expectedResult: func(err error) { @@ -198,18 +207,21 @@ func TestIncorrectBadEncodingFraudProof(t *testing.T) { } func TestBEFP_ValidateOutOfOrderShares(t *testing.T) { - // skipping it for now because `malicious` package has a small issue: Constructor does not apply - // passed options, so it's not possible to store shares and thus get proofs for them. - // should be ok once app team will fix it. - t.Skip() - eds := edstest.RandEDS(t, 16) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*30) + t.Cleanup(cancel) + + size := 4 + eds := edstest.RandEDS(t, size) + shares := eds.Flattened() - shares[0], shares[1] = shares[1], shares[0] // corrupting eds - bServ := ipld.NewMemBlockservice() - batchAddr := ipld.NewNmtNodeAdder(context.Background(), bServ, ipld.MaxSizeBatchOption(16*2)) + shares[0], shares[4] = shares[4], shares[0] // corrupting eds + + bServ := newNamespacedBlockService() + batchAddr := ipld.NewNmtNodeAdder(ctx, bServ, ipld.MaxSizeBatchOption(size*2)) + eds, err := rsmt2d.ImportExtendedDataSquare(shares, share.DefaultRSMT2DCodec(), - malicious.NewConstructor(16, nmt.NodeVisitor(batchAddr.Visit)), + malicious.NewConstructor(uint64(size), nmt.NodeVisitor(batchAddr.Visit)), ) require.NoError(t, err, "failure to recompute the extended data square") @@ -223,9 +235,80 @@ func TestBEFP_ValidateOutOfOrderShares(t *testing.T) { err = eds.Repair(dah.RowRoots, dah.ColumnRoots) require.ErrorAs(t, err, &errRsmt2d) - errByz := NewErrByzantine(context.Background(), bServ, &dah, errRsmt2d) + byzantine := NewErrByzantine(ctx, bServ, &dah, errRsmt2d) + var errByz *ErrByzantine + require.ErrorAs(t, byzantine, &errByz) befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) err = befp.Validate(&header.ExtendedHeader{DAH: &dah}) - require.Error(t, err) + require.NoError(t, err) +} + +// namespacedBlockService wraps `BlockService` and extends the verification part +// to avoid returning blocks that has out of order namespaces. +type namespacedBlockService struct { + blockservice.BlockService + // the data structure that is used on the networking level, in order + // to verify the order of the namespaces + prefix *cid.Prefix +} + +func newNamespacedBlockService() *namespacedBlockService { + sha256NamespaceFlagged := uint64(0x7701) + // register the nmt hasher to validate the order of namespaces + mhcore.Register(sha256NamespaceFlagged, func() hash.Hash { + nh := nmt.NewNmtHasher(sha256.New(), share.NamespaceSize, true) + nh.Reset() + return nh + }) + + bs := &namespacedBlockService{} + bs.BlockService = ipld.NewMemBlockservice() + + bs.prefix = &cid.Prefix{ + Version: 1, + Codec: sha256NamespaceFlagged, + MhType: sha256NamespaceFlagged, + // equals to NmtHasher.Size() + MhLength: sha256.New().Size() + 2*share.NamespaceSize, + } + return bs +} + +func (n *namespacedBlockService) GetBlock(ctx context.Context, c cid.Cid) (blocks.Block, error) { + block, err := n.BlockService.GetBlock(ctx, c) + if err != nil { + return nil, err + } + + _, err = n.prefix.Sum(block.RawData()) + if err != nil { + return nil, err + } + return block, nil +} + +func (n *namespacedBlockService) GetBlocks(ctx context.Context, cids []cid.Cid) <-chan blocks.Block { + blockCh := n.BlockService.GetBlocks(ctx, cids) + resultCh := make(chan blocks.Block) + + go func() { + for { + select { + case <-ctx.Done(): + close(resultCh) + return + case block, ok := <-blockCh: + if !ok { + close(resultCh) + return + } + if _, err := n.prefix.Sum(block.RawData()); err != nil { + continue + } + resultCh <- block + } + } + }() + return resultCh } diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index dfdf681f04..e7aec28959 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -4,11 +4,9 @@ import ( "context" "fmt" - "github.com/ipfs/boxo/blockservice" - "golang.org/x/sync/errgroup" - "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" + "github.com/ipfs/boxo/blockservice" "github.com/celestiaorg/celestia-node/share/ipld" ) @@ -35,7 +33,7 @@ func NewErrByzantine( bGetter blockservice.BlockGetter, dah *da.DataAvailabilityHeader, errByz *rsmt2d.ErrByzantineData, -) *ErrByzantine { +) error { // changing the order to collect proofs against an orthogonal axis roots := [][][]byte{ dah.ColumnRoots, @@ -43,41 +41,41 @@ func NewErrByzantine( }[errByz.Axis] sharesWithProof := make([]*ShareWithProof, len(errByz.Shares)) - sharesAmount := 0 - errGr, ctx := errgroup.WithContext(ctx) + type result struct { + share *ShareWithProof + index int + } + resultCh := make(chan *result) for index, share := range errByz.Shares { - // skip further shares if we already requested half of them, which is enough to recompute the row - // or col - if sharesAmount == len(dah.RowRoots)/2 { - break - } - if share == nil { continue } - sharesAmount++ index := index - errGr.Go(func() error { + go func() { share, err := getProofsAt( ctx, bGetter, ipld.MustCidFromNamespacedSha256(roots[index]), int(errByz.Index), len(errByz.Shares), ) - sharesWithProof[index] = share - return err - }) + if err != nil { + log.Warn("requesting proof failed", "root", roots[index], "err", err) + return + } + resultCh <- &result{share, index} + }() } - if err := errGr.Wait(); err != nil { - // Fatal as rsmt2d proved that error is byzantine, - // but we cannot properly collect the proof, - // so verification will fail and thus services won't be stopped - // while we still have to stop them. - // TODO(@Wondertan): Find a better way to handle - log.Fatalw("getting proof for ErrByzantine", "err", err) + for i := 0; i < len(dah.RowRoots)/2; i++ { + select { + case t := <-resultCh: + sharesWithProof[t.index] = t.share + case <-ctx.Done(): + return ipld.ErrNodeNotFound + } } + return &ErrByzantine{ Index: uint32(errByz.Index), Shares: sharesWithProof, From 8a39228df21cbdc6fe2356663270e65dafb68564 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:15:46 +0000 Subject: [PATCH 335/388] chore(deps): Bump actions/upload-artifact from 3 to 4 (#3014) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [actions/upload-artifact](https://github.com/actions/upload-artifact) from 3 to 4.
Release notes

Sourced from actions/upload-artifact's releases.

v4.0.0

What's Changed

The release of upload-artifact@v4 and download-artifact@v4 are major changes to the backend architecture of Artifacts. They have numerous performance and behavioral improvements.

For more information, see the @​actions/artifact documentation.

New Contributors

Full Changelog: https://github.com/actions/upload-artifact/compare/v3...v4.0.0

v3.1.3

What's Changed

Full Changelog: https://github.com/actions/upload-artifact/compare/v3...v3.1.3

v3.1.2

  • Update all @actions/* NPM packages to their latest versions- #374
  • Update all dev dependencies to their most recent versions - #375

v3.1.1

  • Update actions/core package to latest version to remove set-output deprecation warning #351

v3.1.0

What's Changed

Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=actions/upload-artifact&package-manager=github_actions&previous-version=3&new-version=4)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/go-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index c14692cd58..b004dcfb59 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -87,7 +87,7 @@ jobs: run: make test-unit ENABLE_VERBOSE=${{ needs.setup.outputs.debug }} - name: Upload unit test output - uses: actions/upload-artifact@v3 + uses: actions/upload-artifact@v4 if: always() && needs.setup.outputs.debug == 'true' with: name: unit-test-output-${{ matrix.os }} From a61ae5c3ad64aa7e6023d5ed6c9c83477e9f3fdf Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:28:17 +0000 Subject: [PATCH 336/388] chore(deps): Bump github.com/hashicorp/go-retryablehttp from 0.7.4 to 0.7.5 (#2924) Bumps [github.com/hashicorp/go-retryablehttp](https://github.com/hashicorp/go-retryablehttp) from 0.7.4 to 0.7.5.
Changelog

Sourced from github.com/hashicorp/go-retryablehttp's changelog.

0.7.5 (Nov 8, 2023)

BUG FIXES

  • client: fixes an issue where the request body is not preserved on temporary redirects or re-established HTTP/2 connections GH-207
Commits
  • 4165cf8 Merge pull request #209 from hashicorp/v0.7.5-changelog
  • 6c37e02 v0.7.5 changelog update
  • 309c58e Merge pull request #207 from hashicorp/sebasslash/handle-go-away
  • f95735f Update workflow to use go v1.18
  • 9bb2062 Sets request's GetBody field on wrapper
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/hashicorp/go-retryablehttp&package-manager=go_modules&previous-version=0.7.4&new-version=0.7.5)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index b4441c6f6b..e2a35a0f49 100644 --- a/go.mod +++ b/go.mod @@ -25,7 +25,7 @@ require ( github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 - github.com/hashicorp/go-retryablehttp v0.7.4 + github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.15.0 diff --git a/go.sum b/go.sum index 962e46067b..2a9279400c 100644 --- a/go.sum +++ b/go.sum @@ -958,8 +958,8 @@ github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHh github.com/hashicorp/go-multierror v1.1.1 h1:H5DkEtf6CXdFp0N0Em5UCwQpXMWke8IA0+lD48awMYo= github.com/hashicorp/go-multierror v1.1.1/go.mod h1:iw975J/qwKPdAO1clOe2L8331t/9/fmwbPZ6JB6eMoM= github.com/hashicorp/go-retryablehttp v0.5.3/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= -github.com/hashicorp/go-retryablehttp v0.7.4 h1:ZQgVdpTdAL7WpMIwLzCfbalOcSUdkDZnpUv3/+BxzFA= -github.com/hashicorp/go-retryablehttp v0.7.4/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= +github.com/hashicorp/go-retryablehttp v0.7.5 h1:bJj+Pj19UZMIweq/iie+1u5YCdGrnxCT9yvm0e+Nd5M= +github.com/hashicorp/go-retryablehttp v0.7.5/go.mod h1:Jy/gPYAdjqffZ/yFGCFV2doI5wjtH1ewM9u8iYVjtX8= github.com/hashicorp/go-rootcerts v1.0.0/go.mod h1:K6zTfqpRlCUIjkwsN4Z+hiSfzSTQa6eBIzfwKfwNnHU= github.com/hashicorp/go-safetemp v1.0.0 h1:2HR189eFNrjHQyENnQMMpCiBAsRxzbTMIgBhEyExpmo= github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoDHxNAB65b+Rj1I= From 7f51c599b2f7576909a19c4487e1a8c46c5a67e2 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 18 Dec 2023 12:38:52 +0000 Subject: [PATCH 337/388] chore(deps): Bump cosmossdk.io/math from 1.1.2 to 1.2.0 (#2923) Bumps [cosmossdk.io/math](https://github.com/cosmos/cosmos-sdk) from 1.1.2 to 1.2.0.
Release notes

Sourced from cosmossdk.io/math's releases.

Cosmovisor v1.2.0

Release Notes

New Features

With the cosmovisor init command, all the necessary folders for using cosmovisor are automatically created. You do not need to manually symlink the chain binary anymore.

We've added a new configuration option: DAEMON_RESTART_DELAY (as env variable). When set, Cosmovisor will wait that delay between the node halt and backup. See the README file for more details.

Bug Fixes

  • Fix Cosmovisor binary usage for pre-upgrade. Cosmovisor was using the wrong binary when running a pre-upgrade command.

Changelog

For more details, please see the CHANGELOG.

Commits
  • c33b4db chore: prepare log 1.2.0 (#17214)
  • c36c860 refactor(cli): remove duplicate --home flag (#17215)
  • 31f7247 feat(cli): status cmd cli support output text (#17184)
  • 095a641 feat(docker): Add debuging dependencies directly into the dockerfile (#17228)
  • f47749b fix: text output format of block-results cmd cli (#17183)
  • 236fe4c docs: additional fields in context.go to be added to context.md (#17199)
  • 382de33 feat(testutil): adding DefaultContextWithKeys test helper (#17216)
  • 0b7d2d3 fix(log): add fallback to Stringer when type do not implement `json.Marshal...
  • 60ead8d ci: skip fix_registration.go when using LINT_DIFF=1 (#17185)
  • edba186 refactor(keys): remove duplicate --home flag (#17197)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=cosmossdk.io/math&package-manager=go_modules&previous-version=1.1.2&new-version=1.2.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e2a35a0f49..e6f715e2b5 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.21.1 require ( cosmossdk.io/errors v1.0.0 - cosmossdk.io/math v1.1.2 + cosmossdk.io/math v1.2.0 github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 diff --git a/go.sum b/go.sum index 2a9279400c..b3d4b91036 100644 --- a/go.sum +++ b/go.sum @@ -195,8 +195,8 @@ cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoIS collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= -cosmossdk.io/math v1.1.2 h1:ORZetZCTyWkI5GlZ6CZS28fMHi83ZYf+A2vVnHNzZBM= -cosmossdk.io/math v1.1.2/go.mod h1:l2Gnda87F0su8a/7FEKJfFdJrM0JZRXQaohlgJeyQh0= +cosmossdk.io/math v1.2.0 h1:8gudhTkkD3NxOP2YyyJIYYmt6dQ55ZfJkDOaxXpy7Ig= +cosmossdk.io/math v1.2.0/go.mod h1:l2Gnda87F0su8a/7FEKJfFdJrM0JZRXQaohlgJeyQh0= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20201218220906-28db891af037/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= From c34b741ae3d70e31dcf7865ba959e1b039c0f076 Mon Sep 17 00:00:00 2001 From: Vlad <13818348+walldiss@users.noreply.github.com> Date: Mon, 18 Dec 2023 16:08:52 +0300 Subject: [PATCH 338/388] feat(share/p2p/peer-manager): limit amount of stored pools in peer-manager (#3005) This PR introduces `amountOfStoredPools` param to Peer manager, that will limit amount of stored pools. Defautlt value is 10 pools. Peer manager will try to keep `amountOfStoredPools` amount of pools only for recent headers. Older blocks don't need to be routed by manual pools and could be routed to any full node, as those are expected to have the block by that time. This will lower memory footprint of peer manager as well as resolve memory leaking issues. Resolves https://github.com/celestiaorg/celestia-node/issues/1781 --- share/getters/shrex.go | 15 ++-- share/p2p/peers/manager.go | 103 +++++++++++++------------ share/p2p/peers/manager_test.go | 129 ++++++++++++++++---------------- share/p2p/peers/metrics.go | 10 +-- share/p2p/peers/pool.go | 9 --- 5 files changed, 122 insertions(+), 144 deletions(-) diff --git a/share/getters/shrex.go b/share/getters/shrex.go index 0586826e22..af48a7f8ab 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -135,9 +135,8 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader utils.SetStatusAndEnd(span, err) }() - dah := header.DAH // short circuit if the data root is empty - if dah.Equals(share.EmptyRoot()) { + if header.DAH.Equals(share.EmptyRoot()) { return share.EmptyExtendedDataSquare(), nil } for { @@ -147,10 +146,10 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader } attempt++ start := time.Now() - peer, setStatus, getErr := sg.peerManager.Peer(ctx, dah.Hash()) + peer, setStatus, getErr := sg.peerManager.Peer(ctx, header.DAH.Hash(), header.Height()) if getErr != nil { log.Debugw("eds: couldn't find peer", - "hash", dah.String(), + "hash", header.DAH.String(), "err", getErr, "finished (s)", time.Since(start)) sg.metrics.recordEDSAttempt(ctx, attempt, false) @@ -159,11 +158,11 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader reqStart := time.Now() reqCtx, cancel := ctxWithSplitTimeout(ctx, sg.minAttemptsCount-attempt+1, sg.minRequestTimeout) - eds, getErr := sg.edsClient.RequestEDS(reqCtx, dah.Hash(), peer) + eds, getErr := sg.edsClient.RequestEDS(reqCtx, header.DAH.Hash(), peer) cancel() switch { case getErr == nil: - setStatus(peers.ResultSynced) + setStatus(peers.ResultNoop) sg.metrics.recordEDSAttempt(ctx, attempt, true) return eds, nil case errors.Is(getErr, context.DeadlineExceeded), @@ -182,7 +181,7 @@ func (sg *ShrexGetter) GetEDS(ctx context.Context, header *header.ExtendedHeader err = errors.Join(err, getErr) } log.Debugw("eds: request failed", - "hash", dah.String(), + "hash", header.DAH.String(), "peer", peer.String(), "attempt", attempt, "err", getErr, @@ -223,7 +222,7 @@ func (sg *ShrexGetter) GetSharesByNamespace( } attempt++ start := time.Now() - peer, setStatus, getErr := sg.peerManager.Peer(ctx, dah.Hash()) + peer, setStatus, getErr := sg.peerManager.Peer(ctx, header.DAH.Hash(), header.Height()) if getErr != nil { log.Debugw("nd: couldn't find peer", "hash", dah.String(), diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index e39a181150..23fa18bcb2 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -27,8 +27,6 @@ import ( const ( // ResultNoop indicates operation was successful and no extra action is required ResultNoop result = "result_noop" - // ResultSynced will save the status of pool as "synced" and will remove peers from it - ResultSynced = "result_synced" // ResultCooldownPeer will put returned peer on cooldown, meaning it won't be available by Peer // method for some time ResultCooldownPeer = "result_cooldown_peer" @@ -39,6 +37,9 @@ const ( // eventbusBufSize is the size of the buffered channel to handle // events in libp2p eventbusBufSize = 32 + + // storedPoolsAmount is the amount of pools for recent headers that will be stored in the peer manager + storedPoolsAmount = 10 ) type result string @@ -56,11 +57,14 @@ type Manager struct { host host.Host connGater *conngater.BasicConnectionGater - // pools collecting peers from shrexSub + // pools collecting peers from shrexSub and stores them by datahash pools map[string]*syncPool - // messages from shrex.Sub with height below initialHeight will be ignored, since we don't need to - // track peers for those headers + + // initialHeight is the height of the first header received from headersub initialHeight atomic.Uint64 + // messages from shrex.Sub with height below storeFrom will be ignored, since we don't need to + // track peers for those headers + storeFrom atomic.Uint64 // fullNodes collects full nodes peer.ID found via discovery fullNodes *pool @@ -85,11 +89,8 @@ type syncPool struct { // isValidatedDataHash indicates if datahash was validated by receiving corresponding extended // header from headerSub isValidatedDataHash atomic.Bool - // headerHeight is the height of header corresponding to syncpool - headerHeight atomic.Uint64 - // isSynced will be true if DoneFunc was called with ResultSynced. It indicates that given datahash - // was synced and peer-manager no longer need to keep peers for it - isSynced atomic.Bool + // height is the height of the header that corresponds to datahash + height uint64 // createdAt is the syncPool creation time createdAt time.Time } @@ -190,16 +191,15 @@ func (m *Manager) Stop(ctx context.Context) error { // full nodes, it will wait until any peer appear in either source or timeout happen. // After fetching data using given peer, caller is required to call returned DoneFunc using // appropriate result value -func (m *Manager) Peer( - ctx context.Context, datahash share.DataHash, +func (m *Manager) Peer(ctx context.Context, datahash share.DataHash, height uint64, ) (peer.ID, DoneFunc, error) { - p := m.validatedPool(datahash.String()) + p := m.validatedPool(datahash.String(), height) // first, check if a peer is available for the given datahash peerID, ok := p.tryGet() if ok { if m.removeIfUnreachable(p, peerID) { - return m.Peer(ctx, datahash) + return m.Peer(ctx, datahash, height) } return m.newPeer(ctx, datahash, peerID, sourceShrexSub, p.len(), 0) } @@ -216,7 +216,7 @@ func (m *Manager) Peer( select { case peerID = <-p.next(ctx): if m.removeIfUnreachable(p, peerID) { - return m.Peer(ctx, datahash) + return m.Peer(ctx, datahash, height) } return m.newPeer(ctx, datahash, peerID, sourceShrexSub, p.len(), time.Since(start)) case peerID = <-m.fullNodes.next(ctx): @@ -270,14 +270,12 @@ func (m *Manager) doneFunc(datahash share.DataHash, peerID peer.ID, source peerS m.metrics.observeDoneResult(source, result) switch result { case ResultNoop: - case ResultSynced: - m.markPoolAsSynced(datahash.String()) case ResultCooldownPeer: if source == sourceFullNodes { m.fullNodes.putOnCooldown(peerID) return } - m.getOrCreatePool(datahash.String()).putOnCooldown(peerID) + m.getPool(datahash.String()).putOnCooldown(peerID) case ResultBlacklistPeer: m.blacklistPeers(reasonMisbehave, peerID) } @@ -298,12 +296,16 @@ func (m *Manager) subscribeHeader(ctx context.Context, headerSub libhead.Subscri log.Errorw("get next header from sub", "err", err) continue } - m.validatedPool(h.DataHash.String()) + m.validatedPool(h.DataHash.String(), h.Height()) // store first header for validation purposes if m.initialHeight.CompareAndSwap(0, h.Height()) { log.Debugw("stored initial height", "height", h.Height()) } + + // update storeFrom if header heigh + m.storeFrom.Store(uint64(max(0, int(h.Height())-storedPoolsAmount))) + log.Debugw("updated lowest stored height", "height", h.Height()) } } @@ -355,22 +357,12 @@ func (m *Manager) Validate(_ context.Context, peerID peer.ID, msg shrexsub.Notif return pubsub.ValidationReject } - if msg.Height == 0 { - logger.Debug("received message with 0 height") - return pubsub.ValidationReject - } - - if msg.Height < m.initialHeight.Load() { - // we can use peers from discovery for headers before the first one from headerSub - // if we allow pool creation for those headers, there is chance the pool will not be validated in - // time and will be false-positively trigger blacklisting of hash and all peers that sent msgs for - // that hash + if msg.Height < m.storeFrom.Load() { logger.Debug("received message for past header") return pubsub.ValidationIgnore } - p := m.getOrCreatePool(msg.DataHash.String()) - p.headerHeight.Store(msg.Height) + p := m.getOrCreatePool(msg.DataHash.String(), msg.Height) logger.Debugw("got hash from shrex-sub") p.add(peerID) @@ -381,13 +373,20 @@ func (m *Manager) Validate(_ context.Context, peerID peer.ID, msg shrexsub.Notif return pubsub.ValidationIgnore } -func (m *Manager) getOrCreatePool(datahash string) *syncPool { +func (m *Manager) getPool(datahash string) *syncPool { + m.lock.Lock() + defer m.lock.Unlock() + return m.pools[datahash] +} + +func (m *Manager) getOrCreatePool(datahash string, height uint64) *syncPool { m.lock.Lock() defer m.lock.Unlock() p, ok := m.pools[datahash] if !ok { p = &syncPool{ + height: height, pool: newPool(m.params.PeerCooldown), createdAt: time.Now(), } @@ -432,8 +431,8 @@ func (m *Manager) isBlacklistedHash(hash share.DataHash) bool { return m.blacklistedHashes[hash.String()] } -func (m *Manager) validatedPool(hashStr string) *syncPool { - p := m.getOrCreatePool(hashStr) +func (m *Manager) validatedPool(hashStr string, height uint64) *syncPool { + p := m.getOrCreatePool(hashStr, height) if p.isValidatedDataHash.CompareAndSwap(false, true) { log.Debugw("pool marked validated", "datahash", hashStr) // if pool is proven to be valid, add all collected peers to full nodes @@ -482,12 +481,24 @@ func (m *Manager) cleanUp() []peer.ID { addToBlackList := make(map[peer.ID]struct{}) for h, p := range m.pools { - if !p.isValidatedDataHash.Load() && time.Since(p.createdAt) > m.params.PoolValidationTimeout { - delete(m.pools, h) - if p.headerHeight.Load() < m.initialHeight.Load() { - // outdated pools could still be valid even if not validated, no need to blacklist - continue + if p.isValidatedDataHash.Load() { + // remove pools that are outdated + if p.height < m.storeFrom.Load() { + delete(m.pools, h) } + continue + } + + // can't validate datahashes below initial height + if p.height < m.initialHeight.Load() { + delete(m.pools, h) + continue + } + + // find pools that are not validated in time + if time.Since(p.createdAt) > m.params.PoolValidationTimeout { + delete(m.pools, h) + log.Debug("blacklisting datahash with all corresponding peers", "hash", h, "peer_list", p.peersList) @@ -507,17 +518,3 @@ func (m *Manager) cleanUp() []peer.ID { } return blacklist } - -func (m *Manager) markPoolAsSynced(datahash string) { - p := m.getOrCreatePool(datahash) - if p.isSynced.CompareAndSwap(false, true) { - p.isSynced.Store(true) - p.reset() - } -} - -func (p *syncPool) add(peers ...peer.ID) { - if !p.isSynced.Load() { - p.pool.add(peers...) - } -} diff --git a/share/p2p/peers/manager_test.go b/share/p2p/peers/manager_test.go index 94ec5d5ea2..d4a188ff56 100644 --- a/share/p2p/peers/manager_test.go +++ b/share/p2p/peers/manager_test.go @@ -2,12 +2,12 @@ package peers import ( "context" - sync2 "sync" + "sync" "testing" "time" "github.com/ipfs/go-datastore" - "github.com/ipfs/go-datastore/sync" + dssync "github.com/ipfs/go-datastore/sync" dht "github.com/libp2p/go-libp2p-kad-dht" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/host" @@ -26,10 +26,9 @@ import ( "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) -// TODO: add broadcast to tests func TestManager(t *testing.T) { t.Run("Validate pool by headerSub", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*50) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) // create headerSub mock @@ -64,21 +63,16 @@ func TestManager(t *testing.T) { result := manager.Validate(ctx, peerID, msg) require.Equal(t, pubsub.ValidationIgnore, result) - pID, done, err := manager.Peer(ctx, h.DataHash.Bytes()) + pID, _, err := manager.Peer(ctx, h.DataHash.Bytes(), h.Height()) require.NoError(t, err) require.Equal(t, peerID, pID) // check pool validation - require.True(t, manager.getOrCreatePool(h.DataHash.String()).isValidatedDataHash.Load()) - - done(ResultSynced) - // pool should not be removed after success - require.Len(t, manager.pools, 1) - require.Len(t, manager.getOrCreatePool(h.DataHash.String()).pool.peersList, 0) + require.True(t, manager.getPool(h.DataHash.String()).isValidatedDataHash.Load()) }) t.Run("validator", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*50) + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) // create headerSub mock @@ -100,7 +94,7 @@ func TestManager(t *testing.T) { require.Equal(t, pubsub.ValidationIgnore, result) // mark peer as misbehaved to blacklist it - pID, done, err := manager.Peer(ctx, h.DataHash.Bytes()) + pID, done, err := manager.Peer(ctx, h.DataHash.Bytes(), h.Height()) require.NoError(t, err) require.Equal(t, peerID, pID) manager.params.EnableBlackListing = true @@ -132,19 +126,21 @@ func TestManager(t *testing.T) { // create unvalidated pool peerID := peer.ID("peer1") msg := shrexsub.Notification{ - DataHash: share.DataHash("datahash1"), + DataHash: share.DataHash("datahash1datahash1datahash1datahash1datahash1"), Height: 2, } manager.Validate(ctx, peerID, msg) // create validated pool validDataHash := share.DataHash("datahash2") - manager.fullNodes.add("full") // add FN to unblock Peer call - manager.Peer(ctx, validDataHash) //nolint:errcheck + manager.fullNodes.add("full") // add FN to unblock Peer call + manager.Peer(ctx, validDataHash, h.Height()) //nolint:errcheck + require.Len(t, manager.pools, 3) // trigger cleanup blacklisted := manager.cleanUp() require.Contains(t, blacklisted, peerID) + require.Len(t, manager.pools, 2) // messages with blacklisted hash should be rejected right away peerID2 := peer.ID("peer2") @@ -172,8 +168,7 @@ func TestManager(t *testing.T) { peers := []peer.ID{"peer1", "peer2", "peer3"} manager.fullNodes.add(peers...) - peerID, done, err := manager.Peer(ctx, h.DataHash.Bytes()) - done(ResultSynced) + peerID, _, err := manager.Peer(ctx, h.DataHash.Bytes(), h.Height()) require.NoError(t, err) require.Contains(t, peers, peerID) @@ -195,7 +190,7 @@ func TestManager(t *testing.T) { // make sure peers are not returned before timeout timeoutCtx, cancel := context.WithTimeout(context.Background(), time.Millisecond) t.Cleanup(cancel) - _, _, err = manager.Peer(timeoutCtx, h.DataHash.Bytes()) + _, _, err = manager.Peer(timeoutCtx, h.DataHash.Bytes(), h.Height()) require.ErrorIs(t, err, context.DeadlineExceeded) peers := []peer.ID{"peer1", "peer2", "peer3"} @@ -204,8 +199,7 @@ func TestManager(t *testing.T) { doneCh := make(chan struct{}) go func() { defer close(doneCh) - peerID, done, err := manager.Peer(ctx, h.DataHash.Bytes()) - done(ResultSynced) + peerID, _, err := manager.Peer(ctx, h.DataHash.Bytes(), h.Height()) require.NoError(t, err) require.Contains(t, peers, peerID) }() @@ -223,38 +217,7 @@ func TestManager(t *testing.T) { stopManager(t, manager) }) - t.Run("mark pool synced", func(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) - t.Cleanup(cancel) - - h := testHeader() - headerSub := newSubLock(h, nil) - - // start test manager - manager, err := testManager(ctx, headerSub) - require.NoError(t, err) - - peerID, msg := peer.ID("peer1"), newShrexSubMsg(h) - result := manager.Validate(ctx, peerID, msg) - require.Equal(t, pubsub.ValidationIgnore, result) - - pID, done, err := manager.Peer(ctx, h.DataHash.Bytes()) - require.NoError(t, err) - require.Equal(t, peerID, pID) - done(ResultSynced) - - // check pool is soft deleted and marked synced - pool := manager.getOrCreatePool(h.DataHash.String()) - require.Len(t, pool.peersList, 0) - require.True(t, pool.isSynced.Load()) - - // add peer on synced pool should be noop - result = manager.Validate(ctx, "peer2", msg) - require.Equal(t, pubsub.ValidationIgnore, result) - require.Len(t, pool.peersList, 0) - }) - - t.Run("shrexSub sends a message lower than first headerSub header height, msg first", func(t *testing.T) { + t.Run("shrexSub sends a message lower than first headerSub header height, headerSub first", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) @@ -278,12 +241,16 @@ func TestManager(t *testing.T) { } result := manager.Validate(ctx, "peer", msg) require.Equal(t, pubsub.ValidationIgnore, result) + // pool will be created for first shrexSub message + require.Len(t, manager.pools, 2) - // amount of pools should not change + blacklisted := manager.cleanUp() + require.Empty(t, blacklisted) + // trigger cleanup and outdated pool should be removed require.Len(t, manager.pools, 1) }) - t.Run("shrexSub sends a message lower than first headerSub header height, headerSub first", func(t *testing.T) { + t.Run("shrexSub sends a message lower than first headerSub header height, shrexSub first", func(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) t.Cleanup(cancel) @@ -303,18 +270,50 @@ func TestManager(t *testing.T) { result := manager.Validate(ctx, "peer", msg) require.Equal(t, pubsub.ValidationIgnore, result) - // unlock header sub after message validator + // pool will be created for first shrexSub message + require.Len(t, manager.pools, 1) + + // unlock headerSub to allow it to send next message require.NoError(t, headerSub.wait(ctx, 1)) - // pool will be created for first headerSub header datahash + // second pool should be created require.Len(t, manager.pools, 2) - // trigger cleanup and check that no peers or hashes were blacklisted - manager.params.PoolValidationTimeout = 0 + // trigger cleanup and outdated pool should be removed blacklisted := manager.cleanUp() + require.Len(t, manager.pools, 1) + + // check that no peers or hashes were blacklisted + manager.params.PoolValidationTimeout = 0 require.Len(t, blacklisted, 0) require.Len(t, manager.blacklistedHashes, 0) + }) + + t.Run("pools store window", func(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + t.Cleanup(cancel) + + h := testHeader() + h.RawHeader.Height = storedPoolsAmount * 2 + headerSub := newSubLock(h, nil) + + // start test manager + manager, err := testManager(ctx, headerSub) + require.NoError(t, err) + + // unlock headerSub to read first header + require.NoError(t, headerSub.wait(ctx, 1)) + // pool will be created for first headerSub header datahash + require.Len(t, manager.pools, 1) + + // create shrexSub msg with height lower than storedPoolsAmount + msg := shrexsub.Notification{ + DataHash: share.DataHash("datahash"), + Height: h.Height() - storedPoolsAmount - 3, + } + result := manager.Validate(ctx, "peer", msg) + require.Equal(t, pubsub.ValidationIgnore, result) - // outdated pool should be removed + // shrexSub message should be discarded and amount of pools should not change require.Len(t, manager.pools, 1) }) } @@ -355,7 +354,7 @@ func TestIntegration(t *testing.T) { })) // FN should get message - gotPeer, _, err := fnPeerManager.Peer(ctx, randHash) + gotPeer, _, err := fnPeerManager.Peer(ctx, randHash, 13) require.NoError(t, err) // check that gotPeer matched bridge node @@ -409,7 +408,7 @@ func TestIntegration(t *testing.T) { require.NoError(t, err) // init peer manager for full node - connGater, err := conngater.NewBasicConnectionGater(sync.MutexWrap(datastore.NewMapDatastore())) + connGater, err := conngater.NewBasicConnectionGater(dssync.MutexWrap(datastore.NewMapDatastore())) require.NoError(t, err) fnPeerManager, err := NewManager( DefaultParameters(), @@ -469,7 +468,7 @@ func testManager(ctx context.Context, headerSub libhead.Subscriber[*header.Exten return nil, err } - connGater, err := conngater.NewBasicConnectionGater(sync.MutexWrap(datastore.NewMapDatastore())) + connGater, err := conngater.NewBasicConnectionGater(dssync.MutexWrap(datastore.NewMapDatastore())) if err != nil { return nil, err } @@ -504,7 +503,7 @@ func testHeader() *header.ExtendedHeader { type subLock struct { next chan struct{} - wg *sync2.WaitGroup + wg *sync.WaitGroup expected []*header.ExtendedHeader } @@ -530,7 +529,7 @@ func (s subLock) release(ctx context.Context) error { } func newSubLock(expected ...*header.ExtendedHeader) *subLock { - wg := &sync2.WaitGroup{} + wg := &sync.WaitGroup{} wg.Add(1) return &subLock{ next: make(chan struct{}), diff --git a/share/p2p/peers/metrics.go b/share/p2p/peers/metrics.go index 95d1ce65d9..3b5913ebb8 100644 --- a/share/p2p/peers/metrics.go +++ b/share/p2p/peers/metrics.go @@ -39,14 +39,11 @@ const ( poolStatusKey = "pool_status" poolStatusCreated poolStatus = "created" poolStatusValidated poolStatus = "validated" - poolStatusSynced poolStatus = "synced" poolStatusBlacklisted poolStatus = "blacklisted" // Pool status model: // created(unvalidated) // / \ - // validated(unsynced) blacklisted - // | - // synced + // validated blacklisted ) var ( @@ -266,11 +263,6 @@ func (m *Manager) shrexPools() map[poolStatus]int64 { continue } - if p.isSynced.Load() { - shrexPools[poolStatusSynced]++ - continue - } - // pool is validated but not synced shrexPools[poolStatusValidated]++ } diff --git a/share/p2p/peers/pool.go b/share/p2p/peers/pool.go index d0cc45ac44..365ef0306d 100644 --- a/share/p2p/peers/pool.go +++ b/share/p2p/peers/pool.go @@ -224,12 +224,3 @@ func (p *pool) len() int { defer p.m.RUnlock() return p.activeCount } - -// reset will reset the pool to its initial state. -func (p *pool) reset() { - lock := &p.m - lock.Lock() - defer lock.Lock() - // swap the pool with an empty one - *p = *newPool(time.Second) -} From 150378fa75333c1a13fac0057788077dcccea668 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Wed, 20 Dec 2023 12:55:42 +0100 Subject: [PATCH 339/388] chore: bump go-libp2p (#2883) A regular bump with multiple minor fixes as well broader optimizations like TCP eyeballing https://github.com/libp2p/go-libp2p/releases/tag/v0.32.0 It may also fix the issue behind #2837, but this has to be confirmed yet Co-authored-by: ramin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index e6f715e2b5..a9253c2e59 100644 --- a/go.mod +++ b/go.mod @@ -36,7 +36,7 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car v0.6.2 - github.com/libp2p/go-libp2p v0.32.0 + github.com/libp2p/go-libp2p v0.32.1 github.com/libp2p/go-libp2p-kad-dht v0.25.1 github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/libp2p/go-libp2p-record v0.2.0 diff --git a/go.sum b/go.sum index b3d4b91036..77fbd1037a 100644 --- a/go.sum +++ b/go.sum @@ -1414,8 +1414,8 @@ github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBx github.com/libp2p/go-libp2p v0.23.4/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= github.com/libp2p/go-libp2p v0.25.0/go.mod h1:vXHmFpcfl+xIGN4qW58Bw3a0/SKGAesr5/T4IuJHE3o= github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= -github.com/libp2p/go-libp2p v0.32.0 h1:86I4B7nBUPIyTgw3+5Ibq6K7DdKRCuZw8URCfPc1hQM= -github.com/libp2p/go-libp2p v0.32.0/go.mod h1:hXXC3kXPlBZ1eu8Q2hptGrMB4mZ3048JUoS4EKaHW5c= +github.com/libp2p/go-libp2p v0.32.1 h1:wy1J4kZIZxOaej6NveTWCZmHiJ/kY7GoAqXgqNCnPps= +github.com/libp2p/go-libp2p v0.32.1/go.mod h1:hXXC3kXPlBZ1eu8Q2hptGrMB4mZ3048JUoS4EKaHW5c= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= From ccf9b56317ae8941f91c5d09f24fe8422bbe64bb Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 21 Dec 2023 12:22:29 +0000 Subject: [PATCH 340/388] chore(deps): Bump golang.org/x/crypto from 0.14.0 to 0.17.0 (#3050) Bumps [golang.org/x/crypto](https://github.com/golang/crypto) from 0.14.0 to 0.17.0.
Commits
  • 9d2ee97 ssh: implement strict KEX protocol changes
  • 4e5a261 ssh: close net.Conn on all NewServerConn errors
  • 152cdb1 x509roots/fallback: update bundle
  • fdfe1f8 ssh: defer channel window adjustment
  • b8ffc16 blake2b: drop Go 1.6, Go 1.8 compatibility
  • 7e6fbd8 ssh: wrap errors from client handshake
  • bda2f3f argon2: avoid clobbering BP
  • 325b735 ssh/test: skip TestSSHCLIAuth on Windows
  • 1eadac5 go.mod: update golang.org/x dependencies
  • b2d7c26 ssh: add (*Client).DialContext method
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=golang.org/x/crypto&package-manager=go_modules&previous-version=0.14.0&new-version=0.17.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 6 +++--- go.sum | 12 ++++++------ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/go.mod b/go.mod index a9253c2e59..cb3550e470 100644 --- a/go.mod +++ b/go.mod @@ -66,7 +66,7 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.1 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.14.0 + golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.5.0 golang.org/x/text v0.14.0 @@ -325,8 +325,8 @@ require ( golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect golang.org/x/oauth2 v0.11.0 // indirect - golang.org/x/sys v0.13.0 // indirect - golang.org/x/term v0.13.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect diff --git a/go.sum b/go.sum index 77fbd1037a..6097b9c309 100644 --- a/go.sum +++ b/go.sum @@ -2510,8 +2510,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= -golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= +golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= +golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2858,8 +2858,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= -golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2868,8 +2868,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.13.0 h1:bb+I9cTfFazGW51MZqBVmZy7+JEJMouUHTUSKVQLBek= -golang.org/x/term v0.13.0/go.mod h1:LTmsnFJwVN6bCy1rVCoS+qHT1HhALEFxKncY3WNNh4U= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= From 43ce44a5089dc3125e010d67dc8f94ac2aa8b3cd Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:12:15 +0000 Subject: [PATCH 341/388] chore(deps): Bump github.com/prometheus/client_golang from 1.17.0 to 1.18.0 (#3064) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/prometheus/client_golang](https://github.com/prometheus/client_golang) from 1.17.0 to 1.18.0.
Release notes

Sourced from github.com/prometheus/client_golang's releases.

v1.18.0

What's Changed

  • [FEATURE] promlint: Allow creation of custom metric validations. #1311
  • [FEATURE] Go programs using client_golang can be built in wasip1 OS. #1350
  • [BUGFIX] histograms: Add timer to reset ASAP after bucket limiting has happened. #1367
  • [BUGFIX] testutil: Fix comparison of metrics with empty Help strings. #1378
  • [ENHANCEMENT] Improved performance of MetricVec.WithLabelValues(...). #1360

New Contributors

Full Changelog: https://github.com/prometheus/client_golang/compare/v1.17.0...v1.18.0

Changelog

Sourced from github.com/prometheus/client_golang's changelog.

1.18.0 / 2023-12-22

  • [FEATURE] promlint: Allow creation of custom metric validations. #1311
  • [FEATURE] Go programs using client_golang can be built in wasip1 OS. #1350
  • [BUGFIX] histograms: Add timer to reset ASAP after bucket limiting has happened. #1367
  • [BUGFIX] testutil: Fix comparison of metrics with empty Help strings. #1378
  • [ENHANCEMENT] Improved performance of MetricVec.WithLabelValues(...). #1360
Commits
  • 53be91d Revert "change api http.client to interface"
  • 1a2d072 Add 1.18 changelog
  • 239b123 Merge pull request #1387 from tsipo/main
  • 3f8bd73 Merge pull request #1370 from prometheus/dependabot/go_modules/tutorial/whats...
  • 5e55b31 Bump google.golang.org/grpc from 1.53.0 to 1.56.3 in /tutorial/whatsup
  • e96fb18 Merge pull request #1401 from prometheus/dependabot/go_modules/golang.org/x/s...
  • 2a8fc90 Bump golang.org/x/sys from 0.13.0 to 0.15.0
  • 24d59e9 change client to interface, allow override by other implementations (e.g. git...
  • 80d3f0b Normalize empty help values in CollectAndCompare (#1378)
  • 3f80cd1 Add example of NewConstMetricWithCreatedTimestamp (#1375)
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/prometheus/client_golang&package-manager=go_modules&previous-version=1.17.0&new-version=1.18.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 12 ++++++------ go.sum | 23 ++++++++++++----------- 2 files changed, 18 insertions(+), 17 deletions(-) diff --git a/go.mod b/go.mod index cb3550e470..2e68e686a7 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 - github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_golang v1.18.0 github.com/pyroscope-io/client v0.7.2 github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/spf13/cobra v1.8.0 @@ -83,6 +83,7 @@ require ( github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/supranational/blst v0.3.11 // indirect go.uber.org/mock v0.3.0 // indirect @@ -251,7 +252,6 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect - github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/miekg/dns v1.1.56 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect @@ -279,9 +279,9 @@ require ( github.com/pkg/errors v0.9.1 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/polydawn/refmt v0.89.0 // indirect - github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 // indirect - github.com/prometheus/common v0.44.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.45.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.4 // indirect @@ -324,7 +324,7 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.11.0 // indirect + golang.org/x/oauth2 v0.12.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/tools v0.14.0 // indirect diff --git a/go.sum b/go.sum index 6097b9c309..ee6e4a0f31 100644 --- a/go.sum +++ b/go.sum @@ -1733,8 +1733,9 @@ github.com/mattn/go-runewidth v0.0.13/go.mod h1:Jdepj2loyihRzMpdS35Xk/zdY8IAYHsh github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-tty v0.0.0-20180907095812-13ff1204f104/go.mod h1:XPvLUNfbS4fJH25nqRHfWLMa1ONC8Amw+mIA639KxkE= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= -github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 h1:jWpvCLoY8Z/e3VKvlsiIGKtc+UG6U5vzxaoagmhXfyg= +github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0/go.mod h1:QUyp042oQthUoa9bqDv0ER0wrtXnBruoNd7aNjkbP+k= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/microcosm-cc/bluemonday v1.0.1/go.mod h1:hsXNsILzKxV+sX77C5b8FSuKF00vh2OMYv+xgHpAMF4= github.com/miekg/dns v1.0.14/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= @@ -2034,8 +2035,8 @@ github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqr github.com/prometheus/client_golang v1.12.1/go.mod h1:3Z9XVyYiZYEO+YQWt3RD2R3jrbd179Rt297l4aS6nDY= github.com/prometheus/client_golang v1.13.0/go.mod h1:vTeo+zgvILHsnnj/39Ou/1fPN5nJFOEMgftOUOmlvYQ= github.com/prometheus/client_golang v1.14.0/go.mod h1:8vpkKitgIVNcqrRBWh1C4TIUQgYNtG/XQE4E/Zae36Y= -github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= -github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= +github.com/prometheus/client_golang v1.18.0 h1:HzFfmkOzH5Q8L8G+kSJKUx5dtG87sewO+FoDDqP5Tbk= +github.com/prometheus/client_golang v1.18.0/go.mod h1:T+GXkCk5wSJyOqMIzVgvvjFDlkOQntgjkJWKrN5txjA= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= @@ -2043,8 +2044,8 @@ github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1: github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.3.0/go.mod h1:LDGWKZIo7rky3hgvBe+caln+Dr3dPggB5dvjtD7w9+w= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16 h1:v7DLqVdK4VrYkVD5diGdl4sxJurKJEMnODWRJlxV9oM= -github.com/prometheus/client_model v0.4.1-0.20230718164431-9a2bf3000d16/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= @@ -2059,8 +2060,8 @@ github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9 github.com/prometheus/common v0.32.1/go.mod h1:vu+V0TpY+O6vW9J44gczi3Ap/oXXR10b+M/gUGO4Hls= github.com/prometheus/common v0.33.0/go.mod h1:gB3sOl7P0TvJabZpLY5uQMpUqRCPPCyRLCZYc7JZTNE= github.com/prometheus/common v0.37.0/go.mod h1:phzohg0JFMnBEFGxTDbfu3QyL5GI8gTQJFhYO5B3mfA= -github.com/prometheus/common v0.44.0 h1:+5BrQJwiBB9xsMygAB3TNvpQKOwlkc25LbISbrdOOfY= -github.com/prometheus/common v0.44.0/go.mod h1:ofAIvZbQ1e/nugmZGz4/qCb9Ap1VoSTIO7x0VV9VvuY= +github.com/prometheus/common v0.45.0 h1:2BGz0eBc2hdMDLnO/8n0jeB3oPrt2D08CekT0lneoxM= +github.com/prometheus/common v0.45.0/go.mod h1:YJmSTw9BoKxJplESWWxlbyttQR4uaEcGyv9MZjVOJsY= github.com/prometheus/procfs v0.0.0-20180725123919-05ee40e3a273/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= @@ -2072,8 +2073,8 @@ github.com/prometheus/procfs v0.3.0/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4O github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= @@ -2693,8 +2694,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.11.0 h1:vPL4xzxBM4niKCW6g9whtaWVXTJf1U5e4aZxxFx/gbU= -golang.org/x/oauth2 v0.11.0/go.mod h1:LdF7O/8bLR/qWK9DrpXmbHLTouvRHK0SgJl0GmDBchk= +golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= +golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= From 87a7a1af3dc414ab60aac2af2c75178a388a3944 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:29:36 +0000 Subject: [PATCH 342/388] chore(deps): Bump github.com/ipfs/boxo from 0.15.0 to 0.16.0 (#3057) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/ipfs/boxo](https://github.com/ipfs/boxo) from 0.15.0 to 0.16.0.
Release notes

Sourced from github.com/ipfs/boxo's releases.

v0.16.0

Changed

  • 🛠 boxo/namesys: now fails when multiple valid DNSLink entries are found for the same domain. This used to cause undefined behavior before. Now, we return an error, according to the specification.

Removed

  • 🛠 boxo/gateway: removed support for undocumented legacy ipfs-404.html. Use _redirects instead.
  • 🛠 boxo/namesys: removed support for legacy DNSLink entries at the root of the domain. Use _dnslink. TXT record instead.
  • 🛠 boxo/coreapi, an intrinsic part of Kubo, has been removed and moved to kubo/core/coreiface.

Fixed

  • boxo/gateway
    • a panic (which is recovered) could sporadically be triggered inside a CAR request, if the right conditions were met.
    • no longer emits http: superfluous response.WriteHeader warnings when an error happens.

What's Changed

New Contributors

Full Changelog: https://github.com/ipfs/boxo/compare/v0.15.0...v0.16.0

Changelog

Sourced from github.com/ipfs/boxo's changelog.

[v0.16.0]

Changed

  • 🛠 boxo/namesys: now fails when multiple valid DNSLink entries are found for the same domain. This used to cause undefined behavior before. Now, we return an error, according to the specification.

Removed

  • 🛠 boxo/gateway: removed support for undocumented legacy ipfs-404.html. Use _redirects instead.
  • 🛠 boxo/namesys: removed support for legacy DNSLink entries at the root of the domain. Use _dnslink. TXT record instead.
  • 🛠 boxo/coreapi, an intrinsic part of Kubo, has been removed and moved to kubo/core/coreiface.

Fixed

  • boxo/gateway
    • a panic (which is recovered) could sporadically be triggered inside a CAR request, if the right conditions were met.
    • no longer emits http: superfluous response.WriteHeader warnings when an error happens.
Commits
  • 709c7c6 Merge pull request #518 from ipfs/release-v0.16.0
  • df09068 chore: bump to 0.16.0
  • 9afc46a changelog: prepare 0.16.0
  • 08959f2 coreiface: remove and move to Kubo
  • d06f7ff fix(gateway): no duplicate WriteHeader calls (#515)
  • 438b8a6 bitswap/client: explain what the options do
  • fe55533 fix: all go-log import to v2
  • 50f8a08 test: positive and negative test for multi dnslink
  • 006c1c9 changelog: add namesys entries
  • 7b5207a feat!: remove support for legacy root dnslink, plumb errors
  • Additional commits viewable in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/ipfs/boxo&package-manager=go_modules&previous-version=0.15.0&new-version=0.16.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 2e68e686a7..0ee3f8c797 100644 --- a/go.mod +++ b/go.mod @@ -28,7 +28,7 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 - github.com/ipfs/boxo v0.15.0 + github.com/ipfs/boxo v0.16.0 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index ee6e4a0f31..99fe996ada 100644 --- a/go.sum +++ b/go.sum @@ -1038,8 +1038,8 @@ github.com/influxdata/usage-client v0.0.0-20160829180054-6d3895376368/go.mod h1: github.com/ipfs/bbloom v0.0.1/go.mod h1:oqo8CVWsJFMOZqTglBG4wydCE4IQA/G2/SEofB0rjUI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.15.0 h1:BriLydj2nlK1nKeJQHxcKSuG5ZXcoutzhBklOtxC5pk= -github.com/ipfs/boxo v0.15.0/go.mod h1:X5ulcbR5Nh7sm3Db8+08AApUo6FsGC5mb23QDKAoB/M= +github.com/ipfs/boxo v0.16.0 h1:A9dUmef5a+mEFki6kbyG7el5gl65CiUBzrDeZxzTWKY= +github.com/ipfs/boxo v0.16.0/go.mod h1:jAgpNQn7T7BnibUeReXcKU9Ha1xmYNyOlwVEl193ow0= github.com/ipfs/go-bitfield v1.0.0/go.mod h1:N/UiujQy+K+ceU1EF5EkVd1TNqevLrCQMIcAEPrdtus= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= From 5605f15e1de5df0deb1bcc64bf4e06fe33ecb533 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:36:51 +0000 Subject: [PATCH 343/388] chore(deps): Bump github.com/libp2p/go-libp2p-kad-dht from 0.25.1 to 0.25.2 (#3058) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit [//]: # (dependabot-start) ⚠️ **Dependabot is rebasing this PR** ⚠️ Rebasing might not happen immediately, so don't worry if this takes some time. Note: if you make any changes to this PR yourself, they will take precedence over the rebase. --- [//]: # (dependabot-end) Bumps [github.com/libp2p/go-libp2p-kad-dht](https://github.com/libp2p/go-libp2p-kad-dht) from 0.25.1 to 0.25.2.
Release notes

Sourced from github.com/libp2p/go-libp2p-kad-dht's releases.

v0.25.2

What's Changed

New Contributors

Full Changelog: https://github.com/libp2p/go-libp2p-kad-dht/compare/v0.25.1...v0.25.2

Commits
  • bdca144 Merge pull request #961 from libp2p/release-v0.25.2
  • bea8743 release v0.25.2
  • ef7d2b1 Merge pull request #960 from godeamon/bug/send-request-read-msg-ctx-err-check
  • ae0987e reset stream and return err when ctx canceled
  • 097cb9a return canceled err not retry
  • 12fcd27 add ctx canceled err check
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/libp2p/go-libp2p-kad-dht&package-manager=go_modules&previous-version=0.25.1&new-version=0.25.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0ee3f8c797..db247c2e71 100644 --- a/go.mod +++ b/go.mod @@ -37,7 +37,7 @@ require ( github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car v0.6.2 github.com/libp2p/go-libp2p v0.32.1 - github.com/libp2p/go-libp2p-kad-dht v0.25.1 + github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/libp2p/go-libp2p-record v0.2.0 github.com/libp2p/go-libp2p-routing-helpers v0.7.3 diff --git a/go.sum b/go.sum index 99fe996ada..f6401b2eb3 100644 --- a/go.sum +++ b/go.sum @@ -1472,8 +1472,8 @@ github.com/libp2p/go-libp2p-discovery v0.3.0/go.mod h1:o03drFnz9BVAZdzC/QUQ+NeQO github.com/libp2p/go-libp2p-discovery v0.5.0/go.mod h1:+srtPIU9gDaBNu//UHvcdliKBIcr4SfDcm0/PfPJLug= github.com/libp2p/go-libp2p-kad-dht v0.19.0/go.mod h1:qPIXdiZsLczhV4/+4EO1jE8ae0YCW4ZOogc4WVIyTEU= github.com/libp2p/go-libp2p-kad-dht v0.21.0/go.mod h1:Bhm9diAFmc6qcWAr084bHNL159srVZRKADdp96Qqd1I= -github.com/libp2p/go-libp2p-kad-dht v0.25.1 h1:ofFNrf6MMEy4vi3R1VbJ7LOcTn3Csh0cDcaWHTxtWNA= -github.com/libp2p/go-libp2p-kad-dht v0.25.1/go.mod h1:6za56ncRHYXX4Nc2vn8z7CZK0P4QiMcrn77acKLM2Oo= +github.com/libp2p/go-libp2p-kad-dht v0.25.2 h1:FOIk9gHoe4YRWXTu8SY9Z1d0RILol0TrtApsMDPjAVQ= +github.com/libp2p/go-libp2p-kad-dht v0.25.2/go.mod h1:6za56ncRHYXX4Nc2vn8z7CZK0P4QiMcrn77acKLM2Oo= github.com/libp2p/go-libp2p-kbucket v0.3.1/go.mod h1:oyjT5O7tS9CQurok++ERgc46YLwEpuGoFq9ubvoUOio= github.com/libp2p/go-libp2p-kbucket v0.5.0/go.mod h1:zGzGCpQd78b5BNTDGHNDLaTt9aDK/A02xeZp9QeFC4U= github.com/libp2p/go-libp2p-kbucket v0.6.3 h1:p507271wWzpy2f1XxPzCQG9NiN6R6lHL9GiSErbQQo0= From 53e46c699be5e8a4eb6209f7fbc6ffabf4f65484 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 10:48:22 +0000 Subject: [PATCH 344/388] chore(deps): Bump google.golang.org/protobuf from 1.31.0 to 1.32.0 (#3059) Bumps google.golang.org/protobuf from 1.31.0 to 1.32.0. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/protobuf&package-manager=go_modules&previous-version=1.31.0&new-version=1.32.0)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index db247c2e71..4df8844bd0 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,7 @@ require ( golang.org/x/sync v0.5.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.59.0 - google.golang.org/protobuf v1.31.0 + google.golang.org/protobuf v1.32.0 ) require ( diff --git a/go.sum b/go.sum index f6401b2eb3..ffdffebd39 100644 --- a/go.sum +++ b/go.sum @@ -3236,8 +3236,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc= google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= google.golang.org/protobuf v1.28.1/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= -google.golang.org/protobuf v1.31.0 h1:g0LDEJHgrBl9N9r17Ru3sqWhkIx2NB67okBHPwC7hs8= -google.golang.org/protobuf v1.31.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I= +google.golang.org/protobuf v1.32.0 h1:pPC6BG5ex8PDFnkbrGU3EixyhKcQ2aDuBS36lqK/C7I= +google.golang.org/protobuf v1.32.0/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos= gopkg.in/alecthomas/kingpin.v2 v2.2.6/go.mod h1:FMv+mEhP44yOT+4EoQTLFTRgOQ1FBLkstjWtayDeSgw= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= From 77cc973ba5523e86dbfdd75a9a52660741feddae Mon Sep 17 00:00:00 2001 From: Alireza <50639733+cyneptic@users.noreply.github.com> Date: Tue, 2 Jan 2024 14:44:16 +0330 Subject: [PATCH 345/388] chore(share|das|libs/utils): DRY context resetting in the event of an error for metrics (#2935) this creates a function in the utils/resetctx.go that takes a context as an input and resets it if there is an error. it then uses the function to refactor all the instances of the repeating pattern as requested in the issue - fixes #2928 --------- Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> Co-authored-by: ramin --- core/exchange_metrics.go | 6 ++-- core/listener_metrics.go | 6 ++-- das/metrics.go | 19 +++++-------- libs/utils/resetctx.go | 12 ++++++++ share/eds/metrics.go | 50 ++++++++++------------------------ share/getters/shrex.go | 8 ++---- share/p2p/discovery/metrics.go | 18 ++++-------- share/p2p/metrics.go | 6 ++-- share/p2p/peers/metrics.go | 13 +++------ 9 files changed, 54 insertions(+), 84 deletions(-) create mode 100644 libs/utils/resetctx.go diff --git a/core/exchange_metrics.go b/core/exchange_metrics.go index 0b454a6e4d..4e5bf5956c 100644 --- a/core/exchange_metrics.go +++ b/core/exchange_metrics.go @@ -5,6 +5,8 @@ import ( "time" "go.opentelemetry.io/otel/metric" + + "github.com/celestiaorg/celestia-node/libs/utils" ) type exchangeMetrics struct { @@ -31,9 +33,7 @@ func (m *exchangeMetrics) observe(ctx context.Context, observeFn func(ctx contex return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) observeFn(ctx) } diff --git a/core/listener_metrics.go b/core/listener_metrics.go index a325149b5d..f17903a91a 100644 --- a/core/listener_metrics.go +++ b/core/listener_metrics.go @@ -6,6 +6,8 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/metric" + + "github.com/celestiaorg/celestia-node/libs/utils" ) var meter = otel.Meter("core") @@ -53,9 +55,7 @@ func (m *listenerMetrics) observe(ctx context.Context, observeFn func(ctx contex return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) observeFn(ctx) } diff --git a/das/metrics.go b/das/metrics.go index 42b472d909..6454e9d138 100644 --- a/das/metrics.go +++ b/das/metrics.go @@ -11,6 +11,7 @@ import ( "go.opentelemetry.io/otel/metric" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/libs/utils" ) const ( @@ -19,9 +20,7 @@ const ( failedLabel = "failed" ) -var ( - meter = otel.Meter("das") -) +var meter = otel.Meter("das") type metrics struct { sampled metric.Int64Counter @@ -146,9 +145,9 @@ func (m *metrics) observeSample( if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + + ctx = utils.ResetContextOnError(ctx) + m.sampleTime.Record(ctx, sampleTime.Seconds(), metric.WithAttributes( attribute.Bool(failedLabel, err != nil), @@ -171,9 +170,7 @@ func (m *metrics) observeGetHeader(ctx context.Context, d time.Duration) { if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.getHeaderTime.Record(ctx, d.Seconds()) } @@ -182,8 +179,6 @@ func (m *metrics) observeNewHead(ctx context.Context) { if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.newHead.Add(ctx, 1) } diff --git a/libs/utils/resetctx.go b/libs/utils/resetctx.go new file mode 100644 index 0000000000..3014ba81db --- /dev/null +++ b/libs/utils/resetctx.go @@ -0,0 +1,12 @@ +package utils + +import "context" + +// ResetContextOnError returns a fresh context if the given context has an error. +func ResetContextOnError(ctx context.Context) context.Context { + if ctx.Err() != nil { + ctx = context.Background() + } + + return ctx +} diff --git a/share/eds/metrics.go b/share/eds/metrics.go index cbebf8321a..8d69a3ec41 100644 --- a/share/eds/metrics.go +++ b/share/eds/metrics.go @@ -7,6 +7,8 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/celestiaorg/celestia-node/libs/utils" ) const ( @@ -27,9 +29,7 @@ const ( dagstoreShardStatusKey = "shard_status" ) -var ( - meter = otel.Meter("eds_store") -) +var meter = otel.Meter("eds_store") type putResult string @@ -163,9 +163,7 @@ func (m *metrics) observeGCtime(ctx context.Context, dur time.Duration, failed b if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.gcTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) } @@ -174,9 +172,7 @@ func (m *metrics) observeShardFailure(ctx context.Context, shardKey string) { if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.shardFailureCount.Add(ctx, 1, metric.WithAttributes(attribute.String("shard_key", shardKey))) } @@ -185,9 +181,7 @@ func (m *metrics) observePut(ctx context.Context, dur time.Duration, result putR if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.putTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.String(putResultKey, string(result)), @@ -198,9 +192,7 @@ func (m *metrics) observeLongOp(ctx context.Context, opName string, dur time.Dur if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.longOpTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.String(opNameKey, opName), @@ -211,9 +203,7 @@ func (m *metrics) observeGetCAR(ctx context.Context, dur time.Duration, failed b if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.getCARTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) @@ -223,9 +213,7 @@ func (m *metrics) observeCARBlockstore(ctx context.Context, dur time.Duration, f if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.getCARBlockstoreTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) @@ -235,9 +223,7 @@ func (m *metrics) observeGetDAH(ctx context.Context, dur time.Duration, failed b if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.getDAHTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) @@ -247,9 +233,7 @@ func (m *metrics) observeRemove(ctx context.Context, dur time.Duration, failed b if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.removeTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) @@ -259,9 +243,7 @@ func (m *metrics) observeGet(ctx context.Context, dur time.Duration, failed bool if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.getTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) @@ -271,9 +253,7 @@ func (m *metrics) observeHas(ctx context.Context, dur time.Duration, failed bool if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.hasTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) @@ -283,9 +263,7 @@ func (m *metrics) observeList(ctx context.Context, dur time.Duration, failed boo if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.listTime.Record(ctx, dur.Seconds(), metric.WithAttributes( attribute.Bool(failedKey, failed))) diff --git a/share/getters/shrex.go b/share/getters/shrex.go index af48a7f8ab..826c6b1a10 100644 --- a/share/getters/shrex.go +++ b/share/getters/shrex.go @@ -43,9 +43,7 @@ func (m *metrics) recordEDSAttempt(ctx context.Context, attemptCount int, succes if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.edsAttempts.Record(ctx, int64(attemptCount), metric.WithAttributes( attribute.Bool("success", success))) @@ -55,9 +53,7 @@ func (m *metrics) recordNDAttempt(ctx context.Context, attemptCount int, success if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.ndAttempts.Record(ctx, int64(attemptCount), metric.WithAttributes( attribute.Bool("success", success))) diff --git a/share/p2p/discovery/metrics.go b/share/p2p/discovery/metrics.go index d0be1c219d..78b62a7d97 100644 --- a/share/p2p/discovery/metrics.go +++ b/share/p2p/discovery/metrics.go @@ -8,6 +8,8 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/celestiaorg/celestia-node/libs/utils" ) const ( @@ -24,9 +26,7 @@ const ( advertiseFailedKey = "failed" ) -var ( - meter = otel.Meter("share_discovery") -) +var meter = otel.Meter("share_discovery") type handlePeerResult string @@ -118,9 +118,7 @@ func (m *metrics) observeFindPeers(ctx context.Context, isEnoughPeers bool) { if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.discoveryResult.Add(ctx, 1, metric.WithAttributes( @@ -131,9 +129,7 @@ func (m *metrics) observeHandlePeer(ctx context.Context, result handlePeerResult if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.handlePeerResult.Add(ctx, 1, metric.WithAttributes( @@ -144,9 +140,7 @@ func (m *metrics) observeAdvertise(ctx context.Context, err error) { if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.advertise.Add(ctx, 1, metric.WithAttributes( diff --git a/share/p2p/metrics.go b/share/p2p/metrics.go index fee3b12413..55aefda81d 100644 --- a/share/p2p/metrics.go +++ b/share/p2p/metrics.go @@ -7,6 +7,8 @@ import ( "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + + "github.com/celestiaorg/celestia-node/libs/utils" ) var meter = otel.Meter("shrex/eds") @@ -35,9 +37,7 @@ func (m *Metrics) ObserveRequests(ctx context.Context, count int64, status statu if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.totalRequestCounter.Add(ctx, count, metric.WithAttributes( attribute.String("status", string(status)), diff --git a/share/p2p/peers/metrics.go b/share/p2p/peers/metrics.go index 3b5913ebb8..098610c595 100644 --- a/share/p2p/peers/metrics.go +++ b/share/p2p/peers/metrics.go @@ -12,6 +12,7 @@ import ( "go.opentelemetry.io/otel/attribute" "go.opentelemetry.io/otel/metric" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -46,9 +47,7 @@ const ( // validated blacklisted ) -var ( - meter = otel.Meter("shrex_peer_manager") -) +var meter = otel.Meter("shrex_peer_manager") type blacklistPeerReason string @@ -169,9 +168,7 @@ func (m *metrics) observeGetPeer( if m == nil { return } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.getPeer.Add(ctx, 1, metric.WithAttributes( attribute.String(sourceKey, string(source)), @@ -222,9 +219,7 @@ func (m *metrics) validationObserver(validator shrexsub.ValidatorFn) shrexsub.Va resStr = "unknown" } - if ctx.Err() != nil { - ctx = context.Background() - } + ctx = utils.ResetContextOnError(ctx) m.validationResult.Add(ctx, 1, metric.WithAttributes( From b97175c2ceffa29452031a3e5583f7d015e150df Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 2 Jan 2024 13:11:47 +0100 Subject: [PATCH 346/388] chore(go.mod): bump go-header (#3052) Bumps go-header to the latest RC to pull in the fix to the `peerTracker` in https://github.com/celestiaorg/go-header/pull/139. It's suspected that the `peerTracker`'s aggressive scoring mechanism is causing header syncing to stall after periods of poor connectivity. --------- Co-authored-by: ramin --- go.mod | 2 +- go.sum | 4 ++-- nodebuilder/header/constructors.go | 14 ++++++-------- share/eds/byzantine/byzantine.go | 3 ++- 4 files changed, 11 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index 4df8844bd0..ebeff2c41b 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/celestia-app v1.4.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.4.1 + github.com/celestiaorg/go-header v0.5.0-rc1 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index ffdffebd39..a4c1c0866b 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.4.1 h1:bjbUcKDnhrJJ9EoE7vtPpgleNLVjc2S+cB4/qe8nQmo= -github.com/celestiaorg/go-header v0.4.1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.5.0-rc1 h1:DAcVW8V76VI5VU4fOAdXePpq15UFblwZIMZeHCAVr0w= +github.com/celestiaorg/go-header v0.5.0-rc1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= diff --git a/nodebuilder/header/constructors.go b/nodebuilder/header/constructors.go index 15d2da09b1..a78d609d8e 100644 --- a/nodebuilder/header/constructors.go +++ b/nodebuilder/header/constructors.go @@ -99,16 +99,14 @@ func newInitStore[H libhead.Header[H]]( ds datastore.Batching, ex libhead.Exchange[H], ) (libhead.Store[H], error) { - s, err := store.NewStore[H](ds, store.WithParams(cfg.Store)) - if err != nil { - return nil, err + opts := []store.Option{store.WithParams(cfg.Store)} + if MetricsEnabled { + opts = append(opts, store.WithMetrics()) } - if MetricsEnabled { - err = libhead.WithMetrics[H](s) - if err != nil { - return nil, err - } + s, err := store.NewStore[H](ds, opts...) + if err != nil { + return nil, err } trustedHash, err := cfg.trustedHash(net) diff --git a/share/eds/byzantine/byzantine.go b/share/eds/byzantine/byzantine.go index e7aec28959..d20b56deed 100644 --- a/share/eds/byzantine/byzantine.go +++ b/share/eds/byzantine/byzantine.go @@ -4,9 +4,10 @@ import ( "context" "fmt" + "github.com/ipfs/boxo/blockservice" + "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/rsmt2d" - "github.com/ipfs/boxo/blockservice" "github.com/celestiaorg/celestia-node/share/ipld" ) From b4878302f7da390fcfceb99233f1e51e9619f150 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Tue, 2 Jan 2024 12:21:40 +0000 Subject: [PATCH 347/388] chore(deps): Bump google.golang.org/grpc from 1.59.0 to 1.60.1 (#3060) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [google.golang.org/grpc](https://github.com/grpc/grpc-go) from 1.59.0 to 1.60.1.
Release notes

Sourced from google.golang.org/grpc's releases.

Release v1.60.1

Bug Fixes

  • server: fix two bugs that could lead to panics at shutdown when using NumStreamWorkers (experimental feature).

Release 1.60.0

Security

  • credentials/tls: if not set, set TLS MinVersion to 1.2 and CipherSuites according to supported suites not forbidden by RFC7540.
    • This is a behavior change to bring us into better alignment with RFC 7540.

API Changes

  • resolver: remove deprecated and experimental ClientConn.NewServiceConfig (#6784)
  • client: remove deprecated grpc.WithServiceConfig DialOption (#6800)

Bug Fixes

  • client: fix race that could cause a deadlock while entering idle mode and receiving a name resolver update (#6804)
  • client: always enable TCP keepalives with OS defaults (#6834)
  • credentials/alts: fix a bug preventing ALTS from connecting to the metadata server if the default scheme is overridden (#6686)

Behavior Changes

  • server: Do not return from Stop() or GracefulStop() until all resources are released (#6489)

Documentation

  • codes: clarify that only codes defined by this package are valid and that users should not cast other values to codes.Code (#6701)
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=google.golang.org/grpc&package-manager=go_modules&previous-version=1.59.0&new-version=1.60.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: ramin Co-authored-by: Ryan --- go.mod | 8 ++++---- go.sum | 15 ++++++++------- 2 files changed, 12 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index ebeff2c41b..458c41c9df 100644 --- a/go.mod +++ b/go.mod @@ -70,7 +70,7 @@ require ( golang.org/x/exp v0.0.0-20231006140011-7918f672742d golang.org/x/sync v0.5.0 golang.org/x/text v0.14.0 - google.golang.org/grpc v1.59.0 + google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.32.0 ) @@ -324,16 +324,16 @@ require ( go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.13.0 // indirect golang.org/x/net v0.17.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sys v0.15.0 // indirect golang.org/x/term v0.15.0 // indirect golang.org/x/tools v0.14.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.128.0 // indirect - google.golang.org/appengine v1.6.7 // indirect + google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect diff --git a/go.sum b/go.sum index a4c1c0866b..1ee176676a 100644 --- a/go.sum +++ b/go.sum @@ -2694,8 +2694,8 @@ golang.org/x/oauth2 v0.0.0-20220822191816-0ebed06d0094/go.mod h1:h4gKUeWbJ4rQPri golang.org/x/oauth2 v0.0.0-20220909003341-f21342109be1/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.0.0-20221014153046-6fdb5e3db783/go.mod h1:h4gKUeWbJ4rQPri7E0u6Gs4e9Ri2zaLxzw5DI5XGrYg= golang.org/x/oauth2 v0.1.0/go.mod h1:G9FE4dLTsbXUu90h/Pf85g4w1D+SSAgR+q46nJZ8M4A= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/perf v0.0.0-20180704124530-6e6d33e29852/go.mod h1:JLpeXjPJfIyPr5TlbXLkXWLhP8nz10XfvxElABhCtcw= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -3052,8 +3052,9 @@ google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7 google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= google.golang.org/appengine v1.6.5/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= google.golang.org/appengine v1.6.6/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= -google.golang.org/appengine v1.6.7 h1:FZR1q0exgwxzPzp/aF+VccGrSfxfPpkBqjIIEq3ru6c= google.golang.org/appengine v1.6.7/go.mod h1:8WjMMxjGQR8xUklV/ARdw2HLXBOI7O7uCIDZVag1xfc= +google.golang.org/appengine v1.6.8 h1:IhEN5q69dyKagZPYMSdIjS2HqprW324FRQZJcGqPAsM= +google.golang.org/appengine v1.6.8/go.mod h1:1jJ3jBArFh5pcgW8gCtRJnepW8FzD1V44FJffLiz/Ds= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20180831171423-11092d34479b/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181029155118-b69ba1387ce2/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= @@ -3169,8 +3170,8 @@ google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13 h1:U7+wNaVuSTaUqNvK2+osJ9ejEZxbjHHk8F2b6Hpx0AE= -google.golang.org/genproto/googleapis/api v0.0.0-20230920204549-e6e6cdab5c13/go.mod h1:RdyHbowztCGQySiCvQPgWQWgWhGnouTdCflKoDBt32U= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= +google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= @@ -3218,8 +3219,8 @@ google.golang.org/grpc v1.48.0/go.mod h1:vN9eftEi1UMyUsIF80+uQXhHjbXYbm0uXoFCACu google.golang.org/grpc v1.49.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.0/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= google.golang.org/grpc v1.50.1/go.mod h1:ZgQEeidpAuNRZ8iRrlBKXZQP1ghovWIVhdJRyCDK+GI= -google.golang.org/grpc v1.59.0 h1:Z5Iec2pjwb+LEOqzpB2MR12/eKFhDPhuqW91O+4bwUk= -google.golang.org/grpc v1.59.0/go.mod h1:aUPDwccQo6OTjy7Hct4AfBPD1GptF4fyUjIkQ9YtF98= +google.golang.org/grpc v1.60.1 h1:26+wFr+cNqSGFcOXcabYC0lUVJVRa2Sb2ortSK7VrEU= +google.golang.org/grpc v1.60.1/go.mod h1:OlCHIeLYqSSsLi6i49B5QGdzaMZK9+M7LXN2FKz4eGM= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= From 6f5fef83daf343116480c06a62755d0eefcd3e06 Mon Sep 17 00:00:00 2001 From: 00x-dx <145253945+00x-dx@users.noreply.github.com> Date: Tue, 2 Jan 2024 22:30:18 +0800 Subject: [PATCH 348/388] chore(p2p/flags): Handle empty p2p.network flag. (#3004) Fixes #3003. --------- Co-authored-by: ramin --- nodebuilder/p2p/flags.go | 36 +++++++++++----------- nodebuilder/p2p/flags_test.go | 56 +++++++++++++++++++++++++++++++++++ 2 files changed, 75 insertions(+), 17 deletions(-) diff --git a/nodebuilder/p2p/flags.go b/nodebuilder/p2p/flags.go index 0faf15e3ca..8e7c0f8bc0 100644 --- a/nodebuilder/p2p/flags.go +++ b/nodebuilder/p2p/flags.go @@ -32,10 +32,11 @@ Peers must bidirectionally point to each other. (Format: multiformats.io/multiad ) flags.String( networkFlag, - "", - "The name of the network to connect to, e.g. "+ - listProvidedNetworks()+ - ". Must be passed on both init and start to take effect.", + DefaultNetwork.String(), + fmt.Sprintf("The name of the network to connect to, e.g. %s. Must be passed on "+ + "both init and start to take effect. Assumes mainnet (%s) unless otherwise specified.", + listProvidedNetworks(), + DefaultNetwork.String()), ) return flags @@ -67,22 +68,23 @@ func ParseFlags( // ParseNetwork tries to parse the network from the flags and environment, // and returns either the parsed network or the build's default network func ParseNetwork(cmd *cobra.Command) (Network, error) { - parsed := cmd.Flag(networkFlag).Value.String() - // no network set through the flags, so check if there is an override in the env - if parsed == "" { - envNetwork, err := parseNetworkFromEnv() - // no network found in env, so use the default network - if envNetwork == "" { - return DefaultNetwork, err - } + if envNetwork, err := parseNetworkFromEnv(); envNetwork != "" { return envNetwork, err } - // check if user provided the actual network value - // or an alias - if net, err := Network(parsed).Validate(); err == nil { - return net, nil + parsed := cmd.Flag(networkFlag).Value.String() + switch parsed { + case "": + return "", fmt.Errorf("no network provided, allowed values: %s", listProvidedNetworks()) + + case DefaultNetwork.String(): + return DefaultNetwork, nil + + default: + if net, err := Network(parsed).Validate(); err == nil { + return net, nil + } + return "", fmt.Errorf("invalid network specified: %s, allowed values: %s", parsed, listProvidedNetworks()) } - return "", fmt.Errorf("invalid network specified: %s", parsed) } // parseNetworkFromEnv tries to parse the network from the environment. diff --git a/nodebuilder/p2p/flags_test.go b/nodebuilder/p2p/flags_test.go index bec49f6074..cfbb5fed5d 100644 --- a/nodebuilder/p2p/flags_test.go +++ b/nodebuilder/p2p/flags_test.go @@ -69,3 +69,59 @@ func createCmdWithNetworkFlag() *cobra.Command { cmd.Flags().AddFlagSet(flags) return cmd } + +// Set empty network flag and ensure error returned +func TestParseNetwork_emptyFlag(t *testing.T) { + cmd := createCmdWithNetworkFlag() + + err := cmd.Flags().Set(networkFlag, "") + require.NoError(t, err) + + _, err = ParseNetwork(cmd) + assert.Error(t, err) + +} + +// Set empty network flag and ensure error returned +func TestParseNetwork_emptyEnvEmptyFlag(t *testing.T) { + + t.Setenv(EnvCustomNetwork, "") + + cmd := createCmdWithNetworkFlag() + err := cmd.Flags().Set(networkFlag, "") + require.NoError(t, err) + + _, err = ParseNetwork(cmd) + require.Error(t, err) + +} + +// Env overrides empty flag to take precedence +func TestParseNetwork_envOverridesEmptyFlag(t *testing.T) { + + t.Setenv(EnvCustomNetwork, "custom-network") + + cmd := createCmdWithNetworkFlag() + err := cmd.Flags().Set(networkFlag, "") + require.NoError(t, err) + + network, err := ParseNetwork(cmd) + require.NoError(t, err) + assert.Equal(t, Network("custom-network"), network) + +} + +// Explicitly set flag but env should still override +func TestParseNetwork_envOverridesFlag(t *testing.T) { + + t.Setenv(EnvCustomNetwork, "custom-network") + + cmd := createCmdWithNetworkFlag() + err := cmd.Flags().Set(networkFlag, string(Mocha)) + require.NoError(t, err) + + network, err := ParseNetwork(cmd) + require.NoError(t, err) + assert.Equal(t, Network("custom-network"), network) + +} From d517e3c872523c27f9917597e3c13256c248334e Mon Sep 17 00:00:00 2001 From: smuu <18609909+smuu@users.noreply.github.com> Date: Wed, 3 Jan 2024 11:42:30 +0100 Subject: [PATCH 349/388] chore(nodebuilder/p2p)!: add arabica-11 (#3066) Signed-off-by: Smuu <18609909+Smuu@users.noreply.github.com> Co-authored-by: rene <41963722+renaynay@users.noreply.github.com> --- nodebuilder/p2p/bootstrap.go | 8 ++++---- nodebuilder/p2p/genesis.go | 2 +- nodebuilder/p2p/network.go | 2 +- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/nodebuilder/p2p/bootstrap.go b/nodebuilder/p2p/bootstrap.go index 3e9da1d77d..8e1856f6fb 100644 --- a/nodebuilder/p2p/bootstrap.go +++ b/nodebuilder/p2p/bootstrap.go @@ -49,10 +49,10 @@ var bootstrapList = map[Network][]string{ "/dns4/da-full-3.celestia-bootstrap.net/tcp/2121/p2p/12D3KooWK6Ftsd4XsWCsQZgZPNhTrE5urwmkoo5P61tGvnKmNVyv", }, Arabica: { - "/dns4/da-bridge.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWM3e9MWtyc8GkP8QRt74Riu17QuhGfZMytB2vq5NwkWAu", - "/dns4/da-bridge-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWKj8mcdiBGxQRe1jqhaMnh2tGoC3rPDmr5UH2q8H4WA9M", - "/dns4/da-full-1.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWBWkgmN7kmJSFovVrCjkeG47FkLGq7yEwJ2kEqNKCsBYk", - "/dns4/da-full-2.celestia-arabica-10.com/tcp/2121/p2p/12D3KooWRByRF67a2kVM2j4MP5Po3jgTw7H2iL2Spu8aUwPkrRfP", + "/dns4/da-bridge-1.celestia-arabica-11.com/tcp/2121/p2p/12D3KooWGqwzdEqM54Dce6LXzfFr97Bnhvm6rN7KM7MFwdomfm4S", + "/dns4/da-bridge-2.celestia-arabica-11.com/tcp/2121/p2p/12D3KooWCMGM5eZWVfCN9ZLAViGfLUWAfXP5pCm78NFKb9jpBtua", + "/dns4/da-bridge-3.celestia-arabica-11.com/tcp/2121/p2p/12D3KooWEWuqrjULANpukDFGVoHW3RoeUU53Ec9t9v5cwW3MkVdQ", + "/dns4/da-bridge-4.celestia-arabica-11.com/tcp/2121/p2p/12D3KooWLT1ysSrD7XWSBjh7tU1HQanF5M64dHV6AuM6cYEJxMPk", }, Mocha: { "/dns4/da-bridge-mocha-4.celestia-mocha.com/tcp/2121/p2p/12D3KooWCBAbQbJSpCpCGKzqz3rAN4ixYbc63K68zJg9aisuAajg", diff --git a/nodebuilder/p2p/genesis.go b/nodebuilder/p2p/genesis.go index dcc19dfa49..e35ca9bf29 100644 --- a/nodebuilder/p2p/genesis.go +++ b/nodebuilder/p2p/genesis.go @@ -24,7 +24,7 @@ func GenesisFor(net Network) (string, error) { // NOTE: Every time we add a new long-running network, its genesis hash has to be added here. var genesisList = map[Network]string{ Mainnet: "6BE39EFD10BA412A9DB5288488303F5DD32CF386707A5BEF33617F4C43301872", - Arabica: "5904E55478BA4B3002EE885621E007A2A6A2399662841912219AECD5D5CBE393", + Arabica: "27122593765E07329BC348E8D16E92DCB4C75B34CCCB35C640FD7A4484D4C711", Mocha: "B93BBE20A0FBFDF955811B6420F8433904664D45DB4BF51022BE4200C1A1680D", Private: "", } diff --git a/nodebuilder/p2p/network.go b/nodebuilder/p2p/network.go index a7f9ff7236..53893eff7c 100644 --- a/nodebuilder/p2p/network.go +++ b/nodebuilder/p2p/network.go @@ -12,7 +12,7 @@ const ( // DefaultNetwork is the default network of the current build. DefaultNetwork = Mainnet // Arabica testnet. See: celestiaorg/networks. - Arabica Network = "arabica-10" + Arabica Network = "arabica-11" // Mocha testnet. See: celestiaorg/networks. Mocha Network = "mocha-4" // Private can be used to set up any private network, including local testing setups. From f75e255999d3e9c590b03b83f068b58cc32d2548 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 4 Jan 2024 11:27:56 +0100 Subject: [PATCH 350/388] chore(go.mod): Bump header to latest release (#3067) --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 458c41c9df..6609aeba05 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/celestia-app v1.4.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.5.0-rc1 + github.com/celestiaorg/go-header v0.5.1 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 1ee176676a..a8ad8ad148 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.5.0-rc1 h1:DAcVW8V76VI5VU4fOAdXePpq15UFblwZIMZeHCAVr0w= -github.com/celestiaorg/go-header v0.5.0-rc1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.5.1 h1:1s1lw4fcCHalNK0qw/0a3cxg3ezx3Hl020znIxPZvtk= +github.com/celestiaorg/go-header v0.5.1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= From 45e1847513fde2c1a680fd1b384c4efadc108e51 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Thu, 4 Jan 2024 16:37:21 +0100 Subject: [PATCH 351/388] feat(nodebuilder/prune)!: Enable sampling window for light nodes (#2991) This PR enforces a sampling window of 30 days' worth of seconds for light nodes. The DASer will now skip over sampling headers that fall outside this range. I've labeled this PR as breaking as it contains a behavioural break (sampling constrained to a 30-day window rather than all historical headers). --- das/daser.go | 2 ++ das/daser_test.go | 37 +++++++++++++++++++++++++++++++++++++ nodebuilder/prune/module.go | 2 +- pruner/light/window.go | 2 ++ 4 files changed, 42 insertions(+), 1 deletion(-) diff --git a/das/daser.go b/das/daser.go index 40eee3d316..7d569f7e0b 100644 --- a/das/daser.go +++ b/das/daser.go @@ -151,6 +151,8 @@ func (d *DASer) sample(ctx context.Context, h *header.ExtendedHeader) error { // short-circuit if pruning is enabled and the header is outside the // availability window if !d.isWithinSamplingWindow(h) { + log.Debugw("skipping header outside sampling window", "height", h.Height(), + "time", h.Time()) return nil } diff --git a/das/daser_test.go b/das/daser_test.go index fd1eb39f7d..9eec6392cc 100644 --- a/das/daser_test.go +++ b/das/daser_test.go @@ -2,6 +2,7 @@ package das import ( "context" + "strconv" "testing" "time" @@ -244,6 +245,42 @@ func TestDASerSampleTimeout(t *testing.T) { } } +// TestDASer_SamplingWindow tests the sampling window determination +// for headers. +func TestDASer_SamplingWindow(t *testing.T) { + ds := ds_sync.MutexWrap(datastore.NewMapDatastore()) + sub := new(headertest.Subscriber) + fserv := &fraudtest.DummyService[*header.ExtendedHeader]{} + getter := getterStub{} + avail := mocks.NewMockAvailability(gomock.NewController(t)) + + // create and start DASer + daser, err := NewDASer(avail, sub, getter, ds, fserv, newBroadcastMock(1), + WithSamplingWindow(time.Second)) + require.NoError(t, err) + + var tests = []struct { + timestamp time.Time + withinWindow bool + }{ + {timestamp: time.Now().Add(-(time.Second * 5)), withinWindow: false}, + {timestamp: time.Now().Add(-(time.Millisecond * 800)), withinWindow: true}, + {timestamp: time.Now().Add(-(time.Hour)), withinWindow: false}, + {timestamp: time.Now().Add(-(time.Hour * 24 * 30)), withinWindow: false}, + {timestamp: time.Now(), withinWindow: true}, + } + + for i, tt := range tests { + t.Run(strconv.Itoa(i), func(t *testing.T) { + eh := headertest.RandExtendedHeader(t) + eh.RawHeader.Time = tt.timestamp + + assert.Equal(t, tt.withinWindow, daser.isWithinSamplingWindow(eh)) + }) + } + +} + // createDASerSubcomponents takes numGetter (number of headers // to store in mockGetter) and numSub (number of headers to store // in the mock header.Subscriber), returning a newly instantiated diff --git a/nodebuilder/prune/module.go b/nodebuilder/prune/module.go index 330ef21cdc..2141b74bf1 100644 --- a/nodebuilder/prune/module.go +++ b/nodebuilder/prune/module.go @@ -39,7 +39,7 @@ func ConstructModule(tp node.Type) fx.Option { fx.Provide(func() pruner.Pruner { return light.NewPruner() }), - fx.Supply(archival.Window), // TODO @renaynay: turn this into light.Window in following PR + fx.Supply(light.Window), ) default: panic("unknown node type") diff --git a/pruner/light/window.go b/pruner/light/window.go index 53bfe4a163..dc1a9e4444 100644 --- a/pruner/light/window.go +++ b/pruner/light/window.go @@ -6,4 +6,6 @@ import ( "github.com/celestiaorg/celestia-node/pruner" ) +// Window is the availability window for light nodes in the Celestia +// network (30 days). const Window = pruner.AvailabilityWindow(time.Second * 86400 * 30) From ade25d7362aa318b7600ebb78b84493bee17f48b Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 8 Jan 2024 11:13:26 +0000 Subject: [PATCH 352/388] refactor(makefile + ci): rename swamp -> integration (#3065) Renames `swamp` to `integration` in Makefile and where called in github actions workflows Fixes #2104 --- .github/workflows/go-ci.yml | 8 ++++---- Makefile | 18 +++++++++--------- nodebuilder/tests/README.md | 4 ++-- 3 files changed, 15 insertions(+), 15 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index b004dcfb59..e4cd0e83cc 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -133,8 +133,8 @@ jobs: with: go-version: ${{ inputs.go-version }} - - name: Swamp Tests - run: make test-swamp + - name: Integration Tests + run: make test-integration - - name: Swamp Tests with Race Detector - run: make test-swamp-race + - name: Integration Tests with Race Detector + run: make test-integration-race diff --git a/Makefile b/Makefile index 9e1635308b..b3b3739cb7 100644 --- a/Makefile +++ b/Makefile @@ -121,19 +121,19 @@ test-unit-race: @go test -race `go list ./... | grep -v nodebuilder/tests` .PHONY: test-unit-race -## test-swamp: Running swamp tests located in nodebuilder/tests -test-swamp: - @echo "--> Running swamp tests" +## test-integration: Running /integration tests located in nodebuilder/tests +test-integration: + @echo "--> Running integrations tests" @go test ./nodebuilder/tests -.PHONY: test-swamp +.PHONY: test-integration -## test-swamp-race: Running swamp tests with data race detector located in node/tests -test-swamp-race: - @echo "--> Running swamp tests with data race detector" +## test-integration-race: Running integration tests with data race detector located in node/tests +test-integration-race: + @echo "--> Running integration tests with data race detector" @go test -race ./nodebuilder/tests -.PHONY: test-swamp-race +.PHONY: test-integration-race -## test: Running both unit and swamp tests +## test: Running both unit and integrations tests test: @echo "--> Running all tests without data race detector" @go test ./... diff --git a/nodebuilder/tests/README.md b/nodebuilder/tests/README.md index 176ee2ba21..dd2040ab42 100644 --- a/nodebuilder/tests/README.md +++ b/nodebuilder/tests/README.md @@ -1,6 +1,6 @@ # Swamp: In-Memory Test Tool -Swamp is a testing tool that creates an environment for deploying `celestia-node` and testing instances against each other. +Swamp is our integration testing tool that creates an environment for deploying `celestia-node` and testing instances against each other. While the swamp takes care of setting up networking and initial configuration of node types, the user can focus on tailoring test scenarios. ## Usage @@ -38,7 +38,7 @@ require.NoError(t, err) light := sw.NewLightClient(node.WithTrustedPeer(addrs[0].String())) ``` -## Concenptual overview +## Conceptual overview Each of the test scenario requires flexibility in network topology. The user can define the necessary amount of each type of node and be able to control each of them. From 1afe0a14b77f52db2742ef580e839222bccd97a9 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 8 Jan 2024 11:56:55 +0000 Subject: [PATCH 353/388] chore(deps): update otel group (#3068) reduced myself to fixing dependabot's attempts to upgrade us at https://github.com/celestiaorg/celestia-node/pull/3055 was not working quite right --- go.mod | 23 +++++++++++------------ go.sum | 54 +++++++++++++++++++++++++++--------------------------- 2 files changed, 38 insertions(+), 39 deletions(-) diff --git a/go.mod b/go.mod index 6609aeba05..2a832e3352 100644 --- a/go.mod +++ b/go.mod @@ -49,20 +49,20 @@ require ( github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.18.0 github.com/pyroscope-io/client v0.7.2 - github.com/pyroscope-io/otel-profiling-go v0.4.0 + github.com/pyroscope-io/otel-profiling-go v0.5.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 - go.opentelemetry.io/otel v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 - go.opentelemetry.io/otel/metric v1.19.0 - go.opentelemetry.io/otel/sdk v1.19.0 - go.opentelemetry.io/otel/sdk/metric v1.19.0 - go.opentelemetry.io/otel/trace v1.19.0 + go.opentelemetry.io/otel v1.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 + go.opentelemetry.io/otel/metric v1.21.0 + go.opentelemetry.io/otel/sdk v1.21.0 + go.opentelemetry.io/otel/sdk/metric v1.21.0 + go.opentelemetry.io/otel/trace v1.21.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.1 go.uber.org/zap v1.26.0 @@ -157,7 +157,7 @@ require ( github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.2.4 // indirect + github.com/go-logr/logr v1.3.0 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -175,7 +175,7 @@ require ( github.com/golang/snappy v0.0.5-0.20220116011046-fa5810519dcb // indirect github.com/google/btree v1.1.2 // indirect github.com/google/flatbuffers v1.12.1 // indirect - github.com/google/go-cmp v0.5.9 // indirect + github.com/google/go-cmp v0.6.0 // indirect github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect @@ -317,7 +317,6 @@ require ( github.com/zondax/ledger-go v0.14.3 // indirect go.etcd.io/bbolt v1.3.6 // indirect go.opencensus.io v0.24.0 // indirect - go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect diff --git a/go.sum b/go.sum index a8ad8ad148..24303315da 100644 --- a/go.sum +++ b/go.sum @@ -688,8 +688,8 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.2.4 h1:g01GSCwiDw2xSZfjJ2/T9M+S6pFdcNtFYsp+Y43HYDQ= -github.com/go-logr/logr v1.2.4/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= +github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -823,8 +823,9 @@ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= github.com/google/go-cmp v0.5.8/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck= github.com/google/gofuzz v0.0.0-20170612174753-24818f796faf/go.mod h1:HP5RmnzzSNb993RKQDq4+1A4ia9nllfqcQFTQJedwGI= @@ -2080,8 +2081,8 @@ github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAa github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= -github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic= -github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= +github.com/pyroscope-io/otel-profiling-go v0.5.0 h1:LsTP9VuQ5TgeSiyY2gPHy1de/q3jbFyGWE1v3LtHzMk= +github.com/pyroscope-io/otel-profiling-go v0.5.0/go.mod h1:jUUUXTTgntvGJKS8p5uzypXwTyuGnQP31VnWauH/lUg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= @@ -2117,8 +2118,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= -github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= -github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= +github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= +github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= @@ -2382,36 +2383,35 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 h1:2JydY5UiDpqvj2p7sO9bgHuhTy4hgTZ0ymehdq/Ob0Q= go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0/go.mod h1:ch3a5QxOqVWxas4CzjCFFOOQe+7HgAXC/N1oVxS9DK4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= -go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0 h1:ZtfnDL+tUrs1F0Pzfwbg2d59Gru9NCH3bgSHBM6LDwU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric v0.42.0/go.mod h1:hG4Fj/y8TR/tlEDREo8tWstl9fO9gcFkn4xrx0Io8xU= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0 h1:wNMDy/LVGLj2h3p6zg4d0gypKfWKSWI14E1C4smOgl8= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.42.0/go.mod h1:YfbDdXAAkemWJK3H/DshvlrxqFB2rtW4rY6ky/3x/H0= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0 h1:IeMeyr1aBvBiPVYihXIaeIZba6b8E1bYp7lbdxK8CQg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.19.0/go.mod h1:oVdCUtjq9MK9BlS7TtucsQwUcXcymNiEDjgDD2jMtZU= +go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 h1:bflGWrfYyuulcdxf14V6n9+CoQcu5SAAdHmDPAJnlps= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= -go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= -go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= -go.opentelemetry.io/otel/sdk/metric v1.19.0 h1:EJoTO5qysMsYCa+w4UghwFV/ptQgqSL/8Ni+hx+8i1k= -go.opentelemetry.io/otel/sdk/metric v1.19.0/go.mod h1:XjG0jQyFJrv2PbMvwND7LwCEhsJzCzV5210euduKcKY= +go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= +go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= +go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= -go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= +go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= From f8f4af735b2eed39ad83889c15fafaf28bd8300e Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 8 Jan 2024 13:05:14 +0000 Subject: [PATCH 354/388] test: cleanup of PR 2998 to simplify (#3051) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit in kind replacement of https://github.com/celestiaorg/celestia-node/pull/2998 with the simplified require assertion --------- Co-authored-by: Håvard Anda Estensen --- libs/utils/address.go | 5 ++++- libs/utils/address_test.go | 5 ++++- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/libs/utils/address.go b/libs/utils/address.go index ae52a03b16..c20d11ad06 100644 --- a/libs/utils/address.go +++ b/libs/utils/address.go @@ -1,11 +1,14 @@ package utils import ( + "errors" "fmt" "net" "strings" ) +var ErrInvalidIP = errors.New("invalid IP address or hostname given") + // SanitizeAddr trims leading protocol scheme and port from the given // IP address or hostname if present. func SanitizeAddr(addr string) (string, error) { @@ -16,7 +19,7 @@ func SanitizeAddr(addr string) (string, error) { addr = strings.TrimSuffix(addr, "/") addr = strings.Split(addr, ":")[0] if addr == "" { - return "", fmt.Errorf("invalid IP address or hostname given: %s", original) + return "", fmt.Errorf("%w: %s", ErrInvalidIP, original) } return addr, nil } diff --git a/libs/utils/address_test.go b/libs/utils/address_test.go index 15452f4d1b..48a7747a4a 100644 --- a/libs/utils/address_test.go +++ b/libs/utils/address_test.go @@ -11,6 +11,7 @@ func TestSanitizeAddr(t *testing.T) { var tests = []struct { addr string want string + err error }{ // Testcase: trims protocol prefix {addr: "http://celestia.org", want: "celestia.org"}, @@ -20,13 +21,15 @@ func TestSanitizeAddr(t *testing.T) { {addr: "tcp://192.168.42.42:5050/", want: "192.168.42.42"}, // Testcase: invariant ip {addr: "192.168.42.42", want: "192.168.42.42"}, + // Testcase: empty addr + {addr: "", want: "", err: ErrInvalidIP}, } for _, tt := range tests { t.Run(tt.addr, func(t *testing.T) { got, err := SanitizeAddr(tt.addr) - require.NoError(t, err) require.Equal(t, tt.want, got) + require.ErrorIs(t, err, tt.err) }) } } From 73336eda96510652881080c7b339d0eca7bc4ca1 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 9 Jan 2024 11:36:22 +0100 Subject: [PATCH 355/388] fix(core): Use `Verify` from go-header pkg in `CoreExchange` (#3077) We should have been using the high-level `Verify` check from go-header inside of the `CoreExchange` instead of the `ExtendedHeader`-based implementation of Verify. Co-authored-by: ramin --- core/exchange.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/core/exchange.go b/core/exchange.go index 79f3d6337a..cf889a38bb 100644 --- a/core/exchange.go +++ b/core/exchange.go @@ -78,7 +78,7 @@ func (ce *Exchange) GetRangeByHeight( ce.metrics.requestDurationPerHeader(ctx, time.Since(start), amount) for _, h := range headers { - err := from.Verify(h) + err := libhead.Verify[*header.ExtendedHeader](from, h, libhead.DefaultHeightThreshold) if err != nil { return nil, fmt.Errorf("verifying next header against last verified height: %d: %w", from.Height(), err) From 49894c0b9ca322884ce4a2ead6f4e9f6834da55d Mon Sep 17 00:00:00 2001 From: Chirag <56735482+Chirag018@users.noreply.github.com> Date: Wed, 10 Jan 2024 22:35:56 +0530 Subject: [PATCH 356/388] chore(modstate): Remove IsStopped endpoint from StateModule (#2912) @Wondertan is this what you expected? Closes #2906 --- api/gateway/middleware.go | 20 -------------------- nodebuilder/state/mocks/api.go | 14 -------------- nodebuilder/state/state.go | 10 +++------- nodebuilder/state/stub.go | 4 ---- state/core_access.go | 4 ---- 5 files changed, 3 insertions(+), 49 deletions(-) diff --git a/api/gateway/middleware.go b/api/gateway/middleware.go index 2c88b34185..4b669113dd 100644 --- a/api/gateway/middleware.go +++ b/api/gateway/middleware.go @@ -2,13 +2,8 @@ package gateway import ( "context" - "errors" "net/http" "time" - - "github.com/gorilla/mux" - - "github.com/celestiaorg/celestia-node/nodebuilder/state" ) const timeout = time.Minute @@ -16,7 +11,6 @@ const timeout = time.Minute func (h *Handler) RegisterMiddleware(srv *Server) { srv.RegisterMiddleware( setContentType, - checkPostDisabled(h.state), wrapRequestContext, enableCors, ) @@ -36,20 +30,6 @@ func setContentType(next http.Handler) http.Handler { }) } -// checkPostDisabled ensures that context was canceled and prohibit POST requests. -func checkPostDisabled(state state.Module) mux.MiddlewareFunc { - return func(next http.Handler) http.Handler { - return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { - // check if state service was halted and deny the transaction - if r.Method == http.MethodPost && state.IsStopped(r.Context()) { - writeError(w, http.StatusMethodNotAllowed, r.URL.Path, errors.New("not possible to submit data")) - return - } - next.ServeHTTP(w, r) - }) - } -} - // wrapRequestContext ensures we implement a deadline on serving requests // via the gateway server-side to prevent context leaks. func wrapRequestContext(next http.Handler) http.Handler { diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index 754920dee2..6499a6dfd8 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -131,20 +131,6 @@ func (mr *MockModuleMockRecorder) Delegate(arg0, arg1, arg2, arg3, arg4 interfac return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Delegate", reflect.TypeOf((*MockModule)(nil).Delegate), arg0, arg1, arg2, arg3, arg4) } -// IsStopped mocks base method. -func (m *MockModule) IsStopped(arg0 context.Context) bool { - m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "IsStopped", arg0) - ret0, _ := ret[0].(bool) - return ret0 -} - -// IsStopped indicates an expected call of IsStopped. -func (mr *MockModuleMockRecorder) IsStopped(arg0 interface{}) *gomock.Call { - mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "IsStopped", reflect.TypeOf((*MockModule)(nil).IsStopped), arg0) -} - // QueryDelegation mocks base method. func (m *MockModule) QueryDelegation(arg0 context.Context, arg1 types.ValAddress) (*types0.QueryDelegationResponse, error) { m.ctrl.T.Helper() diff --git a/nodebuilder/state/state.go b/nodebuilder/state/state.go index 83408680da..624f8dcd3f 100644 --- a/nodebuilder/state/state.go +++ b/nodebuilder/state/state.go @@ -16,9 +16,8 @@ var _ Module = (*API)(nil) // messages to the Celestia network. // //go:generate mockgen -destination=mocks/api.go -package=mocks . Module +//nolint:dupl type Module interface { - // IsStopped checks if the Module's context has been stopped - IsStopped(ctx context.Context) bool // AccountAddress retrieves the address of the node's account/signer AccountAddress(ctx context.Context) (state.Address, error) @@ -97,10 +96,11 @@ type Module interface { // API is a wrapper around Module for the RPC. // TODO(@distractedm1nd): These structs need to be autogenerated. +// +//nolint:dupl type API struct { Internal struct { AccountAddress func(ctx context.Context) (state.Address, error) `perm:"read"` - IsStopped func(ctx context.Context) bool `perm:"read"` Balance func(ctx context.Context) (*state.Balance, error) `perm:"read"` BalanceForAddress func(ctx context.Context, addr state.Address) (*state.Balance, error) `perm:"read"` Transfer func( @@ -167,10 +167,6 @@ func (api *API) AccountAddress(ctx context.Context) (state.Address, error) { return api.Internal.AccountAddress(ctx) } -func (api *API) IsStopped(ctx context.Context) bool { - return api.Internal.IsStopped(ctx) -} - func (api *API) BalanceForAddress(ctx context.Context, addr state.Address) (*state.Balance, error) { return api.Internal.BalanceForAddress(ctx, addr) } diff --git a/nodebuilder/state/stub.go b/nodebuilder/state/stub.go index 8d17d651dd..30a431aba5 100644 --- a/nodebuilder/state/stub.go +++ b/nodebuilder/state/stub.go @@ -17,10 +17,6 @@ var ErrNoStateAccess = errors.New("node is running without state access. run wit // to a core endpoint. type stubbedStateModule struct{} -func (s stubbedStateModule) IsStopped(context.Context) bool { - return true -} - func (s stubbedStateModule) AccountAddress(context.Context) (state.Address, error) { return state.Address{}, ErrNoStateAccess } diff --git a/state/core_access.go b/state/core_access.go index 2a49e70a03..358457b4f0 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -573,10 +573,6 @@ func (ca *CoreAccessor) queryMinimumGasPrice( return coins.AmountOf(app.BondDenom).MustFloat64(), nil } -func (ca *CoreAccessor) IsStopped(context.Context) bool { - return ca.ctx.Err() != nil -} - func withFee(fee Int) apptypes.TxBuilderOption { gasFee := sdktypes.NewCoins(sdktypes.NewCoin(app.BondDenom, fee)) return apptypes.SetFeeAmount(gasFee) From 9b2d82a5d07a0cff52b901fbacdaa402cb6af891 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 10 Jan 2024 18:49:56 +0100 Subject: [PATCH 357/388] chore(deps): Bump deps (#3091) Can close out #3075 --- go.mod | 4 ++-- go.sum | 8 ++++---- 2 files changed, 6 insertions(+), 6 deletions(-) diff --git a/go.mod b/go.mod index 2a832e3352..4ad8bb9ba8 100644 --- a/go.mod +++ b/go.mod @@ -43,7 +43,7 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.12.0 + github.com/multiformats/go-multiaddr v0.12.1 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 @@ -68,7 +68,7 @@ require ( go.uber.org/zap v1.26.0 golang.org/x/crypto v0.17.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d - golang.org/x/sync v0.5.0 + golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.60.1 google.golang.org/protobuf v1.32.0 diff --git a/go.sum b/go.sum index 24303315da..f1daccb66c 100644 --- a/go.sum +++ b/go.sum @@ -1830,8 +1830,8 @@ github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= -github.com/multiformats/go-multiaddr v0.12.0 h1:1QlibTFkoXJuDjjYsMHhE73TnzJQl8FSWatk/0gxGzE= -github.com/multiformats/go-multiaddr v0.12.0/go.mod h1:WmZXgObOQOYp9r3cslLlppkrz1FYSHmE834dfz/lWu8= +github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk= +github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= @@ -2712,8 +2712,8 @@ golang.org/x/sync v0.0.0-20220601150217-0de741cfad7f/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220929204114-8fcdb60fdcc0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.5.0 h1:60k92dhOjHxJkrqnwsfl8KuaHbn/5dl0lUPUklKo3qE= -golang.org/x/sync v0.5.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.6.0 h1:5BMeUDZ7vkXGfEr1x9B4bRcTH4lpkTkpdh0T/J+qjbQ= +golang.org/x/sync v0.6.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180810173357-98c5dad5d1a0/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= From 0ce8241d568662b4d203a09b6905cfbe61b96ce1 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Fri, 12 Jan 2024 14:17:51 +0100 Subject: [PATCH 358/388] chore(deps): Bump go-header (#3096) Bumping go-header. --- go.mod | 26 +++++++++++++------------- go.sum | 51 ++++++++++++++++++++++++++------------------------- 2 files changed, 39 insertions(+), 38 deletions(-) diff --git a/go.mod b/go.mod index 4ad8bb9ba8..dbec09d01a 100644 --- a/go.mod +++ b/go.mod @@ -11,7 +11,7 @@ require ( github.com/celestiaorg/celestia-app v1.4.0 github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.5.1 + github.com/celestiaorg/go-header v0.5.2 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 @@ -36,7 +36,7 @@ require ( github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 github.com/ipld/go-car v0.6.2 - github.com/libp2p/go-libp2p v0.32.1 + github.com/libp2p/go-libp2p v0.32.2 github.com/libp2p/go-libp2p-kad-dht v0.25.2 github.com/libp2p/go-libp2p-pubsub v0.10.0 github.com/libp2p/go-libp2p-record v0.2.0 @@ -66,8 +66,8 @@ require ( go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.1 go.uber.org/zap v1.26.0 - golang.org/x/crypto v0.17.0 - golang.org/x/exp v0.0.0-20231006140011-7918f672742d + golang.org/x/crypto v0.18.0 + golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e golang.org/x/sync v0.6.0 golang.org/x/text v0.14.0 google.golang.org/grpc v1.60.1 @@ -157,7 +157,7 @@ require ( github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect github.com/go-logfmt/logfmt v0.5.1 // indirect - github.com/go-logr/logr v1.3.0 // indirect + github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect github.com/go-openapi/jsonpointer v0.19.5 // indirect @@ -180,7 +180,7 @@ require ( github.com/google/orderedcode v0.0.1 // indirect github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect github.com/google/s2a-go v0.1.4 // indirect - github.com/google/uuid v1.3.1 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect @@ -232,7 +232,7 @@ require ( github.com/jmhodges/levigo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/klauspost/compress v1.17.2 // indirect - github.com/klauspost/cpuid/v2 v2.2.5 // indirect + github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect github.com/lib/pq v1.10.7 // indirect @@ -285,7 +285,7 @@ require ( github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.4 // indirect - github.com/quic-go/quic-go v0.39.3 // indirect + github.com/quic-go/quic-go v0.39.4 // indirect github.com/quic-go/webtransport-go v0.6.0 // indirect github.com/rakyll/statik v0.1.7 // indirect github.com/raulk/go-watchdog v1.3.0 // indirect @@ -321,12 +321,12 @@ require ( go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect go.uber.org/multierr v1.11.0 // indirect - golang.org/x/mod v0.13.0 // indirect - golang.org/x/net v0.17.0 // indirect + golang.org/x/mod v0.14.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect - golang.org/x/sys v0.15.0 // indirect - golang.org/x/term v0.15.0 // indirect - golang.org/x/tools v0.14.0 // indirect + golang.org/x/sys v0.16.0 // indirect + golang.org/x/term v0.16.0 // indirect + golang.org/x/tools v0.16.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.128.0 // indirect diff --git a/go.sum b/go.sum index f1daccb66c..05e4c0bfcf 100644 --- a/go.sum +++ b/go.sum @@ -372,8 +372,8 @@ github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXv github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.5.1 h1:1s1lw4fcCHalNK0qw/0a3cxg3ezx3Hl020znIxPZvtk= -github.com/celestiaorg/go-header v0.5.1/go.mod h1:H8xhnDLDLbkpwmWPhCaZyTnIV3dlVxBHPnxNXS2Qu6c= +github.com/celestiaorg/go-header v0.5.2 h1:CFsTAXcs1o38JVd8YN1Naq/Yzs6m9orMPEPNpLEgFJA= +github.com/celestiaorg/go-header v0.5.2/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= @@ -688,8 +688,9 @@ github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KE github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= -github.com/go-logr/logr v1.3.0 h1:2y3SDp0ZXuc6/cjLSZ+Q3ir+QB9T/iG5yYRXqsagWSY= github.com/go-logr/logr v1.3.0/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= +github.com/go-logr/logr v1.4.1 h1:pKouT5E8xu9zeFC39JXRDukb6JFQPXM5p5I91188VAQ= +github.com/go-logr/logr v1.4.1/go.mod h1:9T104GzyrTigFIr8wt5mBrctHMim0Nb2HLGrmQ40KvY= github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-ole/go-ole v1.2.1/go.mod h1:7FAglXiTm7HKlQRDeOQ6ZNUHidzCWXuZWq/1dTyBNF8= @@ -873,8 +874,8 @@ github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.3.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= -github.com/google/uuid v1.3.1 h1:KjJaJ9iWZ3jOFZIf1Lqf4laDRCasjl0BCmnEGxkdLb4= -github.com/google/uuid v1.3.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= @@ -1345,8 +1346,8 @@ github.com/klauspost/cpuid/v2 v2.1.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8t github.com/klauspost/cpuid/v2 v2.1.2/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.1/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= github.com/klauspost/cpuid/v2 v2.2.3/go.mod h1:RVVoqg1df56z8g3pUjL/3lE5UfnlrJX8tyFgg4nqhuY= -github.com/klauspost/cpuid/v2 v2.2.5 h1:0E5MSMDEoAulmXNFquVs//DdoomxaoTY1kUhbc/qbZg= -github.com/klauspost/cpuid/v2 v2.2.5/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= +github.com/klauspost/cpuid/v2 v2.2.6 h1:ndNyv040zDGIDh8thGkXYjnFtiN02M1PVVF+JE/48xc= +github.com/klauspost/cpuid/v2 v2.2.6/go.mod h1:Lcz8mBdAVJIBVzewtcLocK12l3Y+JytZYpaMropDUws= github.com/klauspost/crc32 v0.0.0-20161016154125-cb6bfca970f6/go.mod h1:+ZoRqAPRLkC4NPOvfYeR5KNOrY6TD+/sAC3HXPZgDYg= github.com/klauspost/pgzip v1.0.2-0.20170402124221-0bf5dcad4ada/go.mod h1:Ch1tH69qFZu15pkjo5kYi6mth2Zzwzt50oCQKQE9RUs= github.com/klauspost/reedsolomon v1.11.8 h1:s8RpUW5TK4hjr+djiOpbZJB4ksx+TdYbRH7vHQpwPOY= @@ -1415,8 +1416,8 @@ github.com/libp2p/go-libp2p v0.22.0/go.mod h1:UDolmweypBSjQb2f7xutPnwZ/fxioLbMBx github.com/libp2p/go-libp2p v0.23.4/go.mod h1:s9DEa5NLR4g+LZS+md5uGU4emjMWFiqkZr6hBTY8UxI= github.com/libp2p/go-libp2p v0.25.0/go.mod h1:vXHmFpcfl+xIGN4qW58Bw3a0/SKGAesr5/T4IuJHE3o= github.com/libp2p/go-libp2p v0.25.1/go.mod h1:xnK9/1d9+jeQCVvi/f1g12KqtVi/jP/SijtKV1hML3g= -github.com/libp2p/go-libp2p v0.32.1 h1:wy1J4kZIZxOaej6NveTWCZmHiJ/kY7GoAqXgqNCnPps= -github.com/libp2p/go-libp2p v0.32.1/go.mod h1:hXXC3kXPlBZ1eu8Q2hptGrMB4mZ3048JUoS4EKaHW5c= +github.com/libp2p/go-libp2p v0.32.2 h1:s8GYN4YJzgUoyeYNPdW7JZeZ5Ee31iNaIBfGYMAY4FQ= +github.com/libp2p/go-libp2p v0.32.2/go.mod h1:E0LKe+diV/ZVJVnOJby8VC5xzHF0660osg71skcxJvk= github.com/libp2p/go-libp2p-asn-util v0.1.0/go.mod h1:wu+AnM9Ii2KgO5jMmS1rz9dvzTdj8BXqsPR9HR0XB7I= github.com/libp2p/go-libp2p-asn-util v0.2.0/go.mod h1:WoaWxbHKBymSN41hWSq/lGKJEca7TNm58+gGJi2WsLI= github.com/libp2p/go-libp2p-asn-util v0.3.0 h1:gMDcMyYiZKkocGXDQ5nsUQyquC9+H+iLEQHwOCZ7s8s= @@ -2091,8 +2092,8 @@ github.com/quic-go/qtls-go1-20 v0.1.0/go.mod h1:JKtK6mjbAVcUTN/9jZpvLbGxvdWIKS8u github.com/quic-go/qtls-go1-20 v0.3.4 h1:MfFAPULvst4yoMgY9QmtpYmfij/em7O8UUi+bNVm7Cg= github.com/quic-go/qtls-go1-20 v0.3.4/go.mod h1:X9Nh97ZL80Z+bX/gUXMbipO6OxdiDi58b/fMC9mAL+k= github.com/quic-go/quic-go v0.32.0/go.mod h1:/fCsKANhQIeD5l76c2JFU+07gVE3KaA0FP+0zMWwfwo= -github.com/quic-go/quic-go v0.39.3 h1:o3YB6t2SR+HU/pgwF29kJ6g4jJIJEwEZ8CKia1h1TKg= -github.com/quic-go/quic-go v0.39.3/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= +github.com/quic-go/quic-go v0.39.4 h1:PelfiuG7wXEffUT2yceiqz5V6Pc0TA5ruOd1LcmFc1s= +github.com/quic-go/quic-go v0.39.4/go.mod h1:T09QsDQWjLiQ74ZmacDfqZmhY/NLnw5BC40MANNNZ1Q= github.com/quic-go/webtransport-go v0.5.1/go.mod h1:OhmmgJIzTTqXK5xvtuX0oBpLV2GkLWNDA+UeTGJXErU= github.com/quic-go/webtransport-go v0.6.0 h1:CvNsKqc4W2HljHJnoT+rMmbRJybShZ0YPFDD3NxaZLY= github.com/quic-go/webtransport-go v0.6.0/go.mod h1:9KjU4AEBqEQidGHNDkZrb8CAa1abRaosM2yGOyiikEc= @@ -2511,8 +2512,8 @@ golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw golang.org/x/crypto v0.4.0/go.mod h1:3quD/ATkf6oY+rnes5c3ExXTbLc8mueNue5/DoinL80= golang.org/x/crypto v0.5.0/go.mod h1:NK/OQwhpMQP3MwtdjgLlYHnH9ebylxKWv3e0fK+mkQU= golang.org/x/crypto v0.6.0/go.mod h1:OFC/31mSvZgRz0V1QTNCzfAI1aIRzbiufJtkMIlEp58= -golang.org/x/crypto v0.17.0 h1:r8bRNjWL3GshPW3gkd+RpvzWrZAwPS49OmTGZ/uhM4k= -golang.org/x/crypto v0.17.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= +golang.org/x/crypto v0.18.0 h1:PGVlW0xEltQnzFZ55hkuX5+KLyrMYhHld1YHO4AKcdc= +golang.org/x/crypto v0.18.0/go.mod h1:R0j02AL6hcrfOiy9T4ZYp/rcWeMxM3L6QYxlOuEG1mg= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20180807140117-3d87b88a115f/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -2535,8 +2536,8 @@ golang.org/x/exp v0.0.0-20220916125017-b168a2c6b86b/go.mod h1:cyybsKvd6eL0RnXn6p golang.org/x/exp v0.0.0-20221205204356-47842c84f3db/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230129154200-a960b3787bd2/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= golang.org/x/exp v0.0.0-20230213192124-5e25df0256eb/go.mod h1:CxIveKay+FTh1D0yPZemJVgC/95VzuuOLq5Qi4xnoYc= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= -golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e h1:723BNChdd0c2Wk6WOE320qGBiPtYx0F0Bbm1kriShfE= +golang.org/x/exp v0.0.0-20240110193028-0dcbfd608b1e/go.mod h1:iRJReGqOEeBhDZGkGbynYwcHlctCvnjTYIamk7uXpHI= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= @@ -2572,8 +2573,8 @@ golang.org/x/mod v0.6.0-dev.0.20220106191415-9b9b3d81d5e3/go.mod h1:3p9vT2HGsQu2 golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= golang.org/x/mod v0.6.0/go.mod h1:4mET923SAdbXp2ki8ey+zGs1SLqsuM2Y0uvdZR/fUNI= golang.org/x/mod v0.7.0/go.mod h1:iBbtSCu2XBx23ZKBPSOrRkjjQPZFPuis4dIYUhu/chs= -golang.org/x/mod v0.13.0 h1:I/DsJXRlw/8l/0c24sM9yb0T4z9liZTduXvdAWYiysY= -golang.org/x/mod v0.13.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= +golang.org/x/mod v0.14.0 h1:dGoOF9QVLYng8IHTm7BAyWqCqSheQ5pYWGhzW00YJr0= +golang.org/x/mod v0.14.0/go.mod h1:hTbmBsO62+eylJbnUtE2MGJUyE7QWk4xUqPFrRgJ+7c= golang.org/x/net v0.0.0-20180719180050-a680a1efc54d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -2665,8 +2666,8 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= -golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2859,8 +2860,8 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= -golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= +golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= @@ -2869,8 +2870,8 @@ golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= golang.org/x/term v0.3.0/go.mod h1:q750SLmJuPmVoN1blW3UFBPREJfb1KmY3vwxfr+nFDA= golang.org/x/term v0.4.0/go.mod h1:9P2UbLfCdcvo3p/nzKvsmas4TnlujnuoV9hGgYzW1lQ= golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= -golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= -golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= +golang.org/x/term v0.16.0 h1:m+B6fahuftsE9qjo0VWp2FW0mB3MTJvR0BaMQrq0pmE= +golang.org/x/term v0.16.0/go.mod h1:yn7UURbUtPyrVJPGPq404EukNFxcm/foM+bV/bfcDsY= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -2971,8 +2972,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.14.0 h1:jvNa2pY0M4r62jkRQ6RwEZZyPcymeL9XZMLBbV7U2nc= -golang.org/x/tools v0.14.0/go.mod h1:uYBEerGOWcJyEORxN+Ek8+TT266gXkNlHdJBwexUsBg= +golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= +golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From a7c49d20062327b636e7e08d859802a8f5393ac0 Mon Sep 17 00:00:00 2001 From: ramin Date: Fri, 12 Jan 2024 14:33:29 +0000 Subject: [PATCH 359/388] ci: add continue-on-error temporarily (#3100) Adding `continue-on-error` to the flakey components so we can see that the entire pipeline w/ generated binaries will complete ASAP / next release, will be replaced with separating/fixing the flakey tests next. --- .github/workflows/go-ci.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index e4cd0e83cc..3e73290500 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -108,6 +108,7 @@ jobs: needs: [lint, go_mod_tidy_check] name: Run Unit Tests with Race Detector runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 @@ -124,6 +125,7 @@ jobs: needs: [lint, go_mod_tidy_check] name: Run Integration Tests runs-on: ubuntu-latest + continue-on-error: true steps: - uses: actions/checkout@v4 From 73771ecd6b3fb4600804b18f04947eebe944b789 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 16 Jan 2024 12:39:01 +0100 Subject: [PATCH 360/388] fix: clarifying blob.GetAll godoc (#3090) The previous godoc led to some confusion because it got understood as getting all blobs *under* the given height. Co-authored-by: Viacheslav --- nodebuilder/blob/blob.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index 5e29d3b90c..47ab255ac6 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -19,7 +19,7 @@ type Module interface { Submit(_ context.Context, _ []*blob.Blob, _ *blob.SubmitOptions) (height uint64, _ error) // Get retrieves the blob by commitment under the given namespace and height. Get(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Blob, error) - // GetAll returns all blobs under the given namespaces and height. + // GetAll returns all blobs at the given height under the given namespaces. GetAll(_ context.Context, height uint64, _ []share.Namespace) ([]*blob.Blob, error) // GetProof retrieves proofs in the given namespaces at the given height by commitment. GetProof(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Proof, error) From 4083ea82e87e2bffaf6b97e973e5b909804abf90 Mon Sep 17 00:00:00 2001 From: Jorropo Date: Tue, 16 Jan 2024 12:47:57 +0100 Subject: [PATCH 361/388] chore: cleanup go.mod file require sections (#3103) This PR does not change any of the dependency, it only reorders the go.mod file into a direct and undirect import section, as go versions do since 1.17. The reason your go.mod file got so many require sections is because during the 1.16 to 1.17 go.mod styling transition you had different contributors updating the go.mod file with pre 1.17 and post 1.17 releases of go. See https://go.dev/doc/modules/gomod-ref: > At go 1.17 or higher: section for a description of what changed. Co-authored-by: ramin --- go.mod | 33 +++++++++++++++------------------ 1 file changed, 15 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index dbec09d01a..bbbec88f46 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/cosmos/cosmos-sdk v0.46.14 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 + github.com/dgraph-io/badger/v4 v4.2.1-0.20231013074411-fb1b00959581 github.com/etclabscore/go-openrpc-reflect v0.0.37 github.com/filecoin-project/dagstore v0.5.6 github.com/filecoin-project/go-jsonrpc v0.3.1 @@ -30,6 +31,7 @@ require ( github.com/imdario/mergo v0.3.16 github.com/ipfs/boxo v0.16.0 github.com/ipfs/go-block-format v0.2.0 + github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 github.com/ipfs/go-ipld-cbor v0.1.0 @@ -74,22 +76,6 @@ require ( google.golang.org/protobuf v1.32.0 ) -require ( - github.com/Microsoft/go-winio v0.6.1 // indirect - github.com/bits-and-blooms/bitset v1.7.0 // indirect - github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2 // indirect - github.com/consensys/bavard v0.1.13 // indirect - github.com/consensys/gnark-crypto v0.12.1 // indirect - github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect - github.com/ethereum/c-kzg-4844 v0.3.1 // indirect - github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect - github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect - github.com/mmcloughlin/addchain v0.4.0 // indirect - github.com/supranational/blst v0.3.11 // indirect - go.uber.org/mock v0.3.0 // indirect - rsc.io/tmplfunc v0.0.3 // indirect -) - require ( cloud.google.com/go v0.110.8 // indirect cloud.google.com/go/compute v1.23.0 // indirect @@ -101,6 +87,7 @@ require ( github.com/99designs/keyring v1.2.1 // indirect github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect github.com/Jorropo/jsync v1.0.1 // indirect + github.com/Microsoft/go-winio v0.6.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578 // indirect github.com/StackExchange/wmi v1.2.1 // indirect @@ -110,8 +97,10 @@ require ( github.com/beorn7/perks v1.0.1 // indirect github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect github.com/bgentry/speakeasy v0.1.0 // indirect + github.com/bits-and-blooms/bitset v1.7.0 // indirect github.com/btcsuite/btcd/btcec/v2 v2.3.2 // indirect github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 // indirect + github.com/celestiaorg/quantum-gravity-bridge/v2 v2.1.2 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash v1.1.0 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect @@ -120,6 +109,8 @@ require ( github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect github.com/cometbft/cometbft-db v0.7.0 // indirect github.com/confio/ics23/go v0.9.1 // indirect + github.com/consensys/bavard v0.1.13 // indirect + github.com/consensys/gnark-crypto v0.12.1 // indirect github.com/containerd/cgroups v1.1.0 // indirect github.com/coreos/go-systemd/v22 v22.5.0 // indirect github.com/cosmos/btcutil v1.0.5 // indirect @@ -130,6 +121,7 @@ require ( github.com/cosmos/iavl v0.19.6 // indirect github.com/cosmos/ibc-go/v6 v6.2.0 // indirect github.com/cosmos/ledger-cosmos-go v0.13.2 // indirect + github.com/crate-crypto/go-kzg-4844 v0.3.0 // indirect github.com/creachadair/taskgroup v0.3.2 // indirect github.com/cskr/pubsub v1.0.2 // indirect github.com/danieljoos/wincred v1.1.2 // indirect @@ -140,7 +132,6 @@ require ( github.com/deepmap/oapi-codegen v1.8.2 // indirect github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect - github.com/dgraph-io/badger/v4 v4.2.1-0.20231013074411-fb1b00959581 github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/docker/go-units v0.5.0 // indirect @@ -148,6 +139,7 @@ require ( github.com/dvsekhvalnov/jose2go v1.5.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect + github.com/ethereum/c-kzg-4844 v0.3.1 // indirect github.com/ethereum/go-ethereum v1.13.2 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect @@ -199,6 +191,7 @@ require ( github.com/hashicorp/go-safetemp v1.0.0 // indirect github.com/hashicorp/go-version v1.6.0 // indirect github.com/hashicorp/golang-lru v1.0.2 // indirect + github.com/hashicorp/golang-lru/arc/v2 v2.0.5 // indirect github.com/hashicorp/hcl v1.0.0 // indirect github.com/hdevalence/ed25519consensus v0.0.0-20220222234857-c00d1f31bab3 // indirect github.com/holiman/uint256 v1.2.3 // indirect @@ -209,7 +202,6 @@ require ( github.com/influxdata/influxdb-client-go/v2 v2.12.2 // indirect github.com/influxdata/line-protocol v0.0.0-20210311194329-9aa0e372d097 // indirect github.com/ipfs/bbloom v0.0.4 // indirect - github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-ipfs-blockstore v1.3.1 // indirect github.com/ipfs/go-ipfs-delay v0.0.1 // indirect github.com/ipfs/go-ipfs-ds-help v1.1.1 // indirect @@ -252,6 +244,7 @@ require ( github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.20 // indirect + github.com/matttproud/golang_protobuf_extensions/v2 v2.0.0 // indirect github.com/miekg/dns v1.1.56 // indirect github.com/mikioh/tcpinfo v0.0.0-20190314235526-30a79bb1804b // indirect github.com/mikioh/tcpopt v0.0.0-20190314235656-172688c1accc // indirect @@ -260,6 +253,7 @@ require ( github.com/minio/sha256-simd v1.0.1 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/mapstructure v1.5.0 // indirect + github.com/mmcloughlin/addchain v0.4.0 // indirect github.com/mr-tron/base58 v1.2.0 // indirect github.com/mtibben/percent v0.2.1 // indirect github.com/multiformats/go-base36 v0.2.0 // indirect @@ -302,6 +296,7 @@ require ( github.com/spf13/jwalterweatherman v1.1.0 // indirect github.com/spf13/viper v1.14.0 // indirect github.com/subosito/gotenv v1.4.1 // indirect + github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect @@ -320,6 +315,7 @@ require ( go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect go.uber.org/dig v1.17.1 // indirect + go.uber.org/mock v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.14.0 // indirect golang.org/x/net v0.19.0 // indirect @@ -339,6 +335,7 @@ require ( gopkg.in/yaml.v3 v3.0.1 // indirect lukechampine.com/blake3 v1.2.1 // indirect nhooyr.io/websocket v1.8.7 // indirect + rsc.io/tmplfunc v0.0.3 // indirect sigs.k8s.io/yaml v1.3.0 // indirect ) From ad92edbfa874c746a661433e13f4b0e7d808e920 Mon Sep 17 00:00:00 2001 From: Halimao <1065621723@qq.com> Date: Tue, 16 Jan 2024 22:55:19 +0800 Subject: [PATCH 362/388] feat(nodebuilder): log node version on start (#3089) Closes #2382 --- ## Screenshot ![image](https://github.com/celestiaorg/celestia-node/assets/25278203/2c04a166-0d41-45df-8194-20cc51ef1ba1) --------- Co-authored-by: ramin --- nodebuilder/node.go | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/nodebuilder/node.go b/nodebuilder/node.go index 3e6950a6ae..d5d0ab2016 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -108,8 +108,12 @@ func (n *Node) Start(ctx context.Context) error { return fmt.Errorf("node: failed to start: %w", err) } - log.Infof("\n\n/_____/ /_____/ /_____/ /_____/ /_____/ \n\nStarted celestia DA node \nnode "+ - "type: %s\nnetwork: %s\n\n/_____/ /_____/ /_____/ /_____/ /_____/ \n", strings.ToLower(n.Type.String()), + log.Infof("\n\n/_____/ /_____/ /_____/ /_____/ /_____/ \n\n"+ + "Started celestia DA node \n"+ + "node version: %s\nnode type: %s\nnetwork: %s\n\n"+ + "/_____/ /_____/ /_____/ /_____/ /_____/ \n", + node.GetBuildInfo().SemanticVersion, + strings.ToLower(n.Type.String()), n.Network) addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(n.Host)) From 36205ccf88c1e41fec80e7ef21ca84d87b970f8b Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Wed, 17 Jan 2024 12:51:52 +0100 Subject: [PATCH 363/388] fix(core): Do not propagate blocks if subscribed to blocks from incorrect chain (#3086) Right now, it is possible for a **bridge** node that was initialised and started on one chain (e.g. `mocha-4`) to be stopped and restarted on a different chain (e.g. `mainnet`) and propagate headers from the old chain (`mocha-4`) into the different network (`mainnet`). This PR fixes this issue by causing the listener to fatal if the listener recognises it is receiving blocks from a different chain to that which it expects. Error will look like this: ``` 2024-01-10T16:06:22.001+0100 ERROR core core/listener.go:175 listener: received block with unexpected chain ID: expected arabica-11, received mocha-4 2024-01-10T16:06:22.001+0100 INFO core core/listener.go:177 listener: listening stopped 2024-01-10T16:06:22.001+0100 FATAL core core/listener.go:126 listener: invalid subscription ``` Resolves #3071 --- core/exchange_test.go | 4 ++- core/listener.go | 20 +++++++++++--- core/listener_test.go | 45 +++++++++++++++++++++++++++++--- core/option.go | 10 +++++++ libs/utils/resetctx.go | 4 ++- nodebuilder/core/module.go | 3 ++- nodebuilder/tests/swamp/swamp.go | 1 + 7 files changed, 77 insertions(+), 10 deletions(-) diff --git a/core/exchange_test.go b/core/exchange_test.go index c43084c57d..95c7f83385 100644 --- a/core/exchange_test.go +++ b/core/exchange_test.go @@ -20,7 +20,9 @@ func TestCoreExchange_RequestHeaders(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) t.Cleanup(cancel) - fetcher, _ := createCoreFetcher(t, DefaultTestConfig()) + cfg := DefaultTestConfig() + cfg.ChainID = networkID + fetcher, _ := createCoreFetcher(t, cfg) // generate 10 blocks generateBlocks(t, fetcher) diff --git a/core/listener.go b/core/listener.go index 8447733506..10255fc4cc 100644 --- a/core/listener.go +++ b/core/listener.go @@ -23,6 +23,8 @@ import ( var ( tracer = otel.Tracer("core/listener") retrySubscriptionDelay = 5 * time.Second + + errInvalidSubscription = errors.New("invalid subscription") ) // Listener is responsible for listening to Core for @@ -41,11 +43,12 @@ type Listener struct { headerBroadcaster libhead.Broadcaster[*header.ExtendedHeader] hashBroadcaster shrexsub.BroadcastFn - listenerTimeout time.Duration - metrics *listenerMetrics - cancel context.CancelFunc + chainID string + + listenerTimeout time.Duration + cancel context.CancelFunc } func NewListener( @@ -81,6 +84,7 @@ func NewListener( store: store, listenerTimeout: 5 * blocktime, metrics: metrics, + chainID: p.chainID, }, nil } @@ -117,6 +121,10 @@ func (cl *Listener) runSubscriber(ctx context.Context, sub <-chan types.EventDat // listener stopped because external context was canceled return } + if errors.Is(err, errInvalidSubscription) { + // stop node if there is a critical issue with the block subscription + log.Fatalf("listener: %v", err) + } log.Warnw("listener: subscriber error, resubscribing...", "err", err) sub = cl.resubscribe(ctx) @@ -163,6 +171,12 @@ func (cl *Listener) listen(ctx context.Context, sub <-chan types.EventDataSigned return errors.New("underlying subscription was closed") } + if cl.chainID != "" && b.Header.ChainID != cl.chainID { + log.Errorf("listener: received block with unexpected chain ID: expected %s,"+ + " received %s", cl.chainID, b.Header.ChainID) + return errInvalidSubscription + } + log.Debugw("listener: new block from core", "height", b.Header.Height) err := cl.handleNewSignedBlock(ctx, b) diff --git a/core/listener_test.go b/core/listener_test.go index bf84c07b41..5ddcd5541d 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -47,10 +47,14 @@ func TestListener(t *testing.T) { t.Cleanup(subs.Cancel) // create one block to store as Head in local store and then unsubscribe from block events - fetcher, _ := createCoreFetcher(t, DefaultTestConfig()) + cfg := DefaultTestConfig() + cfg.ChainID = networkID + fetcher, _ := createCoreFetcher(t, cfg) + eds := createEdsPubSub(ctx, t) + // create Listener and start listening - cl := createListener(ctx, t, fetcher, ps0, eds, createStore(t)) + cl := createListener(ctx, t, fetcher, ps0, eds, createStore(t), networkID) err = cl.Start(ctx) require.NoError(t, err) @@ -80,6 +84,7 @@ func TestListenerWithNonEmptyBlocks(t *testing.T) { // create one block to store as Head in local store and then unsubscribe from block events cfg := DefaultTestConfig() + cfg.ChainID = networkID fetcher, cctx := createCoreFetcher(t, cfg) eds := createEdsPubSub(ctx, t) @@ -92,7 +97,7 @@ func TestListenerWithNonEmptyBlocks(t *testing.T) { }) // create Listener and start listening - cl := createListener(ctx, t, fetcher, ps0, eds, store) + cl := createListener(ctx, t, fetcher, ps0, eds, store, networkID) err = cl.Start(ctx) require.NoError(t, err) @@ -124,6 +129,36 @@ func TestListenerWithNonEmptyBlocks(t *testing.T) { require.Nil(t, cl.cancel) } +func TestListenerWithWrongChainRPC(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + + // create mocknet with two pubsub endpoints + ps0, _ := createMocknetWithTwoPubsubEndpoints(ctx, t) + + // create one block to store as Head in local store and then unsubscribe from block events + cfg := DefaultTestConfig() + cfg.ChainID = networkID + fetcher, _ := createCoreFetcher(t, cfg) + eds := createEdsPubSub(ctx, t) + + store := createStore(t) + err := store.Start(ctx) + require.NoError(t, err) + t.Cleanup(func() { + err = store.Stop(ctx) + require.NoError(t, err) + }) + + // create Listener and start listening + cl := createListener(ctx, t, fetcher, ps0, eds, store, "wrong-chain-rpc") + sub, err := cl.fetcher.SubscribeNewBlockEvent(ctx) + require.NoError(t, err) + + err = cl.listen(ctx, sub) + assert.ErrorIs(t, err, errInvalidSubscription) +} + func createMocknetWithTwoPubsubEndpoints(ctx context.Context, t *testing.T) (*pubsub.PubSub, *pubsub.PubSub) { net, err := mocknet.FullMeshLinked(2) require.NoError(t, err) @@ -166,6 +201,7 @@ func createListener( ps *pubsub.PubSub, edsSub *shrexsub.PubSub, store *eds.Store, + chainID string, ) *Listener { p2pSub, err := p2p.NewSubscriber[*header.ExtendedHeader](ps, header.MsgID, p2p.WithSubscriberNetworkID(networkID)) require.NoError(t, err) @@ -180,7 +216,8 @@ func createListener( require.NoError(t, p2pSub.Stop(ctx)) }) - listener, err := NewListener(p2pSub, fetcher, edsSub.Broadcast, header.MakeExtendedHeader, store, nodep2p.BlockTime) + listener, err := NewListener(p2pSub, fetcher, edsSub.Broadcast, header.MakeExtendedHeader, + store, nodep2p.BlockTime, WithChainID(nodep2p.Network(chainID))) require.NoError(t, err) return listener } diff --git a/core/option.go b/core/option.go index 6e06fade48..6916ced4d8 100644 --- a/core/option.go +++ b/core/option.go @@ -1,9 +1,13 @@ package core +import "github.com/celestiaorg/celestia-node/nodebuilder/p2p" + type Option func(*params) type params struct { metrics bool + + chainID string } // WithMetrics is a functional option that enables metrics @@ -13,3 +17,9 @@ func WithMetrics() Option { p.metrics = true } } + +func WithChainID(id p2p.Network) Option { + return func(p *params) { + p.chainID = id.String() + } +} diff --git a/libs/utils/resetctx.go b/libs/utils/resetctx.go index 3014ba81db..a108cc27b4 100644 --- a/libs/utils/resetctx.go +++ b/libs/utils/resetctx.go @@ -1,6 +1,8 @@ package utils -import "context" +import ( + "context" +) // ResetContextOnError returns a fresh context if the given context has an error. func ResetContextOnError(ctx context.Context) context.Context { diff --git a/nodebuilder/core/module.go b/nodebuilder/core/module.go index fec7c14b1b..7c5c9e6bfd 100644 --- a/nodebuilder/core/module.go +++ b/nodebuilder/core/module.go @@ -56,8 +56,9 @@ func ConstructModule(tp node.Type, cfg *Config, options ...fx.Option) fx.Option pubsub *shrexsub.PubSub, construct header.ConstructFn, store *eds.Store, + chainID p2p.Network, ) (*core.Listener, error) { - var opts []core.Option + opts := []core.Option{core.WithChainID(chainID)} if MetricsEnabled { opts = append(opts, core.WithMetrics()) } diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index e3ac3ad4f2..617fe76151 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -78,6 +78,7 @@ func NewSwamp(t *testing.T, options ...Option) *Swamp { // Now, we are making an assumption that consensus mechanism is already tested out // so, we are not creating bridge nodes with each one containing its own core client // instead we are assigning all created BNs to 1 Core from the swamp + ic.WithChainID("private") cctx := core.StartTestNodeWithConfig(t, ic) swp := &Swamp{ t: t, From 2c7266209bdcda448bf8d0b31d5b5b2e43c8fc99 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 18 Jan 2024 13:55:15 +0000 Subject: [PATCH 364/388] chore(deps): Bump cosmossdk.io/errors from 1.0.0 to 1.0.1 (#3105) Bumps [cosmossdk.io/errors](https://github.com/cosmos/cosmos-sdk) from 1.0.0 to 1.0.1.
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=cosmossdk.io/errors&package-manager=go_modules&previous-version=1.0.0&new-version=1.0.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 20 ++++++++++---------- go.sum | 41 ++++++++++++++++++++--------------------- 2 files changed, 30 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index bbbec88f46..78e627b68f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/celestiaorg/celestia-node go 1.21.1 require ( - cosmossdk.io/errors v1.0.0 + cosmossdk.io/errors v1.0.1 cosmossdk.io/math v1.2.0 github.com/BurntSushi/toml v1.3.2 github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b @@ -77,10 +77,10 @@ require ( ) require ( - cloud.google.com/go v0.110.8 // indirect - cloud.google.com/go/compute v1.23.0 // indirect + cloud.google.com/go v0.110.10 // indirect + cloud.google.com/go/compute v1.23.3 // indirect cloud.google.com/go/compute/metadata v0.2.3 // indirect - cloud.google.com/go/iam v1.1.2 // indirect + cloud.google.com/go/iam v1.1.5 // indirect cloud.google.com/go/storage v1.30.1 // indirect filippo.io/edwards25519 v1.0.0-rc.1 // indirect github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect @@ -171,9 +171,9 @@ require ( github.com/google/gopacket v1.1.19 // indirect github.com/google/orderedcode v0.0.1 // indirect github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b // indirect - github.com/google/s2a-go v0.1.4 // indirect + github.com/google/s2a-go v0.1.7 // indirect github.com/google/uuid v1.5.0 // indirect - github.com/googleapis/enterprise-certificate-proxy v0.2.4 // indirect + github.com/googleapis/enterprise-certificate-proxy v0.3.2 // indirect github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect @@ -325,11 +325,11 @@ require ( golang.org/x/tools v0.16.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect - google.golang.org/api v0.128.0 // indirect + google.golang.org/api v0.149.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c // indirect + google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 // indirect + google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 // indirect gopkg.in/ini.v1 v1.67.0 // indirect gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect diff --git a/go.sum b/go.sum index 05e4c0bfcf..3e05a3e503 100644 --- a/go.sum +++ b/go.sum @@ -36,8 +36,8 @@ cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w9 cloud.google.com/go v0.102.0/go.mod h1:oWcCzKlqJ5zgHQt9YsaeTY9KzIvjyy0ArmiBUgpQ+nc= cloud.google.com/go v0.102.1/go.mod h1:XZ77E9qnTEnrgEOvr4xzfdX5TRo7fB4T2F4O6+34hIU= cloud.google.com/go v0.104.0/go.mod h1:OO6xxXdJyvuJPcEPBLN9BJPD+jep5G1+2U5B5gkRYtA= -cloud.google.com/go v0.110.8 h1:tyNdfIxjzaWctIiLYOTalaLKZ17SI44SKFW26QbOhME= -cloud.google.com/go v0.110.8/go.mod h1:Iz8AkXJf1qmxC3Oxoep8R1T36w8B92yU29PcBhHO5fk= +cloud.google.com/go v0.110.10 h1:LXy9GEO+timppncPIAZoOj3l58LIU9k+kn48AN7IO3Y= +cloud.google.com/go v0.110.10/go.mod h1:v1OoFqYxiBkUrruItNM3eT4lLByNjxmJSV/xDKJNnic= cloud.google.com/go/aiplatform v1.22.0/go.mod h1:ig5Nct50bZlzV6NvKaTwmplLLddFx0YReh9WfTO5jKw= cloud.google.com/go/aiplatform v1.24.0/go.mod h1:67UUvRBKG6GTayHKV8DBv2RtR1t93YRu5B1P3x99mYY= cloud.google.com/go/analytics v0.11.0/go.mod h1:DjEWCu41bVbYcKyvlws9Er60YE4a//bK6mnhWvQeFNI= @@ -75,8 +75,8 @@ cloud.google.com/go/compute v1.6.0/go.mod h1:T29tfhtVbq1wvAPo0E3+7vhgmkOYeXjhFvz cloud.google.com/go/compute v1.6.1/go.mod h1:g85FgpzFvNULZ+S8AYq87axRKuf2Kh7deLqV/jJ3thU= cloud.google.com/go/compute v1.7.0/go.mod h1:435lt8av5oL9P3fv1OEzSbSUe+ybHXGMPQHHZWZxy9U= cloud.google.com/go/compute v1.10.0/go.mod h1:ER5CLbMxl90o2jtNbGSbtfOpQKR0t15FOtRsugnLrlU= -cloud.google.com/go/compute v1.23.0 h1:tP41Zoavr8ptEqaW6j+LQOnyBBhO7OkOMAGrgLopTwY= -cloud.google.com/go/compute v1.23.0/go.mod h1:4tCnrn48xsqlwSAiLf1HXMQk8CONslYbdiEZc9FEIbM= +cloud.google.com/go/compute v1.23.3 h1:6sVlXXBmbd7jNX0Ipq0trII3e4n1/MsADLK6a+aiVlk= +cloud.google.com/go/compute v1.23.3/go.mod h1:VCgBUoMnIVIR0CscqQiPJLAG25E3ZRZMzcFZeQ+h8CI= cloud.google.com/go/compute/metadata v0.2.3 h1:mg4jlk7mCAj6xXp9UJ4fjI9VUI5rubuGBW5aJ7UnBMY= cloud.google.com/go/compute/metadata v0.2.3/go.mod h1:VAV5nSsACxMJvgaAuX6Pk2AawlZn8kiOGuCv6gTkwuA= cloud.google.com/go/containeranalysis v0.5.1/go.mod h1:1D92jd8gRR/c0fGMlymRgxWD3Qw9C1ff6/T7mLgVL8I= @@ -116,8 +116,8 @@ cloud.google.com/go/gkehub v0.10.0/go.mod h1:UIPwxI0DsrpsVoWpLB0stwKCP+WFVG9+y97 cloud.google.com/go/grafeas v0.2.0/go.mod h1:KhxgtF2hb0P191HlY5besjYm6MqTSTj3LSI+M+ByZHc= cloud.google.com/go/iam v0.3.0/go.mod h1:XzJPvDayI+9zsASAFO68Hk07u3z+f+JrT2xXNdp4bnY= cloud.google.com/go/iam v0.5.0/go.mod h1:wPU9Vt0P4UmCux7mqtRu6jcpPAb74cP1fh50J3QpkUc= -cloud.google.com/go/iam v1.1.2 h1:gacbrBdWcoVmGLozRuStX45YKvJtzIjJdAolzUs1sm4= -cloud.google.com/go/iam v1.1.2/go.mod h1:A5avdyVL2tCppe4unb0951eI9jreack+RJ0/d+KUZOU= +cloud.google.com/go/iam v1.1.5 h1:1jTsCu4bcsNsE4iiqNT5SHwrDRCfRmIaaaVFhRveTJI= +cloud.google.com/go/iam v1.1.5/go.mod h1:rB6P/Ic3mykPbFio+vo7403drjlgvoWfYpJhMXEbzv8= cloud.google.com/go/language v1.4.0/go.mod h1:F9dRpNFQmJbkaop6g0JhSBXCNlO90e1KWx5iDdxbWic= cloud.google.com/go/language v1.6.0/go.mod h1:6dJ8t3B+lUYfStgls25GusK04NLh3eDLQnWM3mdEbhI= cloud.google.com/go/lifesciences v0.5.0/go.mod h1:3oIKy8ycWGPUyZDR/8RNnTOYevhaMLqh5vLUXs9zvT8= @@ -193,8 +193,8 @@ cloud.google.com/go/webrisk v1.5.0/go.mod h1:iPG6fr52Tv7sGk0H6qUFzmL3HHZev1htXuW cloud.google.com/go/workflows v1.6.0/go.mod h1:6t9F5h/unJz41YqfBmqSASJSXccBLtD1Vwf+KmJENM0= cloud.google.com/go/workflows v1.7.0/go.mod h1:JhSrZuVZWuiDfKEFxU0/F1PQjmpnpcoISEXH2bcHC3M= collectd.org v0.3.0/go.mod h1:A/8DzQBkF6abtvrT2j/AU/4tiBgJWYyh0y/oB/4MlWE= -cosmossdk.io/errors v1.0.0 h1:nxF07lmlBbB8NKQhtJ+sJm6ef5uV1XkvPXG2bUntb04= -cosmossdk.io/errors v1.0.0/go.mod h1:+hJZLuhdDE0pYN8HkOrVNwrIOYvUGnn6+4fjnJs/oV0= +cosmossdk.io/errors v1.0.1 h1:bzu+Kcr0kS/1DuPBtUFdWjzLqyUuCiyHjyJB6srBV/0= +cosmossdk.io/errors v1.0.1/go.mod h1:MeelVSZThMi4bEakzhhhE/CKqVv3nOJDA25bIqRDu/U= cosmossdk.io/math v1.2.0 h1:8gudhTkkD3NxOP2YyyJIYYmt6dQ55ZfJkDOaxXpy7Ig= cosmossdk.io/math v1.2.0/go.mod h1:l2Gnda87F0su8a/7FEKJfFdJrM0JZRXQaohlgJeyQh0= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= @@ -866,8 +866,8 @@ github.com/google/pprof v0.0.0-20221203041831-ce31453925ec/go.mod h1:dDKJzRmX4S3 github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b h1:RMpPgZTSApbPf7xaVel+QkoGPRLFLrwFO89uDUHEGf0= github.com/google/pprof v0.0.0-20231023181126-ff6d637d2a7b/go.mod h1:czg5+yv1E0ZGTi6S6vVK1mke0fV+FaUhNGcd6VRS9Ik= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/s2a-go v0.1.4 h1:1kZ/sQM3srePvKs3tXAvQzo66XfcReoqFpIpIccE7Oc= -github.com/google/s2a-go v0.1.4/go.mod h1:Ej+mSEMGRnqRzjc7VtF+jdBwYG5fuJfiZ8ELkjEwM0A= +github.com/google/s2a-go v0.1.7 h1:60BLSyTrOV4/haCDW4zb1guZItoSq8foHCXrAnjBo/o= +github.com/google/s2a-go v0.1.7/go.mod h1:50CgR4k1jNlWBu4UfS4AcfhVe1r6pdZPygJ3R8F0Qdw= github.com/google/subcommands v1.2.0/go.mod h1:ZjhPrFU+Olkh9WazFPsl27BQ4UPiG37m3yTrtFlrHVk= github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= @@ -879,8 +879,8 @@ github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+ github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.1.0/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/enterprise-certificate-proxy v0.2.0/go.mod h1:8C0jb7/mgJe/9KK8Lm7X9ctZC2t60YyIpYEI16jx0Qg= -github.com/googleapis/enterprise-certificate-proxy v0.2.4 h1:uGy6JWR/uMIILU8wbf+OkstIrNiMjGpEIyhx8f6W7s4= -github.com/googleapis/enterprise-certificate-proxy v0.2.4/go.mod h1:AwSRAtLfXpU5Nm3pW+v7rGDHp09LsPtGY9MduiEsR9k= +github.com/googleapis/enterprise-certificate-proxy v0.3.2 h1:Vie5ybvEvT75RniqhfFxPRy3Bf7vr3h0cechB90XaQs= +github.com/googleapis/enterprise-certificate-proxy v0.3.2/go.mod h1:VLSiSSBs/ksPL8kq3OBOQ6WRI2QnaFynd1DCjZ62+V0= github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY= github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= @@ -2503,7 +2503,6 @@ golang.org/x/crypto v0.0.0-20210513164829-c07d793c2f9a/go.mod h1:P+XmwS30IXTQdn5 golang.org/x/crypto v0.0.0-20210711020723-a769d52b0f97/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20211108221036-ceb1ce70b4fa/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= -golang.org/x/crypto v0.0.0-20220314234659-1baeb1ce4c0b/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220411220226-7b82a4e95df4/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220427172511-eb4f295cb31f/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= @@ -3043,8 +3042,8 @@ google.golang.org/api v0.96.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ google.golang.org/api v0.97.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.98.0/go.mod h1:w7wJQLTM+wvQpNf5JyEcBoxK0RH7EDrh/L4qfsuJ13s= google.golang.org/api v0.100.0/go.mod h1:ZE3Z2+ZOr87Rx7dqFsdRQkRBk36kDtp/h+QpHbB7a70= -google.golang.org/api v0.128.0 h1:RjPESny5CnQRn9V6siglged+DZCgfu9l6mO9dkX9VOg= -google.golang.org/api v0.128.0/go.mod h1:Y611qgqaE92On/7g65MQgxYul3c0rEB894kniWLY750= +google.golang.org/api v0.149.0 h1:b2CqT6kG+zqJIVKRQ3ELJVLN1PwHZ6DJ3dW8yl82rgY= +google.golang.org/api v0.149.0/go.mod h1:Mwn1B7JTXrzXtnvmzQE2BD6bYZQ8DShKZDZbeN9I7qI= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/appengine v1.2.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= google.golang.org/appengine v1.3.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= @@ -3169,12 +3168,12 @@ google.golang.org/genproto v0.0.0-20221010155953-15ba04fc1c0e/go.mod h1:3526vdqw google.golang.org/genproto v0.0.0-20221014173430-6e2ab493f96b/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221014213838-99cd37c6964a/go.mod h1:1vXfmgAz9N9Jx0QA82PqRVauvCz1SGSz739p0f183jM= google.golang.org/genproto v0.0.0-20221025140454-527a21cfbd71/go.mod h1:9qHF0xnpdSfF6knlcsnpzUu5y+rpwgbvsyGAZPBMg4s= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97 h1:SeZZZx0cP0fqUyA+oRzP9k7cSwJlvDFiROO72uwD6i0= -google.golang.org/genproto v0.0.0-20231002182017-d307bd883b97/go.mod h1:t1VqOqqvce95G3hIDCT5FeO3YUc6Q4Oe24L/+rNMxRk= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97 h1:W18sezcAYs+3tDZX4F80yctqa12jcP1PUS2gQu1zTPU= -google.golang.org/genproto/googleapis/api v0.0.0-20231002182017-d307bd883b97/go.mod h1:iargEX0SFPm3xcfMI0d1domjg0ZF4Aa0p2awqyxhvF0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c h1:jHkCUWkseRf+W+edG5hMzr/Uh1xkDREY4caybAq4dpY= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231009173412-8bfb1ae86b6c/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3 h1:1hfbdAfFbkmpg41000wDVqr7jUpK/Yo+LPnIxxGzmkg= +google.golang.org/genproto v0.0.0-20231211222908-989df2bf70f3/go.mod h1:5RBcpGRxr25RbDzY5w+dmaqpSEvl8Gwl1x2CICf60ic= +google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f h1:2yNACc1O40tTnrsbk9Cv6oxiW8pxI/pXj0wRtdlYmgY= +google.golang.org/genproto/googleapis/api v0.0.0-20231120223509-83a465c0220f/go.mod h1:Uy9bTZJqmfrw2rIBxgGLnamc78euZULUBrLZ9XTITKI= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0 h1:/jFB8jK5R3Sq3i/lmeZO0cATSzFfZaJq1J2Euan3XKU= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231212172506-995d672761c0/go.mod h1:FUoWkonphQm3RhTS+kOEhF8h0iDpm4tdXolVCeZ9KKA= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= From 65c6b651df7ddbe8bc48deec189032f3745acbaf Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Fri, 19 Jan 2024 11:32:37 +0100 Subject: [PATCH 365/388] chore: unfork badger and ds-badger4 (#3110) * badger merged leak fix, so we don't need to depend on our fork * We still have to depend on a particular commit as it hasn't been released * we maintained go-ds-badger4, as there was no IPFS one. Now it exists, so we can use it and archive our go-ds-badger4 --- go.mod | 6 ++---- go.sum | 8 ++++---- nodebuilder/store.go | 5 ++--- share/eds/inverted_index.go | 3 +-- share/eds/store_test.go | 2 +- share/getters/getter_test.go | 2 +- 6 files changed, 11 insertions(+), 15 deletions(-) diff --git a/go.mod b/go.mod index 78e627b68f..c75a4023cb 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,6 @@ require ( github.com/alecthomas/jsonschema v0.0.0-20220216202328-9eeeec9d044b github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.4.0 - github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 github.com/celestiaorg/go-fraud v0.2.0 github.com/celestiaorg/go-header v0.5.2 github.com/celestiaorg/go-libp2p-messenger v0.2.0 @@ -18,7 +17,7 @@ require ( github.com/cosmos/cosmos-sdk v0.46.14 github.com/cosmos/cosmos-sdk/api v0.1.0 github.com/cristalhq/jwt v1.2.0 - github.com/dgraph-io/badger/v4 v4.2.1-0.20231013074411-fb1b00959581 + github.com/dgraph-io/badger/v4 v4.2.1-0.20240106094458-1c417aa3799c github.com/etclabscore/go-openrpc-reflect v0.0.37 github.com/filecoin-project/dagstore v0.5.6 github.com/filecoin-project/go-jsonrpc v0.3.1 @@ -34,6 +33,7 @@ require ( github.com/ipfs/go-blockservice v0.5.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 + github.com/ipfs/go-ds-badger4 v0.1.5 github.com/ipfs/go-ipld-cbor v0.1.0 github.com/ipfs/go-ipld-format v0.6.0 github.com/ipfs/go-log/v2 v2.5.1 @@ -347,5 +347,3 @@ replace ( github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 ) - -replace github.com/dgraph-io/badger/v4 => github.com/celestiaorg/badger/v4 v4.0.0-20231125230536-2b9e13346f75 diff --git a/go.sum b/go.sum index 3e05a3e503..30c8df771e 100644 --- a/go.sum +++ b/go.sum @@ -358,8 +358,6 @@ github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7 github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/badger/v4 v4.0.0-20231125230536-2b9e13346f75 h1:dfArr6rKfcB46+/sR/ZZuiBbcB1ZQcpZQT4E4HHcIuE= -github.com/celestiaorg/badger/v4 v4.0.0-20231125230536-2b9e13346f75/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/celestiaorg/celestia-app v1.4.0 h1:hTId3xL8GssN5sHSHPP7svHi/iWp+XVxphStiR7ADiY= github.com/celestiaorg/celestia-app v1.4.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= @@ -368,8 +366,6 @@ github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 h1:+Te28r5Zp4Vp69f82kcON9 github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14/go.mod h1:Og5KKgoBEiQlI6u56lDLG191pfknIdXctFn3COWLQP8= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= -github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5 h1:MJgXvhJP1Au8rXTvMMlBXodu9jplEK1DxiLtMnEphOs= -github.com/celestiaorg/go-ds-badger4 v0.0.0-20230712104058-7ede1c814ac5/go.mod h1:r6xB3nvGotmlTACpAr3SunxtoXeesbqb57elgMJqflY= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= github.com/celestiaorg/go-header v0.5.2 h1:CFsTAXcs1o38JVd8YN1Naq/Yzs6m9orMPEPNpLEgFJA= @@ -551,6 +547,8 @@ github.com/dgraph-io/badger v1.6.2 h1:mNw0qs90GVgGGWylh0umH5iag1j6n/PeJtNvL6KY/x github.com/dgraph-io/badger v1.6.2/go.mod h1:JW2yswe3V058sS0kZ2h/AXeDSqFjxnZcRrVH//y2UQE= github.com/dgraph-io/badger/v2 v2.2007.4 h1:TRWBQg8UrlUhaFdco01nO2uXwzKS7zd+HVdwV/GHc4o= github.com/dgraph-io/badger/v2 v2.2007.4/go.mod h1:vSw/ax2qojzbN6eXHIx6KPKtCSHJN/Uz0X0VPruTIhk= +github.com/dgraph-io/badger/v4 v4.2.1-0.20240106094458-1c417aa3799c h1:Z9rm0wkQBM+VF7vpyrbKnCcSbww0PKygLoptTpkX3d4= +github.com/dgraph-io/badger/v4 v4.2.1-0.20240106094458-1c417aa3799c/go.mod h1:T/uWAYxrXdaXw64ihI++9RMbKTCpKd/yE9+saARew7k= github.com/dgraph-io/ristretto v0.0.2/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3-0.20200630154024-f66de99634de/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= github.com/dgraph-io/ristretto v0.0.3/go.mod h1:KPxhHT9ZxKefz+PCeOGsrHpl1qZ7i70dGTu2u+Ahh6E= @@ -1103,6 +1101,8 @@ github.com/ipfs/go-ds-badger v0.2.1/go.mod h1:Tx7l3aTph3FMFrRS838dcSJh+jjA7cX9Dr github.com/ipfs/go-ds-badger v0.2.3/go.mod h1:pEYw0rgg3FIrywKKnL+Snr+w/LjJZVMTBRn4FS6UHUk= github.com/ipfs/go-ds-badger v0.3.0 h1:xREL3V0EH9S219kFFueOYJJTcjgNSZ2HY1iSvN7U1Ro= github.com/ipfs/go-ds-badger v0.3.0/go.mod h1:1ke6mXNqeV8K3y5Ak2bAA0osoTfmxUdupVCGm4QUIek= +github.com/ipfs/go-ds-badger4 v0.1.5 h1:MwrTsIUJIqH/ChuDdUOzxwxMxHx/Li1ECoSCKsCUxiA= +github.com/ipfs/go-ds-badger4 v0.1.5/go.mod h1:LUU2FbhNdmhAbJmMeoahVRbe4GsduAODSJHWJJh2Vo4= github.com/ipfs/go-ds-leveldb v0.0.1/go.mod h1:feO8V3kubwsEF22n0YRQCffeb79OOYIykR4L04tMOYc= github.com/ipfs/go-ds-leveldb v0.1.0/go.mod h1:hqAW8y4bwX5LWcCtku2rFNX3vjDZCy5LZCg+cSZvYb8= github.com/ipfs/go-ds-leveldb v0.4.1/go.mod h1:jpbku/YqBSsBc1qgME8BkWS4AxzF2cEu1Ii2r79Hh9s= diff --git a/nodebuilder/store.go b/nodebuilder/store.go index f2238c8c3f..7f67a9e782 100644 --- a/nodebuilder/store.go +++ b/nodebuilder/store.go @@ -11,10 +11,9 @@ import ( "github.com/cosmos/cosmos-sdk/crypto/keyring" "github.com/dgraph-io/badger/v4/options" "github.com/ipfs/go-datastore" + dsbadger "github.com/ipfs/go-ds-badger4" "github.com/mitchellh/go-homedir" - dsbadger "github.com/celestiaorg/go-ds-badger4" - "github.com/celestiaorg/celestia-node/libs/fslock" "github.com/celestiaorg/celestia-node/libs/keystore" "github.com/celestiaorg/celestia-node/share" @@ -69,7 +68,7 @@ func OpenStore(path string, ring keyring.Keyring) (Store, error) { ok := IsInit(path) if !ok { - flock.Unlock() //nolint: errcheck + flock.Unlock() //nolint:errcheck return nil, ErrNotInited } diff --git a/share/eds/inverted_index.go b/share/eds/inverted_index.go index dc33c70447..f30d1c37db 100644 --- a/share/eds/inverted_index.go +++ b/share/eds/inverted_index.go @@ -10,9 +10,8 @@ import ( "github.com/filecoin-project/dagstore/index" "github.com/filecoin-project/dagstore/shard" ds "github.com/ipfs/go-datastore" + dsbadger "github.com/ipfs/go-ds-badger4" "github.com/multiformats/go-multihash" - - dsbadger "github.com/celestiaorg/go-ds-badger4" ) const invertedIndexPath = "/inverted_index/" diff --git a/share/eds/store_test.go b/share/eds/store_test.go index 09357347d0..6bc6972bb4 100644 --- a/share/eds/store_test.go +++ b/share/eds/store_test.go @@ -13,12 +13,12 @@ import ( "github.com/ipfs/go-cid" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" + dsbadger "github.com/ipfs/go-ds-badger4" "github.com/ipld/go-car" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - dsbadger "github.com/celestiaorg/go-ds-badger4" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/share" diff --git a/share/getters/getter_test.go b/share/getters/getter_test.go index 8264f6e822..7297766652 100644 --- a/share/getters/getter_test.go +++ b/share/getters/getter_test.go @@ -10,12 +10,12 @@ import ( "github.com/ipfs/boxo/exchange/offline" "github.com/ipfs/go-datastore" ds_sync "github.com/ipfs/go-datastore/sync" + dsbadger "github.com/ipfs/go-ds-badger4" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" "github.com/celestiaorg/celestia-app/pkg/wrapper" - dsbadger "github.com/celestiaorg/go-ds-badger4" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" From ba3800969143901d01e52e4509fd02d68ca7f887 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Fri, 19 Jan 2024 13:21:11 +0200 Subject: [PATCH 366/388] fix(tests/befp): fix panic in befp test (#3108) ShareWithProofs collection happens async and it does not guarantee that Shares at index [1] will be non-nil. Panic happened in `incorrect share with Proof` when `befp.Shares[1] == nil` tested with `go test -v -count=300 -run ^TestBEFP_Validate$ github.com/celestiaorg/celestia-node/share/eds/byzantine` --- share/eds/byzantine/bad_encoding_test.go | 32 +++++++++++------------- 1 file changed, 14 insertions(+), 18 deletions(-) diff --git a/share/eds/byzantine/bad_encoding_test.go b/share/eds/byzantine/bad_encoding_test.go index adb23d3549..e42e3c287c 100644 --- a/share/eds/byzantine/bad_encoding_test.go +++ b/share/eds/byzantine/bad_encoding_test.go @@ -45,7 +45,9 @@ func TestBEFP_Validate(t *testing.T) { var errByz *ErrByzantine require.ErrorAs(t, byzantine, &errByz) - befp := CreateBadEncodingProof([]byte("hash"), 0, errByz) + proof := CreateBadEncodingProof([]byte("hash"), 0, errByz) + befp, ok := proof.(*BadEncodingProof) + require.True(t, ok) var test = []struct { name string prepareFn func() error @@ -54,7 +56,7 @@ func TestBEFP_Validate(t *testing.T) { { name: "valid BEFP", prepareFn: func() error { - return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + return proof.Validate(&header.ExtendedHeader{DAH: &dah}) }, expectedResult: func(err error) { require.NoError(t, err) @@ -88,10 +90,11 @@ func TestBEFP_Validate(t *testing.T) { { name: "incorrect share with Proof", prepareFn: func() error { - befp, ok := befp.(*BadEncodingProof) - require.True(t, ok) - befp.Shares[0].Share = befp.Shares[1].Share - return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + // break the first shareWithProof to test negative case + sh := sharetest.RandShares(t, 2) + nmtProof := nmt.NewInclusionProof(0, 1, nil, false) + befp.Shares[0] = &ShareWithProof{sh[0], &nmtProof} + return proof.Validate(&header.ExtendedHeader{DAH: &dah}) }, expectedResult: func(err error) { require.ErrorIs(t, err, errIncorrectShare) @@ -100,10 +103,8 @@ func TestBEFP_Validate(t *testing.T) { { name: "invalid amount of shares", prepareFn: func() error { - befp, ok := befp.(*BadEncodingProof) - require.True(t, ok) befp.Shares = befp.Shares[0 : len(befp.Shares)/2] - return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + return proof.Validate(&header.ExtendedHeader{DAH: &dah}) }, expectedResult: func(err error) { require.ErrorIs(t, err, errIncorrectAmountOfShares) @@ -112,10 +113,8 @@ func TestBEFP_Validate(t *testing.T) { { name: "not enough shares to recompute the root", prepareFn: func() error { - befp, ok := befp.(*BadEncodingProof) - require.True(t, ok) befp.Shares[0] = nil - return befp.Validate(&header.ExtendedHeader{DAH: &dah}) + return proof.Validate(&header.ExtendedHeader{DAH: &dah}) }, expectedResult: func(err error) { require.ErrorIs(t, err, errIncorrectAmountOfShares) @@ -124,11 +123,8 @@ func TestBEFP_Validate(t *testing.T) { { name: "index out of bounds", prepareFn: func() error { - befp, ok := befp.(*BadEncodingProof) - require.True(t, ok) - befpCopy := *befp - befpCopy.Index = 100 - return befpCopy.Validate(&header.ExtendedHeader{DAH: &dah}) + befp.Index = 100 + return proof.Validate(&header.ExtendedHeader{DAH: &dah}) }, expectedResult: func(err error) { require.ErrorIs(t, err, errIncorrectIndex) @@ -137,7 +133,7 @@ func TestBEFP_Validate(t *testing.T) { { name: "heights mismatch", prepareFn: func() error { - return befp.Validate(&header.ExtendedHeader{ + return proof.Validate(&header.ExtendedHeader{ RawHeader: core.Header{ Height: 42, }, From 2294805ce4146058e9777976de08600354762ad4 Mon Sep 17 00:00:00 2001 From: Artem Bogomaz Date: Fri, 19 Jan 2024 22:59:38 +0700 Subject: [PATCH 367/388] chore(nodebuilder/state): SubmitTx endpoint takes read-level permissions (#3072) refactor: changes SubmitTX permissions from write to read. Following the issue requirement, this PR changes SubmitTX permissions from _write_ to _read_ Changes: * Updated nodebuilder/state/state.go from **write** to **read** permissions * Updated following tests (api/rpc_test.go) and added extra check for SubmitTX rpc call (read permissions check). closes #2958 --------- Co-authored-by: [NODERS]TEAM Co-authored-by: ramin --- api/rpc_test.go | 9 ++++++--- nodebuilder/state/state.go | 2 +- 2 files changed, 7 insertions(+), 4 deletions(-) diff --git a/api/rpc_test.go b/api/rpc_test.go index 9ff35bf1e4..ec308a2320 100644 --- a/api/rpc_test.go +++ b/api/rpc_test.go @@ -186,12 +186,15 @@ func TestAuthedRPC(t *testing.T) { // 2. Test method with write-level permissions expectedResp := &state.TxResponse{} if tt.perm > 2 { - server.State.EXPECT().SubmitTx(gomock.Any(), gomock.Any()).Return(expectedResp, nil) - txResp, err := rpcClient.State.SubmitTx(ctx, []byte{}) + server.State.EXPECT().Delegate(gomock.Any(), gomock.Any(), + gomock.Any(), gomock.Any(), gomock.Any()).Return(expectedResp, nil) + txResp, err := rpcClient.State.Delegate(ctx, + state.ValAddress{}, state.Int{}, state.Int{}, 0) require.NoError(t, err) require.Equal(t, expectedResp, txResp) } else { - _, err := rpcClient.State.SubmitTx(ctx, []byte{}) + _, err := rpcClient.State.Delegate(ctx, + state.ValAddress{}, state.Int{}, state.Int{}, 0) require.Error(t, err) require.ErrorContains(t, err, "missing permission") } diff --git a/nodebuilder/state/state.go b/nodebuilder/state/state.go index 624f8dcd3f..52a2317445 100644 --- a/nodebuilder/state/state.go +++ b/nodebuilder/state/state.go @@ -110,7 +110,7 @@ type API struct { fee state.Int, gasLimit uint64, ) (*state.TxResponse, error) `perm:"write"` - SubmitTx func(ctx context.Context, tx state.Tx) (*state.TxResponse, error) `perm:"write"` + SubmitTx func(ctx context.Context, tx state.Tx) (*state.TxResponse, error) `perm:"read"` SubmitPayForBlob func( ctx context.Context, fee state.Int, From 8f768a71ee11614b49001490347fc33cdd2997a6 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 22 Jan 2024 10:39:06 -0600 Subject: [PATCH 368/388] chore(deps): Bump the otel group with 4 updates (#3121) Bumps the otel group with 4 updates: [go.opentelemetry.io/otel](https://github.com/open-telemetry/opentelemetry-go), [go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp](https://github.com/open-telemetry/opentelemetry-go), [go.opentelemetry.io/otel/exporters/otlp/otlptrace](https://github.com/open-telemetry/opentelemetry-go) and [go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp](https://github.com/open-telemetry/opentelemetry-go). Updates `go.opentelemetry.io/otel` from 1.21.0 to 1.22.0
Changelog

Sourced from go.opentelemetry.io/otel's changelog.

[1.22.0/0.45.0] 2024-01-17

Added

  • The go.opentelemetry.io/otel/semconv/v1.22.0 package. The package contains semantic conventions from the v1.22.0 version of the OpenTelemetry Semantic Conventions. (#4735)
  • The go.opentelemetry.io/otel/semconv/v1.23.0 package. The package contains semantic conventions from the v1.23.0 version of the OpenTelemetry Semantic Conventions. (#4746)
  • The go.opentelemetry.io/otel/semconv/v1.23.1 package. The package contains semantic conventions from the v1.23.1 version of the OpenTelemetry Semantic Conventions. (#4749)
  • The go.opentelemetry.io/otel/semconv/v1.24.0 package. The package contains semantic conventions from the v1.24.0 version of the OpenTelemetry Semantic Conventions. (#4770)
  • Add WithResourceAsConstantLabels option to apply resource attributes for every metric emitted by the Prometheus exporter. (#4733)
  • Experimental cardinality limiting is added to the metric SDK. See metric documentation for more information about this feature and how to enable it. (#4457)
  • Add NewMemberRaw and NewKeyValuePropertyRaw in go.opentelemetry.io/otel/baggage. (#4804)

Changed

  • Upgrade all use of go.opentelemetry.io/otel/semconv to use v1.24.0. (#4754)
  • Update transformations in go.opentelemetry.io/otel/exporters/zipkin to follow v1.19.0 version of the OpenTelemetry specification. (#4754)
  • Record synchronous measurements when the passed context is canceled instead of dropping in go.opentelemetry.io/otel/sdk/metric. If you do not want to make a measurement when the context is cancelled, you need to handle it yourself (e.g if ctx.Err() != nil). (#4671)
  • Improve go.opentelemetry.io/otel/trace.TraceState's performance. (#4722)
  • Improve go.opentelemetry.io/otel/propagation.TraceContext's performance. (#4721)
  • Improve go.opentelemetry.io/otel/baggage performance. (#4743)
  • Improve performance of the (*Set).Filter method in go.opentelemetry.io/otel/attribute when the passed filter does not filter out any attributes from the set. (#4774)
  • Member.String in go.opentelemetry.io/otel/baggage percent-encodes only when necessary. (#4775)
  • Improve go.opentelemetry.io/otel/trace.Span's performance when adding multiple attributes. (#4818)
  • Property.Value in go.opentelemetry.io/otel/baggage now returns a raw string instead of a percent-encoded value. (#4804)

Fixed

  • Fix Parse in go.opentelemetry.io/otel/baggage to validate member value before percent-decoding. (#4755)
  • Fix whitespace encoding of Member.String in go.opentelemetry.io/otel/baggage. (#4756)
  • Fix observable not registered error when the asynchronous instrument has a drop aggregation in go.opentelemetry.io/otel/sdk/metric. (#4772)
  • Fix baggage item key so that it is not canonicalized in go.opentelemetry.io/otel/bridge/opentracing. (#4776)
  • Fix go.opentelemetry.io/otel/bridge/opentracing to properly handle baggage values that requires escaping during propagation. (#4804)
  • Fix a bug where using multiple readers resulted in incorrect asynchronous counter values in go.opentelemetry.io/otel/sdk/metric. (#4742)
Commits
  • 279c549 Release v1.22.0/v0.45.0 (#4821)
  • 237ed37 Fix link changes from instrumentation to languages (#4828)
  • 5ed29d9 Bump lycheeverse/lychee-action from 1.9.0 to 1.9.1 (#4824)
  • 4491b39 sdk/trace: use slices.Grow() to avoid excessive runtime.growslice() (#4818)
  • 19622d3 chore(docs): explicitly mark lizthegrey emeritus (#4822)
  • 7fa7d1b sdk/metric: Fix observable not registered error when the asynchronous instrum...
  • 01472db Upgrade use of semconv to v1.24.0 (#4754)
  • 259143a baggage: Add NewMemberRaw and NewKeyValuePropertyRaw (#4804)
  • 6ead8d8 Bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 in /internal/tools (#4815)
  • deddec3 Optimize (attribute.Set).Filter for no filtered case (#4774)
  • Additional commits viewable in compare view

Updates `go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp` from 0.44.0 to 0.45.0
Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp's changelog.

[1.22.0/0.45.0] 2024-01-17

Added

  • The go.opentelemetry.io/otel/semconv/v1.22.0 package. The package contains semantic conventions from the v1.22.0 version of the OpenTelemetry Semantic Conventions. (#4735)
  • The go.opentelemetry.io/otel/semconv/v1.23.0 package. The package contains semantic conventions from the v1.23.0 version of the OpenTelemetry Semantic Conventions. (#4746)
  • The go.opentelemetry.io/otel/semconv/v1.23.1 package. The package contains semantic conventions from the v1.23.1 version of the OpenTelemetry Semantic Conventions. (#4749)
  • The go.opentelemetry.io/otel/semconv/v1.24.0 package. The package contains semantic conventions from the v1.24.0 version of the OpenTelemetry Semantic Conventions. (#4770)
  • Add WithResourceAsConstantLabels option to apply resource attributes for every metric emitted by the Prometheus exporter. (#4733)
  • Experimental cardinality limiting is added to the metric SDK. See metric documentation for more information about this feature and how to enable it. (#4457)
  • Add NewMemberRaw and NewKeyValuePropertyRaw in go.opentelemetry.io/otel/baggage. (#4804)

Changed

  • Upgrade all use of go.opentelemetry.io/otel/semconv to use v1.24.0. (#4754)
  • Update transformations in go.opentelemetry.io/otel/exporters/zipkin to follow v1.19.0 version of the OpenTelemetry specification. (#4754)
  • Record synchronous measurements when the passed context is canceled instead of dropping in go.opentelemetry.io/otel/sdk/metric. If you do not want to make a measurement when the context is cancelled, you need to handle it yourself (e.g if ctx.Err() != nil). (#4671)
  • Improve go.opentelemetry.io/otel/trace.TraceState's performance. (#4722)
  • Improve go.opentelemetry.io/otel/propagation.TraceContext's performance. (#4721)
  • Improve go.opentelemetry.io/otel/baggage performance. (#4743)
  • Improve performance of the (*Set).Filter method in go.opentelemetry.io/otel/attribute when the passed filter does not filter out any attributes from the set. (#4774)
  • Member.String in go.opentelemetry.io/otel/baggage percent-encodes only when necessary. (#4775)
  • Improve go.opentelemetry.io/otel/trace.Span's performance when adding multiple attributes. (#4818)
  • Property.Value in go.opentelemetry.io/otel/baggage now returns a raw string instead of a percent-encoded value. (#4804)

Fixed

  • Fix Parse in go.opentelemetry.io/otel/baggage to validate member value before percent-decoding. (#4755)
  • Fix whitespace encoding of Member.String in go.opentelemetry.io/otel/baggage. (#4756)
  • Fix observable not registered error when the asynchronous instrument has a drop aggregation in go.opentelemetry.io/otel/sdk/metric. (#4772)
  • Fix baggage item key so that it is not canonicalized in go.opentelemetry.io/otel/bridge/opentracing. (#4776)
  • Fix go.opentelemetry.io/otel/bridge/opentracing to properly handle baggage values that requires escaping during propagation. (#4804)
  • Fix a bug where using multiple readers resulted in incorrect asynchronous counter values in go.opentelemetry.io/otel/sdk/metric. (#4742)
Commits
  • 279c549 Release v1.22.0/v0.45.0 (#4821)
  • 237ed37 Fix link changes from instrumentation to languages (#4828)
  • 5ed29d9 Bump lycheeverse/lychee-action from 1.9.0 to 1.9.1 (#4824)
  • 4491b39 sdk/trace: use slices.Grow() to avoid excessive runtime.growslice() (#4818)
  • 19622d3 chore(docs): explicitly mark lizthegrey emeritus (#4822)
  • 7fa7d1b sdk/metric: Fix observable not registered error when the asynchronous instrum...
  • 01472db Upgrade use of semconv to v1.24.0 (#4754)
  • 259143a baggage: Add NewMemberRaw and NewKeyValuePropertyRaw (#4804)
  • 6ead8d8 Bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 in /internal/tools (#4815)
  • deddec3 Optimize (attribute.Set).Filter for no filtered case (#4774)
  • Additional commits viewable in compare view

Updates `go.opentelemetry.io/otel/exporters/otlp/otlptrace` from 1.21.0 to 1.22.0
Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlptrace's changelog.

[1.22.0/0.45.0] 2024-01-17

Added

  • The go.opentelemetry.io/otel/semconv/v1.22.0 package. The package contains semantic conventions from the v1.22.0 version of the OpenTelemetry Semantic Conventions. (#4735)
  • The go.opentelemetry.io/otel/semconv/v1.23.0 package. The package contains semantic conventions from the v1.23.0 version of the OpenTelemetry Semantic Conventions. (#4746)
  • The go.opentelemetry.io/otel/semconv/v1.23.1 package. The package contains semantic conventions from the v1.23.1 version of the OpenTelemetry Semantic Conventions. (#4749)
  • The go.opentelemetry.io/otel/semconv/v1.24.0 package. The package contains semantic conventions from the v1.24.0 version of the OpenTelemetry Semantic Conventions. (#4770)
  • Add WithResourceAsConstantLabels option to apply resource attributes for every metric emitted by the Prometheus exporter. (#4733)
  • Experimental cardinality limiting is added to the metric SDK. See metric documentation for more information about this feature and how to enable it. (#4457)
  • Add NewMemberRaw and NewKeyValuePropertyRaw in go.opentelemetry.io/otel/baggage. (#4804)

Changed

  • Upgrade all use of go.opentelemetry.io/otel/semconv to use v1.24.0. (#4754)
  • Update transformations in go.opentelemetry.io/otel/exporters/zipkin to follow v1.19.0 version of the OpenTelemetry specification. (#4754)
  • Record synchronous measurements when the passed context is canceled instead of dropping in go.opentelemetry.io/otel/sdk/metric. If you do not want to make a measurement when the context is cancelled, you need to handle it yourself (e.g if ctx.Err() != nil). (#4671)
  • Improve go.opentelemetry.io/otel/trace.TraceState's performance. (#4722)
  • Improve go.opentelemetry.io/otel/propagation.TraceContext's performance. (#4721)
  • Improve go.opentelemetry.io/otel/baggage performance. (#4743)
  • Improve performance of the (*Set).Filter method in go.opentelemetry.io/otel/attribute when the passed filter does not filter out any attributes from the set. (#4774)
  • Member.String in go.opentelemetry.io/otel/baggage percent-encodes only when necessary. (#4775)
  • Improve go.opentelemetry.io/otel/trace.Span's performance when adding multiple attributes. (#4818)
  • Property.Value in go.opentelemetry.io/otel/baggage now returns a raw string instead of a percent-encoded value. (#4804)

Fixed

  • Fix Parse in go.opentelemetry.io/otel/baggage to validate member value before percent-decoding. (#4755)
  • Fix whitespace encoding of Member.String in go.opentelemetry.io/otel/baggage. (#4756)
  • Fix observable not registered error when the asynchronous instrument has a drop aggregation in go.opentelemetry.io/otel/sdk/metric. (#4772)
  • Fix baggage item key so that it is not canonicalized in go.opentelemetry.io/otel/bridge/opentracing. (#4776)
  • Fix go.opentelemetry.io/otel/bridge/opentracing to properly handle baggage values that requires escaping during propagation. (#4804)
  • Fix a bug where using multiple readers resulted in incorrect asynchronous counter values in go.opentelemetry.io/otel/sdk/metric. (#4742)
Commits
  • 279c549 Release v1.22.0/v0.45.0 (#4821)
  • 237ed37 Fix link changes from instrumentation to languages (#4828)
  • 5ed29d9 Bump lycheeverse/lychee-action from 1.9.0 to 1.9.1 (#4824)
  • 4491b39 sdk/trace: use slices.Grow() to avoid excessive runtime.growslice() (#4818)
  • 19622d3 chore(docs): explicitly mark lizthegrey emeritus (#4822)
  • 7fa7d1b sdk/metric: Fix observable not registered error when the asynchronous instrum...
  • 01472db Upgrade use of semconv to v1.24.0 (#4754)
  • 259143a baggage: Add NewMemberRaw and NewKeyValuePropertyRaw (#4804)
  • 6ead8d8 Bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 in /internal/tools (#4815)
  • deddec3 Optimize (attribute.Set).Filter for no filtered case (#4774)
  • Additional commits viewable in compare view

Updates `go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp` from 1.21.0 to 1.22.0
Changelog

Sourced from go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp's changelog.

[1.22.0/0.45.0] 2024-01-17

Added

  • The go.opentelemetry.io/otel/semconv/v1.22.0 package. The package contains semantic conventions from the v1.22.0 version of the OpenTelemetry Semantic Conventions. (#4735)
  • The go.opentelemetry.io/otel/semconv/v1.23.0 package. The package contains semantic conventions from the v1.23.0 version of the OpenTelemetry Semantic Conventions. (#4746)
  • The go.opentelemetry.io/otel/semconv/v1.23.1 package. The package contains semantic conventions from the v1.23.1 version of the OpenTelemetry Semantic Conventions. (#4749)
  • The go.opentelemetry.io/otel/semconv/v1.24.0 package. The package contains semantic conventions from the v1.24.0 version of the OpenTelemetry Semantic Conventions. (#4770)
  • Add WithResourceAsConstantLabels option to apply resource attributes for every metric emitted by the Prometheus exporter. (#4733)
  • Experimental cardinality limiting is added to the metric SDK. See metric documentation for more information about this feature and how to enable it. (#4457)
  • Add NewMemberRaw and NewKeyValuePropertyRaw in go.opentelemetry.io/otel/baggage. (#4804)

Changed

  • Upgrade all use of go.opentelemetry.io/otel/semconv to use v1.24.0. (#4754)
  • Update transformations in go.opentelemetry.io/otel/exporters/zipkin to follow v1.19.0 version of the OpenTelemetry specification. (#4754)
  • Record synchronous measurements when the passed context is canceled instead of dropping in go.opentelemetry.io/otel/sdk/metric. If you do not want to make a measurement when the context is cancelled, you need to handle it yourself (e.g if ctx.Err() != nil). (#4671)
  • Improve go.opentelemetry.io/otel/trace.TraceState's performance. (#4722)
  • Improve go.opentelemetry.io/otel/propagation.TraceContext's performance. (#4721)
  • Improve go.opentelemetry.io/otel/baggage performance. (#4743)
  • Improve performance of the (*Set).Filter method in go.opentelemetry.io/otel/attribute when the passed filter does not filter out any attributes from the set. (#4774)
  • Member.String in go.opentelemetry.io/otel/baggage percent-encodes only when necessary. (#4775)
  • Improve go.opentelemetry.io/otel/trace.Span's performance when adding multiple attributes. (#4818)
  • Property.Value in go.opentelemetry.io/otel/baggage now returns a raw string instead of a percent-encoded value. (#4804)

Fixed

  • Fix Parse in go.opentelemetry.io/otel/baggage to validate member value before percent-decoding. (#4755)
  • Fix whitespace encoding of Member.String in go.opentelemetry.io/otel/baggage. (#4756)
  • Fix observable not registered error when the asynchronous instrument has a drop aggregation in go.opentelemetry.io/otel/sdk/metric. (#4772)
  • Fix baggage item key so that it is not canonicalized in go.opentelemetry.io/otel/bridge/opentracing. (#4776)
  • Fix go.opentelemetry.io/otel/bridge/opentracing to properly handle baggage values that requires escaping during propagation. (#4804)
  • Fix a bug where using multiple readers resulted in incorrect asynchronous counter values in go.opentelemetry.io/otel/sdk/metric. (#4742)
Commits
  • 279c549 Release v1.22.0/v0.45.0 (#4821)
  • 237ed37 Fix link changes from instrumentation to languages (#4828)
  • 5ed29d9 Bump lycheeverse/lychee-action from 1.9.0 to 1.9.1 (#4824)
  • 4491b39 sdk/trace: use slices.Grow() to avoid excessive runtime.growslice() (#4818)
  • 19622d3 chore(docs): explicitly mark lizthegrey emeritus (#4822)
  • 7fa7d1b sdk/metric: Fix observable not registered error when the asynchronous instrum...
  • 01472db Upgrade use of semconv to v1.24.0 (#4754)
  • 259143a baggage: Add NewMemberRaw and NewKeyValuePropertyRaw (#4804)
  • 6ead8d8 Bump github.com/cloudflare/circl from 1.3.3 to 1.3.7 in /internal/tools (#4815)
  • deddec3 Optimize (attribute.Set).Filter for no filtered case (#4774)
  • Additional commits viewable in compare view

Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 16 ++++++++-------- go.sum | 32 ++++++++++++++++---------------- 2 files changed, 24 insertions(+), 24 deletions(-) diff --git a/go.mod b/go.mod index c75a4023cb..0291091e53 100644 --- a/go.mod +++ b/go.mod @@ -57,14 +57,14 @@ require ( github.com/stretchr/testify v1.8.4 github.com/tendermint/tendermint v0.34.28 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 - go.opentelemetry.io/otel v1.21.0 - go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 - go.opentelemetry.io/otel/metric v1.21.0 - go.opentelemetry.io/otel/sdk v1.21.0 - go.opentelemetry.io/otel/sdk/metric v1.21.0 - go.opentelemetry.io/otel/trace v1.21.0 + go.opentelemetry.io/otel v1.22.0 + go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 + go.opentelemetry.io/otel/metric v1.22.0 + go.opentelemetry.io/otel/sdk v1.22.0 + go.opentelemetry.io/otel/sdk/metric v1.22.0 + go.opentelemetry.io/otel/trace v1.22.0 go.opentelemetry.io/proto/otlp v1.0.0 go.uber.org/fx v1.20.1 go.uber.org/zap v1.26.0 diff --git a/go.sum b/go.sum index 30c8df771e..bfa5181462 100644 --- a/go.sum +++ b/go.sum @@ -2387,32 +2387,32 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= -go.opentelemetry.io/otel v1.21.0 h1:hzLeKBZEL7Okw2mGzZ0cc4k/A7Fta0uoPgaJCr8fsFc= -go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0 h1:bflGWrfYyuulcdxf14V6n9+CoQcu5SAAdHmDPAJnlps= -go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.44.0/go.mod h1:qcTO4xHAxZLaLxPd60TdE88rxtItPHgHWqOhOGRr0as= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0 h1:cl5P5/GIfFh4t6xyruOgJP5QiA1pw4fYYdv6nc6CBWw= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.21.0/go.mod h1:zgBdWWAu7oEEMC06MMKc5NLbA/1YDXV1sMpSqEeLQLg= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0 h1:digkEZCJWobwBqMwC0cwCq8/wkkRy/OowZg5OArWZrM= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.21.0/go.mod h1:/OpE/y70qVkndM0TrxT4KBoN3RsFZP0QaofcfYrj76I= +go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= +go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 h1:+RbSCde0ERway5FwKvXR3aRJIFeDu9rtwC6E7BC6uoM= +go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0/go.mod h1:zcI8u2EJxbLPyoZ3SkVAAcQPgYb1TDRzW93xLFnsggU= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0 h1:9M3+rhx7kZCIQQhQRYaZCdNu1V73tm4TvXs2ntl98C4= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.22.0/go.mod h1:noq80iT8rrHP1SfybmPiRGc9dc5M8RPmGvtwo7Oo7tc= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0 h1:FyjCyI9jVEfqhUh2MoSkmolPjfh5fp2hnV0b0irxH4Q= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h1:hYwym2nDEeZfG/motx0p7L7J1N1vyzIThemQsb4g2qY= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= -go.opentelemetry.io/otel/metric v1.21.0 h1:tlYWfeo+Bocx5kLEloTjbcDwBuELRrIFxwdQ36PlJu4= -go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= +go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= +go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= -go.opentelemetry.io/otel/sdk v1.21.0 h1:FTt8qirL1EysG6sTQRZ5TokkU8d0ugCj8htOgThZXQ8= -go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= -go.opentelemetry.io/otel/sdk/metric v1.21.0 h1:smhI5oD714d6jHE6Tie36fPx4WDFIg+Y6RfAY4ICcR0= -go.opentelemetry.io/otel/sdk/metric v1.21.0/go.mod h1:FJ8RAsoPGv/wYMgBdUJXOm+6pzFY3YdljnXtv1SBE8Q= +go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= +go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= +go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI= +go.opentelemetry.io/otel/sdk/metric v1.22.0/go.mod h1:KjQGeMIDlBNEOo6HvjhxIec1p/69/kULDcp4gr0oLQQ= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= -go.opentelemetry.io/otel/trace v1.21.0 h1:WD9i5gzvoUPuXIXH24ZNBudiarZDKuekPqi/E8fpfLc= -go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= +go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= +go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= From c173a1e34710fd33c485e5b8ada10222fd98669f Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 23 Jan 2024 10:31:55 +0000 Subject: [PATCH 369/388] test(gateway): slight refactor of some presentation and add some endpoint registration tests (#2939) goals: readability, test coverage Appreciate that the gateway is v unimportant but is exists. I was starting at the top just looking for places to give a clean up and add some unit test coverage and found this. Mostly trying to make it cleaner to read, then added some tests, also wanted to add a `/status/health` endpoint just for ease, i would want this as a node operator if i was running with `--gateway` for whatever reason Also took liberty to take some error handling code out of `writeError` that would never execute. In trying to write a test to trigger the json.Marshal error case, i realized that as err.Error() always returns a string it could never trigger json.Unmarshal to fail without panic before even getting there, so removed that code path. i might revisit this and give it another refactor at some point soon, unless we're planning to remove this completely? on to next area --- api/gateway/bindings.go | 73 +++++++++++++++++++++ api/gateway/bindings_test.go | 119 +++++++++++++++++++++++++++++++++++ api/gateway/endpoints.go | 32 ---------- api/gateway/health.go | 16 +++++ api/gateway/server.go | 4 ++ api/gateway/util.go | 16 +++-- api/gateway/util_test.go | 24 +++++++ cmd/celestia/cmd_test.go | 4 +- 8 files changed, 245 insertions(+), 43 deletions(-) create mode 100644 api/gateway/bindings.go create mode 100644 api/gateway/bindings_test.go delete mode 100644 api/gateway/endpoints.go create mode 100644 api/gateway/health.go create mode 100644 api/gateway/util_test.go diff --git a/api/gateway/bindings.go b/api/gateway/bindings.go new file mode 100644 index 0000000000..c01bd2da47 --- /dev/null +++ b/api/gateway/bindings.go @@ -0,0 +1,73 @@ +package gateway + +import ( + "fmt" + "net/http" +) + +func (h *Handler) RegisterEndpoints(rpc *Server) { + // state endpoints + rpc.RegisterHandlerFunc( + fmt.Sprintf("%s/{%s}", balanceEndpoint, addrKey), + h.handleBalanceRequest, + http.MethodGet, + ) + + rpc.RegisterHandlerFunc( + submitTxEndpoint, + h.handleSubmitTx, + http.MethodPost, + ) + + rpc.RegisterHandlerFunc( + healthEndpoint, + h.handleHealthRequest, + http.MethodGet, + ) + + // share endpoints + rpc.RegisterHandlerFunc( + fmt.Sprintf( + "%s/{%s}/height/{%s}", + namespacedSharesEndpoint, + namespaceKey, + heightKey, + ), + h.handleSharesByNamespaceRequest, + http.MethodGet, + ) + + rpc.RegisterHandlerFunc( + fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, namespaceKey), + h.handleSharesByNamespaceRequest, + http.MethodGet, + ) + + rpc.RegisterHandlerFunc( + fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, namespaceKey, heightKey), + h.handleDataByNamespaceRequest, + http.MethodGet, + ) + + rpc.RegisterHandlerFunc( + fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, namespaceKey), + h.handleDataByNamespaceRequest, + http.MethodGet, + ) + + // DAS endpoints + rpc.RegisterHandlerFunc( + fmt.Sprintf("%s/{%s}", heightAvailabilityEndpoint, heightKey), + h.handleHeightAvailabilityRequest, + http.MethodGet, + ) + + // header endpoints + rpc.RegisterHandlerFunc( + fmt.Sprintf("%s/{%s}", headerByHeightEndpoint, heightKey), + h.handleHeaderRequest, + http.MethodGet, + ) + + rpc.RegisterHandlerFunc(headEndpoint, h.handleHeadRequest, http.MethodGet) +} diff --git a/api/gateway/bindings_test.go b/api/gateway/bindings_test.go new file mode 100644 index 0000000000..5d27d5e4c7 --- /dev/null +++ b/api/gateway/bindings_test.go @@ -0,0 +1,119 @@ +package gateway + +import ( + "fmt" + "net/http" + "testing" + + "github.com/gorilla/mux" + "github.com/stretchr/testify/require" +) + +func TestRegisterEndpoints(t *testing.T) { + handler := &Handler{} + rpc := NewServer("localhost", "6969") + + handler.RegisterEndpoints(rpc) + + testCases := []struct { + name string + path string + method string + expected bool + }{ + { + name: "Get balance endpoint", + path: fmt.Sprintf("%s/{%s}", balanceEndpoint, addrKey), + method: http.MethodGet, + expected: true, + }, + { + name: "Submit transaction endpoint", + path: submitTxEndpoint, + method: http.MethodPost, + expected: true, + }, + { + name: "Get namespaced shares by height endpoint", + path: fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, namespaceKey, heightKey), + method: http.MethodGet, + expected: true, + }, + { + name: "Get namespaced shares endpoint", + path: fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, namespaceKey), + method: http.MethodGet, + expected: true, + }, + { + name: "Get namespaced data by height endpoint", + path: fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, namespaceKey, heightKey), + method: http.MethodGet, + expected: true, + }, + { + name: "Get namespaced data endpoint", + path: fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, namespaceKey), + method: http.MethodGet, + expected: true, + }, + { + name: "Get health endpoint", + path: "/status/health", + method: http.MethodGet, + expected: true, + }, + + // Going forward, we can add previously deprecated and since + // removed endpoints here to ensure we don't accidentally re-enable + // them in the future and accidentally expand surface area + { + name: "example totally bogus endpoint", + path: fmt.Sprintf("/wutang/{%s}/%s", "chambers", "36"), + method: http.MethodGet, + expected: false, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + require.Equal( + t, + tc.expected, + hasEndpointRegistered(rpc.Router(), tc.path, tc.method), + "Endpoint registration mismatch for: %s %s %s", tc.name, tc.method, tc.path) + }) + } +} + +func hasEndpointRegistered(router *mux.Router, path string, method string) bool { + var registered bool + err := router.Walk(func(route *mux.Route, router *mux.Router, ancestors []*mux.Route) error { + template, err := route.GetPathTemplate() + if err != nil { + return err + } + + if template == path { + methods, err := route.GetMethods() + if err != nil { + return err + } + + for _, m := range methods { + if m == method { + registered = true + return nil + } + } + } + return nil + }) + + if err != nil { + fmt.Println("Error walking through routes:", err) + return false + } + + return registered +} diff --git a/api/gateway/endpoints.go b/api/gateway/endpoints.go deleted file mode 100644 index 104d01b053..0000000000 --- a/api/gateway/endpoints.go +++ /dev/null @@ -1,32 +0,0 @@ -package gateway - -import ( - "fmt" - "net/http" -) - -func (h *Handler) RegisterEndpoints(rpc *Server) { - // state endpoints - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", balanceEndpoint, addrKey), h.handleBalanceRequest, - http.MethodGet) - rpc.RegisterHandlerFunc(submitTxEndpoint, h.handleSubmitTx, http.MethodPost) - - // share endpoints - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedSharesEndpoint, namespaceKey, heightKey), - h.handleSharesByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedSharesEndpoint, namespaceKey), - h.handleSharesByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}/height/{%s}", namespacedDataEndpoint, namespaceKey, heightKey), - h.handleDataByNamespaceRequest, http.MethodGet) - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", namespacedDataEndpoint, namespaceKey), - h.handleDataByNamespaceRequest, http.MethodGet) - - // DAS endpoints - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", heightAvailabilityEndpoint, heightKey), - h.handleHeightAvailabilityRequest, http.MethodGet) - - // header endpoints - rpc.RegisterHandlerFunc(fmt.Sprintf("%s/{%s}", headerByHeightEndpoint, heightKey), h.handleHeaderRequest, - http.MethodGet) - rpc.RegisterHandlerFunc(headEndpoint, h.handleHeadRequest, http.MethodGet) -} diff --git a/api/gateway/health.go b/api/gateway/health.go new file mode 100644 index 0000000000..0ddb56bb67 --- /dev/null +++ b/api/gateway/health.go @@ -0,0 +1,16 @@ +package gateway + +import "net/http" + +const ( + healthEndpoint = "/status/health" +) + +func (h *Handler) handleHealthRequest(w http.ResponseWriter, _ *http.Request) { + _, err := w.Write([]byte("ok")) + if err != nil { + log.Errorw("serving request", "endpoint", healthEndpoint, "err", err) + writeError(w, http.StatusBadGateway, healthEndpoint, err) + return + } +} diff --git a/api/gateway/server.go b/api/gateway/server.go index 181bfdfe55..7eab7c7bf9 100644 --- a/api/gateway/server.go +++ b/api/gateway/server.go @@ -36,6 +36,10 @@ func NewServer(address, port string) *Server { return server } +func (s *Server) Router() *mux.Router { + return s.srvMux +} + // Start starts the gateway Server, listening on the given address. func (s *Server) Start(context.Context) error { couldStart := s.started.CompareAndSwap(false, true) diff --git a/api/gateway/util.go b/api/gateway/util.go index bffd7ebc88..d3739f9e9c 100644 --- a/api/gateway/util.go +++ b/api/gateway/util.go @@ -1,7 +1,6 @@ package gateway import ( - "encoding/json" "net/http" ) @@ -9,13 +8,12 @@ func writeError(w http.ResponseWriter, statusCode int, endpoint string, err erro log.Debugw("serving request", "endpoint", endpoint, "err", err) w.WriteHeader(statusCode) - errBody, jerr := json.Marshal(err.Error()) - if jerr != nil { - log.Errorw("serializing error", "endpoint", endpoint, "err", jerr) - return - } - _, werr := w.Write(errBody) - if werr != nil { - log.Errorw("writing error response", "endpoint", endpoint, "err", werr) + + errorMessage := err.Error() // Get the error message as a string + errorBytes := []byte(errorMessage) + + _, err = w.Write(errorBytes) + if err != nil { + log.Errorw("writing error response", "endpoint", endpoint, "err", err) } } diff --git a/api/gateway/util_test.go b/api/gateway/util_test.go new file mode 100644 index 0000000000..d41b0918d2 --- /dev/null +++ b/api/gateway/util_test.go @@ -0,0 +1,24 @@ +package gateway + +import ( + "errors" + "net/http" + "net/http/httptest" + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestWriteError(t *testing.T) { + t.Run("writeError", func(t *testing.T) { + // Create a mock HTTP response writer + w := httptest.NewRecorder() + + testErr := errors.New("test error") + + writeError(w, http.StatusInternalServerError, "/api/endpoint", testErr) + assert.Equal(t, http.StatusInternalServerError, w.Code) + responseBody := w.Body.Bytes() + assert.Equal(t, testErr.Error(), string(responseBody)) + }) +} diff --git a/cmd/celestia/cmd_test.go b/cmd/celestia/cmd_test.go index 9c26489e14..94dd3625b8 100644 --- a/cmd/celestia/cmd_test.go +++ b/cmd/celestia/cmd_test.go @@ -33,7 +33,7 @@ func TestCompletionHelpString(t *testing.T) { } methods := reflect.VisibleFields(reflect.TypeOf(TestFields{})) for i, method := range methods { - require.Equal(t, testOutputs[i], parseSignatureForHelpstring(method)) + require.Equal(t, testOutputs[i], parseSignatureForHelpString(method)) } } @@ -129,7 +129,7 @@ func TestBridge(t *testing.T) { */ } -func parseSignatureForHelpstring(methodSig reflect.StructField) string { +func parseSignatureForHelpString(methodSig reflect.StructField) string { simplifiedSignature := "(" in, out := methodSig.Type.NumIn(), methodSig.Type.NumOut() for i := 1; i < in; i++ { From dd18169ec8a0fa088df39998ed47149500197c23 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 23 Jan 2024 13:01:56 +0200 Subject: [PATCH 370/388] feat(blob/service): cover getByCommitment with benchmark (#3116) Added the benchmark for `getByCommitment` `BenchmarkGetByCommitment-12 3139 380827 ns/op 701647 B/op 4990 allocs/op` [profile.pdf](https://github.com/celestiaorg/celestia-node/files/13987571/profile.pdf) --- blob/service_test.go | 23 ++++++++++++++++++++++- header/headertest/testing.go | 2 +- 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/blob/service_test.go b/blob/service_test.go index 6777084eb4..3e22f887af 100644 --- a/blob/service_test.go +++ b/blob/service_test.go @@ -414,7 +414,28 @@ func TestService_GetAllWithoutPadding(t *testing.T) { require.NoError(t, err) } -func createService(ctx context.Context, t *testing.T, blobs []*Blob) *Service { +// BenchmarkGetByCommitment-12 3139 380827 ns/op 701647 B/op 4990 allocs/op +func BenchmarkGetByCommitment(b *testing.B) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*5) + b.Cleanup(cancel) + appBlobs, err := blobtest.GenerateV0Blobs([]int{32, 32}, true) + require.NoError(b, err) + + blobs, err := convertBlobs(appBlobs...) + require.NoError(b, err) + + service := createService(ctx, b, blobs) + b.ResetTimer() + for i := 0; i < b.N; i++ { + b.ReportAllocs() + _, _, err = service.getByCommitment( + ctx, 1, blobs[1].Namespace(), blobs[1].Commitment, + ) + require.NoError(b, err) + } +} + +func createService(ctx context.Context, t testing.TB, blobs []*Blob) *Service { bs := ipld.NewMemBlockservice() batching := ds_sync.MutexWrap(ds.NewMapDatastore()) headerStore, err := store.NewStore[*header.ExtendedHeader](batching) diff --git a/header/headertest/testing.go b/header/headertest/testing.go index 9907fd7eb4..e97f7f7825 100644 --- a/header/headertest/testing.go +++ b/header/headertest/testing.go @@ -297,7 +297,7 @@ func RandBlockID(testing.TB) types.BlockID { return bid } -func ExtendedHeaderFromEDS(t *testing.T, height uint64, eds *rsmt2d.ExtendedDataSquare) *header.ExtendedHeader { +func ExtendedHeaderFromEDS(t testing.TB, height uint64, eds *rsmt2d.ExtendedDataSquare) *header.ExtendedHeader { valSet, vals := RandValidatorSet(10, 10) gen := RandRawHeader(t) dah, err := share.NewRoot(eds) From ba969f9fb8e52b3ad1fc0831abed71ed5305dcae Mon Sep 17 00:00:00 2001 From: ramin Date: Tue, 23 Jan 2024 14:16:10 +0000 Subject: [PATCH 371/388] chore: make CI green (#3114) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Building off @vgonkivs's [work to skip tests](https://github.com/celestiaorg/celestia-node/pull/2802), this attempts to get a baseline "green" ci for us. There are still a couple of tests that VERY intermittently flake in CI, but this way they will stand out WHEN they happen and we can track down vs entire thing being red always. This does quite a few things - introduces build tags on swamp tests in each named file to allow us to run parts of the swamp/integration tests independently (ie: `go test ./... -tags=blob` - adds an `integration` tag to allow running all still - splits the integration tests into it's own workflow file (`integration-tests.yml`) which is now triggered from `go-ci.yml` - splits each swamp/integration tests to run as its own job so we can see which are failing/flakey more explicitly - utilizes go's -short test flag to Skip a few swamp/integration tests that are consistently failing. Current we are skipping `TestFullReconstructFromFulls`, `TestFullReconstructFromLights`, `TestSyncStartStopLightWithBridge` which is less than we were originally skipping in https://github.com/celestiaorg/celestia-node/pull/2802/ - plugs in our verbose/debug stuff to integration tests in addition to unit which we had before Unit tests - splits some of the unit tests that were "race flakey" into `*_no_race_test.go` and adds `!race` tag to some others to get to pass consistently when running unit tests with -race flag - macos-latest unit tests still fail on race test ONLY in GitHub actions CI 🤷‍♂️ Next Steps - create issues for each short run / skipped - create issues for any tests that fail race that are NOT the race issue on upstream cosmos-sdk - create issue for macos-latest in GitHub race fail - create issue for macos-latest intermittent fail I think we let this run for a while, then once we see it be consistently green for a bit and isolate any more flakes that pop up, we can toggle on more branch requirements, then in fixing the above issues, we can be green. [Being green is not easy](https://www.youtube.com/watch?v=51BQfPeSK8k). --- .github/workflows/go-ci.yml | 29 ++---- .github/workflows/integration-tests.yml | 115 ++++++++++++++++++++++++ Makefile | 27 +++--- core/fetcher_no_race_test.go | 55 ++++++++++++ core/fetcher_test.go | 43 --------- core/listener_no_race_test.go | 71 +++++++++++++++ core/listener_test.go | 59 ------------ nodebuilder/node_test.go | 2 + nodebuilder/store_test.go | 2 + nodebuilder/tests/api_test.go | 20 ++--- nodebuilder/tests/blob_test.go | 2 + nodebuilder/tests/fraud_test.go | 2 + nodebuilder/tests/helpers_test.go | 35 ++++++++ nodebuilder/tests/nd_test.go | 2 + nodebuilder/tests/p2p_test.go | 7 +- nodebuilder/tests/reconstruct_test.go | 13 ++- nodebuilder/tests/sync_test.go | 12 ++- share/eds/retriever_no_race_test.go | 55 ++++++++++++ share/eds/retriever_test.go | 36 -------- share/p2p/discovery/discovery_test.go | 2 + state/core_access_test.go | 2 + 21 files changed, 390 insertions(+), 201 deletions(-) create mode 100644 .github/workflows/integration-tests.yml create mode 100644 core/fetcher_no_race_test.go create mode 100644 core/listener_no_race_test.go create mode 100644 nodebuilder/tests/helpers_test.go create mode 100644 share/eds/retriever_no_race_test.go diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 3e73290500..5da90de33e 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -14,6 +14,7 @@ concurrency: jobs: setup: + name: Setup runs-on: ubuntu-latest outputs: debug: ${{ steps.debug.outputs.debug }} @@ -104,11 +105,10 @@ jobs: file: ./coverage.txt name: coverage-${{ matrix.os }} - unit_race_test: + unit_test_race: needs: [lint, go_mod_tidy_check] - name: Run Unit Tests with Race Detector + name: Unit Tests with Race Detector (ubuntu-latest) runs-on: ubuntu-latest - continue-on-error: true steps: - uses: actions/checkout@v4 @@ -119,24 +119,11 @@ jobs: go-version: ${{ inputs.go-version }} - name: execute test run - run: make test-unit-race + run: make test-unit-race ENABLE_VERBOSE=${{ needs.setup.outputs.debug }} integration_test: + name: Integration Tests needs: [lint, go_mod_tidy_check] - name: Run Integration Tests - runs-on: ubuntu-latest - continue-on-error: true - - steps: - - uses: actions/checkout@v4 - - - name: set up go - uses: actions/setup-go@v5 - with: - go-version: ${{ inputs.go-version }} - - - name: Integration Tests - run: make test-integration - - - name: Integration Tests with Race Detector - run: make test-integration-race + uses: ./.github/workflows/integration-tests.yml + with: + go-version: ${{ inputs.go-version }} diff --git a/.github/workflows/integration-tests.yml b/.github/workflows/integration-tests.yml new file mode 100644 index 0000000000..cd0fc62385 --- /dev/null +++ b/.github/workflows/integration-tests.yml @@ -0,0 +1,115 @@ +name: Integration Tests + +on: + workflow_call: + inputs: + go-version: + description: 'Go version' + required: true + type: string + +jobs: + api_tests: + name: Integration Tests API + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run API tests + run: make test-integration TAGS=api + + blob_tests: + name: Integration Tests Blob + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run blob tests + run: make test-integration TAGS=blob + + fraud_tests: + name: Integration Tests Fraud + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run fraud tests + run: make test-integration TAGS=fraud + + nd_tests: + name: Integration Tests ND + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run nd tests + run: make test-integration TAGS=nd + + p2p_tests: + name: Integration Tests p2p + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run p2p tests + run: make test-integration TAGS=p2p + + reconstruction_tests: + name: Integration Tests Reconstruction + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run reconstruction tests + run: make test-integration SHORT=true TAGS=reconstruction + + sync_tests: + name: Integration Tests Sync + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + + - name: set up go + uses: actions/setup-go@v5 + with: + go-version: ${{ inputs.go-version }} + + - name: run sync tests + run: make test-integration SHORT=true TAGS=sync diff --git a/Makefile b/Makefile index b3b3739cb7..87bcbbf61c 100644 --- a/Makefile +++ b/Makefile @@ -3,6 +3,8 @@ PROJECTNAME=$(shell basename "$(PWD)") DIR_FULLPATH=$(shell pwd) versioningPath := "github.com/celestiaorg/celestia-node/nodebuilder/node" LDFLAGS=-ldflags="-X '$(versioningPath).buildTime=$(shell date)' -X '$(versioningPath).lastCommit=$(shell git rev-parse HEAD)' -X '$(versioningPath).semanticVersion=$(shell git describe --tags --dirty=-dev 2>/dev/null || git rev-parse --abbrev-ref HEAD)'" +TAGS=integration +SHORT= ifeq (${PREFIX},) PREFIX := /usr/local endif @@ -13,6 +15,11 @@ else VERBOSE = LOG_AND_FILTER = endif +ifeq ($(SHORT),true) + INTEGRATION_RUN_LENGTH = -short +else + INTEGRATION_RUN_LENGTH = +endif ## help: Get more info on make commands. help: Makefile @echo " Choose a command run in "$(PROJECTNAME)":" @@ -118,29 +125,21 @@ test-unit: ## test-unit-race: Running unit tests with data race detector test-unit-race: @echo "--> Running unit tests with data race detector" - @go test -race `go list ./... | grep -v nodebuilder/tests` + @go test $(VERBOSE) -race -covermode=atomic -coverprofile=coverage.txt `go list ./... | grep -v nodebuilder/tests` $(LOG_AND_FILTER) .PHONY: test-unit-race ## test-integration: Running /integration tests located in nodebuilder/tests test-integration: - @echo "--> Running integrations tests" - @go test ./nodebuilder/tests + @echo "--> Running integrations tests $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH)" + @go test $(VERBOSE) -tags=$(TAGS) $(INTEGRATION_RUN_LENGTH) ./nodebuilder/tests .PHONY: test-integration ## test-integration-race: Running integration tests with data race detector located in node/tests test-integration-race: - @echo "--> Running integration tests with data race detector" - @go test -race ./nodebuilder/tests + @echo "--> Running integration tests with data race detector -tags=$(TAGS)" + @go test -race -tags=$(TAGS) ./nodebuilder/tests .PHONY: test-integration-race -## test: Running both unit and integrations tests -test: - @echo "--> Running all tests without data race detector" - @go test ./... - @echo "--> Running all tests with data race detector" - @go test -race ./... -.PHONY: test - ## benchmark: Running all benchmarks benchmark: @echo "--> Running benchmarks" @@ -164,7 +163,6 @@ pb-gen: done; .PHONY: pb-gen - ## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api openrpc-gen: @echo "--> Generating OpenRPC spec" @@ -221,7 +219,6 @@ goreleaser-release: .PHONY: goreleaser-release # Copied from https://github.com/dgraph-io/badger/blob/main/Makefile - USER_ID = $(shell id -u) HAS_JEMALLOC = $(shell test -f /usr/local/lib/libjemalloc.a && echo "jemalloc") JEMALLOC_URL = "https://github.com/jemalloc/jemalloc/releases/download/5.2.1/jemalloc-5.2.1.tar.bz2" diff --git a/core/fetcher_no_race_test.go b/core/fetcher_no_race_test.go new file mode 100644 index 0000000000..890b7c35c1 --- /dev/null +++ b/core/fetcher_no_race_test.go @@ -0,0 +1,55 @@ +//go:build !race + +package core + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + "github.com/tendermint/tendermint/types" +) + +// TestBlockFetcherHeaderValues tests that both the Commit and ValidatorSet +// endpoints are working as intended. +func TestBlockFetcherHeaderValues(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + t.Cleanup(cancel) + + client := StartTestNode(t).Client + fetcher := NewBlockFetcher(client) + + // generate some blocks + newBlockChan, err := fetcher.SubscribeNewBlockEvent(ctx) + require.NoError(t, err) + // read once from channel to generate next block + var h int64 + select { + case evt := <-newBlockChan: + h = evt.Header.Height + case <-ctx.Done(): + require.NoError(t, ctx.Err()) + } + // get Commit from current height + commit, err := fetcher.Commit(ctx, &h) + require.NoError(t, err) + // get ValidatorSet from current height + valSet, err := fetcher.ValidatorSet(ctx, &h) + require.NoError(t, err) + // get next block + var nextBlock types.EventDataSignedBlock + select { + case nextBlock = <-newBlockChan: + case <-ctx.Done(): + require.NoError(t, ctx.Err()) + } + // compare LastCommit from next block to Commit from first block height + assert.Equal(t, nextBlock.Header.LastCommitHash, commit.Hash()) + assert.Equal(t, nextBlock.Header.Height, commit.Height+1) + // compare ValidatorSet hash to the ValidatorsHash from first block height + hexBytes := valSet.Hash() + assert.Equal(t, nextBlock.ValidatorSet.Hash(), hexBytes) + require.NoError(t, fetcher.UnsubscribeNewBlockEvent(ctx)) +} diff --git a/core/fetcher_test.go b/core/fetcher_test.go index 3380dbb402..261b84d78c 100644 --- a/core/fetcher_test.go +++ b/core/fetcher_test.go @@ -7,7 +7,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "github.com/tendermint/tendermint/types" ) func TestBlockFetcher_GetBlock_and_SubscribeNewBlockEvent(t *testing.T) { @@ -38,45 +37,3 @@ func TestBlockFetcher_GetBlock_and_SubscribeNewBlockEvent(t *testing.T) { } require.NoError(t, fetcher.UnsubscribeNewBlockEvent(ctx)) } - -// TestBlockFetcherHeaderValues tests that both the Commit and ValidatorSet -// endpoints are working as intended. -func TestBlockFetcherHeaderValues(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - t.Cleanup(cancel) - - client := StartTestNode(t).Client - fetcher := NewBlockFetcher(client) - - // generate some blocks - newBlockChan, err := fetcher.SubscribeNewBlockEvent(ctx) - require.NoError(t, err) - // read once from channel to generate next block - var h int64 - select { - case evt := <-newBlockChan: - h = evt.Header.Height - case <-ctx.Done(): - require.NoError(t, ctx.Err()) - } - // get Commit from current height - commit, err := fetcher.Commit(ctx, &h) - require.NoError(t, err) - // get ValidatorSet from current height - valSet, err := fetcher.ValidatorSet(ctx, &h) - require.NoError(t, err) - // get next block - var nextBlock types.EventDataSignedBlock - select { - case nextBlock = <-newBlockChan: - case <-ctx.Done(): - require.NoError(t, ctx.Err()) - } - // compare LastCommit from next block to Commit from first block height - assert.Equal(t, nextBlock.Header.LastCommitHash, commit.Hash()) - assert.Equal(t, nextBlock.Header.Height, commit.Height+1) - // compare ValidatorSet hash to the ValidatorsHash from first block height - hexBytes := valSet.Hash() - assert.Equal(t, nextBlock.ValidatorSet.Hash(), hexBytes) - require.NoError(t, fetcher.UnsubscribeNewBlockEvent(ctx)) -} diff --git a/core/listener_no_race_test.go b/core/listener_no_race_test.go new file mode 100644 index 0000000000..eac12785ee --- /dev/null +++ b/core/listener_no_race_test.go @@ -0,0 +1,71 @@ +//go:build !race + +package core + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/cosmos/cosmos-sdk/client/flags" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-node/share" +) + +// TestListenerWithNonEmptyBlocks ensures that non-empty blocks are actually +// stored to eds.Store. +func TestListenerWithNonEmptyBlocks(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + t.Cleanup(cancel) + + // create mocknet with two pubsub endpoints + ps0, _ := createMocknetWithTwoPubsubEndpoints(ctx, t) + + // create one block to store as Head in local store and then unsubscribe from block events + cfg := DefaultTestConfig() + cfg.ChainID = networkID + fetcher, cctx := createCoreFetcher(t, cfg) + eds := createEdsPubSub(ctx, t) + + store := createStore(t) + err := store.Start(ctx) + require.NoError(t, err) + t.Cleanup(func() { + err = store.Stop(ctx) + require.NoError(t, err) + }) + + // create Listener and start listening + cl := createListener(ctx, t, fetcher, ps0, eds, store, networkID) + err = cl.Start(ctx) + require.NoError(t, err) + + // listen for eds hashes broadcasted through eds-sub and ensure store has + // already stored them + sub, err := eds.Subscribe() + require.NoError(t, err) + t.Cleanup(sub.Cancel) + + empty := share.EmptyRoot() + // TODO extract 16 + for i := 0; i < 16; i++ { + _, err := cctx.FillBlock(16, cfg.Accounts, flags.BroadcastBlock) + require.NoError(t, err) + msg, err := sub.Next(ctx) + require.NoError(t, err) + + if bytes.Equal(empty.Hash(), msg.DataHash) { + continue + } + + has, err := store.Has(ctx, msg.DataHash) + require.NoError(t, err) + require.True(t, has) + } + + err = cl.Stop(ctx) + require.NoError(t, err) + require.Nil(t, cl.cancel) +} diff --git a/core/listener_test.go b/core/listener_test.go index 5ddcd5541d..b3ed11e571 100644 --- a/core/listener_test.go +++ b/core/listener_test.go @@ -1,12 +1,10 @@ package core import ( - "bytes" "context" "testing" "time" - "github.com/cosmos/cosmos-sdk/client/flags" pubsub "github.com/libp2p/go-libp2p-pubsub" "github.com/libp2p/go-libp2p/core/event" mocknet "github.com/libp2p/go-libp2p/p2p/net/mock" @@ -17,7 +15,6 @@ import ( "github.com/celestiaorg/celestia-node/header" nodep2p "github.com/celestiaorg/celestia-node/nodebuilder/p2p" - "github.com/celestiaorg/celestia-node/share" "github.com/celestiaorg/celestia-node/share/eds" "github.com/celestiaorg/celestia-node/share/p2p/shrexsub" ) @@ -73,62 +70,6 @@ func TestListener(t *testing.T) { require.Nil(t, cl.cancel) } -// TestListenerWithNonEmptyBlocks ensures that non-empty blocks are actually -// stored to eds.Store. -func TestListenerWithNonEmptyBlocks(t *testing.T) { - ctx, cancel := context.WithTimeout(context.Background(), time.Minute) - t.Cleanup(cancel) - - // create mocknet with two pubsub endpoints - ps0, _ := createMocknetWithTwoPubsubEndpoints(ctx, t) - - // create one block to store as Head in local store and then unsubscribe from block events - cfg := DefaultTestConfig() - cfg.ChainID = networkID - fetcher, cctx := createCoreFetcher(t, cfg) - eds := createEdsPubSub(ctx, t) - - store := createStore(t) - err := store.Start(ctx) - require.NoError(t, err) - t.Cleanup(func() { - err = store.Stop(ctx) - require.NoError(t, err) - }) - - // create Listener and start listening - cl := createListener(ctx, t, fetcher, ps0, eds, store, networkID) - err = cl.Start(ctx) - require.NoError(t, err) - - // listen for eds hashes broadcasted through eds-sub and ensure store has - // already stored them - sub, err := eds.Subscribe() - require.NoError(t, err) - t.Cleanup(sub.Cancel) - - empty := share.EmptyRoot() - // TODO extract 16 - for i := 0; i < 16; i++ { - _, err := cctx.FillBlock(16, cfg.Accounts, flags.BroadcastBlock) - require.NoError(t, err) - msg, err := sub.Next(ctx) - require.NoError(t, err) - - if bytes.Equal(empty.Hash(), msg.DataHash) { - continue - } - - has, err := store.Has(ctx, msg.DataHash) - require.NoError(t, err) - require.True(t, has) - } - - err = cl.Stop(ctx) - require.NoError(t, err) - require.Nil(t, cl.cancel) -} - func TestListenerWithWrongChainRPC(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), time.Minute) t.Cleanup(cancel) diff --git a/nodebuilder/node_test.go b/nodebuilder/node_test.go index 1d0f3c4fad..41eff32fab 100644 --- a/nodebuilder/node_test.go +++ b/nodebuilder/node_test.go @@ -1,3 +1,5 @@ +//go:build !race + package nodebuilder import ( diff --git a/nodebuilder/store_test.go b/nodebuilder/store_test.go index bd179c1258..51bd89c5a7 100644 --- a/nodebuilder/store_test.go +++ b/nodebuilder/store_test.go @@ -1,3 +1,5 @@ +//go:build !race + package nodebuilder import ( diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 2fd4b2d3da..a793ba7b33 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -1,3 +1,5 @@ +//go:build api || integration + package tests import ( @@ -13,26 +15,14 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/blob" "github.com/celestiaorg/celestia-node/blob/blobtest" - "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/node" "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" ) -func getAdminClient(ctx context.Context, nd *nodebuilder.Node, t *testing.T) *client.Client { - t.Helper() - - signer := nd.AdminSigner - listenAddr := "ws://" + nd.RPCServer.ListenAddr() - - jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) - require.NoError(t, err) - - client, err := client.NewClient(ctx, listenAddr, jwt) - require.NoError(t, err) - - return client -} +const ( + btime = time.Millisecond * 300 +) func TestNodeModule(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index 627dd850d0..a30e91b2e6 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -1,3 +1,5 @@ +//go:build blob || integration + package tests import ( diff --git a/nodebuilder/tests/fraud_test.go b/nodebuilder/tests/fraud_test.go index cb07dbb73a..6496cdbb53 100644 --- a/nodebuilder/tests/fraud_test.go +++ b/nodebuilder/tests/fraud_test.go @@ -1,3 +1,5 @@ +//go:build fraud || integration + package tests import ( diff --git a/nodebuilder/tests/helpers_test.go b/nodebuilder/tests/helpers_test.go new file mode 100644 index 0000000000..1e7f14d823 --- /dev/null +++ b/nodebuilder/tests/helpers_test.go @@ -0,0 +1,35 @@ +//nolint:unused +package tests + +import ( + "context" + "testing" + "time" + + "github.com/celestiaorg/celestia-node/api/rpc/client" + "github.com/celestiaorg/celestia-node/libs/authtoken" + "github.com/celestiaorg/celestia-node/nodebuilder" + + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/stretchr/testify/require" +) + +func getAdminClient(ctx context.Context, nd *nodebuilder.Node, t *testing.T) *client.Client { + t.Helper() + + signer := nd.AdminSigner + listenAddr := "ws://" + nd.RPCServer.ListenAddr() + + jwt, err := authtoken.NewSignedJWT(signer, []auth.Permission{"public", "read", "write", "admin"}) + require.NoError(t, err) + + client, err := client.NewClient(ctx, listenAddr, jwt) + require.NoError(t, err) + + return client +} + +func setTimeInterval(cfg *nodebuilder.Config, interval time.Duration) { + cfg.P2P.RoutingTableRefreshPeriod = interval + cfg.Share.Discovery.AdvertiseInterval = interval +} diff --git a/nodebuilder/tests/nd_test.go b/nodebuilder/tests/nd_test.go index f3338ec294..338aa6d0c1 100644 --- a/nodebuilder/tests/nd_test.go +++ b/nodebuilder/tests/nd_test.go @@ -1,3 +1,5 @@ +//go:build nd || integration + package tests import ( diff --git a/nodebuilder/tests/p2p_test.go b/nodebuilder/tests/p2p_test.go index 9fe63fb931..98e9fc15b4 100644 --- a/nodebuilder/tests/p2p_test.go +++ b/nodebuilder/tests/p2p_test.go @@ -1,3 +1,5 @@ +//go:build p2p || integration + package tests import ( @@ -199,8 +201,3 @@ func TestRestartNodeDiscovery(t *testing.T) { require.NoError(t, err) assert.Equal(t, connectedness, network.Connected) } - -func setTimeInterval(cfg *nodebuilder.Config, interval time.Duration) { - cfg.P2P.RoutingTableRefreshPeriod = interval - cfg.Share.Discovery.AdvertiseInterval = interval -} diff --git a/nodebuilder/tests/reconstruct_test.go b/nodebuilder/tests/reconstruct_test.go index 89f4a0171a..d047182669 100644 --- a/nodebuilder/tests/reconstruct_test.go +++ b/nodebuilder/tests/reconstruct_test.go @@ -1,7 +1,4 @@ -// Test with light nodes spawns more goroutines than in the race detectors budget, -// and thus we're disabling the race detector. -// TODO(@Wondertan): Remove this once we move to go1.19 with unlimited race detector -//go:build !race +//go:build reconstruction || integration package tests @@ -89,6 +86,10 @@ Test-Case: Full Node reconstructs blocks from each other, after unsuccessfully s block from LN subnetworks. Analog to TestShareAvailable_DisconnectedFullNodes. */ func TestFullReconstructFromFulls(t *testing.T) { + if testing.Short() { + t.Skip() + } + light.DefaultSampleAmount = 10 // s const ( blocks = 10 @@ -255,6 +256,10 @@ Steps: 9. Check that the FN can retrieve shares from 1 to 20 blocks */ func TestFullReconstructFromLights(t *testing.T) { + if testing.Short() { + t.Skip() + } + eds.RetrieveQuadrantTimeout = time.Millisecond * 100 light.DefaultSampleAmount = 20 const ( diff --git a/nodebuilder/tests/sync_test.go b/nodebuilder/tests/sync_test.go index 8b939749d6..ec8386ea43 100644 --- a/nodebuilder/tests/sync_test.go +++ b/nodebuilder/tests/sync_test.go @@ -1,3 +1,5 @@ +//go:build sync || integration + package tests import ( @@ -16,7 +18,7 @@ import ( const ( numBlocks = 20 bsize = 16 - btime = time.Millisecond * 300 + sbtime = time.Millisecond * 300 ) /* @@ -45,7 +47,7 @@ func TestSyncAgainstBridge_NonEmptyChain(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(sbtime)) // wait for core network to fill 20 blocks fillDn := swamp.FillBlocks(ctx, sw.ClientContext, sw.Accounts, bsize, numBlocks) sw.WaitTillHeight(ctx, numBlocks) @@ -138,7 +140,7 @@ func TestSyncAgainstBridge_EmptyChain(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) - sw := swamp.NewSwamp(t, swamp.WithBlockTime(btime)) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(sbtime)) sw.WaitTillHeight(ctx, numBlocks) // create bridge node and set it as the bootstrapper for the suite @@ -211,6 +213,10 @@ Steps: 9. Check LN is synced to height 40 */ func TestSyncStartStopLightWithBridge(t *testing.T) { + if testing.Short() { + t.Skip("skipping TestSyncStartStopLightWithBridge test in short mode.") + } + ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) defer cancel() diff --git a/share/eds/retriever_no_race_test.go b/share/eds/retriever_no_race_test.go new file mode 100644 index 0000000000..15c6aa2fc4 --- /dev/null +++ b/share/eds/retriever_no_race_test.go @@ -0,0 +1,55 @@ +// go:build !race + +package eds + +import ( + "context" + "testing" + "time" + + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/da" + "github.com/celestiaorg/celestia-app/pkg/wrapper" + "github.com/celestiaorg/nmt" + "github.com/celestiaorg/rsmt2d" + + "github.com/celestiaorg/celestia-node/share" + "github.com/celestiaorg/celestia-node/share/eds/byzantine" + "github.com/celestiaorg/celestia-node/share/eds/edstest" + "github.com/celestiaorg/celestia-node/share/ipld" +) + +func TestRetriever_ByzantineError(t *testing.T) { + const width = 8 + ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) + defer cancel() + + bserv := ipld.NewMemBlockservice() + shares := edstest.RandEDS(t, width).Flattened() + _, err := ipld.ImportShares(ctx, shares, bserv) + require.NoError(t, err) + + // corrupt shares so that eds erasure coding does not match + copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) + + // import corrupted eds + batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) + attackerEDS, err := rsmt2d.ImportExtendedDataSquare( + shares, + share.DefaultRSMT2DCodec(), + wrapper.NewConstructor(uint64(width), + nmt.NodeVisitor(batchAdder.Visit)), + ) + require.NoError(t, err) + err = batchAdder.Commit() + require.NoError(t, err) + + // ensure we rcv an error + dah, err := da.NewDataAvailabilityHeader(attackerEDS) + require.NoError(t, err) + r := NewRetriever(bserv) + _, err = r.Retrieve(ctx, &dah) + var errByz *byzantine.ErrByzantine + require.ErrorAs(t, err, &errByz) +} diff --git a/share/eds/retriever_test.go b/share/eds/retriever_test.go index f3f7ccca64..95da345d17 100644 --- a/share/eds/retriever_test.go +++ b/share/eds/retriever_test.go @@ -12,8 +12,6 @@ import ( "github.com/stretchr/testify/require" "github.com/celestiaorg/celestia-app/pkg/da" - "github.com/celestiaorg/celestia-app/pkg/wrapper" - "github.com/celestiaorg/nmt" "github.com/celestiaorg/rsmt2d" "github.com/celestiaorg/celestia-node/header" @@ -67,40 +65,6 @@ func TestRetriever_Retrieve(t *testing.T) { } } -func TestRetriever_ByzantineError(t *testing.T) { - const width = 8 - ctx, cancel := context.WithTimeout(context.Background(), time.Second*10) - defer cancel() - - bserv := ipld.NewMemBlockservice() - shares := edstest.RandEDS(t, width).Flattened() - _, err := ipld.ImportShares(ctx, shares, bserv) - require.NoError(t, err) - - // corrupt shares so that eds erasure coding does not match - copy(shares[14][share.NamespaceSize:], shares[15][share.NamespaceSize:]) - - // import corrupted eds - batchAdder := ipld.NewNmtNodeAdder(ctx, bserv, ipld.MaxSizeBatchOption(width*2)) - attackerEDS, err := rsmt2d.ImportExtendedDataSquare( - shares, - share.DefaultRSMT2DCodec(), - wrapper.NewConstructor(uint64(width), - nmt.NodeVisitor(batchAdder.Visit)), - ) - require.NoError(t, err) - err = batchAdder.Commit() - require.NoError(t, err) - - // ensure we rcv an error - dah, err := da.NewDataAvailabilityHeader(attackerEDS) - require.NoError(t, err) - r := NewRetriever(bserv) - _, err = r.Retrieve(ctx, &dah) - var errByz *byzantine.ErrByzantine - require.ErrorAs(t, err, &errByz) -} - // TestRetriever_MultipleRandQuadrants asserts that reconstruction succeeds // when any three random quadrants requested. func TestRetriever_MultipleRandQuadrants(t *testing.T) { diff --git a/share/p2p/discovery/discovery_test.go b/share/p2p/discovery/discovery_test.go index c02931e1a4..1d0078196f 100644 --- a/share/p2p/discovery/discovery_test.go +++ b/share/p2p/discovery/discovery_test.go @@ -1,3 +1,5 @@ +// go:build !race + package discovery import ( diff --git a/state/core_access_test.go b/state/core_access_test.go index 68d05a955a..ad7b916ea3 100644 --- a/state/core_access_test.go +++ b/state/core_access_test.go @@ -1,3 +1,5 @@ +//go:build !race + package state import ( From af417b098d4fb29971d18ae8de976cf30d3c4c33 Mon Sep 17 00:00:00 2001 From: rene <41963722+renaynay@users.noreply.github.com> Date: Tue, 23 Jan 2024 11:52:01 -0600 Subject: [PATCH 372/388] fix(libs/pidstore): Detect corruption on startup and reset pidstore (#3132) The recovery of an existing pidstore is not critical to node functionality and should not block the node's start-up if it was corrupted during an ungraceful shutdown. If corruption is detected, a fresh pidstore can be instantiated and used by the node. Found by conduit. --- libs/pidstore/pidstore.go | 20 ++++++++++++++++++++ libs/pidstore/pidstore_test.go | 22 ++++++++++++++++++++++ 2 files changed, 42 insertions(+) diff --git a/libs/pidstore/pidstore.go b/libs/pidstore/pidstore.go index 7e416d98a0..17241aa4a9 100644 --- a/libs/pidstore/pidstore.go +++ b/libs/pidstore/pidstore.go @@ -28,6 +28,7 @@ func NewPeerIDStore(ctx context.Context, ds datastore.Datastore) (*PeerIDStore, pidstore := &PeerIDStore{ ds: namespace.Wrap(ds, storePrefix), } + // check if pidstore is already initialized, and if not, // initialize the pidstore exists, err := pidstore.ds.Has(ctx, peersKey) @@ -37,6 +38,14 @@ func NewPeerIDStore(ctx context.Context, ds datastore.Datastore) (*PeerIDStore, if !exists { return pidstore, pidstore.Put(ctx, []peer.ID{}) } + + // if pidstore exists, ensure its contents are uncorrupted + _, err = pidstore.Load(ctx) + if err != nil { + log.Warn("pidstore: corrupted pidstore detected, resetting...", "err", err) + return pidstore, pidstore.reset(ctx) + } + return pidstore, nil } @@ -75,3 +84,14 @@ func (p *PeerIDStore) Put(ctx context.Context, peers []peer.ID) error { log.Infow("Persisted peers successfully", "amount", len(peers)) return nil } + +// reset resets the pidstore in case of corruption. +func (p *PeerIDStore) reset(ctx context.Context) error { + log.Warn("pidstore: resetting the pidstore...") + err := p.ds.Delete(ctx, peersKey) + if err != nil { + return fmt.Errorf("pidstore: error resetting datastore: %w", err) + } + + return p.Put(ctx, []peer.ID{}) +} diff --git a/libs/pidstore/pidstore_test.go b/libs/pidstore/pidstore_test.go index d8d214c83e..56f8a308ec 100644 --- a/libs/pidstore/pidstore_test.go +++ b/libs/pidstore/pidstore_test.go @@ -8,6 +8,7 @@ import ( "time" "github.com/ipfs/go-datastore" + "github.com/ipfs/go-datastore/namespace" "github.com/ipfs/go-datastore/sync" "github.com/libp2p/go-libp2p/core/crypto" "github.com/libp2p/go-libp2p/core/peer" @@ -29,6 +30,27 @@ func TestPutLoad(t *testing.T) { }) } +// TestCorruptedPidstore tests whether a pidstore can detect +// corruption and reset itself on construction. +func TestCorruptedPidstore(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), time.Minute) + defer t.Cleanup(cancel) + + ds := sync.MutexWrap(datastore.NewMapDatastore()) + + // intentionally corrupt the store + wrappedDS := namespace.Wrap(ds, storePrefix) + err := wrappedDS.Put(ctx, peersKey, []byte("corrupted")) + require.NoError(t, err) + + pidstore, err := NewPeerIDStore(ctx, ds) + require.NoError(t, err) + + got, err := pidstore.Load(ctx) + require.NoError(t, err) + assert.Equal(t, []peer.ID{}, got) +} + func testPutLoad(ctx context.Context, ds datastore.Datastore, t *testing.T) { peerstore, err := NewPeerIDStore(ctx, ds) require.NoError(t, err) From debad37cdd8db3858651b2c0506f7ef24fe43293 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 23 Jan 2024 17:06:50 -0600 Subject: [PATCH 373/388] refactor(blob)!: replace SubmitOptions with a GasPrice parameter (#3094) This PR modifies the blob interface to replace SubmitOptions with GasLimit. The reasoning here is that setting a gas price is a lot more intuitive/easier than calculating the fee and gas limit, which take cosmossdk and celestia-app imports to calculate without much effort. Related: https://github.com/rollkit/go-da/issues/30 Based on a suggestion from @vgonkivs we alias float64 to be able to provide a default value, but now that its there I think we can just have float64 and the same constructor, as it doesn't change that much. Breaks the API, cc @jcstein, @tuxcanfly, and @Ferret-san . Closes #3053. --- blob/service.go | 32 +++++++++++++++++++----- nodebuilder/blob/blob.go | 8 +++--- nodebuilder/blob/cmd/blob.go | 29 +++++++-------------- nodebuilder/blob/mocks/api.go | 5 ++-- nodebuilder/das/mocks/api.go | 3 +-- nodebuilder/fraud/mocks/api.go | 3 +-- nodebuilder/header/mocks/api.go | 3 +-- nodebuilder/node/mocks/api.go | 3 +-- nodebuilder/share/mocks/api.go | 13 +++++----- nodebuilder/state/mocks/api.go | 5 ++-- nodebuilder/tests/api_test.go | 2 +- nodebuilder/tests/blob_test.go | 6 ++--- share/availability/mocks/availability.go | 3 +-- share/mocks/getter.go | 3 +-- 14 files changed, 59 insertions(+), 59 deletions(-) diff --git a/blob/service.go b/blob/service.go index 79e7dd7937..6a70772797 100644 --- a/blob/service.go +++ b/blob/service.go @@ -4,13 +4,17 @@ import ( "context" "errors" "fmt" + "math" "sync" - "cosmossdk.io/math" + sdkmath "cosmossdk.io/math" "github.com/cosmos/cosmos-sdk/types" + auth "github.com/cosmos/cosmos-sdk/x/auth/types" logging "github.com/ipfs/go-log/v2" + "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" + blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/header" "github.com/celestiaorg/celestia-node/share" @@ -23,11 +27,21 @@ var ( log = logging.Logger("blob") ) +// GasPrice represents the amount to be paid per gas unit. Fee is set by +// multiplying GasPrice by GasLimit, which is determined by the blob sizes. +type GasPrice float64 + +// DefaultGasPrice returns the default gas price, letting node automatically +// determine the Fee based on the passed blob sizes. +func DefaultGasPrice() GasPrice { + return -1.0 +} + // Submitter is an interface that allows submitting blobs to the celestia-core. It is used to // avoid a circular dependency between the blob and the state package, since the state package needs // the blob.Blob type for this signature. type Submitter interface { - SubmitPayForBlob(ctx context.Context, fee math.Int, gasLim uint64, blobs []*Blob) (*types.TxResponse, error) + SubmitPayForBlob(ctx context.Context, fee sdkmath.Int, gasLim uint64, blobs []*Blob) (*types.TxResponse, error) } type Service struct { @@ -66,15 +80,21 @@ func DefaultSubmitOptions() *SubmitOptions { } } -// Submit sends PFB transaction and reports the height in which it was included. +// Submit sends PFB transaction and reports the height at which it was included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. // Handles gas estimation and fee calculation. -func (s *Service) Submit(ctx context.Context, blobs []*Blob, options *SubmitOptions) (uint64, error) { +func (s *Service) Submit(ctx context.Context, blobs []*Blob, gasPrice GasPrice) (uint64, error) { log.Debugw("submitting blobs", "amount", len(blobs)) - if options == nil { - options = DefaultSubmitOptions() + options := DefaultSubmitOptions() + if gasPrice >= 0 { + blobSizes := make([]uint32, len(blobs)) + for i, blob := range blobs { + blobSizes[i] = uint32(len(blob.Data)) + } + options.GasLimit = blobtypes.EstimateGas(blobSizes, appconsts.DefaultGasPerBlobByte, auth.DefaultTxSizeCostPerByte) + options.Fee = types.NewInt(int64(math.Ceil(float64(gasPrice) * float64(options.GasLimit)))).Int64() } resp, err := s.blobSubmitter.SubmitPayForBlob(ctx, types.NewInt(options.Fee), options.GasLimit, blobs) diff --git a/nodebuilder/blob/blob.go b/nodebuilder/blob/blob.go index 47ab255ac6..f87105541a 100644 --- a/nodebuilder/blob/blob.go +++ b/nodebuilder/blob/blob.go @@ -16,7 +16,7 @@ type Module interface { // Submit sends Blobs and reports the height in which they were included. // Allows sending multiple Blobs atomically synchronously. // Uses default wallet registered on the Node. - Submit(_ context.Context, _ []*blob.Blob, _ *blob.SubmitOptions) (height uint64, _ error) + Submit(_ context.Context, _ []*blob.Blob, _ blob.GasPrice) (height uint64, _ error) // Get retrieves the blob by commitment under the given namespace and height. Get(_ context.Context, height uint64, _ share.Namespace, _ blob.Commitment) (*blob.Blob, error) // GetAll returns all blobs at the given height under the given namespaces. @@ -30,7 +30,7 @@ type Module interface { type API struct { Internal struct { - Submit func(context.Context, []*blob.Blob, *blob.SubmitOptions) (uint64, error) `perm:"write"` + Submit func(context.Context, []*blob.Blob, blob.GasPrice) (uint64, error) `perm:"write"` Get func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Blob, error) `perm:"read"` GetAll func(context.Context, uint64, []share.Namespace) ([]*blob.Blob, error) `perm:"read"` GetProof func(context.Context, uint64, share.Namespace, blob.Commitment) (*blob.Proof, error) `perm:"read"` @@ -38,8 +38,8 @@ type API struct { } } -func (api *API) Submit(ctx context.Context, blobs []*blob.Blob, options *blob.SubmitOptions) (uint64, error) { - return api.Internal.Submit(ctx, blobs, options) +func (api *API) Submit(ctx context.Context, blobs []*blob.Blob, gasPrice blob.GasPrice) (uint64, error) { + return api.Internal.Submit(ctx, blobs, gasPrice) } func (api *API) Get( diff --git a/nodebuilder/blob/cmd/blob.go b/nodebuilder/blob/cmd/blob.go index f3510edfe1..7862eaa204 100644 --- a/nodebuilder/blob/cmd/blob.go +++ b/nodebuilder/blob/cmd/blob.go @@ -16,8 +16,7 @@ import ( var ( base64Flag bool - fee int64 - gasLimit uint64 + gasPrice float64 ) func init() { @@ -37,23 +36,13 @@ func init() { "printed blob's data as a base64 string", ) - submitCmd.PersistentFlags().Int64Var( - &fee, - "fee", - -1, - "specifies fee (in utia) for blob submission.\n"+ - "Fee will be automatically calculated if negative value is passed [optional]", + submitCmd.PersistentFlags().Float64Var( + &gasPrice, + "gas.price", + float64(blob.DefaultGasPrice()), + "specifies gas price (in utia) for blob submission.\n"+ + "Gas price will be set to default (0.002) if no value is passed", ) - - submitCmd.PersistentFlags().Uint64Var( - &gasLimit, - "gas.limit", - 0, - "sets the amount of gas that is consumed during blob submission [optional]", - ) - - // unset the default value to avoid users confusion - submitCmd.PersistentFlags().Lookup("fee").DefValue = "0" } var Cmd = &cobra.Command{ @@ -135,7 +124,7 @@ var submitCmd = &cobra.Command{ Short: "Submit the blob at the given namespace.\n" + "Note:\n" + "* only one blob is allowed to submit through the RPC.\n" + - "* fee and gas.limit params will be calculated automatically if they are not provided as arguments", + "* fee and gas limit params will be calculated automatically.\n", RunE: func(cmd *cobra.Command, args []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) if err != nil { @@ -156,7 +145,7 @@ var submitCmd = &cobra.Command{ height, err := client.Blob.Submit( cmd.Context(), []*blob.Blob{parsedBlob}, - &blob.SubmitOptions{Fee: fee, GasLimit: gasLimit}, + blob.GasPrice(gasPrice), ) response := struct { diff --git a/nodebuilder/blob/mocks/api.go b/nodebuilder/blob/mocks/api.go index 40bf299da1..0898e70459 100644 --- a/nodebuilder/blob/mocks/api.go +++ b/nodebuilder/blob/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - blob "github.com/celestiaorg/celestia-node/blob" share "github.com/celestiaorg/celestia-node/share" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -98,7 +97,7 @@ func (mr *MockModuleMockRecorder) Included(arg0, arg1, arg2, arg3, arg4 interfac } // Submit mocks base method. -func (m *MockModule) Submit(arg0 context.Context, arg1 []*blob.Blob, arg2 *blob.SubmitOptions) (uint64, error) { +func (m *MockModule) Submit(arg0 context.Context, arg1 []*blob.Blob, arg2 blob.GasPrice) (uint64, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "Submit", arg0, arg1, arg2) ret0, _ := ret[0].(uint64) diff --git a/nodebuilder/das/mocks/api.go b/nodebuilder/das/mocks/api.go index 68ffaf3c8c..c4046e90e8 100644 --- a/nodebuilder/das/mocks/api.go +++ b/nodebuilder/das/mocks/api.go @@ -8,9 +8,8 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - das "github.com/celestiaorg/celestia-node/das" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/fraud/mocks/api.go b/nodebuilder/fraud/mocks/api.go index 10111b81a8..fcc7a58231 100644 --- a/nodebuilder/fraud/mocks/api.go +++ b/nodebuilder/fraud/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - fraud "github.com/celestiaorg/celestia-node/nodebuilder/fraud" fraud0 "github.com/celestiaorg/go-fraud" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/header/mocks/api.go b/nodebuilder/header/mocks/api.go index 9b15f2242e..b0d2b961d9 100644 --- a/nodebuilder/header/mocks/api.go +++ b/nodebuilder/header/mocks/api.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - header "github.com/celestiaorg/celestia-node/header" header0 "github.com/celestiaorg/go-header" sync "github.com/celestiaorg/go-header/sync" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/node/mocks/api.go b/nodebuilder/node/mocks/api.go index 14357316dc..d8789a771c 100644 --- a/nodebuilder/node/mocks/api.go +++ b/nodebuilder/node/mocks/api.go @@ -8,10 +8,9 @@ import ( context "context" reflect "reflect" + node "github.com/celestiaorg/celestia-node/nodebuilder/node" auth "github.com/filecoin-project/go-jsonrpc/auth" gomock "github.com/golang/mock/gomock" - - node "github.com/celestiaorg/celestia-node/nodebuilder/node" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/share/mocks/api.go b/nodebuilder/share/mocks/api.go index 87f98e4f73..4e21cecae0 100644 --- a/nodebuilder/share/mocks/api.go +++ b/nodebuilder/share/mocks/api.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - - da "github.com/celestiaorg/celestia-app/pkg/da" + header "github.com/celestiaorg/celestia-node/header" share "github.com/celestiaorg/celestia-node/share" rsmt2d "github.com/celestiaorg/rsmt2d" + gomock "github.com/golang/mock/gomock" ) // MockModule is a mock of Module interface. @@ -39,7 +38,7 @@ func (m *MockModule) EXPECT() *MockModuleMockRecorder { } // GetEDS mocks base method. -func (m *MockModule) GetEDS(arg0 context.Context, arg1 *da.DataAvailabilityHeader) (*rsmt2d.ExtendedDataSquare, error) { +func (m *MockModule) GetEDS(arg0 context.Context, arg1 *header.ExtendedHeader) (*rsmt2d.ExtendedDataSquare, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetEDS", arg0, arg1) ret0, _ := ret[0].(*rsmt2d.ExtendedDataSquare) @@ -54,7 +53,7 @@ func (mr *MockModuleMockRecorder) GetEDS(arg0, arg1 interface{}) *gomock.Call { } // GetShare mocks base method. -func (m *MockModule) GetShare(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2, arg3 int) ([]byte, error) { +func (m *MockModule) GetShare(arg0 context.Context, arg1 *header.ExtendedHeader, arg2, arg3 int) ([]byte, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetShare", arg0, arg1, arg2, arg3) ret0, _ := ret[0].([]byte) @@ -69,7 +68,7 @@ func (mr *MockModuleMockRecorder) GetShare(arg0, arg1, arg2, arg3 interface{}) * } // GetSharesByNamespace mocks base method. -func (m *MockModule) GetSharesByNamespace(arg0 context.Context, arg1 *da.DataAvailabilityHeader, arg2 share.Namespace) (share.NamespacedShares, error) { +func (m *MockModule) GetSharesByNamespace(arg0 context.Context, arg1 *header.ExtendedHeader, arg2 share.Namespace) (share.NamespacedShares, error) { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "GetSharesByNamespace", arg0, arg1, arg2) ret0, _ := ret[0].(share.NamespacedShares) @@ -84,7 +83,7 @@ func (mr *MockModuleMockRecorder) GetSharesByNamespace(arg0, arg1, arg2 interfac } // SharesAvailable mocks base method. -func (m *MockModule) SharesAvailable(arg0 context.Context, arg1 *da.DataAvailabilityHeader) error { +func (m *MockModule) SharesAvailable(arg0 context.Context, arg1 *header.ExtendedHeader) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "SharesAvailable", arg0, arg1) ret0, _ := ret[0].(error) diff --git a/nodebuilder/state/mocks/api.go b/nodebuilder/state/mocks/api.go index 6499a6dfd8..1861a86e66 100644 --- a/nodebuilder/state/mocks/api.go +++ b/nodebuilder/state/mocks/api.go @@ -9,13 +9,12 @@ import ( reflect "reflect" math "cosmossdk.io/math" + blob "github.com/celestiaorg/celestia-node/blob" + state "github.com/celestiaorg/celestia-node/state" types "github.com/cosmos/cosmos-sdk/types" types0 "github.com/cosmos/cosmos-sdk/x/staking/types" gomock "github.com/golang/mock/gomock" types1 "github.com/tendermint/tendermint/types" - - blob "github.com/celestiaorg/celestia-node/blob" - state "github.com/celestiaorg/celestia-node/state" ) // MockModule is a mock of Module interface. diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index a793ba7b33..960ce38875 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -113,7 +113,7 @@ func TestBlobRPC(t *testing.T) { ) require.NoError(t, err) - height, err := rpcClient.Blob.Submit(ctx, []*blob.Blob{newBlob}, nil) + height, err := rpcClient.Blob.Submit(ctx, []*blob.Blob{newBlob}, blob.DefaultGasPrice()) require.NoError(t, err) require.True(t, height != 0) } diff --git a/nodebuilder/tests/blob_test.go b/nodebuilder/tests/blob_test.go index a30e91b2e6..d0aeefd568 100644 --- a/nodebuilder/tests/blob_test.go +++ b/nodebuilder/tests/blob_test.go @@ -60,7 +60,7 @@ func TestBlobModule(t *testing.T) { fullClient := getAdminClient(ctx, fullNode, t) lightClient := getAdminClient(ctx, lightNode, t) - height, err := fullClient.Blob.Submit(ctx, blobs, nil) + height, err := fullClient.Blob.Submit(ctx, blobs, blob.DefaultGasPrice()) require.NoError(t, err) _, err = fullClient.Header.WaitForHeight(ctx, height) @@ -143,7 +143,7 @@ func TestBlobModule(t *testing.T) { ) require.NoError(t, err) - height, err := fullClient.Blob.Submit(ctx, []*blob.Blob{b, b}, nil) + height, err := fullClient.Blob.Submit(ctx, []*blob.Blob{b, b}, blob.DefaultGasPrice()) require.NoError(t, err) _, err = fullClient.Header.WaitForHeight(ctx, height) @@ -172,7 +172,7 @@ func TestBlobModule(t *testing.T) { // different pfbs. name: "Submit the same blob in different pfb", doFn: func(t *testing.T) { - h, err := fullClient.Blob.Submit(ctx, []*blob.Blob{blobs[0]}, nil) + h, err := fullClient.Blob.Submit(ctx, []*blob.Blob{blobs[0]}, blob.DefaultGasPrice()) require.NoError(t, err) _, err = fullClient.Header.WaitForHeight(ctx, h) diff --git a/share/availability/mocks/availability.go b/share/availability/mocks/availability.go index bb1c01bf85..fc68d3d2bc 100644 --- a/share/availability/mocks/availability.go +++ b/share/availability/mocks/availability.go @@ -8,9 +8,8 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - header "github.com/celestiaorg/celestia-node/header" + gomock "github.com/golang/mock/gomock" ) // MockAvailability is a mock of Availability interface. diff --git a/share/mocks/getter.go b/share/mocks/getter.go index 2adfa50cfe..738e2b246c 100644 --- a/share/mocks/getter.go +++ b/share/mocks/getter.go @@ -8,11 +8,10 @@ import ( context "context" reflect "reflect" - gomock "github.com/golang/mock/gomock" - header "github.com/celestiaorg/celestia-node/header" share "github.com/celestiaorg/celestia-node/share" rsmt2d "github.com/celestiaorg/rsmt2d" + gomock "github.com/golang/mock/gomock" ) // MockGetter is a mock of Getter interface. From 4850c901675696095e2df4af234b5421a270e680 Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 29 Jan 2024 02:25:51 -0600 Subject: [PATCH 374/388] feat!(rpc): rpc.disable-auth flag (#3117) In many setups, nodes are running in trusted environments and setting the jwt token is not necessary and very cumbersome. This PR enables a quick fix by providing a flag/config option to the RPC to disable auth. Breaking config. New flag: `--rpc.skip-auth`. New default config: ```toml [RPC] Address = "localhost" Port = "26658" SkipAuth = false ``` _______________ Comes from a discussion with @joroshiba at Astria. --- api/rpc/server.go | 15 ++++++++++----- nodebuilder/rpc/config.go | 8 +++++--- nodebuilder/rpc/constructors.go | 2 +- nodebuilder/rpc/flags.go | 16 ++++++++++++++++ 4 files changed, 32 insertions(+), 9 deletions(-) diff --git a/api/rpc/server.go b/api/rpc/server.go index 3357140e68..3bf58870bc 100644 --- a/api/rpc/server.go +++ b/api/rpc/server.go @@ -20,16 +20,17 @@ import ( var log = logging.Logger("rpc") type Server struct { - srv *http.Server - rpc *jsonrpc.RPCServer - listener net.Listener + srv *http.Server + rpc *jsonrpc.RPCServer + listener net.Listener + authDisabled bool started atomic.Bool auth jwt.Signer } -func NewServer(address, port string, secret jwt.Signer) *Server { +func NewServer(address, port string, authDisabled bool, secret jwt.Signer) *Server { rpc := jsonrpc.NewServer() srv := &Server{ rpc: rpc, @@ -38,7 +39,8 @@ func NewServer(address, port string, secret jwt.Signer) *Server { // the amount of time allowed to read request headers. set to the default 2 seconds ReadHeaderTimeout: 2 * time.Second, }, - auth: secret, + auth: secret, + authDisabled: authDisabled, } srv.srv.Handler = &auth.Handler{ Verify: srv.verifyAuth, @@ -51,6 +53,9 @@ func NewServer(address, port string, secret jwt.Signer) *Server { // reached if a token is provided in the header of the request, otherwise only // methods with `read` permissions are accessible. func (s *Server) verifyAuth(_ context.Context, token string) ([]auth.Permission, error) { + if s.authDisabled { + return perms.AllPerms, nil + } return authtoken.ExtractSignedPermissions(s.auth, token) } diff --git a/nodebuilder/rpc/config.go b/nodebuilder/rpc/config.go index a270768646..d6031082a8 100644 --- a/nodebuilder/rpc/config.go +++ b/nodebuilder/rpc/config.go @@ -8,15 +8,17 @@ import ( ) type Config struct { - Address string - Port string + Address string + Port string + SkipAuth bool } func DefaultConfig() Config { return Config{ Address: defaultBindAddress, // do NOT expose the same port as celestia-core by default so that both can run on the same machine - Port: defaultPort, + Port: defaultPort, + SkipAuth: false, } } diff --git a/nodebuilder/rpc/constructors.go b/nodebuilder/rpc/constructors.go index ca30af6305..194dea8a03 100644 --- a/nodebuilder/rpc/constructors.go +++ b/nodebuilder/rpc/constructors.go @@ -37,5 +37,5 @@ func registerEndpoints( } func server(cfg *Config, auth jwt.Signer) *rpc.Server { - return rpc.NewServer(cfg.Address, cfg.Port, auth) + return rpc.NewServer(cfg.Address, cfg.Port, cfg.SkipAuth, auth) } diff --git a/nodebuilder/rpc/flags.go b/nodebuilder/rpc/flags.go index b7bad333df..d37014004d 100644 --- a/nodebuilder/rpc/flags.go +++ b/nodebuilder/rpc/flags.go @@ -3,13 +3,16 @@ package rpc import ( "fmt" + logging "github.com/ipfs/go-log/v2" "github.com/spf13/cobra" flag "github.com/spf13/pflag" ) var ( + log = logging.Logger("rpc") addrFlag = "rpc.addr" portFlag = "rpc.port" + authFlag = "rpc.skip-auth" ) // Flags gives a set of hardcoded node/rpc package flags. @@ -26,6 +29,11 @@ func Flags() *flag.FlagSet { "", fmt.Sprintf("Set a custom RPC port (default: %s)", defaultPort), ) + flags.Bool( + authFlag, + false, + "Skips authentication for RPC requests", + ) return flags } @@ -40,4 +48,12 @@ func ParseFlags(cmd *cobra.Command, cfg *Config) { if port != "" { cfg.Port = port } + ok, err := cmd.Flags().GetBool(authFlag) + if err != nil { + panic(err) + } + if ok { + log.Warn("RPC authentication is disabled") + cfg.SkipAuth = true + } } From 9484e15125d43cffedac5f284cfbd7397a5bb59c Mon Sep 17 00:00:00 2001 From: Ryan Date: Mon, 29 Jan 2024 06:58:00 -0600 Subject: [PATCH 375/388] feat(rpc/node): readiness check (#3118) Implements a readiness check on the node module. Question: I have set the permission level to public, but remember that we removed all public methods in favor of `read`. Is this fine or should this method also be `read`? ____________ Comes from a conversation with @joroshiba from Astria --------- Co-authored-by: ramin --- nodebuilder/module.go | 2 +- nodebuilder/node/admin.go | 8 ++++++++ nodebuilder/node/node.go | 8 ++++++++ nodebuilder/tests/api_test.go | 4 ++++ 4 files changed, 21 insertions(+), 1 deletion(-) diff --git a/nodebuilder/module.go b/nodebuilder/module.go index 26dc1d9e33..f9948b2011 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -50,7 +50,6 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store state.ConstructModule(tp, &cfg.State, &cfg.Core), modhead.ConstructModule[*header.ExtendedHeader](tp, &cfg.Header), share.ConstructModule(tp, &cfg.Share), - rpc.ConstructModule(tp, &cfg.RPC), gateway.ConstructModule(tp, &cfg.Gateway), core.ConstructModule(tp, &cfg.Core), das.ConstructModule(tp, &cfg.DASer), @@ -58,6 +57,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store blob.ConstructModule(), node.ConstructModule(tp), prune.ConstructModule(tp), + rpc.ConstructModule(tp, &cfg.RPC), ) return fx.Module( diff --git a/nodebuilder/node/admin.go b/nodebuilder/node/admin.go index 8063835f1d..f71af66081 100644 --- a/nodebuilder/node/admin.go +++ b/nodebuilder/node/admin.go @@ -38,6 +38,14 @@ func (m *module) Info(context.Context) (Info, error) { }, nil } +func (m *module) Ready(context.Context) (bool, error) { + // Because the node uses FX to provide the RPC last, all services' lifecycles have been started by + // the point this endpoint is available. It is not 100% guaranteed at this point that all services + // are fully ready, but it is very high probability and all endpoints are available at this point + // regardless. + return true, nil +} + func (m *module) LogLevelSet(_ context.Context, name, level string) error { return logging.SetLogLevel(name, level) } diff --git a/nodebuilder/node/node.go b/nodebuilder/node/node.go index 18ce93615b..b2bc7dac31 100644 --- a/nodebuilder/node/node.go +++ b/nodebuilder/node/node.go @@ -14,6 +14,9 @@ type Module interface { // Info returns administrative information about the node. Info(context.Context) (Info, error) + // Ready returns true once the node's RPC is ready to accept requests. + Ready(context.Context) (bool, error) + // LogLevelSet sets the given component log level to the given level. LogLevelSet(ctx context.Context, name, level string) error @@ -28,6 +31,7 @@ var _ Module = (*API)(nil) type API struct { Internal struct { Info func(context.Context) (Info, error) `perm:"admin"` + Ready func(context.Context) (bool, error) `perm:"read"` LogLevelSet func(ctx context.Context, name, level string) error `perm:"admin"` AuthVerify func(ctx context.Context, token string) ([]auth.Permission, error) `perm:"admin"` AuthNew func(ctx context.Context, perms []auth.Permission) (string, error) `perm:"admin"` @@ -38,6 +42,10 @@ func (api *API) Info(ctx context.Context) (Info, error) { return api.Internal.Info(ctx) } +func (api *API) Ready(ctx context.Context) (bool, error) { + return api.Internal.Ready(ctx) +} + func (api *API) LogLevelSet(ctx context.Context, name, level string) error { return api.Internal.LogLevelSet(ctx, name, level) } diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 960ce38875..751eaa63e5 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -48,6 +48,10 @@ func TestNodeModule(t *testing.T) { require.NoError(t, err) require.Equal(t, info.APIVersion, node.APIVersion) + ready, err := client.Node.Ready(ctx) + require.NoError(t, err) + require.True(t, ready) + perms, err := client.Node.AuthVerify(ctx, jwt) require.NoError(t, err) require.Equal(t, perms, adminPerms) From 1bae0600307a64bc0a94279ca50eab08ab9770b6 Mon Sep 17 00:00:00 2001 From: ramin Date: Mon, 29 Jan 2024 14:01:11 +0000 Subject: [PATCH 376/388] chore(spelling): fix all stray typos across comments and tests (#3142) Used a tool to find stray spellings then fix them, hopefully this will prevent future small PRs for little fixes emerging and we can encourage more significant / valuable PRs from external contributors --- libs/pidstore/pidstore_test.go | 2 +- nodebuilder/p2p/cmd/p2p.go | 2 +- nodebuilder/tests/api_test.go | 2 +- nodebuilder/tests/swamp/swamp.go | 4 ++-- share/getters/shrex_test.go | 4 ++-- share/ipld/corrupted_data_test.go | 8 ++++---- share/p2p/peers/manager.go | 2 +- state/core_access.go | 2 +- 8 files changed, 13 insertions(+), 13 deletions(-) diff --git a/libs/pidstore/pidstore_test.go b/libs/pidstore/pidstore_test.go index 56f8a308ec..4a35783db3 100644 --- a/libs/pidstore/pidstore_test.go +++ b/libs/pidstore/pidstore_test.go @@ -22,7 +22,7 @@ func TestPutLoad(t *testing.T) { ds := sync.MutexWrap(datastore.NewMapDatastore()) - t.Run("unitialized-pidstore", func(t *testing.T) { + t.Run("uninitialized-pidstore", func(t *testing.T) { testPutLoad(ctx, ds, t) }) t.Run("initialized-pidstore", func(t *testing.T) { diff --git a/nodebuilder/p2p/cmd/p2p.go b/nodebuilder/p2p/cmd/p2p.go index 5951595fa4..8b44802947 100644 --- a/nodebuilder/p2p/cmd/p2p.go +++ b/nodebuilder/p2p/cmd/p2p.go @@ -224,7 +224,7 @@ var connectednessCmd = &cobra.Command{ var natStatusCmd = &cobra.Command{ Use: "nat-status", - Short: "Gets the currrent NAT status", + Short: "Gets the current NAT status", Args: cobra.NoArgs, RunE: func(cmd *cobra.Command, args []string) error { client, err := cmdnode.ParseClientFromCtx(cmd.Context()) diff --git a/nodebuilder/tests/api_test.go b/nodebuilder/tests/api_test.go index 751eaa63e5..a3b99a750b 100644 --- a/nodebuilder/tests/api_test.go +++ b/nodebuilder/tests/api_test.go @@ -93,7 +93,7 @@ func TestGetByHeight(t *testing.T) { require.ErrorContains(t, err, "given height is from the future") } -// TestBlobRPC ensures that blobs can be submited via rpc +// TestBlobRPC ensures that blobs can be submitted via rpc func TestBlobRPC(t *testing.T) { ctx, cancel := context.WithTimeout(context.Background(), swamp.DefaultTestTimeout) t.Cleanup(cancel) diff --git a/nodebuilder/tests/swamp/swamp.go b/nodebuilder/tests/swamp/swamp.go index 617fe76151..9faf69744d 100644 --- a/nodebuilder/tests/swamp/swamp.go +++ b/nodebuilder/tests/swamp/swamp.go @@ -39,7 +39,7 @@ import ( var blackholeIP6 = net.ParseIP("100::") -// DefaultTestTimeout should be used as the default timout on all the Swamp tests. +// DefaultTestTimeout should be used as the default timeout on all the Swamp tests. // It's generously set to 5 minutes to give enough time for CI. const DefaultTestTimeout = time.Minute * 5 @@ -331,7 +331,7 @@ func (s *Swamp) Disconnect(t *testing.T, peerA, peerB *nodebuilder.Node) { // SetBootstrapper sets the given bootstrappers as the "bootstrappers" for the // Swamp test suite. Every new full or light node created on the suite afterwards // will automatically add the suite's bootstrappers as trusted peers to their config. -// NOTE: Bridge nodes do not automaatically add the bootstrappers as trusted peers. +// NOTE: Bridge nodes do not automatically add the bootstrappers as trusted peers. // NOTE: Use `NewNodeWithStore` to avoid this automatic configuration. func (s *Swamp) SetBootstrapper(t *testing.T, bootstrappers ...*nodebuilder.Node) { for _, trusted := range bootstrappers { diff --git a/share/getters/shrex_test.go b/share/getters/shrex_test.go index b625bb4c10..075735579b 100644 --- a/share/getters/shrex_test.go +++ b/share/getters/shrex_test.go @@ -121,7 +121,7 @@ func TestShrexGetter(t *testing.T) { t.Cleanup(cancel) // generate test data - eds, dah, maxNamesapce := generateTestEDS(t) + eds, dah, maxNamespace := generateTestEDS(t) eh := headertest.RandExtendedHeaderWithRoot(t, dah) require.NoError(t, edsStore.Put(ctx, dah.Hash(), eds)) peerManager.Validate(ctx, srvHost.ID(), shrexsub.Notification{ @@ -129,7 +129,7 @@ func TestShrexGetter(t *testing.T) { Height: 1, }) - namespace, err := addToNamespace(maxNamesapce, 1) + namespace, err := addToNamespace(maxNamespace, 1) require.NoError(t, err) // check for namespace to be not in root require.Len(t, ipld.FilterRootByNamespace(dah, namespace), 0) diff --git a/share/ipld/corrupted_data_test.go b/share/ipld/corrupted_data_test.go index d1d6e6b4d5..0d0af6dd35 100644 --- a/share/ipld/corrupted_data_test.go +++ b/share/ipld/corrupted_data_test.go @@ -25,7 +25,7 @@ func TestNamespaceHasher_CorruptedData(t *testing.T) { t.Cleanup(cancel) net := availability_test.NewTestDAGNet(ctx, t) - requestor := full.Node(net) + requester := full.Node(net) provider, mockBS := availability_test.MockNode(t, net) provider.Availability = full.TestAvailability(t, getters.NewIPLDGetter(provider.BlockService)) net.ConnectAll() @@ -37,15 +37,15 @@ func TestNamespaceHasher_CorruptedData(t *testing.T) { eh := headertest.RandExtendedHeaderWithRoot(t, root) getCtx, cancelGet := context.WithTimeout(ctx, sharesAvailableTimeout) t.Cleanup(cancelGet) - err := requestor.SharesAvailable(getCtx, eh) + err := requester.SharesAvailable(getCtx, eh) require.NoError(t, err) // clear the storage of the requester so that it must retrieve again, then start attacking // we reinitialize the node to clear the eds store - requestor = full.Node(net) + requester = full.Node(net) mockBS.Attacking = true getCtx, cancelGet = context.WithTimeout(ctx, sharesAvailableTimeout) t.Cleanup(cancelGet) - err = requestor.SharesAvailable(getCtx, eh) + err = requester.SharesAvailable(getCtx, eh) require.ErrorIs(t, err, share.ErrNotAvailable) } diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index 23fa18bcb2..964b022249 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -303,7 +303,7 @@ func (m *Manager) subscribeHeader(ctx context.Context, headerSub libhead.Subscri log.Debugw("stored initial height", "height", h.Height()) } - // update storeFrom if header heigh + // update storeFrom if header height m.storeFrom.Store(uint64(max(0, int(h.Height())-storedPoolsAmount))) log.Debugw("updated lowest stored height", "height", h.Height()) } diff --git a/state/core_access.go b/state/core_access.go index 358457b4f0..c3fbd4836a 100644 --- a/state/core_access.go +++ b/state/core_access.go @@ -181,7 +181,7 @@ func (ca *CoreAccessor) constructSignedTx( } // SubmitPayForBlob builds, signs, and synchronously submits a MsgPayForBlob. It blocks until the -// transaction is committed and returns the TxReponse. If gasLim is set to 0, the method will +// transaction is committed and returns the TxResponse. If gasLim is set to 0, the method will // automatically estimate the gas limit. If the fee is negative, the method will use the nodes min // gas price multiplied by the gas limit. func (ca *CoreAccessor) SubmitPayForBlob( From cc087accfdd4420a892f20d6af8fe8d889f56972 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 21:15:23 +0000 Subject: [PATCH 377/388] chore(deps): Bump github.com/multiformats/go-multiaddr from 0.12.1 to 0.12.2 (#3144) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [github.com/multiformats/go-multiaddr](https://github.com/multiformats/go-multiaddr) from 0.12.1 to 0.12.2.
Release notes

Sourced from github.com/multiformats/go-multiaddr's releases.

v0.12.2

What's Changed

New Contributors

Full Changelog: https://github.com/multiformats/go-multiaddr/compare/v0.12.1...v0.12.2

Commits
  • e2ad674 chore: release v0.12.2
  • 77f3edf tests: add round trip equality check to fuzz (#232)
  • ac5c928 fix: correctly parse ports as uint16 and explicitely fail on overflows (#228)
  • 45314d9 replace custom random tests with testing.F (#227)
  • See full diff in compare view

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=github.com/multiformats/go-multiaddr&package-manager=go_modules&previous-version=0.12.1&new-version=0.12.2)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore major version` will close this group update PR and stop Dependabot creating any more for the specific dependency's major version (unless you unignore this specific dependency's major version or upgrade to it yourself) - `@dependabot ignore minor version` will close this group update PR and stop Dependabot creating any more for the specific dependency's minor version (unless you unignore this specific dependency's minor version or upgrade to it yourself) - `@dependabot ignore ` will close this group update PR and stop Dependabot creating any more for the specific dependency (unless you unignore this specific dependency or upgrade to it yourself) - `@dependabot unignore ` will remove all of the ignore conditions of the specified dependency - `@dependabot unignore ` will remove the ignore condition of the specified dependency and ignore conditions
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 0291091e53..66d12f6780 100644 --- a/go.mod +++ b/go.mod @@ -45,7 +45,7 @@ require ( github.com/libp2p/go-libp2p-routing-helpers v0.7.3 github.com/mitchellh/go-homedir v1.1.0 github.com/multiformats/go-base32 v0.1.0 - github.com/multiformats/go-multiaddr v0.12.1 + github.com/multiformats/go-multiaddr v0.12.2 github.com/multiformats/go-multiaddr-dns v0.3.1 github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 diff --git a/go.sum b/go.sum index bfa5181462..d2fcb82e20 100644 --- a/go.sum +++ b/go.sum @@ -1831,8 +1831,8 @@ github.com/multiformats/go-multiaddr v0.5.0/go.mod h1:3KAxNkUqLTJ20AAwN4XVX4kZar github.com/multiformats/go-multiaddr v0.6.0/go.mod h1:F4IpaKZuPP360tOMn2Tpyu0At8w23aRyVqeK0DbFeGM= github.com/multiformats/go-multiaddr v0.7.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= github.com/multiformats/go-multiaddr v0.8.0/go.mod h1:Fs50eBDWvZu+l3/9S6xAE7ZYj6yhxlvaVZjakWN7xRs= -github.com/multiformats/go-multiaddr v0.12.1 h1:vm+BA/WZA8QZDp1pF1FWhi5CT3g1tbi5GJmqpb6wnlk= -github.com/multiformats/go-multiaddr v0.12.1/go.mod h1:7mPkiBMmLeFipt+nNSq9pHZUeJSt8lHBgH6yhj0YQzE= +github.com/multiformats/go-multiaddr v0.12.2 h1:9G9sTY/wCYajKa9lyfWPmpZAwe6oV+Wb1zcmMS1HG24= +github.com/multiformats/go-multiaddr v0.12.2/go.mod h1:GKyaTYjZRdcUhyOetrxTk9z0cW+jA/YrnqTOvKgi44M= github.com/multiformats/go-multiaddr-dns v0.0.1/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.0.2/go.mod h1:9kWcqw/Pj6FwxAwW38n/9403szc57zJPs45fmnznu3Q= github.com/multiformats/go-multiaddr-dns v0.2.0/go.mod h1:TJ5pr5bBO7Y1B18djPuRsVkduhQH2YqYSbxWJzYGdK0= From b57ad60e6eb34fbafd9a28594e614a2b06fc1691 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 29 Jan 2024 21:35:52 +0000 Subject: [PATCH 378/388] chore(deps): Bump alpine from 3.19.0 to 3.19.1 (#3143) Bumps alpine from 3.19.0 to 3.19.1. [![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=alpine&package-manager=docker&previous-version=3.19.0&new-version=3.19.1)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- Dockerfile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Dockerfile b/Dockerfile index e15d7d4f1c..01fccafe2e 100644 --- a/Dockerfile +++ b/Dockerfile @@ -25,7 +25,7 @@ RUN uname -a &&\ CGO_ENABLED=${CGO_ENABLED} GOOS=${TARGETOS} GOARCH=${TARGETARCH} \ make build && make cel-key -FROM docker.io/alpine:3.19.0 +FROM docker.io/alpine:3.19.1 # Read here why UID 10001: https://github.com/hexops/dockerfile/blob/main/README.md#do-not-use-a-uid-below-10000 ARG UID=10001 From f235d1056d0f76e3c55697ebfab41cf7ce0eb0e5 Mon Sep 17 00:00:00 2001 From: Viacheslav Date: Tue, 30 Jan 2024 14:22:00 +0200 Subject: [PATCH 379/388] feat(blob/trace): cover blob service with traces (#3113) Cover Blob service with Otel traces. Partly fixes #3112 --- blob/service.go | 50 +++++++++++++++++++++++++++++++++++++++++++------ 1 file changed, 44 insertions(+), 6 deletions(-) diff --git a/blob/service.go b/blob/service.go index 6a70772797..fc1d630e62 100644 --- a/blob/service.go +++ b/blob/service.go @@ -11,12 +11,17 @@ import ( "github.com/cosmos/cosmos-sdk/types" auth "github.com/cosmos/cosmos-sdk/x/auth/types" logging "github.com/ipfs/go-log/v2" + "go.opentelemetry.io/otel" + "go.opentelemetry.io/otel/attribute" + "go.opentelemetry.io/otel/codes" + "go.opentelemetry.io/otel/trace" "github.com/celestiaorg/celestia-app/pkg/appconsts" "github.com/celestiaorg/celestia-app/pkg/shares" blobtypes "github.com/celestiaorg/celestia-app/x/blob/types" "github.com/celestiaorg/celestia-node/header" + "github.com/celestiaorg/celestia-node/libs/utils" "github.com/celestiaorg/celestia-node/share" ) @@ -24,7 +29,8 @@ var ( ErrBlobNotFound = errors.New("blob: not found") ErrInvalidProof = errors.New("blob: invalid proof") - log = logging.Logger("blob") + log = logging.Logger("blob") + tracer = otel.Tracer("blob/service") ) // GasPrice represents the amount to be paid per gas unit. Fee is set by @@ -185,7 +191,11 @@ func (s *Service) Included( namespace share.Namespace, proof *Proof, com Commitment, -) (bool, error) { +) (_ bool, err error) { + ctx, span := tracer.Start(ctx, "included") + defer func() { + utils.SetStatusAndEnd(span, err) + }() // In the current implementation, LNs will have to download all shares to recompute the commitment. // To achieve 1. we need to modify Proof structure and to store all subtree roots, that were // involved in commitment creation and then call `merkle.HashFromByteSlices`(tendermint package). @@ -213,24 +223,47 @@ func (s *Service) getByCommitment( height uint64, namespace share.Namespace, commitment Commitment, -) (*Blob, *Proof, error) { +) (_ *Blob, _ *Proof, err error) { log.Infow("requesting blob", "height", height, "namespace", namespace.String()) - header, err := s.headerGetter(ctx, height) + ctx, span := tracer.Start(ctx, "get-by-commitment") + defer func() { + utils.SetStatusAndEnd(span, err) + }() + span.SetAttributes( + attribute.Int64("height", int64(height)), + attribute.String("commitment", string(commitment)), + ) + + getCtx, headerGetterSpan := tracer.Start(ctx, "header-getter") + + header, err := s.headerGetter(getCtx, height) if err != nil { + headerGetterSpan.SetStatus(codes.Error, err.Error()) return nil, nil, err } - namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) + headerGetterSpan.SetStatus(codes.Ok, "") + headerGetterSpan.AddEvent("received eds", trace.WithAttributes( + attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) + + getCtx, getSharesSpan := tracer.Start(ctx, "get-shares-by-namespace") + + namespacedShares, err := s.shareGetter.GetSharesByNamespace(getCtx, header, namespace) if err != nil { if errors.Is(err, share.ErrNotFound) { err = ErrBlobNotFound } + getSharesSpan.SetStatus(codes.Error, err.Error()) return nil, nil, err } + getSharesSpan.SetStatus(codes.Ok, "") + getSharesSpan.AddEvent("received shares", trace.WithAttributes( + attribute.Int64("eds-size", int64(len(header.DAH.RowRoots))))) + var ( rawShares = make([]shares.Share, 0) proofs = make(Proof, 0) @@ -260,6 +293,7 @@ func (s *Service) getByCommitment( } for _, b := range blobs { if b.Commitment.Equal(commitment) { + span.AddEvent("blob reconstructed") return b, &proofs, nil } // Falling under this flag means that the data from the last row @@ -296,7 +330,11 @@ func (s *Service) getBlobs( ctx context.Context, namespace share.Namespace, header *header.ExtendedHeader, -) ([]*Blob, error) { +) (_ []*Blob, err error) { + ctx, span := tracer.Start(ctx, "get-blobs") + defer func() { + utils.SetStatusAndEnd(span, err) + }() namespacedShares, err := s.shareGetter.GetSharesByNamespace(ctx, header, namespace) if err != nil { return nil, err From d7c7697b7003b3b803a61d72e46073d947672504 Mon Sep 17 00:00:00 2001 From: ramin Date: Wed, 31 Jan 2024 12:51:47 +0000 Subject: [PATCH 380/388] ci(unit tests): YOINK unit tests with race detector temporarily (#3147) suggesting we YEET the `unit test with race` temporarily as the job intermittently fails across a handful of tests pretty regularly and we don't require this job to pass for a PR merge anyway. Lets remove, i'll get it working and also enable for `macos-latest` which fails ALWAYS on github then we can / replace the normal unit tests with unit tests with race after we get it all running and keep CI green vs just ignoring a red x most of the time --- .github/workflows/go-ci.yml | 36 +++++++++++++++++++++--------------- 1 file changed, 21 insertions(+), 15 deletions(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index 5da90de33e..cb8187b82e 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -105,21 +105,27 @@ jobs: file: ./coverage.txt name: coverage-${{ matrix.os }} - unit_test_race: - needs: [lint, go_mod_tidy_check] - name: Unit Tests with Race Detector (ubuntu-latest) - runs-on: ubuntu-latest - - steps: - - uses: actions/checkout@v4 - - - name: set up go - uses: actions/setup-go@v5 - with: - go-version: ${{ inputs.go-version }} - - - name: execute test run - run: make test-unit-race ENABLE_VERBOSE=${{ needs.setup.outputs.debug }} + # @ramin - Temporarily removed while we figure out getting + # these unit tests consistently running on ubuntu-latest + # and then enabled for macos-latest. We aren't requiring + # unit_race_test to pass for PRs so lets remove and reintroduce + # once green + # + # unit_test_race: + # needs: [lint, go_mod_tidy_check] + # name: Unit Tests with Race Detector (ubuntu-latest) + # runs-on: ubuntu-latest + + # steps: + # - uses: actions/checkout@v4 + + # - name: set up go + # uses: actions/setup-go@v5 + # with: + # go-version: ${{ inputs.go-version }} + + # - name: execute test run + # run: make test-unit-race ENABLE_VERBOSE=${{ needs.setup.outputs.debug }} integration_test: name: Integration Tests From c32ccf5933abef5fc7631a62d3c404151d3fee48 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Thu, 1 Feb 2024 10:33:21 +0000 Subject: [PATCH 381/388] chore(deps): Bump codecov/codecov-action from 3.1.4 to 3.1.6 (#3148) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Bumps [codecov/codecov-action](https://github.com/codecov/codecov-action) from 3.1.4 to 3.1.6.
Release notes

Sourced from codecov/codecov-action's releases.

v3.1.6

Full Changelog: https://github.com/codecov/codecov-action/compare/v3.1.5...v3.1.6

v3.1.5

What's Changed

New Contributors

Full Changelog: https://github.com/codecov/codecov-action/compare/v3.1.4...v3.1.5

Changelog

Sourced from codecov/codecov-action's changelog.

4.0.0-beta.2

Fixes

  • #1085 not adding -n if empty to do-upload command

4.0.0-beta.1

v4 represents a move from the universal uploader to the Codecov CLI. Although this will unlock new features for our users, the CLI is not yet at feature parity with the universal uploader.

Breaking Changes

  • No current support for aarch64 and alpine architectures.
  • Tokenless uploading is unsuported
  • Various arguments to the Action have been removed
Commits

[![Dependabot compatibility score](https://dependabot-badges.githubapp.com/badges/compatibility_score?dependency-name=codecov/codecov-action&package-manager=github_actions&previous-version=3.1.4&new-version=3.1.6)](https://docs.github.com/en/github/managing-security-vulnerabilities/about-dependabot-security-updates#about-compatibility-scores) Dependabot will resolve any conflicts with this PR as long as you don't alter it yourself. You can also trigger a rebase manually by commenting `@dependabot rebase`. [//]: # (dependabot-automerge-start) [//]: # (dependabot-automerge-end) ---
Dependabot commands and options
You can trigger Dependabot actions by commenting on this PR: - `@dependabot rebase` will rebase this PR - `@dependabot recreate` will recreate this PR, overwriting any edits that have been made to it - `@dependabot merge` will merge this PR after your CI passes on it - `@dependabot squash and merge` will squash and merge this PR after your CI passes on it - `@dependabot cancel merge` will cancel a previously requested merge and block automerging - `@dependabot reopen` will reopen this PR if it is closed - `@dependabot close` will close this PR and stop Dependabot recreating it. You can achieve the same result by closing it manually - `@dependabot show ignore conditions` will show all of the ignore conditions of the specified dependency - `@dependabot ignore this major version` will close this PR and stop Dependabot creating any more for this major version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this minor version` will close this PR and stop Dependabot creating any more for this minor version (unless you reopen the PR or upgrade to it yourself) - `@dependabot ignore this dependency` will close this PR and stop Dependabot creating any more for this dependency (unless you reopen the PR or upgrade to it yourself)
Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> --- .github/workflows/go-ci.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/go-ci.yml b/.github/workflows/go-ci.yml index cb8187b82e..0a8807f212 100644 --- a/.github/workflows/go-ci.yml +++ b/.github/workflows/go-ci.yml @@ -98,7 +98,7 @@ jobs: retention-days: 5 - name: upload coverage - uses: codecov/codecov-action@v3.1.4 + uses: codecov/codecov-action@v3.1.6 with: env_vars: OS token: ${{ secrets.CODECOV_TOKEN }} From 625cdf3b1f31afe5371cca8c355d59a572ae6ea6 Mon Sep 17 00:00:00 2001 From: Ryan Date: Thu, 1 Feb 2024 17:00:07 +0100 Subject: [PATCH 382/388] feat(rpc): implementing go-da interface (#3146) Integrates go-da directly to eliminate the need for [celestia-da](https://github.com/rollkit/celestia-da). --- Makefile | 2 +- api/rpc/client/client.go | 3 + api/rpc_test.go | 6 + blob/blob.go | 27 +++++ go.mod | 1 + go.sum | 2 + nodebuilder/da/da.go | 54 +++++++++ nodebuilder/da/mocks/api.go | 140 ++++++++++++++++++++++ nodebuilder/da/module.go | 14 +++ nodebuilder/da/service.go | 188 ++++++++++++++++++++++++++++++ nodebuilder/module.go | 2 + nodebuilder/node.go | 2 + nodebuilder/rpc/constructors.go | 3 + nodebuilder/tests/da_test.go | 144 +++++++++++++++++++++++ nodebuilder/tests/helpers_test.go | 6 +- share/p2p/peers/manager.go | 3 +- 16 files changed, 592 insertions(+), 5 deletions(-) create mode 100644 nodebuilder/da/da.go create mode 100644 nodebuilder/da/mocks/api.go create mode 100644 nodebuilder/da/module.go create mode 100644 nodebuilder/da/service.go create mode 100644 nodebuilder/tests/da_test.go diff --git a/Makefile b/Makefile index 87bcbbf61c..a43c917345 100644 --- a/Makefile +++ b/Makefile @@ -166,7 +166,7 @@ pb-gen: ## openrpc-gen: Generate OpenRPC spec for Celestia-Node's RPC api openrpc-gen: @echo "--> Generating OpenRPC spec" - @go run ./cmd/docgen fraud header state share das p2p node blob + @go run ./cmd/docgen fraud header state share das p2p node blob da .PHONY: openrpc-gen ## lint-imports: Lint only Go imports. diff --git a/api/rpc/client/client.go b/api/rpc/client/client.go index 9cd5fe08e3..1d8142048b 100644 --- a/api/rpc/client/client.go +++ b/api/rpc/client/client.go @@ -9,6 +9,7 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc/perms" "github.com/celestiaorg/celestia-node/nodebuilder/blob" + "github.com/celestiaorg/celestia-node/nodebuilder/da" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -33,6 +34,7 @@ type Client struct { P2P p2p.API Node node.API Blob blob.API + DA da.API closer multiClientCloser } @@ -91,5 +93,6 @@ func moduleMap(client *Client) map[string]interface{} { "p2p": &client.P2P.Internal, "node": &client.Node.Internal, "blob": &client.Blob.Internal, + "da": &client.DA.Internal, } } diff --git a/api/rpc_test.go b/api/rpc_test.go index ec308a2320..80dd29ddb4 100644 --- a/api/rpc_test.go +++ b/api/rpc_test.go @@ -22,6 +22,8 @@ import ( "github.com/celestiaorg/celestia-node/nodebuilder" "github.com/celestiaorg/celestia-node/nodebuilder/blob" blobMock "github.com/celestiaorg/celestia-node/nodebuilder/blob/mocks" + "github.com/celestiaorg/celestia-node/nodebuilder/da" + daMock "github.com/celestiaorg/celestia-node/nodebuilder/da/mocks" "github.com/celestiaorg/celestia-node/nodebuilder/das" dasMock "github.com/celestiaorg/celestia-node/nodebuilder/das/mocks" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" @@ -91,6 +93,7 @@ type api struct { Node node.Module P2P p2p.Module Blob blob.Module + DA da.Module } func TestModulesImplementFullAPI(t *testing.T) { @@ -297,6 +300,7 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, * p2pMock.NewMockModule(ctrl), nodeMock.NewMockModule(ctrl), blobMock.NewMockModule(ctrl), + daMock.NewMockModule(ctrl), } // given the behavior of fx.Invoke, this invoke will be called last as it is added at the root @@ -310,6 +314,7 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, * srv.RegisterAuthedService("p2p", mockAPI.P2P, &p2p.API{}) srv.RegisterAuthedService("node", mockAPI.Node, &node.API{}) srv.RegisterAuthedService("blob", mockAPI.Blob, &blob.API{}) + srv.RegisterAuthedService("da", mockAPI.DA, &da.API{}) }) // fx.Replace does not work here, but fx.Decorate does nd := nodebuilder.TestNode(t, node.Full, invokeRPC, fx.Decorate(func() (jwt.Signer, error) { @@ -334,4 +339,5 @@ type mockAPI struct { P2P *p2pMock.MockModule Node *nodeMock.MockModule Blob *blobMock.MockModule + DA *daMock.MockModule } diff --git a/blob/blob.go b/blob/blob.go index 945ffdf587..9843441dd2 100644 --- a/blob/blob.go +++ b/blob/blob.go @@ -35,6 +35,33 @@ type Proof []*nmt.Proof func (p Proof) Len() int { return len(p) } +func (p Proof) MarshalJSON() ([]byte, error) { + proofs := make([]string, 0, len(p)) + for _, proof := range p { + proofBytes, err := proof.MarshalJSON() + if err != nil { + return nil, err + } + proofs = append(proofs, string(proofBytes)) + } + return json.Marshal(proofs) +} + +func (p *Proof) UnmarshalJSON(b []byte) error { + var proofs []string + if err := json.Unmarshal(b, &proofs); err != nil { + return err + } + for _, proof := range proofs { + var nmtProof nmt.Proof + if err := nmtProof.UnmarshalJSON([]byte(proof)); err != nil { + return err + } + *p = append(*p, &nmtProof) + } + return nil +} + // equal is a temporary method that compares two proofs. // should be removed in BlobService V1. func (p Proof) equal(input Proof) error { diff --git a/go.mod b/go.mod index 66d12f6780..1a3f54a6bc 100644 --- a/go.mod +++ b/go.mod @@ -52,6 +52,7 @@ require ( github.com/prometheus/client_golang v1.18.0 github.com/pyroscope-io/client v0.7.2 github.com/pyroscope-io/otel-profiling-go v0.5.0 + github.com/rollkit/go-da v0.4.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 diff --git a/go.sum b/go.sum index d2fcb82e20..0712788b90 100644 --- a/go.sum +++ b/go.sum @@ -2121,6 +2121,8 @@ github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTE github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= github.com/rogpeppe/go-internal v1.11.0 h1:cWPaGQEPrBb5/AsnsZesgZZ9yb1OQ+GOISoDNXVBh4M= github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUzkipdSkR5nkCZA= +github.com/rollkit/go-da v0.4.0 h1:/s7ZrVq7DC2aK8UXIvB7rsXrZ2mVGRw7zrexcxRvhlw= +github.com/rollkit/go-da v0.4.0/go.mod h1:Kef0XI5ecEKd3TXzI8S+9knAUJnZg0svh2DuXoCsPlM= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= diff --git a/nodebuilder/da/da.go b/nodebuilder/da/da.go new file mode 100644 index 0000000000..0d604d769f --- /dev/null +++ b/nodebuilder/da/da.go @@ -0,0 +1,54 @@ +package da + +import ( + "context" + + "github.com/rollkit/go-da" +) + +//go:generate mockgen -destination=mocks/api.go -package=mocks . Module +type Module interface { + da.DA +} + +// API is a wrapper around Module for the RPC. +// TODO(@distractedm1nd): These structs need to be autogenerated. +type API struct { + Internal struct { + MaxBlobSize func(ctx context.Context) (uint64, error) `perm:"read"` + Get func(ctx context.Context, ids []da.ID, ns da.Namespace) ([]da.Blob, error) `perm:"read"` + GetIDs func(ctx context.Context, height uint64, ns da.Namespace) ([]da.ID, error) `perm:"read"` + GetProofs func(ctx context.Context, ids []da.ID, ns da.Namespace) ([]da.Proof, error) `perm:"read"` + Commit func(ctx context.Context, blobs []da.Blob, ns da.Namespace) ([]da.Commitment, error) `perm:"read"` + Validate func(context.Context, []da.ID, []da.Proof, da.Namespace) ([]bool, error) `perm:"read"` + Submit func(context.Context, []da.Blob, float64, da.Namespace) ([]da.ID, error) `perm:"write"` + } +} + +func (api *API) MaxBlobSize(ctx context.Context) (uint64, error) { + return api.Internal.MaxBlobSize(ctx) +} + +func (api *API) Get(ctx context.Context, ids []da.ID, ns da.Namespace) ([]da.Blob, error) { + return api.Internal.Get(ctx, ids, ns) +} + +func (api *API) GetIDs(ctx context.Context, height uint64, ns da.Namespace) ([]da.ID, error) { + return api.Internal.GetIDs(ctx, height, ns) +} + +func (api *API) GetProofs(ctx context.Context, ids []da.ID, ns da.Namespace) ([]da.Proof, error) { + return api.Internal.GetProofs(ctx, ids, ns) +} + +func (api *API) Commit(ctx context.Context, blobs []da.Blob, ns da.Namespace) ([]da.Commitment, error) { + return api.Internal.Commit(ctx, blobs, ns) +} + +func (api *API) Validate(ctx context.Context, ids []da.ID, proofs []da.Proof, ns da.Namespace) ([]bool, error) { + return api.Internal.Validate(ctx, ids, proofs, ns) +} + +func (api *API) Submit(ctx context.Context, blobs []da.Blob, gasPrice float64, ns da.Namespace) ([]da.ID, error) { + return api.Internal.Submit(ctx, blobs, gasPrice, ns) +} diff --git a/nodebuilder/da/mocks/api.go b/nodebuilder/da/mocks/api.go new file mode 100644 index 0000000000..5895240906 --- /dev/null +++ b/nodebuilder/da/mocks/api.go @@ -0,0 +1,140 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/celestiaorg/celestia-node/nodebuilder/da (interfaces: Module) + +// Package mocks is a generated GoMock package. +package mocks + +import ( + context "context" + reflect "reflect" + + gomock "github.com/golang/mock/gomock" +) + +// MockModule is a mock of Module interface. +type MockModule struct { + ctrl *gomock.Controller + recorder *MockModuleMockRecorder +} + +// MockModuleMockRecorder is the mock recorder for MockModule. +type MockModuleMockRecorder struct { + mock *MockModule +} + +// NewMockModule creates a new mock instance. +func NewMockModule(ctrl *gomock.Controller) *MockModule { + mock := &MockModule{ctrl: ctrl} + mock.recorder = &MockModuleMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockModule) EXPECT() *MockModuleMockRecorder { + return m.recorder +} + +// Commit mocks base method. +func (m *MockModule) Commit(arg0 context.Context, arg1 [][]byte, arg2 []byte) ([][]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Commit", arg0, arg1, arg2) + ret0, _ := ret[0].([][]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Commit indicates an expected call of Commit. +func (mr *MockModuleMockRecorder) Commit(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Commit", reflect.TypeOf((*MockModule)(nil).Commit), arg0, arg1, arg2) +} + +// Get mocks base method. +func (m *MockModule) Get(arg0 context.Context, arg1 [][]byte, arg2 []byte) ([][]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Get", arg0, arg1, arg2) + ret0, _ := ret[0].([][]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Get indicates an expected call of Get. +func (mr *MockModuleMockRecorder) Get(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Get", reflect.TypeOf((*MockModule)(nil).Get), arg0, arg1, arg2) +} + +// GetIDs mocks base method. +func (m *MockModule) GetIDs(arg0 context.Context, arg1 uint64, arg2 []byte) ([][]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetIDs", arg0, arg1, arg2) + ret0, _ := ret[0].([][]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetIDs indicates an expected call of GetIDs. +func (mr *MockModuleMockRecorder) GetIDs(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetIDs", reflect.TypeOf((*MockModule)(nil).GetIDs), arg0, arg1, arg2) +} + +// GetProofs mocks base method. +func (m *MockModule) GetProofs(arg0 context.Context, arg1 [][]byte, arg2 []byte) ([][]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetProofs", arg0, arg1, arg2) + ret0, _ := ret[0].([][]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// GetProofs indicates an expected call of GetProofs. +func (mr *MockModuleMockRecorder) GetProofs(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetProofs", reflect.TypeOf((*MockModule)(nil).GetProofs), arg0, arg1, arg2) +} + +// MaxBlobSize mocks base method. +func (m *MockModule) MaxBlobSize(arg0 context.Context) (uint64, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "MaxBlobSize", arg0) + ret0, _ := ret[0].(uint64) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// MaxBlobSize indicates an expected call of MaxBlobSize. +func (mr *MockModuleMockRecorder) MaxBlobSize(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "MaxBlobSize", reflect.TypeOf((*MockModule)(nil).MaxBlobSize), arg0) +} + +// Submit mocks base method. +func (m *MockModule) Submit(arg0 context.Context, arg1 [][]byte, arg2 float64, arg3 []byte) ([][]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Submit", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].([][]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Submit indicates an expected call of Submit. +func (mr *MockModuleMockRecorder) Submit(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Submit", reflect.TypeOf((*MockModule)(nil).Submit), arg0, arg1, arg2, arg3) +} + +// Validate mocks base method. +func (m *MockModule) Validate(arg0 context.Context, arg1, arg2 [][]byte, arg3 []byte) ([]bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Validate", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].([]bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Validate indicates an expected call of Validate. +func (mr *MockModuleMockRecorder) Validate(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Validate", reflect.TypeOf((*MockModule)(nil).Validate), arg0, arg1, arg2, arg3) +} diff --git a/nodebuilder/da/module.go b/nodebuilder/da/module.go new file mode 100644 index 0000000000..b119d11076 --- /dev/null +++ b/nodebuilder/da/module.go @@ -0,0 +1,14 @@ +package da + +import ( + "go.uber.org/fx" +) + +func ConstructModule() fx.Option { + return fx.Module("da", + fx.Provide(NewService), + fx.Provide(func(serv *Service) Module { + return serv + }), + ) +} diff --git a/nodebuilder/da/service.go b/nodebuilder/da/service.go new file mode 100644 index 0000000000..b775e10396 --- /dev/null +++ b/nodebuilder/da/service.go @@ -0,0 +1,188 @@ +package da + +import ( + "context" + "encoding/binary" + "fmt" + "strings" + + logging "github.com/ipfs/go-log/v2" + "github.com/rollkit/go-da" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + + "github.com/celestiaorg/celestia-node/blob" + nodeblob "github.com/celestiaorg/celestia-node/nodebuilder/blob" + "github.com/celestiaorg/celestia-node/share" +) + +var _ da.DA = (*Service)(nil) + +var log = logging.Logger("go-da") + +// heightLen is a length (in bytes) of serialized height. +// +// This is 8 as uint64 consist of 8 bytes. +const heightLen = 8 + +type Service struct { + blobServ nodeblob.Module +} + +func NewService(blobMod nodeblob.Module) *Service { + return &Service{ + blobServ: blobMod, + } +} + +// MaxBlobSize returns the max blob size +func (s *Service) MaxBlobSize(context.Context) (uint64, error) { + return appconsts.DefaultMaxBytes, nil +} + +// Get returns Blob for each given ID, or an error. +func (s *Service) Get(ctx context.Context, ids []da.ID, ns da.Namespace) ([]da.Blob, error) { + blobs := make([]da.Blob, 0, len(ids)) + for _, id := range ids { + height, commitment := SplitID(id) + log.Debugw("getting blob", "height", height, "commitment", commitment, "namespace", share.Namespace(ns)) + currentBlob, err := s.blobServ.Get(ctx, height, ns, commitment) + log.Debugw("got blob", "height", height, "commitment", commitment, "namespace", share.Namespace(ns)) + if err != nil { + return nil, err + } + blobs = append(blobs, currentBlob.Data) + } + return blobs, nil +} + +// GetIDs returns IDs of all Blobs located in DA at given height. +func (s *Service) GetIDs(ctx context.Context, height uint64, namespace da.Namespace) ([]da.ID, error) { + var ids []da.ID //nolint:prealloc + log.Debugw("getting ids", "height", height, "namespace", share.Namespace(namespace)) + blobs, err := s.blobServ.GetAll(ctx, height, []share.Namespace{namespace}) + log.Debugw("got ids", "height", height, "namespace", share.Namespace(namespace)) + if err != nil { + if strings.Contains(err.Error(), blob.ErrBlobNotFound.Error()) { + return nil, nil + } + return nil, err + } + for _, b := range blobs { + ids = append(ids, MakeID(height, b.Commitment)) + } + return ids, nil +} + +// GetProofs returns inclusion Proofs for all Blobs located in DA at given height. +func (s *Service) GetProofs(ctx context.Context, ids []da.ID, namespace da.Namespace) ([]da.Proof, error) { + proofs := make([]da.Proof, len(ids)) + for i, id := range ids { + height, commitment := SplitID(id) + proof, err := s.blobServ.GetProof(ctx, height, namespace, commitment) + if err != nil { + return nil, err + } + proofs[i], err = proof.MarshalJSON() + if err != nil { + return nil, err + } + } + return proofs, nil +} + +// Commit creates a Commitment for each given Blob. +func (s *Service) Commit(_ context.Context, daBlobs []da.Blob, namespace da.Namespace) ([]da.Commitment, error) { + _, commitments, err := s.blobsAndCommitments(daBlobs, namespace) + return commitments, err +} + +// Submit submits the Blobs to Data Availability layer. +func (s *Service) Submit( + ctx context.Context, + daBlobs []da.Blob, + gasPrice float64, + namespace da.Namespace, +) ([]da.ID, error) { + blobs, _, err := s.blobsAndCommitments(daBlobs, namespace) + if err != nil { + return nil, err + } + + height, err := s.blobServ.Submit(ctx, blobs, blob.GasPrice(gasPrice)) + if err != nil { + log.Error("failed to submit blobs", "height", height, "gas price", gasPrice) + return nil, err + } + log.Info("successfully submitted blobs", "height", height, "gas price", gasPrice) + ids := make([]da.ID, len(blobs)) + for i, blob := range blobs { + ids[i] = MakeID(height, blob.Commitment) + } + return ids, nil +} + +// blobsAndCommitments converts []da.Blob to []*blob.Blob and generates corresponding +// []da.Commitment +func (s *Service) blobsAndCommitments( + daBlobs []da.Blob, namespace da.Namespace, +) ([]*blob.Blob, []da.Commitment, error) { + blobs := make([]*blob.Blob, 0, len(daBlobs)) + commitments := make([]da.Commitment, 0, len(daBlobs)) + for _, daBlob := range daBlobs { + b, err := blob.NewBlobV0(namespace, daBlob) + if err != nil { + return nil, nil, err + } + blobs = append(blobs, b) + + commitments = append(commitments, b.Commitment) + } + return blobs, commitments, nil +} + +// Validate validates Commitments against the corresponding Proofs. This should be possible without +// retrieving the Blobs. +func (s *Service) Validate( + ctx context.Context, + ids []da.ID, + daProofs []da.Proof, + namespace da.Namespace, +) ([]bool, error) { + included := make([]bool, len(ids)) + proofs := make([]*blob.Proof, len(ids)) + for i, daProof := range daProofs { + blobProof := &blob.Proof{} + err := blobProof.UnmarshalJSON(daProof) + if err != nil { + return nil, err + } + proofs[i] = blobProof + } + for i, id := range ids { + height, commitment := SplitID(id) + // TODO(tzdybal): for some reason, if proof doesn't match commitment, API returns (false, "blob: + // invalid proof") but analysis of the code in celestia-node implies this should never happen - + // maybe it's caused by openrpc? there is no way of gently handling errors here, but returned + // value is fine for us + fmt.Println("proof", proofs[i] == nil, "commitment", commitment == nil) + isIncluded, _ := s.blobServ.Included(ctx, height, namespace, proofs[i], commitment) + included = append(included, isIncluded) + } + return included, nil +} + +func MakeID(height uint64, commitment da.Commitment) da.ID { + id := make([]byte, heightLen+len(commitment)) + binary.LittleEndian.PutUint64(id, height) + copy(id[heightLen:], commitment) + return id +} + +func SplitID(id da.ID) (uint64, da.Commitment) { + if len(id) <= heightLen { + return 0, nil + } + commitment := id[heightLen:] + return binary.LittleEndian.Uint64(id[:heightLen]), commitment +} diff --git a/nodebuilder/module.go b/nodebuilder/module.go index f9948b2011..ad287b1ac8 100644 --- a/nodebuilder/module.go +++ b/nodebuilder/module.go @@ -9,6 +9,7 @@ import ( "github.com/celestiaorg/celestia-node/libs/fxutil" "github.com/celestiaorg/celestia-node/nodebuilder/blob" "github.com/celestiaorg/celestia-node/nodebuilder/core" + "github.com/celestiaorg/celestia-node/nodebuilder/da" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/gateway" @@ -55,6 +56,7 @@ func ConstructModule(tp node.Type, network p2p.Network, cfg *Config, store Store das.ConstructModule(tp, &cfg.DASer), fraud.ConstructModule(tp), blob.ConstructModule(), + da.ConstructModule(), node.ConstructModule(tp), prune.ConstructModule(tp), rpc.ConstructModule(tp, &cfg.RPC), diff --git a/nodebuilder/node.go b/nodebuilder/node.go index d5d0ab2016..b16a376cc1 100644 --- a/nodebuilder/node.go +++ b/nodebuilder/node.go @@ -22,6 +22,7 @@ import ( "github.com/celestiaorg/celestia-node/api/gateway" "github.com/celestiaorg/celestia-node/api/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/blob" + "github.com/celestiaorg/celestia-node/nodebuilder/da" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -71,6 +72,7 @@ type Node struct { BlobServ blob.Module // not optional DASer das.Module // not optional AdminServ node.Module // not optional + DAMod da.Module // not optional // start and stop control ref internal fx.App lifecycle funcs to be called from Start and Stop start, stop lifecycleFunc diff --git a/nodebuilder/rpc/constructors.go b/nodebuilder/rpc/constructors.go index 194dea8a03..c8e928a98a 100644 --- a/nodebuilder/rpc/constructors.go +++ b/nodebuilder/rpc/constructors.go @@ -5,6 +5,7 @@ import ( "github.com/celestiaorg/celestia-node/api/rpc" "github.com/celestiaorg/celestia-node/nodebuilder/blob" + "github.com/celestiaorg/celestia-node/nodebuilder/da" "github.com/celestiaorg/celestia-node/nodebuilder/das" "github.com/celestiaorg/celestia-node/nodebuilder/fraud" "github.com/celestiaorg/celestia-node/nodebuilder/header" @@ -24,6 +25,7 @@ func registerEndpoints( p2pMod p2p.Module, nodeMod node.Module, blobMod blob.Module, + daMod da.Module, serv *rpc.Server, ) { serv.RegisterAuthedService("fraud", fraudMod, &fraud.API{}) @@ -34,6 +36,7 @@ func registerEndpoints( serv.RegisterAuthedService("p2p", p2pMod, &p2p.API{}) serv.RegisterAuthedService("node", nodeMod, &node.API{}) serv.RegisterAuthedService("blob", blobMod, &blob.API{}) + serv.RegisterAuthedService("da", daMod, &da.API{}) } func server(cfg *Config, auth jwt.Signer) *rpc.Server { diff --git a/nodebuilder/tests/da_test.go b/nodebuilder/tests/da_test.go new file mode 100644 index 0000000000..2925898e1e --- /dev/null +++ b/nodebuilder/tests/da_test.go @@ -0,0 +1,144 @@ +//go:build blob || integration + +package tests + +import ( + "bytes" + "context" + "testing" + "time" + + "github.com/libp2p/go-libp2p/core/host" + "github.com/libp2p/go-libp2p/core/peer" + "github.com/stretchr/testify/require" + + "github.com/celestiaorg/celestia-app/pkg/appconsts" + + "github.com/celestiaorg/celestia-node/blob" + "github.com/celestiaorg/celestia-node/blob/blobtest" + "github.com/celestiaorg/celestia-node/nodebuilder/da" + "github.com/celestiaorg/celestia-node/nodebuilder/node" + "github.com/celestiaorg/celestia-node/nodebuilder/tests/swamp" + "github.com/celestiaorg/celestia-node/share" +) + +func TestDaModule(t *testing.T) { + ctx, cancel := context.WithTimeout(context.Background(), 25*time.Second) + t.Cleanup(cancel) + sw := swamp.NewSwamp(t, swamp.WithBlockTime(time.Second)) + + namespace, err := share.NewBlobNamespaceV0([]byte("namespace")) + require.NoError(t, err) + + appBlobs0, err := blobtest.GenerateV0Blobs([]int{8, 4}, true) + require.NoError(t, err) + appBlobs1, err := blobtest.GenerateV0Blobs([]int{4}, false) + require.NoError(t, err) + blobs := make([]*blob.Blob, 0, len(appBlobs0)+len(appBlobs1)) + daBlobs := make([][]byte, 0, len(appBlobs0)+len(appBlobs1)) + + for _, b := range append(appBlobs0, appBlobs1...) { + blob, err := blob.NewBlob(b.ShareVersion, namespace, b.Data) + require.NoError(t, err) + blobs = append(blobs, blob) + daBlobs = append(daBlobs, blob.Data) + } + + require.NoError(t, err) + bridge := sw.NewBridgeNode() + require.NoError(t, bridge.Start(ctx)) + + addrs, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(bridge.Host)) + require.NoError(t, err) + + fullCfg := sw.DefaultTestConfig(node.Full) + fullCfg.Header.TrustedPeers = append(fullCfg.Header.TrustedPeers, addrs[0].String()) + fullNode := sw.NewNodeWithConfig(node.Full, fullCfg) + require.NoError(t, fullNode.Start(ctx)) + + addrsFull, err := peer.AddrInfoToP2pAddrs(host.InfoFromHost(fullNode.Host)) + require.NoError(t, err) + + lightCfg := sw.DefaultTestConfig(node.Light) + lightCfg.Header.TrustedPeers = append(lightCfg.Header.TrustedPeers, addrsFull[0].String()) + lightNode := sw.NewNodeWithConfig(node.Light, lightCfg) + require.NoError(t, lightNode.Start(ctx)) + + fullClient := getAdminClient(ctx, fullNode, t) + lightClient := getAdminClient(ctx, lightNode, t) + + ids, err := fullClient.DA.Submit(ctx, daBlobs, -1, namespace) + require.NoError(t, err) + + var test = []struct { + name string + doFn func(t *testing.T) + }{ + { + name: "MaxBlobSize", + doFn: func(t *testing.T) { + mbs, err := fullClient.DA.MaxBlobSize(ctx) + require.NoError(t, err) + require.Equal(t, mbs, uint64(appconsts.DefaultMaxBytes)) + }, + }, + { + name: "GetProofs + Validate", + doFn: func(t *testing.T) { + t.Skip() + h, _ := da.SplitID(ids[0]) + lightClient.Header.WaitForHeight(ctx, h) + proofs, err := lightClient.DA.GetProofs(ctx, ids, namespace) + require.NoError(t, err) + require.NotEmpty(t, proofs) + valid, err := fullClient.DA.Validate(ctx, ids, proofs, namespace) + require.NoError(t, err) + for _, v := range valid { + require.True(t, v) + } + }, + }, + { + name: "GetIDs", + doFn: func(t *testing.T) { + height, _ := da.SplitID(ids[0]) + ids2, err := fullClient.DA.GetIDs(ctx, height, namespace) + require.NoError(t, err) + require.EqualValues(t, ids, ids2) + }, + }, + { + name: "Get", + doFn: func(t *testing.T) { + h, _ := da.SplitID(ids[0]) + lightClient.Header.WaitForHeight(ctx, h) + fetched, err := lightClient.DA.Get(ctx, ids, namespace) + require.NoError(t, err) + require.Len(t, fetched, len(ids)) + for i := range fetched { + require.True(t, bytes.Equal(fetched[i], daBlobs[i])) + } + }, + }, + { + name: "Commit", + doFn: func(t *testing.T) { + t.Skip() + fetched, err := fullClient.DA.Commit(ctx, ids, namespace) + require.NoError(t, err) + require.Len(t, fetched, len(ids)) + for i := range fetched { + _, commitment := da.SplitID(ids[i]) + require.EqualValues(t, fetched[i], commitment) + } + }, + }, + } + + for _, tt := range test { + tt := tt + t.Run(tt.name, func(t *testing.T) { + tt.doFn(t) + }) + } +} diff --git a/nodebuilder/tests/helpers_test.go b/nodebuilder/tests/helpers_test.go index 1e7f14d823..978b66553d 100644 --- a/nodebuilder/tests/helpers_test.go +++ b/nodebuilder/tests/helpers_test.go @@ -6,12 +6,12 @@ import ( "testing" "time" + "github.com/filecoin-project/go-jsonrpc/auth" + "github.com/stretchr/testify/require" + "github.com/celestiaorg/celestia-node/api/rpc/client" "github.com/celestiaorg/celestia-node/libs/authtoken" "github.com/celestiaorg/celestia-node/nodebuilder" - - "github.com/filecoin-project/go-jsonrpc/auth" - "github.com/stretchr/testify/require" ) func getAdminClient(ctx context.Context, nd *nodebuilder.Node, t *testing.T) *client.Client { diff --git a/share/p2p/peers/manager.go b/share/p2p/peers/manager.go index 964b022249..1a00059628 100644 --- a/share/p2p/peers/manager.go +++ b/share/p2p/peers/manager.go @@ -38,7 +38,8 @@ const ( // events in libp2p eventbusBufSize = 32 - // storedPoolsAmount is the amount of pools for recent headers that will be stored in the peer manager + // storedPoolsAmount is the amount of pools for recent headers that will be stored in the peer + // manager storedPoolsAmount = 10 ) From a33c80e20da684d656c7213580be7878bcd27cf4 Mon Sep 17 00:00:00 2001 From: Hlib Kanunnikov Date: Thu, 1 Feb 2024 20:26:42 +0100 Subject: [PATCH 383/388] chore: bump go-header (#3158) https://github.com/celestiaorg/go-header/releases/tag/v0.5.3 --- go.mod | 2 +- go.sum | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 1a3f54a6bc..b79e18ff12 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/benbjohnson/clock v1.3.5 github.com/celestiaorg/celestia-app v1.4.0 github.com/celestiaorg/go-fraud v0.2.0 - github.com/celestiaorg/go-header v0.5.2 + github.com/celestiaorg/go-header v0.5.3 github.com/celestiaorg/go-libp2p-messenger v0.2.0 github.com/celestiaorg/nmt v0.20.0 github.com/celestiaorg/rsmt2d v0.11.0 diff --git a/go.sum b/go.sum index 0712788b90..ec8fe9276c 100644 --- a/go.sum +++ b/go.sum @@ -368,8 +368,8 @@ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJ github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403/go.mod h1:cCGM1UoMvyTk8k62mkc+ReVu8iHBCtSBAAL4wYU7KEI= github.com/celestiaorg/go-fraud v0.2.0 h1:aaq2JiW0gTnhEdac3l51UCqSyJ4+VjFGTTpN83V4q7I= github.com/celestiaorg/go-fraud v0.2.0/go.mod h1:lNY1i4K6kUeeE60Z2VK8WXd+qXb8KRzfBhvwPkK6aUc= -github.com/celestiaorg/go-header v0.5.2 h1:CFsTAXcs1o38JVd8YN1Naq/Yzs6m9orMPEPNpLEgFJA= -github.com/celestiaorg/go-header v0.5.2/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= +github.com/celestiaorg/go-header v0.5.3 h1:8CcflT6aIlcQXKNWcMekoBNs3EU50mEmDp17gbn1pP4= +github.com/celestiaorg/go-header v0.5.3/go.mod h1:7BVR6myjRfACbqW1de6s8OjuK66XzHm8MpFNYr0G+nU= github.com/celestiaorg/go-libp2p-messenger v0.2.0 h1:/0MuPDcFamQMbw9xTZ73yImqgTO3jHV7wKHvWD/Irao= github.com/celestiaorg/go-libp2p-messenger v0.2.0/go.mod h1:s9PIhMi7ApOauIsfBcQwbr7m+HBzmVfDIS+QLdgzDSo= github.com/celestiaorg/merkletree v0.0.0-20210714075610-a84dc3ddbbe4 h1:CJdIpo8n5MFP2MwK0gSRcOVlDlFdQJO1p+FqdxYzmvc= From 01585ba111b819a74071e1f1e1cfaf3ef09dbf9f Mon Sep 17 00:00:00 2001 From: Tien Nguyen <116023870+htiennv@users.noreply.github.com> Date: Mon, 5 Feb 2024 19:37:06 +0700 Subject: [PATCH 384/388] chore: add missing stop ticker (#3161) --- share/p2p/discovery/backoff.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/share/p2p/discovery/backoff.go b/share/p2p/discovery/backoff.go index 70581bc519..7294915727 100644 --- a/share/p2p/discovery/backoff.go +++ b/share/p2p/discovery/backoff.go @@ -91,6 +91,8 @@ func (b *backoffConnector) HasBackoff(p peer.ID) bool { // GC is a perpetual GCing loop. func (b *backoffConnector) GC(ctx context.Context) { ticker := time.NewTicker(gcInterval) + defer ticker.Stop() + for { select { case <-ctx.Done(): From e55e1c88708b46839867bcbbed9bcdd8a3ffa830 Mon Sep 17 00:00:00 2001 From: Ryan Date: Tue, 6 Feb 2024 12:23:00 +0100 Subject: [PATCH 385/388] fix: improve rpc.skip-auth (#3152) --- api/rpc/server.go | 13 ++++++------- api/rpc_test.go | 18 +++++++++--------- nodebuilder/rpc/constructors.go | 18 +++++++++--------- 3 files changed, 24 insertions(+), 25 deletions(-) diff --git a/api/rpc/server.go b/api/rpc/server.go index 3bf58870bc..f247682083 100644 --- a/api/rpc/server.go +++ b/api/rpc/server.go @@ -61,15 +61,14 @@ func (s *Server) verifyAuth(_ context.Context, token string) ([]auth.Permission, // RegisterService registers a service onto the RPC server. All methods on the service will then be // exposed over the RPC. -func (s *Server) RegisterService(namespace string, service interface{}) { - s.rpc.Register(namespace, service) -} +func (s *Server) RegisterService(namespace string, service interface{}, out interface{}) { + if s.authDisabled { + s.rpc.Register(namespace, service) + return + } -// RegisterAuthedService registers a service onto the RPC server. All methods on the service will -// then be exposed over the RPC. -func (s *Server) RegisterAuthedService(namespace string, service interface{}, out interface{}) { auth.PermissionedProxy(perms.AllPerms, perms.DefaultPerms, service, getInternalStruct(out)) - s.RegisterService(namespace, out) + s.rpc.Register(namespace, out) } func getInternalStruct(api interface{}) interface{} { diff --git a/api/rpc_test.go b/api/rpc_test.go index 80dd29ddb4..ff38a42045 100644 --- a/api/rpc_test.go +++ b/api/rpc_test.go @@ -306,15 +306,15 @@ func setupNodeWithAuthedRPC(t *testing.T, auth jwt.Signer) (*nodebuilder.Node, * // given the behavior of fx.Invoke, this invoke will be called last as it is added at the root // level module. For further information, check the documentation on fx.Invoke. invokeRPC := fx.Invoke(func(srv *rpc.Server) { - srv.RegisterAuthedService("state", mockAPI.State, &statemod.API{}) - srv.RegisterAuthedService("share", mockAPI.Share, &share.API{}) - srv.RegisterAuthedService("fraud", mockAPI.Fraud, &fraud.API{}) - srv.RegisterAuthedService("header", mockAPI.Header, &header.API{}) - srv.RegisterAuthedService("das", mockAPI.Das, &das.API{}) - srv.RegisterAuthedService("p2p", mockAPI.P2P, &p2p.API{}) - srv.RegisterAuthedService("node", mockAPI.Node, &node.API{}) - srv.RegisterAuthedService("blob", mockAPI.Blob, &blob.API{}) - srv.RegisterAuthedService("da", mockAPI.DA, &da.API{}) + srv.RegisterService("fraud", mockAPI.Fraud, &fraud.API{}) + srv.RegisterService("das", mockAPI.Das, &das.API{}) + srv.RegisterService("header", mockAPI.Header, &header.API{}) + srv.RegisterService("state", mockAPI.State, &statemod.API{}) + srv.RegisterService("share", mockAPI.Share, &share.API{}) + srv.RegisterService("p2p", mockAPI.P2P, &p2p.API{}) + srv.RegisterService("node", mockAPI.Node, &node.API{}) + srv.RegisterService("blob", mockAPI.Blob, &blob.API{}) + srv.RegisterService("da", mockAPI.DA, &da.API{}) }) // fx.Replace does not work here, but fx.Decorate does nd := nodebuilder.TestNode(t, node.Full, invokeRPC, fx.Decorate(func() (jwt.Signer, error) { diff --git a/nodebuilder/rpc/constructors.go b/nodebuilder/rpc/constructors.go index c8e928a98a..43a8055207 100644 --- a/nodebuilder/rpc/constructors.go +++ b/nodebuilder/rpc/constructors.go @@ -28,15 +28,15 @@ func registerEndpoints( daMod da.Module, serv *rpc.Server, ) { - serv.RegisterAuthedService("fraud", fraudMod, &fraud.API{}) - serv.RegisterAuthedService("das", daserMod, &das.API{}) - serv.RegisterAuthedService("header", headerMod, &header.API{}) - serv.RegisterAuthedService("state", stateMod, &state.API{}) - serv.RegisterAuthedService("share", shareMod, &share.API{}) - serv.RegisterAuthedService("p2p", p2pMod, &p2p.API{}) - serv.RegisterAuthedService("node", nodeMod, &node.API{}) - serv.RegisterAuthedService("blob", blobMod, &blob.API{}) - serv.RegisterAuthedService("da", daMod, &da.API{}) + serv.RegisterService("fraud", fraudMod, &fraud.API{}) + serv.RegisterService("das", daserMod, &das.API{}) + serv.RegisterService("header", headerMod, &header.API{}) + serv.RegisterService("state", stateMod, &state.API{}) + serv.RegisterService("share", shareMod, &share.API{}) + serv.RegisterService("p2p", p2pMod, &p2p.API{}) + serv.RegisterService("node", nodeMod, &node.API{}) + serv.RegisterService("blob", blobMod, &blob.API{}) + serv.RegisterService("da", daMod, &da.API{}) } func server(cfg *Config, auth jwt.Signer) *rpc.Server { From 53fbbfe6953a16d87b00aa5cc47f7d78baacfdb4 Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Thu, 22 Feb 2024 10:41:27 +0100 Subject: [PATCH 386/388] bump deps --- go.mod | 52 ++++++++++++-------- go.sum | 149 +++++++++++++++++++++++++++++++++------------------------ 2 files changed, 120 insertions(+), 81 deletions(-) diff --git a/go.mod b/go.mod index b79e18ff12..ceb0466d9c 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,8 @@ module github.com/celestiaorg/celestia-node -go 1.21.1 +go 1.22 + +toolchain go1.22.0 require ( cosmossdk.io/errors v1.0.1 @@ -87,6 +89,7 @@ require ( github.com/99designs/go-keychain v0.0.0-20191008050251-8e49817e8af4 // indirect github.com/99designs/keyring v1.2.1 // indirect github.com/ChainSafe/go-schnorrkel v1.0.0 // indirect + github.com/DataDog/zstd v1.5.0 // indirect github.com/Jorropo/jsync v1.0.1 // indirect github.com/Microsoft/go-winio v0.6.1 // indirect github.com/PuerkitoBio/purell v1.1.1 // indirect @@ -107,8 +110,13 @@ require ( github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/chzyer/readline v1.5.1 // indirect github.com/cockroachdb/apd/v2 v2.0.2 // indirect + github.com/cockroachdb/errors v1.11.1 // indirect + github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b // indirect + github.com/cockroachdb/pebble v1.1.0 // indirect + github.com/cockroachdb/redact v1.1.5 // indirect + github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect github.com/coinbase/rosetta-sdk-go v0.7.9 // indirect - github.com/cometbft/cometbft-db v0.7.0 // indirect + github.com/cometbft/cometbft-db v0.11.0 // indirect github.com/confio/ics23/go v0.9.1 // indirect github.com/consensys/bavard v0.1.13 // indirect github.com/consensys/gnark-crypto v0.12.1 // indirect @@ -136,20 +144,21 @@ require ( github.com/dgraph-io/ristretto v0.1.1 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect github.com/docker/go-units v0.5.0 // indirect - github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac // indirect - github.com/dvsekhvalnov/jose2go v1.5.0 // indirect + github.com/dustin/go-humanize v1.0.1 // indirect + github.com/dvsekhvalnov/jose2go v1.6.0 // indirect github.com/elastic/gosigar v0.14.2 // indirect github.com/etclabscore/go-jsonschema-walk v0.0.6 // indirect - github.com/ethereum/c-kzg-4844 v0.3.1 // indirect + github.com/ethereum/c-kzg-4844 v0.4.0 // indirect github.com/ethereum/go-ethereum v1.13.2 // indirect github.com/felixge/httpsnoop v1.0.1 // indirect github.com/flynn/noise v1.0.0 // indirect github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.6.0 // indirect github.com/gammazero/deque v0.2.0 // indirect + github.com/getsentry/sentry-go v0.18.0 // indirect github.com/go-kit/kit v0.12.0 // indirect github.com/go-kit/log v0.2.1 // indirect - github.com/go-logfmt/logfmt v0.5.1 // indirect + github.com/go-logfmt/logfmt v0.6.0 // indirect github.com/go-logr/logr v1.4.1 // indirect github.com/go-logr/stdr v1.2.2 // indirect github.com/go-ole/go-ole v1.2.6 // indirect @@ -178,6 +187,9 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect + github.com/grafana/otel-profiling-go v0.5.1 // indirect + github.com/grafana/pyroscope-go v1.1.1 // indirect + github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect @@ -224,10 +236,12 @@ require ( github.com/jmespath/go-jmespath v0.4.0 // indirect github.com/jmhodges/levigo v1.0.0 // indirect github.com/josharian/intern v1.0.0 // indirect - github.com/klauspost/compress v1.17.2 // indirect + github.com/klauspost/compress v1.17.3 // indirect github.com/klauspost/cpuid/v2 v2.2.6 // indirect github.com/klauspost/reedsolomon v1.11.8 // indirect github.com/koron/go-ssdp v0.0.4 // indirect + github.com/kr/pretty v0.3.1 // indirect + github.com/kr/text v0.2.0 // indirect github.com/lib/pq v1.10.7 // indirect github.com/libp2p/go-buffer-pool v0.1.0 // indirect github.com/libp2p/go-cidranger v1.1.0 // indirect @@ -239,7 +253,8 @@ require ( github.com/libp2p/go-netroute v0.2.1 // indirect github.com/libp2p/go-reuseport v0.4.0 // indirect github.com/libp2p/go-yamux/v4 v4.0.1 // indirect - github.com/magiconair/properties v1.8.6 // indirect + github.com/linxGnu/grocksdb v1.8.12 // indirect + github.com/magiconair/properties v1.8.7 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/manifoldco/promptui v0.9.0 // indirect github.com/marten-seemann/tcp v0.0.0-20210406111302-dfbc87cc63fd // indirect @@ -267,7 +282,6 @@ require ( github.com/opencontainers/runtime-spec v1.1.0 // indirect github.com/opentracing/opentracing-go v1.2.0 // indirect github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect - github.com/pelletier/go-toml v1.9.5 // indirect github.com/pelletier/go-toml/v2 v2.0.7 // indirect github.com/petar/GoLLRB v0.0.0-20210522233825-ae3b015fd3e9 // indirect github.com/petermattis/goid v0.0.0-20230317030725-371a4b8eda08 // indirect @@ -286,20 +300,19 @@ require ( github.com/raulk/go-watchdog v1.3.0 // indirect github.com/rcrowley/go-metrics v0.0.0-20201227073835-cf1acfcdf475 // indirect github.com/regen-network/cosmos-proto v0.3.1 // indirect - github.com/rivo/uniseg v0.4.4 // indirect - github.com/rs/cors v1.8.2 // indirect + github.com/rogpeppe/go-internal v1.11.0 // indirect + github.com/rs/cors v1.8.3 // indirect github.com/rs/zerolog v1.31.0 // indirect github.com/sasha-s/go-deadlock v0.3.1 // indirect github.com/shirou/gopsutil v3.21.6+incompatible // indirect github.com/spaolacci/murmur3 v1.1.0 // indirect - github.com/spf13/afero v1.9.2 // indirect + github.com/spf13/afero v1.9.3 // indirect github.com/spf13/cast v1.5.0 // indirect github.com/spf13/jwalterweatherman v1.1.0 // indirect - github.com/spf13/viper v1.14.0 // indirect - github.com/subosito/gotenv v1.4.1 // indirect + github.com/spf13/viper v1.15.0 // indirect + github.com/subosito/gotenv v1.4.2 // indirect github.com/supranational/blst v0.3.11 // indirect github.com/syndtr/goleveldb v1.0.1-0.20220721030215-126854af5e6d // indirect - github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c // indirect github.com/tendermint/go-amino v0.16.0 // indirect github.com/tendermint/tm-db v0.6.7 // indirect github.com/tidwall/btree v1.5.0 // indirect @@ -311,7 +324,7 @@ require ( github.com/whyrusleeping/go-keyspace v0.0.0-20160322163242-5b898ac5add1 // indirect github.com/zondax/hid v0.9.2 // indirect github.com/zondax/ledger-go v0.14.3 // indirect - go.etcd.io/bbolt v1.3.6 // indirect + go.etcd.io/bbolt v1.3.8 // indirect go.opencensus.io v0.24.0 // indirect go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 // indirect go.uber.org/atomic v1.11.0 // indirect @@ -319,11 +332,11 @@ require ( go.uber.org/mock v0.3.0 // indirect go.uber.org/multierr v1.11.0 // indirect golang.org/x/mod v0.14.0 // indirect - golang.org/x/net v0.19.0 // indirect + golang.org/x/net v0.20.0 // indirect golang.org/x/oauth2 v0.13.0 // indirect golang.org/x/sys v0.16.0 // indirect golang.org/x/term v0.16.0 // indirect - golang.org/x/tools v0.16.0 // indirect + golang.org/x/tools v0.17.0 // indirect golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect gonum.org/v1/gonum v0.13.0 // indirect google.golang.org/api v0.149.0 // indirect @@ -341,10 +354,11 @@ require ( ) replace ( + github.com/celestiaorg/celestia-app => github.com/faddat/celestia-app v1.6.1-0.20240222001605-69472d965497 github.com/cosmos/cosmos-sdk => github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 github.com/filecoin-project/dagstore => github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 github.com/gogo/protobuf => github.com/regen-network/protobuf v1.3.3-alpha.regen.1 // broken goleveldb needs to be replaced for the cosmos-sdk and celestia-app github.com/syndtr/goleveldb => github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 - github.com/tendermint/tendermint => github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 + github.com/tendermint/tendermint => github.com/faddat/celestia-core v1.29.0-tm-v0.34.29.0.20240222093945-cd36a4de3a4d ) diff --git a/go.sum b/go.sum index ec8fe9276c..54ad13537f 100644 --- a/go.sum +++ b/go.sum @@ -219,8 +219,8 @@ github.com/AndreasBriese/bbloom v0.0.0-20190825152654-46b345b51c96/go.mod h1:bOv github.com/Azure/azure-sdk-for-go/sdk/azcore v0.21.1/go.mod h1:fBF9PQNqB8scdgpZ3ufzaLntG0AG7C1WjPMsiFOmfHM= github.com/Azure/azure-sdk-for-go/sdk/internal v0.8.3/go.mod h1:KLF4gFr6DcKFZwSuH8w8yEK6DpFl3LP5rhdvAb7Yz5I= github.com/Azure/azure-sdk-for-go/sdk/storage/azblob v0.3.0/go.mod h1:tPaiy8S5bQ+S5sOiDlINkp7+Ef339+Nz5L5XO+cnOHo= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1 h1:UQHMgLO+TxOElx5B5HZ4hJQsoJ/PvUvKRhJHDQXO8P8= -github.com/Azure/go-ansiterm v0.0.0-20210617225240-d185dfc1b5a1/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161 h1:L/gRVlceqvL25UVaW/CKtUDjefjrs0SPonmDGUVOYP0= +github.com/Azure/go-ansiterm v0.0.0-20230124172434-306776ec8161/go.mod h1:xomTg63KZ2rFqZQzSB4Vz2SUXa1BpHTVz9L5PTmPC4E= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.3.2 h1:o7IhLm0Msx3BaB+n3Ag7L8EVlByGnpq14C4YWiu/gL8= github.com/BurntSushi/toml v1.3.2/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= @@ -335,8 +335,8 @@ github.com/btcsuite/btcd/btcec/v2 v2.1.2/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJ github.com/btcsuite/btcd/btcec/v2 v2.1.3/go.mod h1:ctjw4H1kknNJmRN4iP1R7bTQ+v3GJkZBd6mui8ZsAZE= github.com/btcsuite/btcd/btcec/v2 v2.3.2 h1:5n0X6hX0Zk+6omWcihdYvdAlGf2DfasC0GMf7DClJ3U= github.com/btcsuite/btcd/btcec/v2 v2.3.2/go.mod h1:zYzJ8etWJQIv1Ogk7OzpWjowwOdXY1W/17j2MW85J04= -github.com/btcsuite/btcd/btcutil v1.1.2 h1:XLMbX8JQEiwMcYft2EGi8zPUkoa0abKIU6/BJSRsjzQ= -github.com/btcsuite/btcd/btcutil v1.1.2/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= +github.com/btcsuite/btcd/btcutil v1.1.3 h1:xfbtw8lwpp0G6NwSHb+UE67ryTFHJAiNuipusjXSohQ= +github.com/btcsuite/btcd/btcutil v1.1.3/go.mod h1:UR7dsSJzJUfMmFiiLlIrMq1lS9jh9EdCV7FStZSnpi0= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.0/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1 h1:q0rUy8C/TYNBQS1+CGKw68tLOFYSNEs0TFnxxnS9+4U= github.com/btcsuite/btcd/chaincfg/chainhash v1.0.1/go.mod h1:7SFka0XMvUgj3hfZtydOrQY2mwhPclbT2snogU7SQQc= @@ -352,16 +352,12 @@ github.com/btcsuite/snappy-go v0.0.0-20151229074030-0bdef8d06723/go.mod h1:8woku github.com/btcsuite/snappy-go v1.0.0/go.mod h1:8woku9dyThutzjeg+3xrA5iCpBRH8XEEg3lh6TiUghc= github.com/btcsuite/websocket v0.0.0-20150119174127-31079b680792/go.mod h1:ghJtEyQwv5/p4Mg4C0fgbePVuGr935/5ddU9Z3TmDRY= github.com/btcsuite/winsvc v1.0.0/go.mod h1:jsenWakMcC0zFBFurPLEAyrnc/teJEM1O46fmI40EZs= -github.com/bufbuild/protocompile v0.1.0 h1:HjgJBI85hY/qmW5tw/66sNDZ7z0UDdVSi/5r40WHw4s= -github.com/bufbuild/protocompile v0.1.0/go.mod h1:ix/MMMdsT3fzxfw91dvbfzKW3fRRnuPCP47kpAm5m/4= +github.com/bufbuild/protocompile v0.5.1 h1:mixz5lJX4Hiz4FpqFREJHIXLfaLBntfaJv1h+/jS+Qg= +github.com/bufbuild/protocompile v0.5.1/go.mod h1:G5iLmavmF4NsYtpZFvE3B/zFch2GIY8+wjsYLR/lc40= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/bwesterb/go-ristretto v1.2.0/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/c-bata/go-prompt v0.2.2/go.mod h1:VzqtzE2ksDBcdln8G7mk2RX9QyGjH+OVqOCSiVIqS34= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= -github.com/celestiaorg/celestia-app v1.4.0 h1:hTId3xL8GssN5sHSHPP7svHi/iWp+XVxphStiR7ADiY= -github.com/celestiaorg/celestia-app v1.4.0/go.mod h1:zhdQIFGFZRRxrDVtFE4OFIT7/12RE8DRyfvNZdW8ceM= -github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29 h1:Fd7ymPUzExPGNl2gZw4i5S74arMw+iDHLE78M/cCxl4= -github.com/celestiaorg/celestia-core v1.29.0-tm-v0.34.29/go.mod h1:xrICN0PBhp3AdTaZ8q4wS5Jvi32V02HNjaC2EsWiEKk= github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14 h1:+Te28r5Zp4Vp69f82kcON9/BIF8f1BNXb0go2+achuc= github.com/celestiaorg/cosmos-sdk v1.18.3-sdk-v0.46.14/go.mod h1:Og5KKgoBEiQlI6u56lDLG191pfknIdXctFn3COWLQP8= github.com/celestiaorg/dagstore v0.0.0-20230824094345-537c012aa403 h1:Lj73O3S+KJx5/hgZ+IeOLEIoLsAveJN/7/ZtQQtPSVw= @@ -427,23 +423,25 @@ github.com/cncf/xds/go v0.0.0-20211011173535-cb28da3451f1/go.mod h1:eXthEFrGJvWH github.com/cockroachdb/apd/v2 v2.0.2 h1:weh8u7Cneje73dDh+2tEVLUvyBc89iwepWCD8b8034E= github.com/cockroachdb/apd/v2 v2.0.2/go.mod h1:DDxRlzC2lo3/vSlmSoS7JkqbbrARPuFOGr0B9pvN3Gw= github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:zn76sxSg3SzpJ0PPJaLDCu+Bu0Lg3sKTORVIj19EIF8= -github.com/cockroachdb/errors v1.8.1 h1:A5+txlVZfOqFBDa4mGz2bUWSp0aHElvHX2bKkdbQu+Y= -github.com/cockroachdb/errors v1.8.1/go.mod h1:qGwQn6JmZ+oMjuLwjWzUNqblqk0xl4CVV3SQbGwK7Ac= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f h1:o/kfcElHqOiXqcou5a3rIlMc7oJbMQkeLk0VQJ7zgqY= -github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f/go.mod h1:i/u985jwjWRlyHXQbwatDASoW0RMlZ/3i9yJHE2xLkI= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06 h1:T+Np/xtzIjYM/P5NAw0e2Rf1FGvzDau1h54MKvx8G7w= -github.com/cockroachdb/pebble v0.0.0-20230906160148-46873a6a7a06/go.mod h1:bynZ3gvVyhlvjLI7PT6dmZ7g76xzJ7HpxfjgkzCGz6s= -github.com/cockroachdb/redact v1.0.8 h1:8QG/764wK+vmEYoOlfobpe12EQcS81ukx/a4hdVMxNw= -github.com/cockroachdb/redact v1.0.8/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2 h1:IKgmqgMQlVJIZj19CdocBeSfSaiCbEBZGKODaixqtHM= -github.com/cockroachdb/sentry-go v0.6.1-cockroachdb.2/go.mod h1:8BT+cPK6xvFOcRlk0R8eg+OTkcqI6baNH4xAkpiYVvQ= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f h1:otljaYPt5hWxV3MUfO5dFPFiOXg9CyG5/kCfayTqsJ4= +github.com/cockroachdb/datadriven v1.0.3-0.20230413201302-be42291fc80f/go.mod h1:a9RdTaap04u637JoCzcUoIcDmvwSUtcUFtT/C3kJlTU= +github.com/cockroachdb/errors v1.11.1 h1:xSEW75zKaKCWzR3OfxXUxgrk/NtT4G1MiOv5lWZazG8= +github.com/cockroachdb/errors v1.11.1/go.mod h1:8MUxA3Gi6b25tYlFEBGLf+D8aISL+M4MIpiWMSNRfxw= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b h1:r6VH0faHjZeQy818SGhaone5OnYfxFR/+AzdY3sf5aE= +github.com/cockroachdb/logtags v0.0.0-20230118201751-21c54148d20b/go.mod h1:Vz9DsVWQQhf3vs21MhPMZpMGSht7O/2vFW2xusFUVOs= +github.com/cockroachdb/pebble v1.1.0 h1:pcFh8CdCIt2kmEpK0OIatq67Ln9uGDYY3d5XnE0LJG4= +github.com/cockroachdb/pebble v1.1.0/go.mod h1:sEHm5NOXxyiAoKWhoFxT8xMgd/f3RA6qUqQ1BXKrh2E= +github.com/cockroachdb/redact v1.1.5 h1:u1PMllDkdFfPWaNGMyLD1+so+aq3uUItthCFqzwPJ30= +github.com/cockroachdb/redact v1.1.5/go.mod h1:BVNblN9mBWFyMyqK1k3AAiSxhvhfK2oOZZ2lK+dpvRg= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 h1:zuQyyAKVxetITBuuhv3BI9cMrmStnpT18zmgmTxunpo= +github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:7nc4anLGjupUW/PeY5qiNYsdNXj7zopG+eqsS7To5IQ= github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/codegangsta/cli v1.20.0/go.mod h1:/qJNoX69yVSKu5o4jLyXAENLRyk1uhi7zkbQ3slBdOA= github.com/coinbase/kryptology v1.8.0/go.mod h1:RYXOAPdzOGUe3qlSFkMGn58i3xUA8hmxYHksuq+8ciI= github.com/coinbase/rosetta-sdk-go v0.7.9 h1:lqllBjMnazTjIqYrOGv8h8jxjg9+hJazIGZr9ZvoCcA= github.com/coinbase/rosetta-sdk-go v0.7.9/go.mod h1:0/knutI7XGVqXmmH4OQD8OckFrbQ8yMsUZTG7FXCR2M= -github.com/cometbft/cometbft-db v0.7.0 h1:uBjbrBx4QzU0zOEnU8KxoDl18dMNgDh+zZRUE0ucsbo= -github.com/cometbft/cometbft-db v0.7.0/go.mod h1:yiKJIm2WKrt6x8Cyxtq9YTEcIMPcEe4XPxhgX59Fzf0= +github.com/cometbft/cometbft-db v0.11.0 h1:M3Lscmpogx5NTbb1EGyGDaFRdsoLWrUWimFEyf7jej8= +github.com/cometbft/cometbft-db v0.11.0/go.mod h1:GDPJAC/iFHNjmZZPN8V8C1yr/eyityhi2W1hz2MGKSc= github.com/confio/ics23/go v0.9.1 h1:3MV46eeWwO3xCauKyAtuAdJYMyPnnchW4iLr2bTw6/U= github.com/confio/ics23/go v0.9.1/go.mod h1:4LPZ2NYqnYIVRklaozjNR1FScgDJ2s5Xrp+e/mYVRak= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= @@ -574,10 +572,10 @@ github.com/dop251/goja v0.0.0-20211011172007-d99e4b8cbf48/go.mod h1:R9ET47fwRVRP github.com/dop251/goja_nodejs v0.0.0-20210225215109-d91c329300e7/go.mod h1:hn7BA7c8pLvoGndExHudxTDKZ84Pyvv+90pbBjbTz0Y= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac h1:opbrjaN/L8gg6Xh5D04Tem+8xVcz6ajZlGCs49mQgyg= -github.com/dustin/go-humanize v1.0.1-0.20200219035652-afde56e7acac/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/dvsekhvalnov/jose2go v1.5.0 h1:3j8ya4Z4kMCwT5nXIKFSV84YS+HdqSSO0VsTQxaLAeM= -github.com/dvsekhvalnov/jose2go v1.5.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= +github.com/dustin/go-humanize v1.0.1 h1:GzkhY7T5VNhEkwH0PVJgjz+fX1rhBrR7pRT3mDkpeCY= +github.com/dustin/go-humanize v1.0.1/go.mod h1:Mu1zIs6XwVuF/gI1OepvI0qD18qycQx+mFykh5fBlto= +github.com/dvsekhvalnov/jose2go v1.6.0 h1:Y9gnSnP4qEI0+/uQkHvFXeD2PLPJeXEL+ySMEA2EjTY= +github.com/dvsekhvalnov/jose2go v1.6.0/go.mod h1:QsHjhyTlD/lAVqn/NSbVZmSCGeDehTB/mPZadG+mhXU= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= @@ -601,8 +599,8 @@ github.com/etclabscore/go-jsonschema-walk v0.0.6 h1:DrNzoKWKd8f8XB5nFGBY00IcjakR github.com/etclabscore/go-jsonschema-walk v0.0.6/go.mod h1:VdfDY72AFAiUhy0ZXEaWSpveGjMT5JcDIm903NGqFwQ= github.com/etclabscore/go-openrpc-reflect v0.0.37 h1:IH0e7JqIvR9OhbbFWi/BHIkXrqbR3Zyia3RJ733eT6c= github.com/etclabscore/go-openrpc-reflect v0.0.37/go.mod h1:0404Ky3igAasAOpyj1eESjstTyneBAIk5PgJFbK4s5E= -github.com/ethereum/c-kzg-4844 v0.3.1 h1:sR65+68+WdnMKxseNWxSJuAv2tsUrihTpVBTfM/U5Zg= -github.com/ethereum/c-kzg-4844 v0.3.1/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= +github.com/ethereum/c-kzg-4844 v0.4.0 h1:3MS1s4JtA868KpJxroZoepdV0ZKBp3u/O5HcZ7R3nlY= +github.com/ethereum/c-kzg-4844 v0.4.0/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0= github.com/ethereum/go-ethereum v1.10.17/go.mod h1:Lt5WzjM07XlXc95YzrhosmR4J9Ahd6X2wyEV2SvGhk0= github.com/ethereum/go-ethereum v1.13.2 h1:g9mCpfPWqCA1OL4e6C98PeVttb0HadfBRuKTGvMnOvw= github.com/ethereum/go-ethereum v1.13.2/go.mod h1:gkQ5Ygi64ZBh9M/4iXY1R8WqoNCx1Ey0CkYn2BD4/fw= @@ -612,9 +610,14 @@ github.com/facebookgo/stack v0.0.0-20160209184415-751773369052 h1:JWuenKqqX8nojt github.com/facebookgo/stack v0.0.0-20160209184415-751773369052/go.mod h1:UbMTZqLaRiH3MsBH8va0n7s1pQYcu3uTb8G4tygF4Zg= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4 h1:7HZCaLC5+BZpmbhCOZJ293Lz68O7PYrF2EzeiFMwCLk= github.com/facebookgo/subset v0.0.0-20200203212716-c811ad88dec4/go.mod h1:5tD+neXqOorC30/tWg0LCSkrqj/AR6gu8yY8/fpw1q0= +github.com/faddat/celestia-app v1.6.1-0.20240222001605-69472d965497 h1:xLSiIAn4PDy9D3bxf+70JrRKJyiEB3AzuwBwZp+Tmbs= +github.com/faddat/celestia-app v1.6.1-0.20240222001605-69472d965497/go.mod h1:zWPznuIdfKWCmfQt27sW29ahZME88fWr7982NSs+A98= +github.com/faddat/celestia-core v1.29.0-tm-v0.34.29.0.20240222093945-cd36a4de3a4d h1:JLYT2ynpFu9mo8ISsxi1fK0zF0wVEKRDBVaIbNphsJM= +github.com/faddat/celestia-core v1.29.0-tm-v0.34.29.0.20240222093945-cd36a4de3a4d/go.mod h1:0f44crIBHrgfMMB7D+Q0wCnOGETDqx1kIXOhM8ox3hE= github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= -github.com/fatih/color v1.13.0 h1:8LOYc1KYPPmyKMuN8QV2DNRWNbLo6LZ0iLs8+mlH53w= github.com/fatih/color v1.13.0/go.mod h1:kLAiJbzzSOZDVNGyDpeOxJ47H46qBXwg5ILebYFFOfk= +github.com/fatih/color v1.15.0 h1:kOqh6YHBtK8aywxGerMG2Eq3H6Qgoqeo13Bk2Mv/nBs= +github.com/fatih/color v1.15.0/go.mod h1:0h5ZqXfHYED7Bhv2ZJamyIOUej9KtShiJESRwBDUSsw= github.com/felixge/httpsnoop v1.0.1 h1:lvB5Jl89CsZtGIWuTcDM1E/vkVs49/Ml7JJe07l8SPQ= github.com/felixge/httpsnoop v1.0.1/go.mod h1:m8KPJKqk1gH5J9DgRY2ASl2lWCfGKXixSwevea8zH2U= github.com/filecoin-project/go-jsonrpc v0.3.1 h1:qwvAUc5VwAkooquKJmfz9R2+F8znhiqcNHYjEp/NM10= @@ -654,18 +657,22 @@ github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff h1:tY80oXqG github.com/gballet/go-libpcsclite v0.0.0-20190607065134-2772fd86a8ff/go.mod h1:x7DCsMOv1taUwEWCzT4cmDeAkigA5/QCwUodaVOe8Ww= github.com/getkin/kin-openapi v0.53.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= github.com/getkin/kin-openapi v0.61.0/go.mod h1:7Yn5whZr5kJi6t+kShccXS8ae1APpYTW6yheSwk8Yi4= +github.com/getsentry/sentry-go v0.18.0 h1:MtBW5H9QgdcJabtZcuJG80BMOwaBpkRDZkxRkNC1sN0= +github.com/getsentry/sentry-go v0.18.0/go.mod h1:Kgon4Mby+FJ7ZWHFUAZgVaIa8sxHtnRJRLTXZr51aKQ= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= -github.com/gin-gonic/gin v1.7.0 h1:jGB9xAJQ12AIGNB4HguylppmDK1Am9ppF7XnGXXJuoU= -github.com/gin-gonic/gin v1.7.0/go.mod h1:jD2toBW3GZUr5UMcdrwQA10I7RuaFOl/SGeDjXkfUtY= +github.com/gin-gonic/gin v1.8.1 h1:4+fr/el88TOO3ewCmQr8cx/CtZ/umlIRIs5M4NTNjf8= +github.com/gin-gonic/gin v1.8.1/go.mod h1:ji8BvRH1azfM+SYow9zQ6SZMvR8qOMZHmsCuWR9tTTk= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-check/check v0.0.0-20180628173108-788fd7840127/go.mod h1:9ES+weclKsC9YodN5RgxqK/VD9HM9JsCSh7rNhMZE98= github.com/go-chi/chi/v5 v5.0.0/go.mod h1:BBug9lr0cqtdAhsu6R4AAdvufI0/XBzAQSsUqJpoZOs= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= +github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= +github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20200222043503-6f7a984d4dc4/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -681,8 +688,9 @@ github.com/go-kit/log v0.2.1/go.mod h1:NwTd00d/i8cPZ3xOwwiv2PO5MOcx78fFErGNcVmBj github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9GBnD5lWE= github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= -github.com/go-logfmt/logfmt v0.5.1 h1:otpy5pqBCBZ1ng9RQ0dPu4PN7ba75Y/aA+UpowDyNVA= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logfmt/logfmt v0.6.0 h1:wGYYu3uicYdqXVgoYbvnkrPVXkuLM1p1ifugDMEdRi4= +github.com/go-logfmt/logfmt v0.6.0/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= github.com/go-logr/logr v0.4.0/go.mod h1:z6/tIYblkpsD+a4lm/fGIIU9mZ+XfAiaFtq7xTgseGU= github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= @@ -712,13 +720,15 @@ github.com/go-openapi/swag v0.19.8/go.mod h1:ao+8BpOPyKdpQz3AOJfbeEVpLmWAvlT1IfT github.com/go-openapi/swag v0.19.11 h1:RFTu/dlFySpyVvJDfp/7674JY4SDglYWKztbiIGFpmc= github.com/go-openapi/swag v0.19.11/go.mod h1:Uc0gKkdR+ojzsEpjh39QChyu92vPgIr72POcgHMAgSY= github.com/go-playground/assert/v2 v2.0.1/go.mod h1:VDjEfimB/XKnb+ZQfWdccd7VUvScMdVu0Titje2rxJ4= -github.com/go-playground/locales v0.13.0 h1:HyWk6mgj5qFqCT5fjGBuRArbVDfE4hi8+e8ceBS/t7Q= github.com/go-playground/locales v0.13.0/go.mod h1:taPMhCMXrRLJO55olJkUXHZBHCxTMfnGwq/HNwmWNS8= -github.com/go-playground/universal-translator v0.17.0 h1:icxd5fm+REJzpZx7ZfpaD876Lmtgy7VtROAbHHXk8no= +github.com/go-playground/locales v0.14.0 h1:u50s323jtVGugKlcYeyzC0etD1HifMjqmJqb8WugfUU= +github.com/go-playground/locales v0.14.0/go.mod h1:sawfccIbzZTqEDETgFXqTho0QybSa7l++s0DH+LDiLs= github.com/go-playground/universal-translator v0.17.0/go.mod h1:UkSxE5sNxxRwHyU+Scu5vgOQjsIJAF8j9muTVoKLVtA= +github.com/go-playground/universal-translator v0.18.0 h1:82dyy6p4OuJq4/CByFNOn/jYrnRPArHwAcmLoJZxyho= +github.com/go-playground/universal-translator v0.18.0/go.mod h1:UvRDBj+xPUEGrFYl+lu/H90nyDXpg0fqeB/AQUGNTVA= github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GOhaH6EGOAJShg8Id5JGkI= -github.com/go-playground/validator/v10 v10.4.1 h1:pH2c5ADXtd66mxoE0Zm9SUhxE20r7aM3F26W0hOn+GE= -github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4= +github.com/go-playground/validator/v10 v10.11.1 h1:prmOlTVv+YjZjmRmNSF3VmspqJIxJWXmqUsHwfTRRkQ= +github.com/go-playground/validator/v10 v10.11.1/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU= github.com/go-sourcemap/sourcemap v2.1.3+incompatible/go.mod h1:F8jJfvm2KbVjc5NqelyYJmf/v5J0dwNLS2mL4sNA1Jg= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= @@ -738,6 +748,8 @@ github.com/gobwas/pool v0.2.1/go.mod h1:q8bcK0KcYlCgd9e7WYLm9LpyS+YeLd8JVDW6Wezm github.com/gobwas/ws v1.0.2/go.mod h1:szmBTxLgaFppYjEmNtny/v3w89xOydFnnZMcgRRu/EM= github.com/gobwas/ws v1.2.1 h1:F2aeBZrm2NDsc7vbovKrWSogd4wvfAxg0FQ89/iqOTk= github.com/gobwas/ws v1.2.1/go.mod h1:hRKAFb8wOxFROYNsT1bqfWnhX+b5MFeJM9r2ZSwg/KY= +github.com/goccy/go-json v0.9.11 h1:/pAaQDLHEoCq/5FFmSKBswWmK6H0e8g4159Kc/X/nqk= +github.com/goccy/go-json v0.9.11/go.mod h1:6MelG93GURQebXPDq3khkgXZkazVtN9CRI+MGFi0w8I= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2 h1:ZpnhV/YsD2/4cESfV5+Hoeu/iUR3ruzNvZ+yQfO03a0= github.com/godbus/dbus v0.0.0-20190726142602-4481cbc300e2/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= @@ -911,6 +923,12 @@ github.com/gorilla/websocket v1.4.1/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/ad github.com/gorilla/websocket v1.4.2/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= github.com/gorilla/websocket v1.5.0 h1:PPwGk2jz7EePpoHN/+ClbZu8SPxiqlu12wZP/3sWmnc= github.com/gorilla/websocket v1.5.0/go.mod h1:YR8l580nyteQvAITg2hZ9XVh4b55+EU/adAjf1fMHhE= +github.com/grafana/otel-profiling-go v0.5.1 h1:stVPKAFZSa7eGiqbYuG25VcqYksR6iWvF3YH66t4qL8= +github.com/grafana/otel-profiling-go v0.5.1/go.mod h1:ftN/t5A/4gQI19/8MoWurBEtC6gFw8Dns1sJZ9W4Tls= +github.com/grafana/pyroscope-go v1.1.1 h1:PQoUU9oWtO3ve/fgIiklYuGilvsm8qaGhlY4Vw6MAcQ= +github.com/grafana/pyroscope-go v1.1.1/go.mod h1:Mw26jU7jsL/KStNSGGuuVYdUq7Qghem5P8aXYXSXG88= +github.com/grafana/pyroscope-go/godeltaprof v0.1.6 h1:nEdZ8louGAplSvIJi1HVp7kWvFvdiiYg3COLlTwJiFo= +github.com/grafana/pyroscope-go/godeltaprof v0.1.6/go.mod h1:Tk376Nbldo4Cha9RgiU7ik8WKFkNpfds98aUzS8omLE= github.com/graph-gophers/graphql-go v1.3.0/go.mod h1:9CQHMSxwO4MprSdzoIEobiHpoLtHm77vfxsvsIN5Vuc= github.com/gregjones/httpcache v0.0.0-20180305231024-9cad4c3443a7/go.mod h1:FecbI9+v66THATjSRHfNgh1IVFe/9kFxbXtjV0ctIMA= github.com/grpc-ecosystem/go-grpc-middleware v1.0.1-0.20190118093823-f849b5445de4/go.mod h1:FiyG127CGDf3tlThmgyCl78X/SZQqEOJBCDaAfeWzPs= @@ -966,8 +984,9 @@ github.com/hashicorp/go-safetemp v1.0.0/go.mod h1:oaerMy3BhqiTbVye6QuFhFtIceqFoD github.com/hashicorp/go-sockaddr v1.0.0/go.mod h1:7Xibr9yA9JjQq1JpNB2Vw7kxv8xerXegt+ozgdvDeDU= github.com/hashicorp/go-syslog v1.0.0/go.mod h1:qPfqrKkXGihmCqbJM2mZgkZGvKG1dFdvsLplgctolz4= github.com/hashicorp/go-uuid v1.0.0/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= -github.com/hashicorp/go-uuid v1.0.1 h1:fv1ep09latC32wFoVwnqcnKJGnMSdBanPczbHAYm1BE= github.com/hashicorp/go-uuid v1.0.1/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= +github.com/hashicorp/go-uuid v1.0.2 h1:cfejS+Tpcp13yd5nYHWDI6qVCny6wyX2Mt5SGur2IGE= +github.com/hashicorp/go-uuid v1.0.2/go.mod h1:6SBZvOh/SIDV7/2o3Jml5SYk/TvGqwFJ/bN7x4byOro= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.6.0 h1:feTTfFNnjP967rlCxM/I9g701jU+RN74YKx2mOkIeek= github.com/hashicorp/go-version v1.6.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= @@ -1334,8 +1353,8 @@ github.com/klauspost/compress v1.15.1/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47e github.com/klauspost/compress v1.15.10/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= github.com/klauspost/compress v1.15.12/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= -github.com/klauspost/compress v1.17.2 h1:RlWWUY/Dr4fL8qk9YG7DTZ7PDgME2V4csBXA8L/ixi4= -github.com/klauspost/compress v1.17.2/go.mod h1:ntbaceVETuRiXiv4DpjP66DpAtAGkEQskQzEyD//IeE= +github.com/klauspost/compress v1.17.3 h1:qkRjuerhUU1EmXLYGkSH6EZL+vPSxIrYjLNAK4slzwA= +github.com/klauspost/compress v1.17.3/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid v0.0.0-20170728055534-ae7887de9fa5/go.mod h1:Pj4uuM528wm8OyEC2QMXAi2YiTZ96dNQPGgoMS4s3ek= github.com/klauspost/cpuid/v2 v2.0.4/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/klauspost/cpuid/v2 v2.0.6/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= @@ -1379,8 +1398,9 @@ github.com/labstack/echo/v4 v4.2.1/go.mod h1:AA49e0DZ8kk5jTOOCKNuPR6oTnBS0dYiM4F github.com/labstack/gommon v0.3.0/go.mod h1:MULnywXg0yavhxWKc+lOruYdAhDwPK9wf0OL7NoOu+k= github.com/leanovate/gopter v0.2.9 h1:fQjYxZaynp97ozCzfOyOuAGOU4aU/z37zf/tOujFk7c= github.com/leanovate/gopter v0.2.9/go.mod h1:U2L/78B+KVFIx2VmW6onHJQzXtFb+p5y3y2Sh+Jxxv8= -github.com/leodido/go-urn v1.2.0 h1:hpXL4XnriNwQ/ABnpepYM/1vCLWNDfUNts8dX3xTG6Y= github.com/leodido/go-urn v1.2.0/go.mod h1:+8+nEpDfqqsY+g338gtMEUOtuK+4dEMhiQEgxpxOKII= +github.com/leodido/go-urn v1.2.1 h1:BqpAaACuzVSgi/VLzGZIobT2z4v53pjosyNd9Yv6n/w= +github.com/leodido/go-urn v1.2.1/go.mod h1:zt4jvISO2HfUBqxjfIshjdMTYS56ZS/qv49ictyFfxY= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.10.7 h1:p7ZhMD+KsSRozJr34udlUrhboJwWAgCg34+/ZZNvZZw= github.com/lib/pq v1.10.7/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= @@ -1664,6 +1684,8 @@ github.com/libp2p/zeroconf/v2 v2.1.1/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0 github.com/libp2p/zeroconf/v2 v2.2.0/go.mod h1:fuJqLnUwZTshS3U/bMRJ3+ow/v9oid1n0DmyYyNO1Xs= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/linxGnu/grocksdb v1.8.12 h1:1/pCztQUOa3BX/1gR3jSZDoaKFpeHFvQ1XrqZpSvZVo= +github.com/linxGnu/grocksdb v1.8.12/go.mod h1:xZCIb5Muw+nhbDK4Y5UJuOrin5MceOuiXkVUR7vp4WY= github.com/lucas-clemente/quic-go v0.19.3/go.mod h1:ADXpNbTQjq1hIzCpB+y/k5iz4n4z4IwqoLb94Kh5Hu8= github.com/lucas-clemente/quic-go v0.23.0/go.mod h1:paZuzjXCE5mj6sikVLMvqXk8lJV2AsqtJ6bDhjEfxx0= github.com/lucas-clemente/quic-go v0.25.0/go.mod h1:YtzP8bxRVCBlO77yRanE264+fY/T2U9ZlW1AaHOsMOg= @@ -1674,8 +1696,8 @@ github.com/lucasjones/reggen v0.0.0-20180717132126-cdb49ff09d77/go.mod h1:5ELEyG github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= github.com/lyft/protoc-gen-validate v0.0.13/go.mod h1:XbGvPuh87YZc5TdIa2/I4pLk0QoUACkjt2znoq26NVQ= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= -github.com/magiconair/properties v1.8.6 h1:5ibWZ6iY0NctNGWo87LalDlEZ6R41TqbbDamhfG/Qzo= -github.com/magiconair/properties v1.8.6/go.mod h1:y3VJvCyxH9uVvJTWEGAELF3aiYNyPKd5NZ3oSwXrF60= +github.com/magiconair/properties v1.8.7 h1:IeQXZAiQcpL9mgcAe1Nu6cX9LLw6ExEHKjN0VQdvPDY= +github.com/magiconair/properties v1.8.7/go.mod h1:Dhd985XPs7jluiymwWYZ0G4Z61jb3vdS329zhj2hYo0= github.com/mailru/easyjson v0.0.0-20180823135443-60711f1a8329/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190312143242-1de009706dbe/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= @@ -1991,8 +2013,6 @@ github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2D github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= -github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= -github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml/v2 v2.0.7 h1:muncTPStnKRos5dpVKULv2FVd4bMOhNePj9CjgDb8Us= github.com/pelletier/go-toml/v2 v2.0.7/go.mod h1:eumQOmlWiOPt5WriQQqoM5y18pDHwha2N+QD+EUNTek= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= @@ -2007,6 +2027,8 @@ github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG github.com/philhofer/fwd v1.1.1/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= +github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= +github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -2124,8 +2146,8 @@ github.com/rogpeppe/go-internal v1.11.0/go.mod h1:ddIwULY96R17DhadqLgMfk9H9tvdUz github.com/rollkit/go-da v0.4.0 h1:/s7ZrVq7DC2aK8UXIvB7rsXrZ2mVGRw7zrexcxRvhlw= github.com/rollkit/go-da v0.4.0/go.mod h1:Kef0XI5ecEKd3TXzI8S+9knAUJnZg0svh2DuXoCsPlM= github.com/rs/cors v1.7.0/go.mod h1:gFx+x8UowdsKA9AchylcLynDq+nNFfI8FkUZdN/jGCU= -github.com/rs/cors v1.8.2 h1:KCooALfAYGs415Cwu5ABvv9n9509fSiG5SQJn/AQo4U= -github.com/rs/cors v1.8.2/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= +github.com/rs/cors v1.8.3 h1:O+qNyWn7Z+F9M0ILBHgMVPuB1xTOucVd5gtaYyXBpRo= +github.com/rs/cors v1.8.3/go.mod h1:XyqrcTp5zjWr1wsJ8PIRZssZ8b/WMcMf71DJnit4EMU= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= github.com/rs/zerolog v1.21.0/go.mod h1:ZPhntP/xmq1nnND05hhpAh2QMhSsA4UN3MGZ6O2J3hM= @@ -2199,8 +2221,8 @@ github.com/spaolacci/murmur3 v0.0.0-20180118202830-f09979ecbc72/go.mod h1:JwIasO github.com/spaolacci/murmur3 v1.1.0 h1:7c1g84S4BPRrfL5Xrdp6fOJ206sU9y293DDHaoy0bLI= github.com/spaolacci/murmur3 v1.1.0/go.mod h1:JwIasOWyU6f++ZhiEuf87xNszmSA2myDM2Kzu9HwQUA= github.com/spf13/afero v1.1.2/go.mod h1:j4pytiNVoe2o6bmDsKpLACNPDBIoEAkihy7loJ1B0CQ= -github.com/spf13/afero v1.9.2 h1:j49Hj62F0n+DaZ1dDCvhABaPNSGNkt32oRFxI33IEMw= -github.com/spf13/afero v1.9.2/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= +github.com/spf13/afero v1.9.3 h1:41FoI0fD7OR7mGcKE/aOiLkGreyf8ifIOQmJANWogMk= +github.com/spf13/afero v1.9.3/go.mod h1:iUV7ddyEEZPO5gA3zD4fJt6iStLlL+Lg4m2cihcDf8Y= github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE= github.com/spf13/cast v1.5.0 h1:rj3WzYc11XZaIZMPKmwP96zkFEnnAmV8s6XbB2aY32w= github.com/spf13/cast v1.5.0/go.mod h1:SpXXQ5YoyJw6s3/6cMTQuxvgRl3PCJiyaX9p6b155UU= @@ -2216,8 +2238,8 @@ github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnIn github.com/spf13/pflag v1.0.5 h1:iy+VFUOCP1a+8yFto/drg2CJ5u0yRoB7fZw3DKv/JXA= github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An2Bg= github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= -github.com/spf13/viper v1.14.0 h1:Rg7d3Lo706X9tHsJMUjdiwMpHB7W8WnSVOssIY+JElU= -github.com/spf13/viper v1.14.0/go.mod h1:WT//axPky3FdvXHzGw33dNdXXXfFQqmEalje+egj8As= +github.com/spf13/viper v1.15.0 h1:js3yy885G8xwJa6iOISGFwd+qlUo5AvyXb7CiihdtiU= +github.com/spf13/viper v1.15.0/go.mod h1:fFcTBJxvhhzSJiZy8n+PeW6t8l+KeT/uTARa0jHOQLA= github.com/src-d/envconfig v1.0.0/go.mod h1:Q9YQZ7BKITldTBnoxsE5gOeB5y66RyPXeue/R4aaNBc= github.com/status-im/keycard-go v0.0.0-20190316090335-8537d3370df4/go.mod h1:RZLeN1LMWmRsyYjvAu+I6Dm9QmlDaIIt+Y+4Kd7Tp+Q= github.com/status-im/keycard-go v0.2.0 h1:QDLFswOQu1r5jsycloeQh3bVU8n/NatHHaZobtDnDzA= @@ -2244,15 +2266,13 @@ github.com/stretchr/testify v1.8.1/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/subosito/gotenv v1.4.1 h1:jyEFiXpy21Wm81FBN71l9VoMMV8H8jG+qIK3GCpY6Qs= -github.com/subosito/gotenv v1.4.1/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= +github.com/subosito/gotenv v1.4.2 h1:X1TuBLAMDFbaTAChgCBLu3DU3UPyELpnF2jjJ2cz/S8= +github.com/subosito/gotenv v1.4.2/go.mod h1:ayKnFf/c6rvx/2iiLrJUk1e6plDbT3edrFNGqEflhK0= github.com/supranational/blst v0.3.11 h1:LyU6FolezeWAhvQk0k6O/d49jqgO52MSDDfYgbeoEm4= github.com/supranational/blst v0.3.11/go.mod h1:jZJtfjgudtNl4en1tzwPIV3KjUnQUvG3/j+w+fVonLw= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7 h1:epCh84lMvA70Z7CTTCmYQn2CKbY8j86K7/FAIr141uY= github.com/syndtr/goleveldb v1.0.1-0.20210819022825-2ae1ddf74ef7/go.mod h1:q4W45IWZaF22tdD+VEXcAWRA037jwmWEB5VWYORlTpc= github.com/tarm/serial v0.0.0-20180830185346-98f6abe2eb07/go.mod h1:kDXzergiv9cbyO7IOYJZWg1U88JhDg3PB6klq9Hg2pA= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c h1:g+WoO5jjkqGAzHWCjJB1zZfXPIAaDpzXIEJ0eS6B5Ok= -github.com/tecbot/gorocksdb v0.0.0-20191217155057-f0fad39f321c/go.mod h1:ahpPrc7HpcfEWDQRZEmnXMzHY03mLDYMCxeDzy46i+8= github.com/tendermint/go-amino v0.16.0 h1:GyhmgQKvqF82e2oZeuMSp9JTN0N09emoSZlb2lyGa2E= github.com/tendermint/go-amino v0.16.0/go.mod h1:TQU0M1i/ImAo+tYpZi73AU3V/dKeCoMC9Sphe2ZwGME= github.com/tendermint/tm-db v0.6.7 h1:fE00Cbl0jayAoqlExN6oyQJ7fR/ZtoVOmvPJ//+shu8= @@ -2293,8 +2313,9 @@ github.com/ucarion/urlpath v0.0.0-20200424170820-7ccc79b76bbb/go.mod h1:ikPs9bRW github.com/ugorji/go v1.1.7 h1:/68gy2h+1mWMrwZFeD1kQialdSzAb432dtpeJ42ovdo= github.com/ugorji/go v1.1.7/go.mod h1:kZn38zHttfInRq0xu/PH0az30d+z6vm202qpg1oXVMw= github.com/ugorji/go/codec v0.0.0-20181204163529-d75b2dcb6bc8/go.mod h1:VFNgLljTbGfSG7qAOspJ7OScBnGdDN/yBr0sguwnwf0= -github.com/ugorji/go/codec v1.1.7 h1:2SvQaVZ1ouYrrKKwoSk2pzd4A9evlKJb9oTL+OaLUSs= github.com/ugorji/go/codec v1.1.7/go.mod h1:Ax+UKWsSmolVDwsd+7N3ZtXu+yMGCf907BLYF3GoBXY= +github.com/ugorji/go/codec v1.2.7 h1:YPXUKf7fYbp/y8xloBqZOw2qaVggbfwMlI8WM3wZUJ0= +github.com/ugorji/go/codec v1.2.7/go.mod h1:WGN1fab3R1fzQlVQTkfxVtIBhWDRqOviHU95kRgeqEY= github.com/ulikunitz/xz v0.5.10 h1:t92gobL9l3HE202wg3rlk19F6X+JOxl9BBrCCMYEYd8= github.com/ulikunitz/xz v0.5.10/go.mod h1:nbz6k7qbPmH4IRqmfOplQw/tblSgqTqBwxkY0oWt/14= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= @@ -2367,8 +2388,8 @@ gitlab.com/NebulousLabs/errors v0.0.0-20200929122200-06c536cf6975/go.mod h1:ZkMZ gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40 h1:dizWJqTWjwyD8KGcMOwgrkqu1JIkofYgKkmDeNE7oAs= gitlab.com/NebulousLabs/fastrand v0.0.0-20181126182046-603482d69e40/go.mod h1:rOnSnoRyxMI3fe/7KIbVcsHRGxe30OONv8dEgo+vCfA= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= -go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= -go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= +go.etcd.io/bbolt v1.3.8 h1:xs88BrvEv273UsB79e0hcVrlUWmS0a8upikMFhSyAtA= +go.etcd.io/bbolt v1.3.8/go.mod h1:N9Mkw9X8x5fupy0IKsmuqVtoGDyxsaDlbk4Rd05IAQw= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.opencensus.io v0.18.0/go.mod h1:vKdFvxhtzZ9onBp9VKHK8z/sRpBMnKAsufL7wlDrCOA= go.opencensus.io v0.20.1/go.mod h1:6WKK9ahsWS3RSO+PY9ZHZUfv2irvY6gN279GOPZjmmk= @@ -2389,6 +2410,7 @@ go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzox go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= +go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 h1:+RbSCde0ERway5FwKvXR3aRJIFeDu9rtwC6E7BC6uoM= @@ -2401,10 +2423,12 @@ go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhC go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= +go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= go.opentelemetry.io/otel/oteltest v0.20.0/go.mod h1:L7bgKf9ZB7qCwT9Up7i9/pn0PWIa9FqQ2IQ8LoxiGnw= go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc= +go.opentelemetry.io/otel/sdk v1.21.0/go.mod h1:Nna6Yv7PWTdgJHVRD9hIYywQBRx7pbox6nwBnZIxl/E= go.opentelemetry.io/otel/sdk v1.22.0 h1:6coWHw9xw7EfClIC/+O31R8IY3/+EiRFHevmHafB2Gw= go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxtLzPvPz1DOc= go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI= @@ -2413,6 +2437,7 @@ go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16g go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= +go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI= @@ -2667,8 +2692,8 @@ golang.org/x/net v0.4.0/go.mod h1:MBQ8lrhLObU/6UmLb4fmbmk5OcyYmqtbGd/9yIeKjEE= golang.org/x/net v0.5.0/go.mod h1:DivGGAXEgPSlEBzxGzZI+ZLohi+xUj054jfeKui00ws= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= -golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= -golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= +golang.org/x/net v0.20.0 h1:aCL9BSgETF1k+blQaYUBx9hJ9LOGP3gAVemcZlf1Kpo= +golang.org/x/net v0.20.0/go.mod h1:z8BVo6PvndSri0LbOE3hAn0apkU+1YvI6E70E9jsnvY= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= @@ -2785,7 +2810,6 @@ golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200814200057-3d37ad5750ed/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200826173525-f9321e4c35a6/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -2861,6 +2885,7 @@ golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.8.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/sys v0.16.0 h1:xWw16ngr6ZMtmxDyKyIgsE93KNKz5HKmMa3b8ALHidU= golang.org/x/sys v0.16.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= @@ -2973,8 +2998,8 @@ golang.org/x/tools v0.1.10/go.mod h1:Uh6Zz+xoGYZom868N8YTex3t7RhtHDBrE8Gzo9bV56E golang.org/x/tools v0.1.12/go.mod h1:hNGJHUnrk76NpqgfD5Aqm5Crs+Hm0VOH/i9J2+nxYbc= golang.org/x/tools v0.2.0/go.mod h1:y4OqIKeOV/fWJetJ8bXPU1sEVniLMIyDAZWeHdV+NTA= golang.org/x/tools v0.3.0/go.mod h1:/rWhSS2+zyEVwoJf8YAX6L2f0ntZ7Kn/mGgAWcipA5k= -golang.org/x/tools v0.16.0 h1:GO788SKMRunPIBCXiQyo2AaexLstOrVhuAL5YwsckQM= -golang.org/x/tools v0.16.0/go.mod h1:kYVVN6I1mBNoB1OX+noeBjbRk4IUEPa7JJ+TJMEooJ0= +golang.org/x/tools v0.17.0 h1:FvmRgNOcs3kOa+T20R1uhfP9F6HgG2mfxDv1vrx1Htc= +golang.org/x/tools v0.17.0/go.mod h1:xsh6VxdV005rRVaS6SSAf9oiAqljS7UZUacMZ8Bnsps= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= From 43c5ef6cef9ad51e64fc8c04ba344ada4a8911f4 Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Thu, 22 Feb 2024 10:49:59 +0100 Subject: [PATCH 387/388] move to local machine: updating pyroscope --- go.mod | 4 ++-- go.sum | 9 ++++----- 2 files changed, 6 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index ceb0466d9c..03ac56064a 100644 --- a/go.mod +++ b/go.mod @@ -53,12 +53,12 @@ require ( github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.18.0 github.com/pyroscope-io/client v0.7.2 - github.com/pyroscope-io/otel-profiling-go v0.5.0 + github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/rollkit/go-da v0.4.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 github.com/stretchr/testify v1.8.4 - github.com/tendermint/tendermint v0.34.28 + github.com/tendermint/tendermint v0.34.29 go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 go.opentelemetry.io/otel v1.22.0 go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp v0.45.0 diff --git a/go.sum b/go.sum index 54ad13537f..98da8b7dd0 100644 --- a/go.sum +++ b/go.sum @@ -2104,8 +2104,8 @@ github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAa github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= -github.com/pyroscope-io/otel-profiling-go v0.5.0 h1:LsTP9VuQ5TgeSiyY2gPHy1de/q3jbFyGWE1v3LtHzMk= -github.com/pyroscope-io/otel-profiling-go v0.5.0/go.mod h1:jUUUXTTgntvGJKS8p5uzypXwTyuGnQP31VnWauH/lUg= +github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic= +github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= @@ -2407,9 +2407,9 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 h1:2JydY5UiDpqvj2p7sO9bgHuhTy4hgTZ0ymehdq/Ob0Q= go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0/go.mod h1:ch3a5QxOqVWxas4CzjCFFOOQe+7HgAXC/N1oVxS9DK4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= +go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= -go.opentelemetry.io/otel v1.20.0/go.mod h1:oUIGj3D77RwJdM6PPZImDpSZGDvkD9fhesHny69JFrs= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= go.opentelemetry.io/otel v1.22.0 h1:xS7Ku+7yTFvDfDraDIJVpw7XPyuHlB9MCiqqX5mcJ6Y= go.opentelemetry.io/otel v1.22.0/go.mod h1:eoV4iAi3Ea8LkAEI9+GFT44O6T/D0GWAVFyZVCC6pMI= @@ -2422,7 +2422,6 @@ go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.22.0/go.mod h go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1 h1:2PunuO5SbkN5MhCbuHCd3tC6qrcaj+uDAkX/qBU5BAs= go.opentelemetry.io/otel/exporters/stdout/stdouttrace v1.15.1/go.mod h1:q8+Tha+5LThjeSU8BW93uUC5w5/+DnYHMKBMpRCsui0= go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU= -go.opentelemetry.io/otel/metric v1.20.0/go.mod h1:90DRw3nfK4D7Sm/75yQ00gTJxtkBxX+wu6YaNymbpVM= go.opentelemetry.io/otel/metric v1.21.0/go.mod h1:o1p3CA8nNHW8j5yuQLdc1eeqEaPfzug24uvsyIEJRWM= go.opentelemetry.io/otel/metric v1.22.0 h1:lypMQnGyJYeuYPhOM/bgjbFM6WE44W1/T45er4d8Hhg= go.opentelemetry.io/otel/metric v1.22.0/go.mod h1:evJGjVpZv0mQ5QBRJoBF64yMuOf4xCWdXjK8pzFvliY= @@ -2434,9 +2433,9 @@ go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxt go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI= go.opentelemetry.io/otel/sdk/metric v1.22.0/go.mod h1:KjQGeMIDlBNEOo6HvjhxIec1p/69/kULDcp4gr0oLQQ= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= +go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= -go.opentelemetry.io/otel/trace v1.20.0/go.mod h1:HJSK7F/hA5RlzpZ0zKDCHCDHm556LCDtKaAo6JmBFUU= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= go.opentelemetry.io/otel/trace v1.22.0 h1:Hg6pPujv0XG9QaVbGOBVHunyuLcCC3jN7WEhPx83XD0= go.opentelemetry.io/otel/trace v1.22.0/go.mod h1:RbbHXVqKES9QhzZq/fE5UnOSILqRt40a21sPw2He1xo= From 4c6e1e03310b731d7d31c52dc94260db17b1b806 Mon Sep 17 00:00:00 2001 From: Jacob Gadikian Date: Thu, 22 Feb 2024 12:01:50 +0200 Subject: [PATCH 388/388] Add Grafana profiling libraries --- cmd/cel-shed/eds_store_stress.go | 2 +- cmd/flags_misc.go | 2 +- go.mod | 7 ++----- go.sum | 8 -------- nodebuilder/settings.go | 4 ++-- 5 files changed, 6 insertions(+), 17 deletions(-) diff --git a/cmd/cel-shed/eds_store_stress.go b/cmd/cel-shed/eds_store_stress.go index 62ea5cb772..d106f73152 100644 --- a/cmd/cel-shed/eds_store_stress.go +++ b/cmd/cel-shed/eds_store_stress.go @@ -10,9 +10,9 @@ import ( "os" "time" + pyroscope "github.com/grafana/pyroscope-go" logging "github.com/ipfs/go-log/v2" "github.com/mitchellh/go-homedir" - "github.com/pyroscope-io/client/pyroscope" "github.com/spf13/cobra" "github.com/celestiaorg/celestia-node/libs/edssser" diff --git a/cmd/flags_misc.go b/cmd/flags_misc.go index cd539bde4c..332d2f8022 100644 --- a/cmd/flags_misc.go +++ b/cmd/flags_misc.go @@ -7,8 +7,8 @@ import ( "net/http/pprof" "strings" + otelpyroscope "github.com/grafana/otel-profiling-go" logging "github.com/ipfs/go-log/v2" - otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "github.com/spf13/cobra" flag "github.com/spf13/pflag" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp" diff --git a/go.mod b/go.mod index 03ac56064a..9f4754f981 100644 --- a/go.mod +++ b/go.mod @@ -27,6 +27,8 @@ require ( github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 github.com/gorilla/mux v1.8.1 + github.com/grafana/otel-profiling-go v0.5.1 + github.com/grafana/pyroscope-go v1.1.1 github.com/hashicorp/go-retryablehttp v0.7.5 github.com/hashicorp/golang-lru/v2 v2.0.7 github.com/imdario/mergo v0.3.16 @@ -52,8 +54,6 @@ require ( github.com/multiformats/go-multihash v0.2.3 github.com/open-rpc/meta-schema v0.0.0-20201029221707-1b72ef2ea333 github.com/prometheus/client_golang v1.18.0 - github.com/pyroscope-io/client v0.7.2 - github.com/pyroscope-io/otel-profiling-go v0.4.0 github.com/rollkit/go-da v0.4.0 github.com/spf13/cobra v1.8.0 github.com/spf13/pflag v1.0.5 @@ -187,8 +187,6 @@ require ( github.com/googleapis/gax-go/v2 v2.12.0 // indirect github.com/gorilla/handlers v1.5.1 // indirect github.com/gorilla/websocket v1.5.0 // indirect - github.com/grafana/otel-profiling-go v0.5.1 // indirect - github.com/grafana/pyroscope-go v1.1.1 // indirect github.com/grafana/pyroscope-go/godeltaprof v0.1.6 // indirect github.com/grpc-ecosystem/go-grpc-middleware v1.3.0 // indirect github.com/grpc-ecosystem/grpc-gateway v1.16.0 // indirect @@ -291,7 +289,6 @@ require ( github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.45.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect - github.com/pyroscope-io/godeltaprof v0.1.2 // indirect github.com/quic-go/qpack v0.4.0 // indirect github.com/quic-go/qtls-go1-20 v0.3.4 // indirect github.com/quic-go/quic-go v0.39.4 // indirect diff --git a/go.sum b/go.sum index 98da8b7dd0..5635e8b0ea 100644 --- a/go.sum +++ b/go.sum @@ -2100,12 +2100,6 @@ github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0ua github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= -github.com/pyroscope-io/client v0.7.2 h1:OX2qdUQsS8RSkn/3C8isD7f/P0YiZQlRbAlecAaj/R8= -github.com/pyroscope-io/client v0.7.2/go.mod h1:FEocnjn+Ngzxy6EtU9ZxXWRvQ0+pffkrBxHLnPpxwi8= -github.com/pyroscope-io/godeltaprof v0.1.2 h1:MdlEmYELd5w+lvIzmZvXGNMVzW2Qc9jDMuJaPOR75g4= -github.com/pyroscope-io/godeltaprof v0.1.2/go.mod h1:psMITXp90+8pFenXkKIpNhrfmI9saQnPbba27VIaiQE= -github.com/pyroscope-io/otel-profiling-go v0.4.0 h1:Hk/rbUqOWoByoWy1tt4r5BX5xoKAvs5drr0511Ki8ic= -github.com/pyroscope-io/otel-profiling-go v0.4.0/go.mod h1:MXaofiWU7PgLP7eISUZJYVO4Z8WYMqpkYgeP4XrPLyg= github.com/quic-go/qpack v0.4.0 h1:Cr9BXA1sQS2SmDUWjSofMPNKmvF6IiIfDRmgU0w1ZCo= github.com/quic-go/qpack v0.4.0/go.mod h1:UZVnYIfi5GRk+zI9UMaCPsmZ2xKJP7XBUvVyT1Knj9A= github.com/quic-go/qtls-go1-18 v0.2.0/go.mod h1:moGulGHK7o6O8lSPSZNoOwcLvJKJ85vVNc7oJFD65bc= @@ -2407,7 +2401,6 @@ go.opencensus.io v0.24.0/go.mod h1:vNK8G9p7aAivkbmorf4v+7Hgx+Zs0yY+0fOtgBfjQKo= go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0 h1:2JydY5UiDpqvj2p7sO9bgHuhTy4hgTZ0ymehdq/Ob0Q= go.opentelemetry.io/contrib/instrumentation/runtime v0.45.0/go.mod h1:ch3a5QxOqVWxas4CzjCFFOOQe+7HgAXC/N1oVxS9DK4= go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo= -go.opentelemetry.io/otel v1.4.1/go.mod h1:StM6F/0fSwpd8dKWDCdRr7uRvEPYdW0hBSlbdTiUde4= go.opentelemetry.io/otel v1.7.0/go.mod h1:5BdUoMIz5WEs0vt0CUEMtSSaTSHBBVwrhnz7+nrD5xk= go.opentelemetry.io/otel v1.13.0/go.mod h1:FH3RtdZCzRkJYFTCsAKDy9l/XYjMdNv6QrkFFB8DvVg= go.opentelemetry.io/otel v1.21.0/go.mod h1:QZzNPQPm1zLX4gZK4cMi+71eaorMSGT3A4znnUvNNEo= @@ -2433,7 +2426,6 @@ go.opentelemetry.io/otel/sdk v1.22.0/go.mod h1:iu7luyVGYovrRpe2fmj3CVKouQNdTOkxt go.opentelemetry.io/otel/sdk/metric v1.22.0 h1:ARrRetm1HCVxq0cbnaZQlfwODYJHo3gFL8Z3tSmHBcI= go.opentelemetry.io/otel/sdk/metric v1.22.0/go.mod h1:KjQGeMIDlBNEOo6HvjhxIec1p/69/kULDcp4gr0oLQQ= go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw= -go.opentelemetry.io/otel/trace v1.4.1/go.mod h1:iYEVbroFCNut9QkwEczV9vMRPHNKSSwYZjulEtsmhFc= go.opentelemetry.io/otel/trace v1.7.0/go.mod h1:fzLSB9nqR2eXzxPXb2JW9IKE+ScyXA48yyE4TNvoHqU= go.opentelemetry.io/otel/trace v1.13.0/go.mod h1:muCvmmO9KKpvuXSf3KKAXXB2ygNYHQ+ZfI5X08d3tds= go.opentelemetry.io/otel/trace v1.21.0/go.mod h1:LGbsEB0f9LGjN+OZaQQ26sohbOmiMR+BaslueVtS/qQ= diff --git a/nodebuilder/settings.go b/nodebuilder/settings.go index 298976fda4..6dc95df285 100644 --- a/nodebuilder/settings.go +++ b/nodebuilder/settings.go @@ -5,10 +5,10 @@ import ( "fmt" "time" + otelpyroscope "github.com/grafana/otel-profiling-go" + pyroscope "github.com/grafana/pyroscope-go" logging "github.com/ipfs/go-log/v2" "github.com/libp2p/go-libp2p/core/peer" - "github.com/pyroscope-io/client/pyroscope" - otelpyroscope "github.com/pyroscope-io/otel-profiling-go" "go.opentelemetry.io/contrib/instrumentation/runtime" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/exporters/otlp/otlpmetric/otlpmetrichttp"