diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7e1c3..02562fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The following emojis are used to highlight certain changes: - boxo [v0.24.3](https://github.com/ipfs/boxo/releases/tag/v0.24.3) - go-libp2p-kad-dht [v0.28.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.28.1) - passing headers that require authorization but are not authorized now results in an HTTP 401 instead of ignoring those headers +- Bitswap settings: Increased default content-discovery limits, with up to 100 in-flight requests. ### Removed diff --git a/go.mod b/go.mod index 4d0b093..11d621c 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.24.3 + github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 @@ -78,6 +78,8 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/gammazero/chanqueue v1.0.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index 89ef865..7da46ea 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o= +github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= @@ -256,8 +260,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c= -github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg= +github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36 h1:/el1Poab8nY9SBAWxjGEPCHOlRfbuxm39Ewcj7b6w+0= +github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= @@ -658,6 +662,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8= +github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= diff --git a/handler_test.go b/handler_test.go index a3cebc5..041d29a 100644 --- a/handler_test.go +++ b/handler_test.go @@ -111,7 +111,7 @@ func TestNoBlockcacheHeader(t *testing.T) { ctx := context.Background() // pacify metrics reporting code ctx = metrics.CtxScope(ctx, "test.bsserver.host") - n := bsnet.NewFromIpfsHost(newHost, nil) + n := bsnet.NewFromIpfsHost(newHost) bs := bsserver.New(ctx, n, gnd.blockstore) n.Start(bs) defer bs.Close() diff --git a/setup_bitswap.go b/setup_bitswap.go index 3029506..5079b27 100644 --- a/setup_bitswap.go +++ b/setup_bitswap.go @@ -4,6 +4,8 @@ import ( "context" "time" + "github.com/ipfs/boxo/routing/providerquerymanager" + "github.com/ipfs/boxo/bitswap" bsclient "github.com/ipfs/boxo/bitswap/client" bsnet "github.com/ipfs/boxo/bitswap/network" @@ -21,7 +23,16 @@ import ( func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routing.ContentRouting, bstore blockstore.Blockstore) exchange.Interface { bsctx := metri.CtxScope(ctx, "ipfs_bitswap") - bn := bsnet.NewFromIpfsHost(h, cr) + + bn := bsnet.NewFromIpfsHost(h) + + // Custom query manager with the content router and the host + // and our custom options to overwrite the default. + pqm, err := providerquerymanager.New(ctx, h, cr, providerquerymanager.WithMaxInProcessRequests(100)) + if err != nil { + panic(err) + } + pqm.Startup() // --- Client Options // bitswap.RebroadcastDelay: default is 1 minute to search for a random @@ -31,6 +42,14 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi // bitswap.ProviderSearchDelay: default is 1 second. providerSearchDelay := 1 * time.Second + // --- Bitswap Client Options + clientOpts := []bsclient.Option{ + bsclient.RebroadcastDelay(rebroadcastDelay), + bsclient.ProviderSearchDelay(providerSearchDelay), + bsclient.WithoutDuplicatedBlockStats(), + bsclient.WithDefaultProviderQueryManager(false), // we pass it in manually + } + // If peering and shared cache are both enabled, we initialize both a // Client and a Server with custom request filter and custom options. // client+server is more expensive but necessary when deployment requires @@ -48,31 +67,28 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi return ok } - // Initialize client+server - bswap := bitswap.New(bsctx, bn, bstore, - // --- Client Options - bitswap.RebroadcastDelay(rebroadcastDelay), - bitswap.ProviderSearchDelay(providerSearchDelay), - bitswap.WithoutDuplicatedBlockStats(), + // turn bitswap clients option into bitswap options + var opts []bitswap.Option + for _, o := range clientOpts { + opts = append(opts, bitswap.WithClientOption(o)) + } - // ---- Server Options + // ---- Server Options + opts = append(opts, bitswap.WithPeerBlockRequestFilter(peerBlockRequestFilter), - bitswap.ProvideEnabled(false), // When we don't have a block, don't reply. This reduces processment. bitswap.SetSendDontHaves(false), bitswap.WithWantHaveReplaceSize(cfg.BitswapWantHaveReplaceSize), ) + + // Initialize client+server + bswap := bitswap.New(bsctx, bn, pqm, bstore, opts...) bn.Start(bswap) return &noNotifyExchange{bswap} } // By default, rainbow runs with bitswap client alone - bswap := bsclient.New(bsctx, bn, bstore, - // --- Client Options - bsclient.RebroadcastDelay(rebroadcastDelay), - bsclient.ProviderSearchDelay(providerSearchDelay), - bsclient.WithoutDuplicatedBlockStats(), - ) + bswap := bsclient.New(bsctx, bn, pqm, bstore, clientOpts...) bn.Start(bswap) return bswap }