From 754e753126ca8b45f8bec05b4f903ad387da1993 Mon Sep 17 00:00:00 2001 From: Adin Schmahmann Date: Wed, 31 Jul 2024 18:17:28 -0400 Subject: [PATCH 1/6] feat: increase routing limits --- go.mod | 2 +- setup_bitswap.go | 40 +++++++++++++++++++++++++++------------- 2 files changed, 28 insertions(+), 14 deletions(-) diff --git a/go.mod b/go.mod index 4d0b093..224f96f 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.24.3 + github.com/ipfs/boxo v0.21.1-0.20240726111146-e95eeb2ae5f1 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/setup_bitswap.go b/setup_bitswap.go index 3029506..323a46e 100644 --- a/setup_bitswap.go +++ b/setup_bitswap.go @@ -4,6 +4,8 @@ import ( "context" "time" + "github.com/ipfs/boxo/routing/providerquerymanager" + "github.com/ipfs/boxo/bitswap" bsclient "github.com/ipfs/boxo/bitswap/client" bsnet "github.com/ipfs/boxo/bitswap/network" @@ -21,6 +23,12 @@ import ( func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routing.ContentRouting, bstore blockstore.Blockstore) exchange.Interface { bsctx := metri.CtxScope(ctx, "ipfs_bitswap") + n := &providerQueryNetwork{cr, h} + pqm, err := providerquerymanager.New(ctx, n, providerquerymanager.WithMaxInProcessRequests(100)) + if err != nil { + panic(err) + } + cr = &wrapProv{pqm: pqm} bn := bsnet.NewFromIpfsHost(h, cr) // --- Client Options @@ -31,6 +39,14 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi // bitswap.ProviderSearchDelay: default is 1 second. providerSearchDelay := 1 * time.Second + // --- Bitswap Client Options + clientOpts := []bsclient.Option{ + bsclient.RebroadcastDelay(rebroadcastDelay), + bsclient.ProviderSearchDelay(providerSearchDelay), + bsclient.WithoutDuplicatedBlockStats(), + bsclient.WithDefaultLookupManagement(false), + } + // If peering and shared cache are both enabled, we initialize both a // Client and a Server with custom request filter and custom options. // client+server is more expensive but necessary when deployment requires @@ -48,31 +64,29 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi return ok } - // Initialize client+server - bswap := bitswap.New(bsctx, bn, bstore, - // --- Client Options - bitswap.RebroadcastDelay(rebroadcastDelay), - bitswap.ProviderSearchDelay(providerSearchDelay), - bitswap.WithoutDuplicatedBlockStats(), + // turn bitswap clients option into bitswap options + var opts []bitswap.Option + for _, o := range clientOpts { + opts = append(opts, bitswap.WithClientOption(o)) + } - // ---- Server Options + // ---- Server Options + opts = append(opts, bitswap.WithPeerBlockRequestFilter(peerBlockRequestFilter), bitswap.ProvideEnabled(false), // When we don't have a block, don't reply. This reduces processment. bitswap.SetSendDontHaves(false), bitswap.WithWantHaveReplaceSize(cfg.BitswapWantHaveReplaceSize), ) + + // Initialize client+server + bswap := bitswap.New(bsctx, bn, bstore, opts...) bn.Start(bswap) return &noNotifyExchange{bswap} } // By default, rainbow runs with bitswap client alone - bswap := bsclient.New(bsctx, bn, bstore, - // --- Client Options - bsclient.RebroadcastDelay(rebroadcastDelay), - bsclient.ProviderSearchDelay(providerSearchDelay), - bsclient.WithoutDuplicatedBlockStats(), - ) + bswap := bsclient.New(bsctx, bn, bstore, clientOpts...) bn.Start(bswap) return bswap } From 2ebfc5b97452ab74b2cbd6b159ea6517eac9b303 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 15 Nov 2024 13:05:16 +0100 Subject: [PATCH 2/6] Cleanup and update to boxo with latest changes --- go.mod | 2 +- go.sum | 6 ++++-- setup_bitswap.go | 18 ++++++++++-------- 3 files changed, 15 insertions(+), 11 deletions(-) diff --git a/go.mod b/go.mod index 224f96f..58f0653 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.21.1-0.20240726111146-e95eeb2ae5f1 + github.com/ipfs/boxo v0.24.4-0.20241126153154-86735605944b github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 89ef865..f0a61b4 100644 --- a/go.sum +++ b/go.sum @@ -256,8 +256,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c= -github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg= +github.com/ipfs/boxo v0.24.4-0.20241126153154-86735605944b h1:ft0uBj8fmb7Eoq1fwBX0BvM3lgZF4ArGYhzeC7cxi4o= +github.com/ipfs/boxo v0.24.4-0.20241126153154-86735605944b/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= @@ -658,6 +658,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw= github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= +github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8= +github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA= github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs= github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= diff --git a/setup_bitswap.go b/setup_bitswap.go index 323a46e..145d04b 100644 --- a/setup_bitswap.go +++ b/setup_bitswap.go @@ -23,13 +23,16 @@ import ( func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routing.ContentRouting, bstore blockstore.Blockstore) exchange.Interface { bsctx := metri.CtxScope(ctx, "ipfs_bitswap") - n := &providerQueryNetwork{cr, h} - pqm, err := providerquerymanager.New(ctx, n, providerquerymanager.WithMaxInProcessRequests(100)) + + bn := bsnet.NewFromIpfsHost(h) + + // Custom query manager with the content router and the host + // and our custom options to overwrite the default. + pqm, err := providerquerymanager.New(ctx, h, cr, providerquerymanager.WithMaxInProcessRequests(100)) + if err != nil { panic(err) } - cr = &wrapProv{pqm: pqm} - bn := bsnet.NewFromIpfsHost(h, cr) // --- Client Options // bitswap.RebroadcastDelay: default is 1 minute to search for a random @@ -44,7 +47,7 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi bsclient.RebroadcastDelay(rebroadcastDelay), bsclient.ProviderSearchDelay(providerSearchDelay), bsclient.WithoutDuplicatedBlockStats(), - bsclient.WithDefaultLookupManagement(false), + bsclient.WithDefaultProviderQueryManager(false), // we pass it in manually } // If peering and shared cache are both enabled, we initialize both a @@ -73,20 +76,19 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi // ---- Server Options opts = append(opts, bitswap.WithPeerBlockRequestFilter(peerBlockRequestFilter), - bitswap.ProvideEnabled(false), // When we don't have a block, don't reply. This reduces processment. bitswap.SetSendDontHaves(false), bitswap.WithWantHaveReplaceSize(cfg.BitswapWantHaveReplaceSize), ) // Initialize client+server - bswap := bitswap.New(bsctx, bn, bstore, opts...) + bswap := bitswap.New(bsctx, bn, pqm, bstore, opts...) bn.Start(bswap) return &noNotifyExchange{bswap} } // By default, rainbow runs with bitswap client alone - bswap := bsclient.New(bsctx, bn, bstore, clientOpts...) + bswap := bsclient.New(bsctx, bn, pqm, bstore, clientOpts...) bn.Start(bswap) return bswap } From e6515bfb5235198d96ad26fbc34272df80435bb8 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Wed, 27 Nov 2024 14:43:17 +0100 Subject: [PATCH 3/6] Fix tests --- handler_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/handler_test.go b/handler_test.go index a3cebc5..041d29a 100644 --- a/handler_test.go +++ b/handler_test.go @@ -111,7 +111,7 @@ func TestNoBlockcacheHeader(t *testing.T) { ctx := context.Background() // pacify metrics reporting code ctx = metrics.CtxScope(ctx, "test.bsserver.host") - n := bsnet.NewFromIpfsHost(newHost, nil) + n := bsnet.NewFromIpfsHost(newHost) bs := bsserver.New(ctx, n, gnd.blockstore) n.Start(bs) defer bs.Close() From 3abcea1146dab7d642c34b0a62b5cfdbe1c3cc5b Mon Sep 17 00:00:00 2001 From: gammazero <11790789+gammazero@users.noreply.github.com> Date: Wed, 27 Nov 2024 14:11:24 -1000 Subject: [PATCH 4/6] depend on latest boxo --- go.mod | 4 +++- go.sum | 8 ++++++-- 2 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index 58f0653..10bbef5 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.24.4-0.20241126153154-86735605944b + github.com/ipfs/boxo v0.24.4-0.20241127202418-33c7b9c236f9 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 @@ -78,6 +78,8 @@ require ( github.com/francoispqt/gojay v1.2.13 // indirect github.com/fsnotify/fsnotify v1.7.0 // indirect github.com/gabriel-vasile/mimetype v1.4.6 // indirect + github.com/gammazero/chanqueue v1.0.0 // indirect + github.com/gammazero/deque v1.0.0 // indirect github.com/getsentry/sentry-go v0.27.0 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect diff --git a/go.sum b/go.sum index f0a61b4..468e140 100644 --- a/go.sum +++ b/go.sum @@ -130,6 +130,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM= github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc= github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc= +github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o= +github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc= +github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34= +github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo= github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY= github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps= github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= @@ -256,8 +260,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241126153154-86735605944b h1:ft0uBj8fmb7Eoq1fwBX0BvM3lgZF4ArGYhzeC7cxi4o= -github.com/ipfs/boxo v0.24.4-0.20241126153154-86735605944b/go.mod h1:Kxk43F+avGAsJSwhJW4isNYrpGwXHRJCvJ19Pt+MQc4= +github.com/ipfs/boxo v0.24.4-0.20241127202418-33c7b9c236f9 h1:B9Wq3bN2zaiUwdh/gWTIXvPi+BkRrYj51PUTEEyQNNY= +github.com/ipfs/boxo v0.24.4-0.20241127202418-33c7b9c236f9/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= From 15c48309d0f558bf29b10e57e020dc77c633b173 Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Thu, 28 Nov 2024 17:31:10 +0100 Subject: [PATCH 5/6] bitswap_setup: Update boxo and call pqm.Startup --- go.mod | 2 +- go.sum | 4 ++-- setup_bitswap.go | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 10bbef5..11d621c 100644 --- a/go.mod +++ b/go.mod @@ -10,7 +10,7 @@ require ( github.com/felixge/httpsnoop v1.0.4 github.com/ipfs-shipyard/nopfs v0.0.12 github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a - github.com/ipfs/boxo v0.24.4-0.20241127202418-33c7b9c236f9 + github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36 github.com/ipfs/go-block-format v0.2.0 github.com/ipfs/go-cid v0.4.1 github.com/ipfs/go-datastore v0.6.0 diff --git a/go.sum b/go.sum index 468e140..7da46ea 100644 --- a/go.sum +++ b/go.sum @@ -260,8 +260,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI= github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs= github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0= -github.com/ipfs/boxo v0.24.4-0.20241127202418-33c7b9c236f9 h1:B9Wq3bN2zaiUwdh/gWTIXvPi+BkRrYj51PUTEEyQNNY= -github.com/ipfs/boxo v0.24.4-0.20241127202418-33c7b9c236f9/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= +github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36 h1:/el1Poab8nY9SBAWxjGEPCHOlRfbuxm39Ewcj7b6w+0= +github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g= github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA= github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU= github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ= diff --git a/setup_bitswap.go b/setup_bitswap.go index 145d04b..5079b27 100644 --- a/setup_bitswap.go +++ b/setup_bitswap.go @@ -29,10 +29,10 @@ func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routi // Custom query manager with the content router and the host // and our custom options to overwrite the default. pqm, err := providerquerymanager.New(ctx, h, cr, providerquerymanager.WithMaxInProcessRequests(100)) - if err != nil { panic(err) } + pqm.Startup() // --- Client Options // bitswap.RebroadcastDelay: default is 1 minute to search for a random From d5b297ad2ff68992c57f3ca6c1412d58bef47a1c Mon Sep 17 00:00:00 2001 From: Hector Sanjuan Date: Fri, 29 Nov 2024 11:27:21 +0100 Subject: [PATCH 6/6] Update changelog for increased routing limits --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cb7e1c3..02562fe 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -22,6 +22,7 @@ The following emojis are used to highlight certain changes: - boxo [v0.24.3](https://github.com/ipfs/boxo/releases/tag/v0.24.3) - go-libp2p-kad-dht [v0.28.1](https://github.com/libp2p/go-libp2p-kad-dht/releases/tag/v0.28.1) - passing headers that require authorization but are not authorized now results in an HTTP 401 instead of ignoring those headers +- Bitswap settings: Increased default content-discovery limits, with up to 100 in-flight requests. ### Removed