Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

feat: increase routing limits #154

Merged
merged 6 commits into from
Dec 2, 2024
+44 −19
Merged

feat: increase routing limits #154

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
754e753
feat: increase routing limits
aschmahmann Jul 31, 2024
2ebfc5b
Cleanup and update to boxo with latest changes
hsanjuan Nov 15, 2024
e6515bf
Fix tests
hsanjuan Nov 27, 2024
3abcea1
depend on latest boxo
gammazero Nov 28, 2024
15c4830
bitswap_setup: Update boxo and call pqm.Startup
hsanjuan Nov 28, 2024
d5b297a
Update changelog for increased routing limits
hsanjuan Nov 29, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 3 additions & 1 deletion go.mod
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ require (
github.com/felixge/httpsnoop v1.0.4
github.com/ipfs-shipyard/nopfs v0.0.12
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a
github.com/ipfs/boxo v0.24.3
github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36
github.com/ipfs/go-block-format v0.2.0
github.com/ipfs/go-cid v0.4.1
github.com/ipfs/go-datastore v0.6.0
Expand Down Expand Up @@ -78,6 +78,8 @@ require (
github.com/francoispqt/gojay v1.2.13 // indirect
github.com/fsnotify/fsnotify v1.7.0 // indirect
github.com/gabriel-vasile/mimetype v1.4.6 // indirect
github.com/gammazero/chanqueue v1.0.0 // indirect
github.com/gammazero/deque v1.0.0 // indirect
github.com/getsentry/sentry-go v0.27.0 // indirect
github.com/go-logr/logr v1.4.2 // indirect
github.com/go-logr/stdr v1.2.2 // indirect
Expand Down
10 changes: 8 additions & 2 deletions go.sum
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,10 @@ github.com/fsnotify/fsnotify v1.7.0 h1:8JEhPFa5W2WU7YfeZzPNqzMP6Lwt7L2715Ggo0nos
github.com/fsnotify/fsnotify v1.7.0/go.mod h1:40Bi/Hjc2AVfZrqy+aj+yEI+/bRxZnMJyTJwOpGvigM=
github.com/gabriel-vasile/mimetype v1.4.6 h1:3+PzJTKLkvgjeTbts6msPJt4DixhT4YtFNf1gtGe3zc=
github.com/gabriel-vasile/mimetype v1.4.6/go.mod h1:JX1qVKqZd40hUPpAfiNTe0Sne7hdfKSbOqqmkq8GCXc=
github.com/gammazero/chanqueue v1.0.0 h1:FER/sMailGFA3DDvFooEkipAMU+3c9Bg3bheloPSz6o=
github.com/gammazero/chanqueue v1.0.0/go.mod h1:fMwpwEiuUgpab0sH4VHiVcEoji1pSi+EIzeG4TPeKPc=
github.com/gammazero/deque v1.0.0 h1:LTmimT8H7bXkkCy6gZX7zNLtkbz4NdS2z8LZuor3j34=
github.com/gammazero/deque v1.0.0/go.mod h1:iflpYvtGfM3U8S8j+sZEKIak3SAKYpA5/SQewgfXDKo=
github.com/garyburd/redigo v1.1.1-0.20170914051019-70e1b1943d4f/go.mod h1:NR3MbYisc3/PwhQ00EMzDiPmrwpPxAn5GI05/YaO1SY=
github.com/getsentry/sentry-go v0.27.0 h1:Pv98CIbtB3LkMWmXi4Joa5OOcwbmnX88sF5qbK3r3Ps=
github.com/getsentry/sentry-go v0.27.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY=
Expand Down Expand Up @@ -256,8 +260,8 @@ github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a h1:MKG
github.com/ipfs-shipyard/nopfs/ipfs v0.13.2-0.20231024163508-120e0c51ee3a/go.mod h1:6EekK/jo+TynwSE/ZOiOJd4eEvRXoavEC3vquKtv4yI=
github.com/ipfs/bbloom v0.0.4 h1:Gi+8EGJ2y5qiD5FbsbpX/TMNcJw8gSqr7eyjHa4Fhvs=
github.com/ipfs/bbloom v0.0.4/go.mod h1:cS9YprKXpoZ9lT0n/Mw/a6/aFV6DTjTLYHeA+gyqMG0=
github.com/ipfs/boxo v0.24.3 h1:gldDPOWdM3Rz0v5LkVLtZu7A7gFNvAlWcmxhCqlHR3c=
github.com/ipfs/boxo v0.24.3/go.mod h1:h0DRzOY1IBFDHp6KNvrJLMFdSXTYID0Zf+q7X05JsNg=
github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36 h1:/el1Poab8nY9SBAWxjGEPCHOlRfbuxm39Ewcj7b6w+0=
github.com/ipfs/boxo v0.24.4-0.20241128163335-b5656aa69b36/go.mod h1:lAoydO+oJhB1e7pUn4ju1Z1fuUIwy+zb0hQXRb/bu2g=
github.com/ipfs/go-bitfield v1.1.0 h1:fh7FIo8bSwaJEh6DdTWbCeZ1eqOaOkKFI74SCnsWbGA=
github.com/ipfs/go-bitfield v1.1.0/go.mod h1:paqf1wjq/D2BBmzfTVFlJQ9IlFOZpg422HL0HqsGWHU=
github.com/ipfs/go-bitswap v0.11.0 h1:j1WVvhDX1yhG32NTC9xfxnqycqYIlhzEzLXG/cU1HyQ=
Expand Down Expand Up @@ -658,6 +662,8 @@ github.com/shurcooL/users v0.0.0-20180125191416-49c67e49c537/go.mod h1:QJTqeLYED
github.com/shurcooL/webdavfs v0.0.0-20170829043945-18c3829fa133/go.mod h1:hKmq5kWdCj2z2KEozexVbfEZIWiTjhE0+UjmZgPqehw=
github.com/sirupsen/logrus v1.2.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo=
github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0=
github.com/slok/go-http-metrics v0.12.0 h1:mAb7hrX4gB4ItU6NkFoKYdBslafg3o60/HbGBRsKaG8=
github.com/slok/go-http-metrics v0.12.0/go.mod h1:Ee/mdT9BYvGrlGzlClkK05pP2hRHmVbRF9dtUVS8LNA=
github.com/smartystreets/assertions v1.2.0 h1:42S6lae5dvLc7BrLu/0ugRtcFVjoJNMC/N3yZFZkDFs=
github.com/smartystreets/assertions v1.2.0/go.mod h1:tcbTF8ujkAEcZ8TElKY+i30BzYlVhC/LOxJk7iOWnoo=
github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg=
Expand Down
2 changes: 1 addition & 1 deletion handler_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -111,7 +111,7 @@ func TestNoBlockcacheHeader(t *testing.T) {
ctx := context.Background()
// pacify metrics reporting code
ctx = metrics.CtxScope(ctx, "test.bsserver.host")
n := bsnet.NewFromIpfsHost(newHost, nil)
n := bsnet.NewFromIpfsHost(newHost)
bs := bsserver.New(ctx, n, gnd.blockstore)
n.Start(bs)
defer bs.Close()
Expand Down
46 changes: 31 additions & 15 deletions setup_bitswap.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@
"context"
"time"

"github.com/ipfs/boxo/routing/providerquerymanager"

"github.com/ipfs/boxo/bitswap"
bsclient "github.com/ipfs/boxo/bitswap/client"
bsnet "github.com/ipfs/boxo/bitswap/network"
Expand All @@ -21,7 +23,16 @@

func setupBitswapExchange(ctx context.Context, cfg Config, h host.Host, cr routing.ContentRouting, bstore blockstore.Blockstore) exchange.Interface {
bsctx := metri.CtxScope(ctx, "ipfs_bitswap")
bn := bsnet.NewFromIpfsHost(h, cr)

bn := bsnet.NewFromIpfsHost(h)

// Custom query manager with the content router and the host
// and our custom options to overwrite the default.
pqm, err := providerquerymanager.New(ctx, h, cr, providerquerymanager.WithMaxInProcessRequests(100))
gammazero marked this conversation as resolved.
Show resolved Hide resolved
if err != nil {
panic(err)

Check warning on line 33 in setup_bitswap.go

View check run for this annotation

Codecov / codecov/patch

setup_bitswap.go#L33 

Added line #L33 was not covered by tests
}
pqm.Startup()
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Unless there is a need to start at some later time, or some modes of operation need a ProviderQueryQanager that has not been started, then I think New should also perform the startup.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Doing so prevents problems where calling Startup is missed.

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ipfs/boxo#734


// --- Client Options
// bitswap.RebroadcastDelay: default is 1 minute to search for a random
Expand All @@ -31,6 +42,14 @@
// bitswap.ProviderSearchDelay: default is 1 second.
providerSearchDelay := 1 * time.Second

// --- Bitswap Client Options
clientOpts := []bsclient.Option{
bsclient.RebroadcastDelay(rebroadcastDelay),
bsclient.ProviderSearchDelay(providerSearchDelay),
bsclient.WithoutDuplicatedBlockStats(),
bsclient.WithDefaultProviderQueryManager(false), // we pass it in manually
}

// If peering and shared cache are both enabled, we initialize both a
// Client and a Server with custom request filter and custom options.
// client+server is more expensive but necessary when deployment requires
Expand All @@ -48,31 +67,28 @@
return ok
}

// Initialize client+server
bswap := bitswap.New(bsctx, bn, bstore,
// --- Client Options
bitswap.RebroadcastDelay(rebroadcastDelay),
bitswap.ProviderSearchDelay(providerSearchDelay),
bitswap.WithoutDuplicatedBlockStats(),
// turn bitswap clients option into bitswap options
var opts []bitswap.Option
for _, o := range clientOpts {
opts = append(opts, bitswap.WithClientOption(o))
}

// ---- Server Options
// ---- Server Options
opts = append(opts,
bitswap.WithPeerBlockRequestFilter(peerBlockRequestFilter),
bitswap.ProvideEnabled(false),
// When we don't have a block, don't reply. This reduces processment.
bitswap.SetSendDontHaves(false),
bitswap.WithWantHaveReplaceSize(cfg.BitswapWantHaveReplaceSize),
)

// Initialize client+server
bswap := bitswap.New(bsctx, bn, pqm, bstore, opts...)
bn.Start(bswap)
return &noNotifyExchange{bswap}
}

// By default, rainbow runs with bitswap client alone
bswap := bsclient.New(bsctx, bn, bstore,
// --- Client Options
bsclient.RebroadcastDelay(rebroadcastDelay),
bsclient.ProviderSearchDelay(providerSearchDelay),
bsclient.WithoutDuplicatedBlockStats(),
)
bswap := bsclient.New(bsctx, bn, pqm, bstore, clientOpts...)
bn.Start(bswap)
return bswap
}
Expand Down
Loading