Merge pull request #747 from ipfs/http-retr2

httpnet: bitswap network for HTTP block retrieval over trustless gateway endpoints.

CHANGELOG.md

@@ -14,6 +14,13 @@ The following emojis are used to highlight certain changes:
 
 ## [Unreleased]
 
+- ✨ `httpnet`: Transparent HTTP-block retrieval support over Trustless Gateways [#747]((https://github.com/ipfs/boxo/pull/747):
+  - Complements Bitswap as a block-retrieval mechanism, implementing `bitswap/network`.
+  - Understands peers found in provider records with `/.../http` endpoints (trustless gateway).
+  - Treats them as "Bitswap" peers, except instead of using Bitswap it makes HTTP/2 requests to discover (`HEAD`) and retrieve (`GET`) individual blocks (`?format=raw`).
+  - A `bitswap/network` proxy implementation allows co-existance with standard `bitswap/network/bsnet`.
+  - `httpnet` is not enabled by default. Upstream implementations may use it by modifying how they create the Bitswap network and initialize bitswap.
+
 ### Added
 
 ### Changed

bitswap/benchmarks_test.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/ipfs/boxo/bitswap"
-	bsnet "github.com/ipfs/boxo/bitswap/network"
+	bsnet "github.com/ipfs/boxo/bitswap/network/bsnet"
 	testinstance "github.com/ipfs/boxo/bitswap/testinstance"
 	tn "github.com/ipfs/boxo/bitswap/testnet"
 	mockrouting "github.com/ipfs/boxo/routing/mock"

bitswap/client/client.go

@@ -1,4 +1,4 @@
-// Package bitswap implements the IPFS exchange interface with the BitSwap
+// Package client implements the IPFS exchange interface with the BitSwap
 // bilateral exchange protocol.
 package client
 
@@ -202,7 +202,7 @@ func New(parent context.Context, network bsnet.BitSwapNetwork, providerFinder ro
 
 	sim := bssim.New()
 	bpm := bsbpm.New()
-	pm := bspm.New(ctx, peerQueueFactory, network.Self())
+	pm := bspm.New(ctx, peerQueueFactory)
 
 	if bs.providerFinder != nil && bs.defaultProviderQueryManager {
 		// network can do dialing.
@@ -241,7 +241,7 @@ func New(parent context.Context, network bsnet.BitSwapNetwork, providerFinder ro
 		return bssession.New(sessctx, sessmgr, id, spm, sessionProvFinder, sim, pm, bpm, notif, provSearchDelay, rebroadcastDelay, self)
 	}
 	sessionPeerManagerFactory := func(ctx context.Context, id uint64) bssession.SessionPeerManager {
-		return bsspm.New(id, network.ConnectionManager())
+		return bsspm.New(id, network)
 	}
 	notif := notifications.New()
 	sm = bssm.New(ctx, sessionFactory, sim, sessionPeerManagerFactory, bpm, pm, notif, network.Self())

bitswap/client/internal/peermanager/peermanager.go

@@ -44,20 +44,17 @@ type PeerManager struct {
 	psLk         sync.RWMutex
 	sessions     map[uint64]Session
 	peerSessions map[peer.ID]map[uint64]struct{}
-
-	self peer.ID
 }
 
 // New creates a new PeerManager, given a context and a peerQueueFactory.
-func New(ctx context.Context, createPeerQueue PeerQueueFactory, self peer.ID) *PeerManager {
+func New(ctx context.Context, createPeerQueue PeerQueueFactory) *PeerManager {
 	wantGauge := metrics.NewCtx(ctx, "wantlist_total", "Number of items in wantlist.").Gauge()
 	wantBlockGauge := metrics.NewCtx(ctx, "want_blocks_total", "Number of want-blocks in wantlist.").Gauge()
 	return &PeerManager{
 		peerQueues:      make(map[peer.ID]PeerQueue),
 		pwm:             newPeerWantManager(wantGauge, wantBlockGauge),
 		createPeerQueue: createPeerQueue,
 		ctx:             ctx,
-		self:            self,
 
 		sessions:     make(map[uint64]Session),
 		peerSessions: make(map[peer.ID]map[uint64]struct{}),

bitswap/client/internal/peermanager/peermanager_test.go

@@ -85,8 +85,8 @@ func TestAddingAndRemovingPeers(t *testing.T) {
 	peerQueueFactory := makePeerQueueFactory(msgs)
 
 	tp := random.Peers(6)
-	self, peer1, peer2, peer3, peer4, peer5 := tp[0], tp[1], tp[2], tp[3], tp[4], tp[5]
-	peerManager := New(ctx, peerQueueFactory, self)
+	peer1, peer2, peer3, peer4, peer5 := tp[0], tp[1], tp[2], tp[3], tp[4]
+	peerManager := New(ctx, peerQueueFactory)
 
 	peerManager.Connected(peer1)
 	peerManager.Connected(peer2)
@@ -128,8 +128,8 @@ func TestBroadcastOnConnect(t *testing.T) {
 	msgs := make(chan msg, 16)
 	peerQueueFactory := makePeerQueueFactory(msgs)
 	tp := random.Peers(2)
-	self, peer1 := tp[0], tp[1]
-	peerManager := New(ctx, peerQueueFactory, self)
+	peer1 := tp[0]
+	peerManager := New(ctx, peerQueueFactory)
 
 	cids := random.Cids(2)
 	peerManager.BroadcastWantHaves(ctx, cids)
@@ -149,8 +149,8 @@ func TestBroadcastWantHaves(t *testing.T) {
 	msgs := make(chan msg, 16)
 	peerQueueFactory := makePeerQueueFactory(msgs)
 	tp := random.Peers(3)
-	self, peer1, peer2 := tp[0], tp[1], tp[2]
-	peerManager := New(ctx, peerQueueFactory, self)
+	peer1, peer2 := tp[0], tp[1]
+	peerManager := New(ctx, peerQueueFactory)
 
 	cids := random.Cids(3)
 
@@ -190,8 +190,8 @@ func TestSendWants(t *testing.T) {
 	msgs := make(chan msg, 16)
 	peerQueueFactory := makePeerQueueFactory(msgs)
 	tp := random.Peers(2)
-	self, peer1 := tp[0], tp[1]
-	peerManager := New(ctx, peerQueueFactory, self)
+	peer1 := tp[0]
+	peerManager := New(ctx, peerQueueFactory)
 	cids := random.Cids(4)
 
 	peerManager.Connected(peer1)
@@ -224,8 +224,8 @@ func TestSendCancels(t *testing.T) {
 	msgs := make(chan msg, 16)
 	peerQueueFactory := makePeerQueueFactory(msgs)
 	tp := random.Peers(3)
-	self, peer1, peer2 := tp[0], tp[1], tp[2]
-	peerManager := New(ctx, peerQueueFactory, self)
+	peer1, peer2 := tp[0], tp[1]
+	peerManager := New(ctx, peerQueueFactory)
 	cids := random.Cids(4)
 
 	// Connect to peer1 and peer2
@@ -285,8 +285,8 @@ func TestSessionRegistration(t *testing.T) {
 	peerQueueFactory := makePeerQueueFactory(msgs)
 
 	tp := random.Peers(3)
-	self, p1, p2 := tp[0], tp[1], tp[2]
-	peerManager := New(ctx, peerQueueFactory, self)
+	p1, p2 := tp[0], tp[1]
+	peerManager := New(ctx, peerQueueFactory)
 
 	id := uint64(1)
 	s := newSess(id)
@@ -344,9 +344,8 @@ func BenchmarkPeerManager(b *testing.B) {
 		return &benchPeerQueue{}
 	}
 
-	self := random.Peers(1)[0]
 	peers := random.Peers(500)
-	peerManager := New(ctx, peerQueueFactory, self)
+	peerManager := New(ctx, peerQueueFactory)
 
 	// Create a bunch of connections
 	connected := 0

bitswap/message/message.go

@@ -416,26 +416,30 @@ func BlockPresenceSize(c cid.Cid) int {
 }
 
 // FromNet generates a new BitswapMessage from incoming data on an io.Reader.
-func FromNet(r io.Reader) (BitSwapMessage, error) {
+func FromNet(r io.Reader) (BitSwapMessage, int, error) {
 	reader := msgio.NewVarintReaderSize(r, network.MessageSizeMax)
 	return FromMsgReader(reader)
 }
 
 // FromPBReader generates a new Bitswap message from a protobuf reader.
-func FromMsgReader(r msgio.Reader) (BitSwapMessage, error) {
+func FromMsgReader(r msgio.Reader) (BitSwapMessage, int, error) {
 	msg, err := r.ReadMsg()
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
 	pb := new(pb.Message)
 	err = proto.Unmarshal(msg, pb)
 	r.ReleaseMsg(msg)
 	if err != nil {
-		return nil, err
+		return nil, 0, err
 	}
 
-	return newMessageFromProto(pb)
+	m, err := newMessageFromProto(pb)
+	if err != nil {
+		return nil, 0, err
+	}
+	return m, len(msg), nil
 }
 
 func (m *impl) ToProtoV0() *pb.Message {

bitswap/message/message_test.go

@@ -116,7 +116,7 @@ func TestToNetFromNetPreservesWantList(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	copied, err := FromNet(buf)
+	copied, _, err := FromNet(buf)
 	if err != nil {
 		t.Fatal(err)
 	}
@@ -149,7 +149,7 @@ func TestToAndFromNetMessage(t *testing.T) {
 		t.Fatal(err)
 	}
 
-	m2, err := FromNet(buf)
+	m2, _, err := FromNet(buf)
 	if err != nil {
 		t.Fatal(err)
 	}

bitswap/network/bsnet/bsnet.go

@@ -0,0 +1,14 @@
+package bsnet
+
+import "github.com/ipfs/boxo/bitswap/network/bsnet/internal"
+
+var (
+	// ProtocolBitswapNoVers is equivalent to the legacy bitswap protocol
+	ProtocolBitswapNoVers = internal.ProtocolBitswapNoVers
+	// ProtocolBitswapOneZero is the prefix for the legacy bitswap protocol
+	ProtocolBitswapOneZero = internal.ProtocolBitswapOneZero
+	// ProtocolBitswapOneOne is the prefix for version 1.1.0
+	ProtocolBitswapOneOne = internal.ProtocolBitswapOneOne
+	// ProtocolBitswap is the current version of the bitswap protocol: 1.2.0
+	ProtocolBitswap = internal.ProtocolBitswap
+)