From 24dc9749061d432fb4498b794192c420098f708f Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 12:42:47 -0700 Subject: [PATCH 01/11] feat(intercept): carry the tunnel over a dev tunnel with --tunnel-protocol=dev-tunnel With --tunnel-protocol=dev-tunnel, --server-addr is api-dev-tunnel's URL and the intercept holds the tunnel's Connect stream itself, as `vc dev --tunnel` does: the deployment's conns reach --app-port, and intercepted outbound conns leave as client Opens, which the deployment connects to by hostname. Flow control, half-close, heartbeats and reconnects follow the relay's protocol. InterceptorService reports the tunnel in GetStatus and takes client tokens through SetTunnelToken, so whoever ran create-session can hand the intercept the tunnel and renew its token. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/go/bridge/v1/interceptor.pb.go | 417 +++++- api/go/bridge/v1/interceptor_grpc.pb.go | 50 +- api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go | 1165 +++++++++++++++++ .../api/dev_tunnel/v1/tunnel_grpc.pb.go | 210 +++ api/ts/bridge/v1/interceptor_pb.d.ts | 181 ++- api/ts/bridge/v1/interceptor_pb.js | 47 +- .../vercel/api/dev_tunnel/v1/tunnel_pb.d.ts | 458 +++++++ api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js | 127 ++ pkg/commands/dev_test.go | 4 + pkg/commands/intercept.go | 227 +++- pkg/commands/intercept_server.go | 26 + pkg/commands/intercept_server_test.go | 27 + pkg/devtunnel/client.go | 543 ++++++++ pkg/devtunnel/client_test.go | 425 ++++++ pkg/devtunnel/conn.go | 234 ++++ pkg/devtunnel/stream.go | 275 ++++ proto/bridge/v1/interceptor.proto | 61 + proto/vercel/api/dev_tunnel/v1/tunnel.proto | 144 ++ 18 files changed, 4507 insertions(+), 114 deletions(-) create mode 100644 api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go create mode 100644 api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go create mode 100644 api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.d.ts create mode 100644 api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js create mode 100644 pkg/devtunnel/client.go create mode 100644 pkg/devtunnel/client_test.go create mode 100644 pkg/devtunnel/conn.go create mode 100644 pkg/devtunnel/stream.go create mode 100644 proto/vercel/api/dev_tunnel/v1/tunnel.proto diff --git a/api/go/bridge/v1/interceptor.pb.go b/api/go/bridge/v1/interceptor.pb.go index 1e818c5..298aec7 100644 --- a/api/go/bridge/v1/interceptor.pb.go +++ b/api/go/bridge/v1/interceptor.pb.go @@ -1,6 +1,6 @@ // Code generated by protoc-gen-go. DO NOT EDIT. // versions: -// protoc-gen-go v1.36.11 +// protoc-gen-go v1.36.12 // protoc (unknown) // source: bridge/v1/interceptor.proto @@ -21,6 +21,71 @@ const ( _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) ) +type TunnelState int32 + +const ( + TunnelState_TUNNEL_STATE_UNSPECIFIED TunnelState = 0 + // No stream holds the tunnel yet, or the last one ended and the next is + // opening. + TunnelState_TUNNEL_STATE_CONNECTING TunnelState = 1 + // A stream holds the tunnel: the deployment's conns reach the app. + TunnelState_TUNNEL_STATE_LIVE TunnelState = 2 + // The relay refused the client token, or there is none yet. Waiting for + // SetTunnelToken. + TunnelState_TUNNEL_STATE_NEEDS_TOKEN TunnelState = 3 + // Another client holds the tunnel. Retrying until it lets go. + TunnelState_TUNNEL_STATE_TAKEN TunnelState = 4 + // The relay refused the tunnel for good; last_error says why. + TunnelState_TUNNEL_STATE_FAILED TunnelState = 5 +) + +// Enum value maps for TunnelState. +var ( + TunnelState_name = map[int32]string{ + 0: "TUNNEL_STATE_UNSPECIFIED", + 1: "TUNNEL_STATE_CONNECTING", + 2: "TUNNEL_STATE_LIVE", + 3: "TUNNEL_STATE_NEEDS_TOKEN", + 4: "TUNNEL_STATE_TAKEN", + 5: "TUNNEL_STATE_FAILED", + } + TunnelState_value = map[string]int32{ + "TUNNEL_STATE_UNSPECIFIED": 0, + "TUNNEL_STATE_CONNECTING": 1, + "TUNNEL_STATE_LIVE": 2, + "TUNNEL_STATE_NEEDS_TOKEN": 3, + "TUNNEL_STATE_TAKEN": 4, + "TUNNEL_STATE_FAILED": 5, + } +) + +func (x TunnelState) Enum() *TunnelState { + p := new(TunnelState) + *p = x + return p +} + +func (x TunnelState) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (TunnelState) Descriptor() protoreflect.EnumDescriptor { + return file_bridge_v1_interceptor_proto_enumTypes[0].Descriptor() +} + +func (TunnelState) Type() protoreflect.EnumType { + return &file_bridge_v1_interceptor_proto_enumTypes[0] +} + +func (x TunnelState) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use TunnelState.Descriptor instead. +func (TunnelState) EnumDescriptor() ([]byte, []int) { + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{0} +} + type ProbeHealth int32 const ( @@ -60,11 +125,11 @@ func (x ProbeHealth) String() string { } func (ProbeHealth) Descriptor() protoreflect.EnumDescriptor { - return file_bridge_v1_interceptor_proto_enumTypes[0].Descriptor() + return file_bridge_v1_interceptor_proto_enumTypes[1].Descriptor() } func (ProbeHealth) Type() protoreflect.EnumType { - return &file_bridge_v1_interceptor_proto_enumTypes[0] + return &file_bridge_v1_interceptor_proto_enumTypes[1] } func (x ProbeHealth) Number() protoreflect.EnumNumber { @@ -73,7 +138,7 @@ func (x ProbeHealth) Number() protoreflect.EnumNumber { // Deprecated: Use ProbeHealth.Descriptor instead. func (ProbeHealth) EnumDescriptor() ([]byte, []int) { - return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{0} + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{1} } type GetStatusRequest struct { @@ -125,7 +190,10 @@ type GetStatusResponse struct { Readiness *ProbeStatus `protobuf:"bytes,3,opt,name=readiness,proto3" json:"readiness,omitempty"` // Startup probe result against the local app. Unset if no startup // probe was configured on the source. - Startup *ProbeStatus `protobuf:"bytes,4,opt,name=startup,proto3" json:"startup,omitempty"` + Startup *ProbeStatus `protobuf:"bytes,4,opt,name=startup,proto3" json:"startup,omitempty"` + // The dev tunnel's stream to its relay. Unset unless + // --tunnel-protocol=dev-tunnel. + Tunnel *TunnelStatus `protobuf:"bytes,5,opt,name=tunnel,proto3" json:"tunnel,omitempty"` unknownFields protoimpl.UnknownFields sizeCache protoimpl.SizeCache } @@ -188,6 +256,219 @@ func (x *GetStatusResponse) GetStartup() *ProbeStatus { return nil } +func (x *GetStatusResponse) GetTunnel() *TunnelStatus { + if x != nil { + return x.Tunnel + } + return nil +} + +// TunnelStatus is the state of a dev tunnel's stream to api-dev-tunnel, and +// the conns it carries. +type TunnelStatus struct { + state protoimpl.MessageState `protogen:"open.v1"` + State TunnelState `protobuf:"varint,1,opt,name=state,proto3,enum=bridge.v1.TunnelState" json:"state,omitempty"` + // Why the last stream ended, or the relay refused one. Empty before any + // has. + LastError string `protobuf:"bytes,2,opt,name=last_error,proto3" json:"last_error,omitempty"` + // Unix seconds since a stream has held the tunnel. Zero unless LIVE. + LiveSinceUnixSeconds int64 `protobuf:"varint,3,opt,name=live_since_unix_seconds,proto3" json:"live_since_unix_seconds,omitempty"` + // Streams opened after the first. + Reconnects int64 `protobuf:"varint,4,opt,name=reconnects,proto3" json:"reconnects,omitempty"` + // Conns open now from the deployment to the app (ingress), and from the + // app out through the deployment (egress). + IngressConns int64 `protobuf:"varint,5,opt,name=ingress_conns,proto3" json:"ingress_conns,omitempty"` + EgressConns int64 `protobuf:"varint,6,opt,name=egress_conns,proto3" json:"egress_conns,omitempty"` + // Conns opened since the intercept started. + IngressConnsTotal int64 `protobuf:"varint,7,opt,name=ingress_conns_total,proto3" json:"ingress_conns_total,omitempty"` + EgressConnsTotal int64 `protobuf:"varint,8,opt,name=egress_conns_total,proto3" json:"egress_conns_total,omitempty"` + // Payload bytes sent to and received from the relay. + BytesSent int64 `protobuf:"varint,9,opt,name=bytes_sent,proto3" json:"bytes_sent,omitempty"` + BytesReceived int64 `protobuf:"varint,10,opt,name=bytes_received,proto3" json:"bytes_received,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *TunnelStatus) Reset() { + *x = TunnelStatus{} + mi := &file_bridge_v1_interceptor_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *TunnelStatus) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*TunnelStatus) ProtoMessage() {} + +func (x *TunnelStatus) ProtoReflect() protoreflect.Message { + mi := &file_bridge_v1_interceptor_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use TunnelStatus.ProtoReflect.Descriptor instead. +func (*TunnelStatus) Descriptor() ([]byte, []int) { + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{2} +} + +func (x *TunnelStatus) GetState() TunnelState { + if x != nil { + return x.State + } + return TunnelState_TUNNEL_STATE_UNSPECIFIED +} + +func (x *TunnelStatus) GetLastError() string { + if x != nil { + return x.LastError + } + return "" +} + +func (x *TunnelStatus) GetLiveSinceUnixSeconds() int64 { + if x != nil { + return x.LiveSinceUnixSeconds + } + return 0 +} + +func (x *TunnelStatus) GetReconnects() int64 { + if x != nil { + return x.Reconnects + } + return 0 +} + +func (x *TunnelStatus) GetIngressConns() int64 { + if x != nil { + return x.IngressConns + } + return 0 +} + +func (x *TunnelStatus) GetEgressConns() int64 { + if x != nil { + return x.EgressConns + } + return 0 +} + +func (x *TunnelStatus) GetIngressConnsTotal() int64 { + if x != nil { + return x.IngressConnsTotal + } + return 0 +} + +func (x *TunnelStatus) GetEgressConnsTotal() int64 { + if x != nil { + return x.EgressConnsTotal + } + return 0 +} + +func (x *TunnelStatus) GetBytesSent() int64 { + if x != nil { + return x.BytesSent + } + return 0 +} + +func (x *TunnelStatus) GetBytesReceived() int64 { + if x != nil { + return x.BytesReceived + } + return 0 +} + +type SetTunnelTokenRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + Token string `protobuf:"bytes,1,opt,name=token,proto3" json:"token,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetTunnelTokenRequest) Reset() { + *x = SetTunnelTokenRequest{} + mi := &file_bridge_v1_interceptor_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetTunnelTokenRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetTunnelTokenRequest) ProtoMessage() {} + +func (x *SetTunnelTokenRequest) ProtoReflect() protoreflect.Message { + mi := &file_bridge_v1_interceptor_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetTunnelTokenRequest.ProtoReflect.Descriptor instead. +func (*SetTunnelTokenRequest) Descriptor() ([]byte, []int) { + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{3} +} + +func (x *SetTunnelTokenRequest) GetToken() string { + if x != nil { + return x.Token + } + return "" +} + +type SetTunnelTokenResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *SetTunnelTokenResponse) Reset() { + *x = SetTunnelTokenResponse{} + mi := &file_bridge_v1_interceptor_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *SetTunnelTokenResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*SetTunnelTokenResponse) ProtoMessage() {} + +func (x *SetTunnelTokenResponse) ProtoReflect() protoreflect.Message { + mi := &file_bridge_v1_interceptor_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use SetTunnelTokenResponse.ProtoReflect.Descriptor instead. +func (*SetTunnelTokenResponse) Descriptor() ([]byte, []int) { + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{4} +} + // ProbeStatus reflects the current health view from a single configured // probe — what the source deployment's probe would have reported if it had // been run against the developer's local app. @@ -212,7 +493,7 @@ type ProbeStatus struct { func (x *ProbeStatus) Reset() { *x = ProbeStatus{} - mi := &file_bridge_v1_interceptor_proto_msgTypes[2] + mi := &file_bridge_v1_interceptor_proto_msgTypes[5] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -224,7 +505,7 @@ func (x *ProbeStatus) String() string { func (*ProbeStatus) ProtoMessage() {} func (x *ProbeStatus) ProtoReflect() protoreflect.Message { - mi := &file_bridge_v1_interceptor_proto_msgTypes[2] + mi := &file_bridge_v1_interceptor_proto_msgTypes[5] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -237,7 +518,7 @@ func (x *ProbeStatus) ProtoReflect() protoreflect.Message { // Deprecated: Use ProbeStatus.ProtoReflect.Descriptor instead. func (*ProbeStatus) Descriptor() ([]byte, []int) { - return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{2} + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{5} } func (x *ProbeStatus) GetHealth() ProbeHealth { @@ -290,7 +571,7 @@ type ProbeRequest struct { func (x *ProbeRequest) Reset() { *x = ProbeRequest{} - mi := &file_bridge_v1_interceptor_proto_msgTypes[3] + mi := &file_bridge_v1_interceptor_proto_msgTypes[6] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -302,7 +583,7 @@ func (x *ProbeRequest) String() string { func (*ProbeRequest) ProtoMessage() {} func (x *ProbeRequest) ProtoReflect() protoreflect.Message { - mi := &file_bridge_v1_interceptor_proto_msgTypes[3] + mi := &file_bridge_v1_interceptor_proto_msgTypes[6] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -315,7 +596,7 @@ func (x *ProbeRequest) ProtoReflect() protoreflect.Message { // Deprecated: Use ProbeRequest.ProtoReflect.Descriptor instead. func (*ProbeRequest) Descriptor() ([]byte, []int) { - return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{3} + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{6} } // ProbeResponse reports the outcome of a synchronous run of each @@ -332,7 +613,7 @@ type ProbeResponse struct { func (x *ProbeResponse) Reset() { *x = ProbeResponse{} - mi := &file_bridge_v1_interceptor_proto_msgTypes[4] + mi := &file_bridge_v1_interceptor_proto_msgTypes[7] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -344,7 +625,7 @@ func (x *ProbeResponse) String() string { func (*ProbeResponse) ProtoMessage() {} func (x *ProbeResponse) ProtoReflect() protoreflect.Message { - mi := &file_bridge_v1_interceptor_proto_msgTypes[4] + mi := &file_bridge_v1_interceptor_proto_msgTypes[7] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -357,7 +638,7 @@ func (x *ProbeResponse) ProtoReflect() protoreflect.Message { // Deprecated: Use ProbeResponse.ProtoReflect.Descriptor instead. func (*ProbeResponse) Descriptor() ([]byte, []int) { - return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{4} + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{7} } func (x *ProbeResponse) GetLiveness() *ProbeCheckResult { @@ -397,7 +678,7 @@ type ProbeCheckResult struct { func (x *ProbeCheckResult) Reset() { *x = ProbeCheckResult{} - mi := &file_bridge_v1_interceptor_proto_msgTypes[5] + mi := &file_bridge_v1_interceptor_proto_msgTypes[8] ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) ms.StoreMessageInfo(mi) } @@ -409,7 +690,7 @@ func (x *ProbeCheckResult) String() string { func (*ProbeCheckResult) ProtoMessage() {} func (x *ProbeCheckResult) ProtoReflect() protoreflect.Message { - mi := &file_bridge_v1_interceptor_proto_msgTypes[5] + mi := &file_bridge_v1_interceptor_proto_msgTypes[8] if x != nil { ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) if ms.LoadMessageInfo() == nil { @@ -422,7 +703,7 @@ func (x *ProbeCheckResult) ProtoReflect() protoreflect.Message { // Deprecated: Use ProbeCheckResult.ProtoReflect.Descriptor instead. func (*ProbeCheckResult) Descriptor() ([]byte, []int) { - return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{5} + return file_bridge_v1_interceptor_proto_rawDescGZIP(), []int{8} } func (x *ProbeCheckResult) GetPassed() bool { @@ -451,12 +732,34 @@ var File_bridge_v1_interceptor_proto protoreflect.FileDescriptor const file_bridge_v1_interceptor_proto_rawDesc = "" + "\n" + "\x1bbridge/v1/interceptor.proto\x12\tbridge.v1\"\x12\n" + - "\x10GetStatusRequest\"\xc5\x01\n" + + "\x10GetStatusRequest\"\xf6\x01\n" + "\x11GetStatusResponse\x12\x14\n" + "\x05ready\x18\x01 \x01(\bR\x05ready\x122\n" + "\bliveness\x18\x02 \x01(\v2\x16.bridge.v1.ProbeStatusR\bliveness\x124\n" + "\treadiness\x18\x03 \x01(\v2\x16.bridge.v1.ProbeStatusR\treadiness\x120\n" + - "\astartup\x18\x04 \x01(\v2\x16.bridge.v1.ProbeStatusR\astartup\"\x9f\x02\n" + + "\astartup\x18\x04 \x01(\v2\x16.bridge.v1.ProbeStatusR\astartup\x12/\n" + + "\x06tunnel\x18\x05 \x01(\v2\x17.bridge.v1.TunnelStatusR\x06tunnel\"\xaa\x03\n" + + "\fTunnelStatus\x12,\n" + + "\x05state\x18\x01 \x01(\x0e2\x16.bridge.v1.TunnelStateR\x05state\x12\x1e\n" + + "\n" + + "last_error\x18\x02 \x01(\tR\n" + + "last_error\x128\n" + + "\x17live_since_unix_seconds\x18\x03 \x01(\x03R\x17live_since_unix_seconds\x12\x1e\n" + + "\n" + + "reconnects\x18\x04 \x01(\x03R\n" + + "reconnects\x12$\n" + + "\ringress_conns\x18\x05 \x01(\x03R\ringress_conns\x12\"\n" + + "\fegress_conns\x18\x06 \x01(\x03R\fegress_conns\x120\n" + + "\x13ingress_conns_total\x18\a \x01(\x03R\x13ingress_conns_total\x12.\n" + + "\x12egress_conns_total\x18\b \x01(\x03R\x12egress_conns_total\x12\x1e\n" + + "\n" + + "bytes_sent\x18\t \x01(\x03R\n" + + "bytes_sent\x12&\n" + + "\x0ebytes_received\x18\n" + + " \x01(\x03R\x0ebytes_received\"-\n" + + "\x15SetTunnelTokenRequest\x12\x14\n" + + "\x05token\x18\x01 \x01(\tR\x05token\"\x18\n" + + "\x16SetTunnelTokenResponse\"\x9f\x02\n" + "\vProbeStatus\x12.\n" + "\x06health\x18\x01 \x01(\x0e2\x16.bridge.v1.ProbeHealthR\x06health\x12\x18\n" + "\amessage\x18\x02 \x01(\tR\amessage\x128\n" + @@ -472,15 +775,23 @@ const file_bridge_v1_interceptor_proto_rawDesc = "" + "\x10ProbeCheckResult\x12\x16\n" + "\x06passed\x18\x01 \x01(\bR\x06passed\x12\x14\n" + "\x05error\x18\x02 \x01(\tR\x05error\x128\n" + - "\x17checked_at_unix_seconds\x18\x03 \x01(\x03R\x17checked_at_unix_seconds*{\n" + + "\x17checked_at_unix_seconds\x18\x03 \x01(\x03R\x17checked_at_unix_seconds*\xae\x01\n" + + "\vTunnelState\x12\x1c\n" + + "\x18TUNNEL_STATE_UNSPECIFIED\x10\x00\x12\x1b\n" + + "\x17TUNNEL_STATE_CONNECTING\x10\x01\x12\x15\n" + + "\x11TUNNEL_STATE_LIVE\x10\x02\x12\x1c\n" + + "\x18TUNNEL_STATE_NEEDS_TOKEN\x10\x03\x12\x16\n" + + "\x12TUNNEL_STATE_TAKEN\x10\x04\x12\x17\n" + + "\x13TUNNEL_STATE_FAILED\x10\x05*{\n" + "\vProbeHealth\x12\x1c\n" + "\x18PROBE_HEALTH_UNSPECIFIED\x10\x00\x12\x18\n" + "\x14PROBE_HEALTH_PENDING\x10\x01\x12\x18\n" + "\x14PROBE_HEALTH_HEALTHY\x10\x02\x12\x1a\n" + - "\x16PROBE_HEALTH_UNHEALTHY\x10\x032\x98\x01\n" + + "\x16PROBE_HEALTH_UNHEALTHY\x10\x032\xef\x01\n" + "\x12InterceptorService\x12F\n" + "\tGetStatus\x12\x1b.bridge.v1.GetStatusRequest\x1a\x1c.bridge.v1.GetStatusResponse\x12:\n" + - "\x05Probe\x12\x17.bridge.v1.ProbeRequest\x1a\x18.bridge.v1.ProbeResponseB\x9a\x01\n" + + "\x05Probe\x12\x17.bridge.v1.ProbeRequest\x1a\x18.bridge.v1.ProbeResponse\x12U\n" + + "\x0eSetTunnelToken\x12 .bridge.v1.SetTunnelTokenRequest\x1a!.bridge.v1.SetTunnelTokenResponseB\x9a\x01\n" + "\rcom.bridge.v1B\x10InterceptorProtoP\x01Z2github.com/vercel/bridge/api/go/bridge/v1;bridgev1\xa2\x02\x03BXX\xaa\x02\tBridge.V1\xca\x02\tBridge\\V1\xe2\x02\x15Bridge\\V1\\GPBMetadata\xea\x02\n" + "Bridge::V1b\x06proto3" @@ -496,34 +807,42 @@ func file_bridge_v1_interceptor_proto_rawDescGZIP() []byte { return file_bridge_v1_interceptor_proto_rawDescData } -var file_bridge_v1_interceptor_proto_enumTypes = make([]protoimpl.EnumInfo, 1) -var file_bridge_v1_interceptor_proto_msgTypes = make([]protoimpl.MessageInfo, 6) +var file_bridge_v1_interceptor_proto_enumTypes = make([]protoimpl.EnumInfo, 2) +var file_bridge_v1_interceptor_proto_msgTypes = make([]protoimpl.MessageInfo, 9) var file_bridge_v1_interceptor_proto_goTypes = []any{ - (ProbeHealth)(0), // 0: bridge.v1.ProbeHealth - (*GetStatusRequest)(nil), // 1: bridge.v1.GetStatusRequest - (*GetStatusResponse)(nil), // 2: bridge.v1.GetStatusResponse - (*ProbeStatus)(nil), // 3: bridge.v1.ProbeStatus - (*ProbeRequest)(nil), // 4: bridge.v1.ProbeRequest - (*ProbeResponse)(nil), // 5: bridge.v1.ProbeResponse - (*ProbeCheckResult)(nil), // 6: bridge.v1.ProbeCheckResult + (TunnelState)(0), // 0: bridge.v1.TunnelState + (ProbeHealth)(0), // 1: bridge.v1.ProbeHealth + (*GetStatusRequest)(nil), // 2: bridge.v1.GetStatusRequest + (*GetStatusResponse)(nil), // 3: bridge.v1.GetStatusResponse + (*TunnelStatus)(nil), // 4: bridge.v1.TunnelStatus + (*SetTunnelTokenRequest)(nil), // 5: bridge.v1.SetTunnelTokenRequest + (*SetTunnelTokenResponse)(nil), // 6: bridge.v1.SetTunnelTokenResponse + (*ProbeStatus)(nil), // 7: bridge.v1.ProbeStatus + (*ProbeRequest)(nil), // 8: bridge.v1.ProbeRequest + (*ProbeResponse)(nil), // 9: bridge.v1.ProbeResponse + (*ProbeCheckResult)(nil), // 10: bridge.v1.ProbeCheckResult } var file_bridge_v1_interceptor_proto_depIdxs = []int32{ - 3, // 0: bridge.v1.GetStatusResponse.liveness:type_name -> bridge.v1.ProbeStatus - 3, // 1: bridge.v1.GetStatusResponse.readiness:type_name -> bridge.v1.ProbeStatus - 3, // 2: bridge.v1.GetStatusResponse.startup:type_name -> bridge.v1.ProbeStatus - 0, // 3: bridge.v1.ProbeStatus.health:type_name -> bridge.v1.ProbeHealth - 6, // 4: bridge.v1.ProbeResponse.liveness:type_name -> bridge.v1.ProbeCheckResult - 6, // 5: bridge.v1.ProbeResponse.readiness:type_name -> bridge.v1.ProbeCheckResult - 6, // 6: bridge.v1.ProbeResponse.startup:type_name -> bridge.v1.ProbeCheckResult - 1, // 7: bridge.v1.InterceptorService.GetStatus:input_type -> bridge.v1.GetStatusRequest - 4, // 8: bridge.v1.InterceptorService.Probe:input_type -> bridge.v1.ProbeRequest - 2, // 9: bridge.v1.InterceptorService.GetStatus:output_type -> bridge.v1.GetStatusResponse - 5, // 10: bridge.v1.InterceptorService.Probe:output_type -> bridge.v1.ProbeResponse - 9, // [9:11] is the sub-list for method output_type - 7, // [7:9] is the sub-list for method input_type - 7, // [7:7] is the sub-list for extension type_name - 7, // [7:7] is the sub-list for extension extendee - 0, // [0:7] is the sub-list for field type_name + 7, // 0: bridge.v1.GetStatusResponse.liveness:type_name -> bridge.v1.ProbeStatus + 7, // 1: bridge.v1.GetStatusResponse.readiness:type_name -> bridge.v1.ProbeStatus + 7, // 2: bridge.v1.GetStatusResponse.startup:type_name -> bridge.v1.ProbeStatus + 4, // 3: bridge.v1.GetStatusResponse.tunnel:type_name -> bridge.v1.TunnelStatus + 0, // 4: bridge.v1.TunnelStatus.state:type_name -> bridge.v1.TunnelState + 1, // 5: bridge.v1.ProbeStatus.health:type_name -> bridge.v1.ProbeHealth + 10, // 6: bridge.v1.ProbeResponse.liveness:type_name -> bridge.v1.ProbeCheckResult + 10, // 7: bridge.v1.ProbeResponse.readiness:type_name -> bridge.v1.ProbeCheckResult + 10, // 8: bridge.v1.ProbeResponse.startup:type_name -> bridge.v1.ProbeCheckResult + 2, // 9: bridge.v1.InterceptorService.GetStatus:input_type -> bridge.v1.GetStatusRequest + 8, // 10: bridge.v1.InterceptorService.Probe:input_type -> bridge.v1.ProbeRequest + 5, // 11: bridge.v1.InterceptorService.SetTunnelToken:input_type -> bridge.v1.SetTunnelTokenRequest + 3, // 12: bridge.v1.InterceptorService.GetStatus:output_type -> bridge.v1.GetStatusResponse + 9, // 13: bridge.v1.InterceptorService.Probe:output_type -> bridge.v1.ProbeResponse + 6, // 14: bridge.v1.InterceptorService.SetTunnelToken:output_type -> bridge.v1.SetTunnelTokenResponse + 12, // [12:15] is the sub-list for method output_type + 9, // [9:12] is the sub-list for method input_type + 9, // [9:9] is the sub-list for extension type_name + 9, // [9:9] is the sub-list for extension extendee + 0, // [0:9] is the sub-list for field type_name } func init() { file_bridge_v1_interceptor_proto_init() } @@ -536,8 +855,8 @@ func file_bridge_v1_interceptor_proto_init() { File: protoimpl.DescBuilder{ GoPackagePath: reflect.TypeOf(x{}).PkgPath(), RawDescriptor: unsafe.Slice(unsafe.StringData(file_bridge_v1_interceptor_proto_rawDesc), len(file_bridge_v1_interceptor_proto_rawDesc)), - NumEnums: 1, - NumMessages: 6, + NumEnums: 2, + NumMessages: 9, NumExtensions: 0, NumServices: 1, }, diff --git a/api/go/bridge/v1/interceptor_grpc.pb.go b/api/go/bridge/v1/interceptor_grpc.pb.go index 6e6869e..d949772 100644 --- a/api/go/bridge/v1/interceptor_grpc.pb.go +++ b/api/go/bridge/v1/interceptor_grpc.pb.go @@ -19,8 +19,9 @@ import ( const _ = grpc.SupportPackageIsVersion9 const ( - InterceptorService_GetStatus_FullMethodName = "/bridge.v1.InterceptorService/GetStatus" - InterceptorService_Probe_FullMethodName = "/bridge.v1.InterceptorService/Probe" + InterceptorService_GetStatus_FullMethodName = "/bridge.v1.InterceptorService/GetStatus" + InterceptorService_Probe_FullMethodName = "/bridge.v1.InterceptorService/Probe" + InterceptorService_SetTunnelToken_FullMethodName = "/bridge.v1.InterceptorService/SetTunnelToken" ) // InterceptorServiceClient is the client API for InterceptorService service. @@ -43,6 +44,11 @@ type InterceptorServiceClient interface { // immediate check passed. Useful as a "is the app healthy right now?" // primitive that ignores any stale state from a previous dev session. Probe(ctx context.Context, in *ProbeRequest, opts ...grpc.CallOption) (*ProbeResponse, error) + // SetTunnelToken gives an intercept on a dev tunnel + // (--tunnel-protocol=dev-tunnel) its client token, from create-session. One + // started without a token, or whose token the relay refused + // (TUNNEL_STATE_NEEDS_TOKEN), connects with it. + SetTunnelToken(ctx context.Context, in *SetTunnelTokenRequest, opts ...grpc.CallOption) (*SetTunnelTokenResponse, error) } type interceptorServiceClient struct { @@ -73,6 +79,16 @@ func (c *interceptorServiceClient) Probe(ctx context.Context, in *ProbeRequest, return out, nil } +func (c *interceptorServiceClient) SetTunnelToken(ctx context.Context, in *SetTunnelTokenRequest, opts ...grpc.CallOption) (*SetTunnelTokenResponse, error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + out := new(SetTunnelTokenResponse) + err := c.cc.Invoke(ctx, InterceptorService_SetTunnelToken_FullMethodName, in, out, cOpts...) + if err != nil { + return nil, err + } + return out, nil +} + // InterceptorServiceServer is the server API for InterceptorService service. // All implementations must embed UnimplementedInterceptorServiceServer // for forward compatibility. @@ -93,6 +109,11 @@ type InterceptorServiceServer interface { // immediate check passed. Useful as a "is the app healthy right now?" // primitive that ignores any stale state from a previous dev session. Probe(context.Context, *ProbeRequest) (*ProbeResponse, error) + // SetTunnelToken gives an intercept on a dev tunnel + // (--tunnel-protocol=dev-tunnel) its client token, from create-session. One + // started without a token, or whose token the relay refused + // (TUNNEL_STATE_NEEDS_TOKEN), connects with it. + SetTunnelToken(context.Context, *SetTunnelTokenRequest) (*SetTunnelTokenResponse, error) mustEmbedUnimplementedInterceptorServiceServer() } @@ -109,6 +130,9 @@ func (UnimplementedInterceptorServiceServer) GetStatus(context.Context, *GetStat func (UnimplementedInterceptorServiceServer) Probe(context.Context, *ProbeRequest) (*ProbeResponse, error) { return nil, status.Error(codes.Unimplemented, "method Probe not implemented") } +func (UnimplementedInterceptorServiceServer) SetTunnelToken(context.Context, *SetTunnelTokenRequest) (*SetTunnelTokenResponse, error) { + return nil, status.Error(codes.Unimplemented, "method SetTunnelToken not implemented") +} func (UnimplementedInterceptorServiceServer) mustEmbedUnimplementedInterceptorServiceServer() {} func (UnimplementedInterceptorServiceServer) testEmbeddedByValue() {} @@ -166,6 +190,24 @@ func _InterceptorService_Probe_Handler(srv interface{}, ctx context.Context, dec return interceptor(ctx, in, info, handler) } +func _InterceptorService_SetTunnelToken_Handler(srv interface{}, ctx context.Context, dec func(interface{}) error, interceptor grpc.UnaryServerInterceptor) (interface{}, error) { + in := new(SetTunnelTokenRequest) + if err := dec(in); err != nil { + return nil, err + } + if interceptor == nil { + return srv.(InterceptorServiceServer).SetTunnelToken(ctx, in) + } + info := &grpc.UnaryServerInfo{ + Server: srv, + FullMethod: InterceptorService_SetTunnelToken_FullMethodName, + } + handler := func(ctx context.Context, req interface{}) (interface{}, error) { + return srv.(InterceptorServiceServer).SetTunnelToken(ctx, req.(*SetTunnelTokenRequest)) + } + return interceptor(ctx, in, info, handler) +} + // InterceptorService_ServiceDesc is the grpc.ServiceDesc for InterceptorService service. // It's only intended for direct use with grpc.RegisterService, // and not to be introspected or modified (even as a copy) @@ -181,6 +223,10 @@ var InterceptorService_ServiceDesc = grpc.ServiceDesc{ MethodName: "Probe", Handler: _InterceptorService_Probe_Handler, }, + { + MethodName: "SetTunnelToken", + Handler: _InterceptorService_SetTunnelToken_Handler, + }, }, Streams: []grpc.StreamDesc{}, Metadata: "bridge/v1/interceptor.proto", diff --git a/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go b/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go new file mode 100644 index 0000000..16c6f12 --- /dev/null +++ b/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go @@ -0,0 +1,1165 @@ +// Vendored from vercel/api, services/api-dev-tunnel/proto/tunnel/v1/tunnel.proto. +// Edit it there and copy it here. + +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.36.12 +// protoc (unknown) +// source: vercel/api/dev_tunnel/v1/tunnel.proto + +package dev_tunnelv1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" + unsafe "unsafe" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type Protocol int32 + +const ( + Protocol_PROTOCOL_UNSPECIFIED Protocol = 0 + Protocol_PROTOCOL_TCP Protocol = 1 +) + +// Enum value maps for Protocol. +var ( + Protocol_name = map[int32]string{ + 0: "PROTOCOL_UNSPECIFIED", + 1: "PROTOCOL_TCP", + } + Protocol_value = map[string]int32{ + "PROTOCOL_UNSPECIFIED": 0, + "PROTOCOL_TCP": 1, + } +) + +func (x Protocol) Enum() *Protocol { + p := new(Protocol) + *p = x + return p +} + +func (x Protocol) String() string { + return protoimpl.X.EnumStringOf(x.Descriptor(), protoreflect.EnumNumber(x)) +} + +func (Protocol) Descriptor() protoreflect.EnumDescriptor { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_enumTypes[0].Descriptor() +} + +func (Protocol) Type() protoreflect.EnumType { + return &file_vercel_api_dev_tunnel_v1_tunnel_proto_enumTypes[0] +} + +func (x Protocol) Number() protoreflect.EnumNumber { + return protoreflect.EnumNumber(x) +} + +// Deprecated: Use Protocol.Descriptor instead. +func (Protocol) EnumDescriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{0} +} + +// CLI to service. +type ConnectRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Frame: + // + // *ConnectRequest_Data + // *ConnectRequest_Close + // *ConnectRequest_Reset_ + // *ConnectRequest_Heartbeat + // *ConnectRequest_Window + // *ConnectRequest_Open + Frame isConnectRequest_Frame `protobuf_oneof:"frame"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConnectRequest) Reset() { + *x = ConnectRequest{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConnectRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectRequest) ProtoMessage() {} + +func (x *ConnectRequest) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[0] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectRequest.ProtoReflect.Descriptor instead. +func (*ConnectRequest) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{0} +} + +func (x *ConnectRequest) GetFrame() isConnectRequest_Frame { + if x != nil { + return x.Frame + } + return nil +} + +func (x *ConnectRequest) GetData() *Data { + if x != nil { + if x, ok := x.Frame.(*ConnectRequest_Data); ok { + return x.Data + } + } + return nil +} + +func (x *ConnectRequest) GetClose() *Close { + if x != nil { + if x, ok := x.Frame.(*ConnectRequest_Close); ok { + return x.Close + } + } + return nil +} + +func (x *ConnectRequest) GetReset_() *Reset { + if x != nil { + if x, ok := x.Frame.(*ConnectRequest_Reset_); ok { + return x.Reset_ + } + } + return nil +} + +func (x *ConnectRequest) GetHeartbeat() *Heartbeat { + if x != nil { + if x, ok := x.Frame.(*ConnectRequest_Heartbeat); ok { + return x.Heartbeat + } + } + return nil +} + +func (x *ConnectRequest) GetWindow() *Window { + if x != nil { + if x, ok := x.Frame.(*ConnectRequest_Window); ok { + return x.Window + } + } + return nil +} + +func (x *ConnectRequest) GetOpen() *Open { + if x != nil { + if x, ok := x.Frame.(*ConnectRequest_Open); ok { + return x.Open + } + } + return nil +} + +type isConnectRequest_Frame interface { + isConnectRequest_Frame() +} + +type ConnectRequest_Data struct { + Data *Data `protobuf:"bytes,1,opt,name=data,proto3,oneof"` +} + +type ConnectRequest_Close struct { + Close *Close `protobuf:"bytes,2,opt,name=close,proto3,oneof"` +} + +type ConnectRequest_Reset_ struct { + Reset_ *Reset `protobuf:"bytes,3,opt,name=reset,proto3,oneof"` +} + +type ConnectRequest_Heartbeat struct { + Heartbeat *Heartbeat `protobuf:"bytes,4,opt,name=heartbeat,proto3,oneof"` +} + +type ConnectRequest_Window struct { + Window *Window `protobuf:"bytes,5,opt,name=window,proto3,oneof"` +} + +type ConnectRequest_Open struct { + Open *Open `protobuf:"bytes,6,opt,name=open,proto3,oneof"` +} + +func (*ConnectRequest_Data) isConnectRequest_Frame() {} + +func (*ConnectRequest_Close) isConnectRequest_Frame() {} + +func (*ConnectRequest_Reset_) isConnectRequest_Frame() {} + +func (*ConnectRequest_Heartbeat) isConnectRequest_Frame() {} + +func (*ConnectRequest_Window) isConnectRequest_Frame() {} + +func (*ConnectRequest_Open) isConnectRequest_Frame() {} + +// Service to CLI. +type ConnectResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Frame: + // + // *ConnectResponse_Open + // *ConnectResponse_Data + // *ConnectResponse_Close + // *ConnectResponse_Reset_ + // *ConnectResponse_Heartbeat + // *ConnectResponse_Window + // *ConnectResponse_Reconnect + Frame isConnectResponse_Frame `protobuf_oneof:"frame"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *ConnectResponse) Reset() { + *x = ConnectResponse{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *ConnectResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ConnectResponse) ProtoMessage() {} + +func (x *ConnectResponse) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[1] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ConnectResponse.ProtoReflect.Descriptor instead. +func (*ConnectResponse) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{1} +} + +func (x *ConnectResponse) GetFrame() isConnectResponse_Frame { + if x != nil { + return x.Frame + } + return nil +} + +func (x *ConnectResponse) GetOpen() *Open { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Open); ok { + return x.Open + } + } + return nil +} + +func (x *ConnectResponse) GetData() *Data { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Data); ok { + return x.Data + } + } + return nil +} + +func (x *ConnectResponse) GetClose() *Close { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Close); ok { + return x.Close + } + } + return nil +} + +func (x *ConnectResponse) GetReset_() *Reset { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Reset_); ok { + return x.Reset_ + } + } + return nil +} + +func (x *ConnectResponse) GetHeartbeat() *Heartbeat { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Heartbeat); ok { + return x.Heartbeat + } + } + return nil +} + +func (x *ConnectResponse) GetWindow() *Window { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Window); ok { + return x.Window + } + } + return nil +} + +func (x *ConnectResponse) GetReconnect() *Reconnect { + if x != nil { + if x, ok := x.Frame.(*ConnectResponse_Reconnect); ok { + return x.Reconnect + } + } + return nil +} + +type isConnectResponse_Frame interface { + isConnectResponse_Frame() +} + +type ConnectResponse_Open struct { + Open *Open `protobuf:"bytes,1,opt,name=open,proto3,oneof"` +} + +type ConnectResponse_Data struct { + Data *Data `protobuf:"bytes,2,opt,name=data,proto3,oneof"` +} + +type ConnectResponse_Close struct { + Close *Close `protobuf:"bytes,3,opt,name=close,proto3,oneof"` +} + +type ConnectResponse_Reset_ struct { + Reset_ *Reset `protobuf:"bytes,4,opt,name=reset,proto3,oneof"` +} + +type ConnectResponse_Heartbeat struct { + Heartbeat *Heartbeat `protobuf:"bytes,5,opt,name=heartbeat,proto3,oneof"` +} + +type ConnectResponse_Window struct { + Window *Window `protobuf:"bytes,6,opt,name=window,proto3,oneof"` +} + +type ConnectResponse_Reconnect struct { + Reconnect *Reconnect `protobuf:"bytes,7,opt,name=reconnect,proto3,oneof"` +} + +func (*ConnectResponse_Open) isConnectResponse_Frame() {} + +func (*ConnectResponse_Data) isConnectResponse_Frame() {} + +func (*ConnectResponse_Close) isConnectResponse_Frame() {} + +func (*ConnectResponse_Reset_) isConnectResponse_Frame() {} + +func (*ConnectResponse_Heartbeat) isConnectResponse_Frame() {} + +func (*ConnectResponse_Window) isConnectResponse_Frame() {} + +func (*ConnectResponse_Reconnect) isConnectResponse_Frame() {} + +// A new conn. From the service, the CLI connects to `port` on the developer's +// machine, or resets the conn if it doesn't serve that port. From the client, +// the deployment connects to `host` and `port`, resolving `host` as its own +// code would, and the service resets the conn if it can't. The service numbers +// its conns from 1; a client numbers its own from 2^63, so the two never meet. +type Open struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConnId uint64 `protobuf:"varint,1,opt,name=conn_id,json=connId,proto3" json:"conn_id,omitempty"` + Protocol Protocol `protobuf:"varint,2,opt,name=protocol,proto3,enum=vercel.api.dev_tunnel.v1.Protocol" json:"protocol,omitempty"` + Port uint32 `protobuf:"varint,3,opt,name=port,proto3" json:"port,omitempty"` + // A hostname or IP address. Only a client's Open has one. + Host string `protobuf:"bytes,4,opt,name=host,proto3" json:"host,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Open) Reset() { + *x = Open{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[2] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Open) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Open) ProtoMessage() {} + +func (x *Open) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[2] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Open.ProtoReflect.Descriptor instead. +func (*Open) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{2} +} + +func (x *Open) GetConnId() uint64 { + if x != nil { + return x.ConnId + } + return 0 +} + +func (x *Open) GetProtocol() Protocol { + if x != nil { + return x.Protocol + } + return Protocol_PROTOCOL_UNSPECIFIED +} + +func (x *Open) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +func (x *Open) GetHost() string { + if x != nil { + return x.Host + } + return "" +} + +// No more than the conn's window allows (see Window). +type Data struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConnId uint64 `protobuf:"varint,1,opt,name=conn_id,json=connId,proto3" json:"conn_id,omitempty"` + Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3" json:"payload,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Data) Reset() { + *x = Data{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[3] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Data) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Data) ProtoMessage() {} + +func (x *Data) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[3] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Data.ProtoReflect.Descriptor instead. +func (*Data) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{3} +} + +func (x *Data) GetConnId() uint64 { + if x != nil { + return x.ConnId + } + return 0 +} + +func (x *Data) GetPayload() []byte { + if x != nil { + return x.Payload + } + return nil +} + +// The sender has nothing more to send (TCP half-close). A conn is over once +// both sides have closed, or either has reset. +type Close struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConnId uint64 `protobuf:"varint,1,opt,name=conn_id,json=connId,proto3" json:"conn_id,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Close) Reset() { + *x = Close{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[4] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Close) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Close) ProtoMessage() {} + +func (x *Close) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[4] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Close.ProtoReflect.Descriptor instead. +func (*Close) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{4} +} + +func (x *Close) GetConnId() uint64 { + if x != nil { + return x.ConnId + } + return 0 +} + +// Abandons the conn in both directions, e.g. connection refused. +type Reset struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConnId uint64 `protobuf:"varint,1,opt,name=conn_id,json=connId,proto3" json:"conn_id,omitempty"` + Reason string `protobuf:"bytes,2,opt,name=reason,proto3" json:"reason,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Reset) Reset() { + *x = Reset{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[5] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Reset) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Reset) ProtoMessage() {} + +func (x *Reset) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[5] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Reset.ProtoReflect.Descriptor instead. +func (*Reset) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{5} +} + +func (x *Reset) GetConnId() uint64 { + if x != nil { + return x.ConnId + } + return 0 +} + +func (x *Reset) GetReason() string { + if x != nil { + return x.Reason + } + return "" +} + +// Lets the peer send `bytes` more of a conn's Data, as the sender of this +// consumes what it received. Each side starts each conn able to send 256 KiB. +// A client that sends past its window, or grants back more than it was sent, +// has the conn reset. +type Window struct { + state protoimpl.MessageState `protogen:"open.v1"` + ConnId uint64 `protobuf:"varint,1,opt,name=conn_id,json=connId,proto3" json:"conn_id,omitempty"` + Bytes uint32 `protobuf:"varint,2,opt,name=bytes,proto3" json:"bytes,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Window) Reset() { + *x = Window{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[6] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Window) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Window) ProtoMessage() {} + +func (x *Window) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[6] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Window.ProtoReflect.Descriptor instead. +func (*Window) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{6} +} + +func (x *Window) GetConnId() uint64 { + if x != nil { + return x.ConnId + } + return 0 +} + +func (x *Window) GetBytes() uint32 { + if x != nil { + return x.Bytes + } + return 0 +} + +// Sent both ways every 30s, inside the ALB's idle timeout. The service drops a +// client it hasn't heard from in three. The service also sends one as soon as +// a Connect stream holds the tunnel, so the CLI knows it's live. +type Heartbeat struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Heartbeat) Reset() { + *x = Heartbeat{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[7] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Heartbeat) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Heartbeat) ProtoMessage() {} + +func (x *Heartbeat) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[7] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Heartbeat.ProtoReflect.Descriptor instead. +func (*Heartbeat) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{7} +} + +// This pod is shutting down. The CLI opens a new stream now, and keeps this one +// until the service ends it: it still carries the conns already open on it. +type Reconnect struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *Reconnect) Reset() { + *x = Reconnect{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[8] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *Reconnect) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*Reconnect) ProtoMessage() {} + +func (x *Reconnect) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[8] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use Reconnect.ProtoReflect.Descriptor instead. +func (*Reconnect) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{8} +} + +type DialRequest struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Frame: + // + // *DialRequest_Target + // *DialRequest_Payload + Frame isDialRequest_Frame `protobuf_oneof:"frame"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DialRequest) Reset() { + *x = DialRequest{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[9] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DialRequest) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DialRequest) ProtoMessage() {} + +func (x *DialRequest) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[9] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DialRequest.ProtoReflect.Descriptor instead. +func (*DialRequest) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{9} +} + +func (x *DialRequest) GetFrame() isDialRequest_Frame { + if x != nil { + return x.Frame + } + return nil +} + +func (x *DialRequest) GetTarget() *DialTarget { + if x != nil { + if x, ok := x.Frame.(*DialRequest_Target); ok { + return x.Target + } + } + return nil +} + +func (x *DialRequest) GetPayload() []byte { + if x != nil { + if x, ok := x.Frame.(*DialRequest_Payload); ok { + return x.Payload + } + } + return nil +} + +type isDialRequest_Frame interface { + isDialRequest_Frame() +} + +type DialRequest_Target struct { + // First message only. + Target *DialTarget `protobuf:"bytes,1,opt,name=target,proto3,oneof"` +} + +type DialRequest_Payload struct { + Payload []byte `protobuf:"bytes,2,opt,name=payload,proto3,oneof"` +} + +func (*DialRequest_Target) isDialRequest_Frame() {} + +func (*DialRequest_Payload) isDialRequest_Frame() {} + +type DialTarget struct { + state protoimpl.MessageState `protogen:"open.v1"` + Protocol Protocol `protobuf:"varint,1,opt,name=protocol,proto3,enum=vercel.api.dev_tunnel.v1.Protocol" json:"protocol,omitempty"` + Port uint32 `protobuf:"varint,2,opt,name=port,proto3" json:"port,omitempty"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DialTarget) Reset() { + *x = DialTarget{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[10] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DialTarget) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DialTarget) ProtoMessage() {} + +func (x *DialTarget) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[10] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DialTarget.ProtoReflect.Descriptor instead. +func (*DialTarget) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{10} +} + +func (x *DialTarget) GetProtocol() Protocol { + if x != nil { + return x.Protocol + } + return Protocol_PROTOCOL_UNSPECIFIED +} + +func (x *DialTarget) GetPort() uint32 { + if x != nil { + return x.Port + } + return 0 +} + +type DialResponse struct { + state protoimpl.MessageState `protogen:"open.v1"` + // Types that are valid to be assigned to Frame: + // + // *DialResponse_Payload + // *DialResponse_End + Frame isDialResponse_Frame `protobuf_oneof:"frame"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DialResponse) Reset() { + *x = DialResponse{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[11] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DialResponse) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DialResponse) ProtoMessage() {} + +func (x *DialResponse) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[11] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DialResponse.ProtoReflect.Descriptor instead. +func (*DialResponse) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{11} +} + +func (x *DialResponse) GetFrame() isDialResponse_Frame { + if x != nil { + return x.Frame + } + return nil +} + +func (x *DialResponse) GetPayload() []byte { + if x != nil { + if x, ok := x.Frame.(*DialResponse_Payload); ok { + return x.Payload + } + } + return nil +} + +func (x *DialResponse) GetEnd() *DialEnd { + if x != nil { + if x, ok := x.Frame.(*DialResponse_End); ok { + return x.End + } + } + return nil +} + +type isDialResponse_Frame interface { + isDialResponse_Frame() +} + +type DialResponse_Payload struct { + Payload []byte `protobuf:"bytes,1,opt,name=payload,proto3,oneof"` +} + +type DialResponse_End struct { + // The developer's side has nothing more to send. Sent once; more payload + // may still go the other way. + End *DialEnd `protobuf:"bytes,2,opt,name=end,proto3,oneof"` +} + +func (*DialResponse_Payload) isDialResponse_Frame() {} + +func (*DialResponse_End) isDialResponse_Frame() {} + +type DialEnd struct { + state protoimpl.MessageState `protogen:"open.v1"` + unknownFields protoimpl.UnknownFields + sizeCache protoimpl.SizeCache +} + +func (x *DialEnd) Reset() { + *x = DialEnd{} + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[12] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) +} + +func (x *DialEnd) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*DialEnd) ProtoMessage() {} + +func (x *DialEnd) ProtoReflect() protoreflect.Message { + mi := &file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[12] + if x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use DialEnd.ProtoReflect.Descriptor instead. +func (*DialEnd) Descriptor() ([]byte, []int) { + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP(), []int{12} +} + +var File_vercel_api_dev_tunnel_v1_tunnel_proto protoreflect.FileDescriptor + +const file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDesc = "" + + "\n" + + "%vercel/api/dev_tunnel/v1/tunnel.proto\x12\x18vercel.api.dev_tunnel.v1\"\xf8\x02\n" + + "\x0eConnectRequest\x124\n" + + "\x04data\x18\x01 \x01(\v2\x1e.vercel.api.dev_tunnel.v1.DataH\x00R\x04data\x127\n" + + "\x05close\x18\x02 \x01(\v2\x1f.vercel.api.dev_tunnel.v1.CloseH\x00R\x05close\x127\n" + + "\x05reset\x18\x03 \x01(\v2\x1f.vercel.api.dev_tunnel.v1.ResetH\x00R\x05reset\x12C\n" + + "\theartbeat\x18\x04 \x01(\v2#.vercel.api.dev_tunnel.v1.HeartbeatH\x00R\theartbeat\x12:\n" + + "\x06window\x18\x05 \x01(\v2 .vercel.api.dev_tunnel.v1.WindowH\x00R\x06window\x124\n" + + "\x04open\x18\x06 \x01(\v2\x1e.vercel.api.dev_tunnel.v1.OpenH\x00R\x04openB\a\n" + + "\x05frame\"\xbe\x03\n" + + "\x0fConnectResponse\x124\n" + + "\x04open\x18\x01 \x01(\v2\x1e.vercel.api.dev_tunnel.v1.OpenH\x00R\x04open\x124\n" + + "\x04data\x18\x02 \x01(\v2\x1e.vercel.api.dev_tunnel.v1.DataH\x00R\x04data\x127\n" + + "\x05close\x18\x03 \x01(\v2\x1f.vercel.api.dev_tunnel.v1.CloseH\x00R\x05close\x127\n" + + "\x05reset\x18\x04 \x01(\v2\x1f.vercel.api.dev_tunnel.v1.ResetH\x00R\x05reset\x12C\n" + + "\theartbeat\x18\x05 \x01(\v2#.vercel.api.dev_tunnel.v1.HeartbeatH\x00R\theartbeat\x12:\n" + + "\x06window\x18\x06 \x01(\v2 .vercel.api.dev_tunnel.v1.WindowH\x00R\x06window\x12C\n" + + "\treconnect\x18\a \x01(\v2#.vercel.api.dev_tunnel.v1.ReconnectH\x00R\treconnectB\a\n" + + "\x05frame\"\x87\x01\n" + + "\x04Open\x12\x17\n" + + "\aconn_id\x18\x01 \x01(\x04R\x06connId\x12>\n" + + "\bprotocol\x18\x02 \x01(\x0e2\".vercel.api.dev_tunnel.v1.ProtocolR\bprotocol\x12\x12\n" + + "\x04port\x18\x03 \x01(\rR\x04port\x12\x12\n" + + "\x04host\x18\x04 \x01(\tR\x04host\"9\n" + + "\x04Data\x12\x17\n" + + "\aconn_id\x18\x01 \x01(\x04R\x06connId\x12\x18\n" + + "\apayload\x18\x02 \x01(\fR\apayload\" \n" + + "\x05Close\x12\x17\n" + + "\aconn_id\x18\x01 \x01(\x04R\x06connId\"8\n" + + "\x05Reset\x12\x17\n" + + "\aconn_id\x18\x01 \x01(\x04R\x06connId\x12\x16\n" + + "\x06reason\x18\x02 \x01(\tR\x06reason\"7\n" + + "\x06Window\x12\x17\n" + + "\aconn_id\x18\x01 \x01(\x04R\x06connId\x12\x14\n" + + "\x05bytes\x18\x02 \x01(\rR\x05bytes\"\v\n" + + "\tHeartbeat\"\v\n" + + "\tReconnect\"r\n" + + "\vDialRequest\x12>\n" + + "\x06target\x18\x01 \x01(\v2$.vercel.api.dev_tunnel.v1.DialTargetH\x00R\x06target\x12\x1a\n" + + "\apayload\x18\x02 \x01(\fH\x00R\apayloadB\a\n" + + "\x05frame\"`\n" + + "\n" + + "DialTarget\x12>\n" + + "\bprotocol\x18\x01 \x01(\x0e2\".vercel.api.dev_tunnel.v1.ProtocolR\bprotocol\x12\x12\n" + + "\x04port\x18\x02 \x01(\rR\x04port\"j\n" + + "\fDialResponse\x12\x1a\n" + + "\apayload\x18\x01 \x01(\fH\x00R\apayload\x125\n" + + "\x03end\x18\x02 \x01(\v2!.vercel.api.dev_tunnel.v1.DialEndH\x00R\x03endB\a\n" + + "\x05frame\"\t\n" + + "\aDialEnd*6\n" + + "\bProtocol\x12\x18\n" + + "\x14PROTOCOL_UNSPECIFIED\x10\x00\x12\x10\n" + + "\fPROTOCOL_TCP\x10\x012\xce\x01\n" + + "\rTunnelService\x12b\n" + + "\aConnect\x12(.vercel.api.dev_tunnel.v1.ConnectRequest\x1a).vercel.api.dev_tunnel.v1.ConnectResponse(\x010\x01\x12Y\n" + + "\x04Dial\x12%.vercel.api.dev_tunnel.v1.DialRequest\x1a&.vercel.api.dev_tunnel.v1.DialResponse(\x010\x01B\xf1\x01\n" + + "\x1ccom.vercel.api.dev_tunnel.v1B\vTunnelProtoP\x01ZEgithub.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1;dev_tunnelv1\xa2\x02\x03VAD\xaa\x02\x17Vercel.Api.DevTunnel.V1\xca\x02\x17Vercel\\Api\\DevTunnel\\V1\xe2\x02#Vercel\\Api\\DevTunnel\\V1\\GPBMetadata\xea\x02\x1aVercel::Api::DevTunnel::V1b\x06proto3" + +var ( + file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescOnce sync.Once + file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescData []byte +) + +func file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescGZIP() []byte { + file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescOnce.Do(func() { + file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescData = protoimpl.X.CompressGZIP(unsafe.Slice(unsafe.StringData(file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDesc), len(file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDesc))) + }) + return file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescData +} + +var file_vercel_api_dev_tunnel_v1_tunnel_proto_enumTypes = make([]protoimpl.EnumInfo, 1) +var file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes = make([]protoimpl.MessageInfo, 13) +var file_vercel_api_dev_tunnel_v1_tunnel_proto_goTypes = []any{ + (Protocol)(0), // 0: vercel.api.dev_tunnel.v1.Protocol + (*ConnectRequest)(nil), // 1: vercel.api.dev_tunnel.v1.ConnectRequest + (*ConnectResponse)(nil), // 2: vercel.api.dev_tunnel.v1.ConnectResponse + (*Open)(nil), // 3: vercel.api.dev_tunnel.v1.Open + (*Data)(nil), // 4: vercel.api.dev_tunnel.v1.Data + (*Close)(nil), // 5: vercel.api.dev_tunnel.v1.Close + (*Reset)(nil), // 6: vercel.api.dev_tunnel.v1.Reset + (*Window)(nil), // 7: vercel.api.dev_tunnel.v1.Window + (*Heartbeat)(nil), // 8: vercel.api.dev_tunnel.v1.Heartbeat + (*Reconnect)(nil), // 9: vercel.api.dev_tunnel.v1.Reconnect + (*DialRequest)(nil), // 10: vercel.api.dev_tunnel.v1.DialRequest + (*DialTarget)(nil), // 11: vercel.api.dev_tunnel.v1.DialTarget + (*DialResponse)(nil), // 12: vercel.api.dev_tunnel.v1.DialResponse + (*DialEnd)(nil), // 13: vercel.api.dev_tunnel.v1.DialEnd +} +var file_vercel_api_dev_tunnel_v1_tunnel_proto_depIdxs = []int32{ + 4, // 0: vercel.api.dev_tunnel.v1.ConnectRequest.data:type_name -> vercel.api.dev_tunnel.v1.Data + 5, // 1: vercel.api.dev_tunnel.v1.ConnectRequest.close:type_name -> vercel.api.dev_tunnel.v1.Close + 6, // 2: vercel.api.dev_tunnel.v1.ConnectRequest.reset:type_name -> vercel.api.dev_tunnel.v1.Reset + 8, // 3: vercel.api.dev_tunnel.v1.ConnectRequest.heartbeat:type_name -> vercel.api.dev_tunnel.v1.Heartbeat + 7, // 4: vercel.api.dev_tunnel.v1.ConnectRequest.window:type_name -> vercel.api.dev_tunnel.v1.Window + 3, // 5: vercel.api.dev_tunnel.v1.ConnectRequest.open:type_name -> vercel.api.dev_tunnel.v1.Open + 3, // 6: vercel.api.dev_tunnel.v1.ConnectResponse.open:type_name -> vercel.api.dev_tunnel.v1.Open + 4, // 7: vercel.api.dev_tunnel.v1.ConnectResponse.data:type_name -> vercel.api.dev_tunnel.v1.Data + 5, // 8: vercel.api.dev_tunnel.v1.ConnectResponse.close:type_name -> vercel.api.dev_tunnel.v1.Close + 6, // 9: vercel.api.dev_tunnel.v1.ConnectResponse.reset:type_name -> vercel.api.dev_tunnel.v1.Reset + 8, // 10: vercel.api.dev_tunnel.v1.ConnectResponse.heartbeat:type_name -> vercel.api.dev_tunnel.v1.Heartbeat + 7, // 11: vercel.api.dev_tunnel.v1.ConnectResponse.window:type_name -> vercel.api.dev_tunnel.v1.Window + 9, // 12: vercel.api.dev_tunnel.v1.ConnectResponse.reconnect:type_name -> vercel.api.dev_tunnel.v1.Reconnect + 0, // 13: vercel.api.dev_tunnel.v1.Open.protocol:type_name -> vercel.api.dev_tunnel.v1.Protocol + 11, // 14: vercel.api.dev_tunnel.v1.DialRequest.target:type_name -> vercel.api.dev_tunnel.v1.DialTarget + 0, // 15: vercel.api.dev_tunnel.v1.DialTarget.protocol:type_name -> vercel.api.dev_tunnel.v1.Protocol + 13, // 16: vercel.api.dev_tunnel.v1.DialResponse.end:type_name -> vercel.api.dev_tunnel.v1.DialEnd + 1, // 17: vercel.api.dev_tunnel.v1.TunnelService.Connect:input_type -> vercel.api.dev_tunnel.v1.ConnectRequest + 10, // 18: vercel.api.dev_tunnel.v1.TunnelService.Dial:input_type -> vercel.api.dev_tunnel.v1.DialRequest + 2, // 19: vercel.api.dev_tunnel.v1.TunnelService.Connect:output_type -> vercel.api.dev_tunnel.v1.ConnectResponse + 12, // 20: vercel.api.dev_tunnel.v1.TunnelService.Dial:output_type -> vercel.api.dev_tunnel.v1.DialResponse + 19, // [19:21] is the sub-list for method output_type + 17, // [17:19] is the sub-list for method input_type + 17, // [17:17] is the sub-list for extension type_name + 17, // [17:17] is the sub-list for extension extendee + 0, // [0:17] is the sub-list for field type_name +} + +func init() { file_vercel_api_dev_tunnel_v1_tunnel_proto_init() } +func file_vercel_api_dev_tunnel_v1_tunnel_proto_init() { + if File_vercel_api_dev_tunnel_v1_tunnel_proto != nil { + return + } + file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[0].OneofWrappers = []any{ + (*ConnectRequest_Data)(nil), + (*ConnectRequest_Close)(nil), + (*ConnectRequest_Reset_)(nil), + (*ConnectRequest_Heartbeat)(nil), + (*ConnectRequest_Window)(nil), + (*ConnectRequest_Open)(nil), + } + file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[1].OneofWrappers = []any{ + (*ConnectResponse_Open)(nil), + (*ConnectResponse_Data)(nil), + (*ConnectResponse_Close)(nil), + (*ConnectResponse_Reset_)(nil), + (*ConnectResponse_Heartbeat)(nil), + (*ConnectResponse_Window)(nil), + (*ConnectResponse_Reconnect)(nil), + } + file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[9].OneofWrappers = []any{ + (*DialRequest_Target)(nil), + (*DialRequest_Payload)(nil), + } + file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes[11].OneofWrappers = []any{ + (*DialResponse_Payload)(nil), + (*DialResponse_End)(nil), + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: unsafe.Slice(unsafe.StringData(file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDesc), len(file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDesc)), + NumEnums: 1, + NumMessages: 13, + NumExtensions: 0, + NumServices: 1, + }, + GoTypes: file_vercel_api_dev_tunnel_v1_tunnel_proto_goTypes, + DependencyIndexes: file_vercel_api_dev_tunnel_v1_tunnel_proto_depIdxs, + EnumInfos: file_vercel_api_dev_tunnel_v1_tunnel_proto_enumTypes, + MessageInfos: file_vercel_api_dev_tunnel_v1_tunnel_proto_msgTypes, + }.Build() + File_vercel_api_dev_tunnel_v1_tunnel_proto = out.File + file_vercel_api_dev_tunnel_v1_tunnel_proto_goTypes = nil + file_vercel_api_dev_tunnel_v1_tunnel_proto_depIdxs = nil +} diff --git a/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go b/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go new file mode 100644 index 0000000..ac0728a --- /dev/null +++ b/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go @@ -0,0 +1,210 @@ +// Vendored from vercel/api, services/api-dev-tunnel/proto/tunnel/v1/tunnel.proto. +// Edit it there and copy it here. + +// Code generated by protoc-gen-go-grpc. DO NOT EDIT. +// versions: +// - protoc-gen-go-grpc v1.6.2 +// - protoc (unknown) +// source: vercel/api/dev_tunnel/v1/tunnel.proto + +package dev_tunnelv1 + +import ( + context "context" + grpc "google.golang.org/grpc" + codes "google.golang.org/grpc/codes" + status "google.golang.org/grpc/status" +) + +// This is a compile-time assertion to ensure that this generated file +// is compatible with the grpc package it is being compiled against. +// Requires gRPC-Go v1.64.0 or later. +const _ = grpc.SupportPackageIsVersion9 + +const ( + TunnelService_Connect_FullMethodName = "/vercel.api.dev_tunnel.v1.TunnelService/Connect" + TunnelService_Dial_FullMethodName = "/vercel.api.dev_tunnel.v1.TunnelService/Dial" +) + +// TunnelServiceClient is the client API for TunnelService service. +// +// For semantics around ctx use and closing/ending streaming RPCs, please refer to https://pkg.go.dev/google.golang.org/grpc/?tab=doc#ClientConn.NewStream. +// +// Relays TCP connections between the tunnel's deployment and the developer's +// machine. Payloads are opaque bytes; nothing here interprets them. +type TunnelServiceClient interface { + // The stream `vc dev --tunnel` holds open, carrying every conn the + // deployment dials and every conn the client opens through the deployment. + // Authenticated with the client token + // (`authorization: Bearer`), which the next create-session replaces: a CLI + // refused with UNAUTHENTICATED on a reconnect asks for a new one. + // + // One client per tunnel: another stream fails with ALREADY_EXISTS, unless it + // carries the same `x-vercel-dev-tunnel-instance` (1 to 64 letters, digits + // or dashes, one per CLI process). Then it replaces the old stream, which + // ends with ABORTED, so a CLI can reconnect while its old stream is still + // open on the service. The CLI sends a Heartbeat at least every 30s; a + // stream silent for three ends with DEADLINE_EXCEEDED, and one ending with + // UNAVAILABLE means reconnect. A CLI that stops reading the stream while + // the service has frames for it has it end with RESOURCE_EXHAUSTED. + // + // A pod shutting down sends Reconnect and refuses new streams with + // UNAVAILABLE. Once the CLI's new stream holds the tunnel elsewhere, the old + // one gets no new conns and ends with ABORTED when its open ones are done. + Connect(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ConnectRequest, ConnectResponse], error) + // One conn from the deployment: a TCP connection to a port on the + // developer's machine. The first message is the target; the rest carry + // payload. Ending the request stream half-closes the conn, and `end` says the + // developer's side has; the call ends once both have, and fails if either + // side resets. With no client connected, it waits up to 5s for one, then + // fails with UNAVAILABLE. A tunnel carries at most 256 conns at once; more + // wait up to 30s for one to end, then fail with RESOURCE_EXHAUSTED, as do + // any past 256 waiting. Authenticated with the server token + // (`authorization: Bearer`). + Dial(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[DialRequest, DialResponse], error) +} + +type tunnelServiceClient struct { + cc grpc.ClientConnInterface +} + +func NewTunnelServiceClient(cc grpc.ClientConnInterface) TunnelServiceClient { + return &tunnelServiceClient{cc} +} + +func (c *tunnelServiceClient) Connect(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[ConnectRequest, ConnectResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TunnelService_ServiceDesc.Streams[0], TunnelService_Connect_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[ConnectRequest, ConnectResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TunnelService_ConnectClient = grpc.BidiStreamingClient[ConnectRequest, ConnectResponse] + +func (c *tunnelServiceClient) Dial(ctx context.Context, opts ...grpc.CallOption) (grpc.BidiStreamingClient[DialRequest, DialResponse], error) { + cOpts := append([]grpc.CallOption{grpc.StaticMethod()}, opts...) + stream, err := c.cc.NewStream(ctx, &TunnelService_ServiceDesc.Streams[1], TunnelService_Dial_FullMethodName, cOpts...) + if err != nil { + return nil, err + } + x := &grpc.GenericClientStream[DialRequest, DialResponse]{ClientStream: stream} + return x, nil +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TunnelService_DialClient = grpc.BidiStreamingClient[DialRequest, DialResponse] + +// TunnelServiceServer is the server API for TunnelService service. +// All implementations must embed UnimplementedTunnelServiceServer +// for forward compatibility. +// +// Relays TCP connections between the tunnel's deployment and the developer's +// machine. Payloads are opaque bytes; nothing here interprets them. +type TunnelServiceServer interface { + // The stream `vc dev --tunnel` holds open, carrying every conn the + // deployment dials and every conn the client opens through the deployment. + // Authenticated with the client token + // (`authorization: Bearer`), which the next create-session replaces: a CLI + // refused with UNAUTHENTICATED on a reconnect asks for a new one. + // + // One client per tunnel: another stream fails with ALREADY_EXISTS, unless it + // carries the same `x-vercel-dev-tunnel-instance` (1 to 64 letters, digits + // or dashes, one per CLI process). Then it replaces the old stream, which + // ends with ABORTED, so a CLI can reconnect while its old stream is still + // open on the service. The CLI sends a Heartbeat at least every 30s; a + // stream silent for three ends with DEADLINE_EXCEEDED, and one ending with + // UNAVAILABLE means reconnect. A CLI that stops reading the stream while + // the service has frames for it has it end with RESOURCE_EXHAUSTED. + // + // A pod shutting down sends Reconnect and refuses new streams with + // UNAVAILABLE. Once the CLI's new stream holds the tunnel elsewhere, the old + // one gets no new conns and ends with ABORTED when its open ones are done. + Connect(grpc.BidiStreamingServer[ConnectRequest, ConnectResponse]) error + // One conn from the deployment: a TCP connection to a port on the + // developer's machine. The first message is the target; the rest carry + // payload. Ending the request stream half-closes the conn, and `end` says the + // developer's side has; the call ends once both have, and fails if either + // side resets. With no client connected, it waits up to 5s for one, then + // fails with UNAVAILABLE. A tunnel carries at most 256 conns at once; more + // wait up to 30s for one to end, then fail with RESOURCE_EXHAUSTED, as do + // any past 256 waiting. Authenticated with the server token + // (`authorization: Bearer`). + Dial(grpc.BidiStreamingServer[DialRequest, DialResponse]) error + mustEmbedUnimplementedTunnelServiceServer() +} + +// UnimplementedTunnelServiceServer must be embedded to have +// forward compatible implementations. +// +// NOTE: this should be embedded by value instead of pointer to avoid a nil +// pointer dereference when methods are called. +type UnimplementedTunnelServiceServer struct{} + +func (UnimplementedTunnelServiceServer) Connect(grpc.BidiStreamingServer[ConnectRequest, ConnectResponse]) error { + return status.Error(codes.Unimplemented, "method Connect not implemented") +} +func (UnimplementedTunnelServiceServer) Dial(grpc.BidiStreamingServer[DialRequest, DialResponse]) error { + return status.Error(codes.Unimplemented, "method Dial not implemented") +} +func (UnimplementedTunnelServiceServer) mustEmbedUnimplementedTunnelServiceServer() {} +func (UnimplementedTunnelServiceServer) testEmbeddedByValue() {} + +// UnsafeTunnelServiceServer may be embedded to opt out of forward compatibility for this service. +// Use of this interface is not recommended, as added methods to TunnelServiceServer will +// result in compilation errors. +type UnsafeTunnelServiceServer interface { + mustEmbedUnimplementedTunnelServiceServer() +} + +func RegisterTunnelServiceServer(s grpc.ServiceRegistrar, srv TunnelServiceServer) { + // If the following call panics, it indicates UnimplementedTunnelServiceServer was + // embedded by pointer and is nil. This will cause panics if an + // unimplemented method is ever invoked, so we test this at initialization + // time to prevent it from happening at runtime later due to I/O. + if t, ok := srv.(interface{ testEmbeddedByValue() }); ok { + t.testEmbeddedByValue() + } + s.RegisterService(&TunnelService_ServiceDesc, srv) +} + +func _TunnelService_Connect_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TunnelServiceServer).Connect(&grpc.GenericServerStream[ConnectRequest, ConnectResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TunnelService_ConnectServer = grpc.BidiStreamingServer[ConnectRequest, ConnectResponse] + +func _TunnelService_Dial_Handler(srv interface{}, stream grpc.ServerStream) error { + return srv.(TunnelServiceServer).Dial(&grpc.GenericServerStream[DialRequest, DialResponse]{ServerStream: stream}) +} + +// This type alias is provided for backwards compatibility with existing code that references the prior non-generic stream type by name. +type TunnelService_DialServer = grpc.BidiStreamingServer[DialRequest, DialResponse] + +// TunnelService_ServiceDesc is the grpc.ServiceDesc for TunnelService service. +// It's only intended for direct use with grpc.RegisterService, +// and not to be introspected or modified (even as a copy) +var TunnelService_ServiceDesc = grpc.ServiceDesc{ + ServiceName: "vercel.api.dev_tunnel.v1.TunnelService", + HandlerType: (*TunnelServiceServer)(nil), + Methods: []grpc.MethodDesc{}, + Streams: []grpc.StreamDesc{ + { + StreamName: "Connect", + Handler: _TunnelService_Connect_Handler, + ServerStreams: true, + ClientStreams: true, + }, + { + StreamName: "Dial", + Handler: _TunnelService_Dial_Handler, + ServerStreams: true, + ClientStreams: true, + }, + }, + Metadata: "vercel/api/dev_tunnel/v1/tunnel.proto", +} diff --git a/api/ts/bridge/v1/interceptor_pb.d.ts b/api/ts/bridge/v1/interceptor_pb.d.ts index fdebca1..224ccd1 100644 --- a/api/ts/bridge/v1/interceptor_pb.d.ts +++ b/api/ts/bridge/v1/interceptor_pb.d.ts @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v2.12.0 with parameter "target=js+dts" +// @generated by protoc-gen-es v2.16.0 with parameter "target=js+dts" // @generated from file bridge/v1/interceptor.proto (package bridge.v1, syntax proto3) /* eslint-disable */ @@ -57,6 +57,14 @@ export declare type GetStatusResponse = Message<"bridge.v1.GetStatusResponse"> & * @generated from field: bridge.v1.ProbeStatus startup = 4; */ startup?: ProbeStatus | undefined; + + /** + * The dev tunnel's stream to its relay. Unset unless + * --tunnel-protocol=dev-tunnel. + * + * @generated from field: bridge.v1.TunnelStatus tunnel = 5; + */ + tunnel?: TunnelStatus | undefined; }; /** @@ -65,6 +73,112 @@ export declare type GetStatusResponse = Message<"bridge.v1.GetStatusResponse"> & */ export declare const GetStatusResponseSchema: GenMessage; +/** + * TunnelStatus is the state of a dev tunnel's stream to api-dev-tunnel, and + * the conns it carries. + * + * @generated from message bridge.v1.TunnelStatus + */ +export declare type TunnelStatus = Message<"bridge.v1.TunnelStatus"> & { + /** + * @generated from field: bridge.v1.TunnelState state = 1; + */ + state: TunnelState; + + /** + * Why the last stream ended, or the relay refused one. Empty before any + * has. + * + * @generated from field: string last_error = 2 [json_name = "last_error"]; + */ + lastError: string; + + /** + * Unix seconds since a stream has held the tunnel. Zero unless LIVE. + * + * @generated from field: int64 live_since_unix_seconds = 3 [json_name = "live_since_unix_seconds"]; + */ + liveSinceUnixSeconds: bigint; + + /** + * Streams opened after the first. + * + * @generated from field: int64 reconnects = 4; + */ + reconnects: bigint; + + /** + * Conns open now from the deployment to the app (ingress), and from the + * app out through the deployment (egress). + * + * @generated from field: int64 ingress_conns = 5 [json_name = "ingress_conns"]; + */ + ingressConns: bigint; + + /** + * @generated from field: int64 egress_conns = 6 [json_name = "egress_conns"]; + */ + egressConns: bigint; + + /** + * Conns opened since the intercept started. + * + * @generated from field: int64 ingress_conns_total = 7 [json_name = "ingress_conns_total"]; + */ + ingressConnsTotal: bigint; + + /** + * @generated from field: int64 egress_conns_total = 8 [json_name = "egress_conns_total"]; + */ + egressConnsTotal: bigint; + + /** + * Payload bytes sent to and received from the relay. + * + * @generated from field: int64 bytes_sent = 9 [json_name = "bytes_sent"]; + */ + bytesSent: bigint; + + /** + * @generated from field: int64 bytes_received = 10 [json_name = "bytes_received"]; + */ + bytesReceived: bigint; +}; + +/** + * Describes the message bridge.v1.TunnelStatus. + * Use `create(TunnelStatusSchema)` to create a new message. + */ +export declare const TunnelStatusSchema: GenMessage; + +/** + * @generated from message bridge.v1.SetTunnelTokenRequest + */ +export declare type SetTunnelTokenRequest = Message<"bridge.v1.SetTunnelTokenRequest"> & { + /** + * @generated from field: string token = 1; + */ + token: string; +}; + +/** + * Describes the message bridge.v1.SetTunnelTokenRequest. + * Use `create(SetTunnelTokenRequestSchema)` to create a new message. + */ +export declare const SetTunnelTokenRequestSchema: GenMessage; + +/** + * @generated from message bridge.v1.SetTunnelTokenResponse + */ +export declare type SetTunnelTokenResponse = Message<"bridge.v1.SetTunnelTokenResponse"> & { +}; + +/** + * Describes the message bridge.v1.SetTunnelTokenResponse. + * Use `create(SetTunnelTokenResponseSchema)` to create a new message. + */ +export declare const SetTunnelTokenResponseSchema: GenMessage; + /** * ProbeStatus reflects the current health view from a single configured * probe — what the source deployment's probe would have reported if it had @@ -200,6 +314,58 @@ export declare type ProbeCheckResult = Message<"bridge.v1.ProbeCheckResult"> & { */ export declare const ProbeCheckResultSchema: GenMessage; +/** + * @generated from enum bridge.v1.TunnelState + */ +export enum TunnelState { + /** + * @generated from enum value: TUNNEL_STATE_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * No stream holds the tunnel yet, or the last one ended and the next is + * opening. + * + * @generated from enum value: TUNNEL_STATE_CONNECTING = 1; + */ + CONNECTING = 1, + + /** + * A stream holds the tunnel: the deployment's conns reach the app. + * + * @generated from enum value: TUNNEL_STATE_LIVE = 2; + */ + LIVE = 2, + + /** + * The relay refused the client token, or there is none yet. Waiting for + * SetTunnelToken. + * + * @generated from enum value: TUNNEL_STATE_NEEDS_TOKEN = 3; + */ + NEEDS_TOKEN = 3, + + /** + * Another client holds the tunnel. Retrying until it lets go. + * + * @generated from enum value: TUNNEL_STATE_TAKEN = 4; + */ + TAKEN = 4, + + /** + * The relay refused the tunnel for good; last_error says why. + * + * @generated from enum value: TUNNEL_STATE_FAILED = 5; + */ + FAILED = 5, +} + +/** + * Describes the enum bridge.v1.TunnelState. + */ +export declare const TunnelStateSchema: GenEnum; + /** * @generated from enum bridge.v1.ProbeHealth */ @@ -272,5 +438,18 @@ export declare const InterceptorService: GenService<{ input: typeof ProbeRequestSchema; output: typeof ProbeResponseSchema; }, + /** + * SetTunnelToken gives an intercept on a dev tunnel + * (--tunnel-protocol=dev-tunnel) its client token, from create-session. One + * started without a token, or whose token the relay refused + * (TUNNEL_STATE_NEEDS_TOKEN), connects with it. + * + * @generated from rpc bridge.v1.InterceptorService.SetTunnelToken + */ + setTunnelToken: { + methodKind: "unary"; + input: typeof SetTunnelTokenRequestSchema; + output: typeof SetTunnelTokenResponseSchema; + }, }>; diff --git a/api/ts/bridge/v1/interceptor_pb.js b/api/ts/bridge/v1/interceptor_pb.js index e47bb91..603f0cc 100644 --- a/api/ts/bridge/v1/interceptor_pb.js +++ b/api/ts/bridge/v1/interceptor_pb.js @@ -1,4 +1,4 @@ -// @generated by protoc-gen-es v2.12.0 with parameter "target=js+dts" +// @generated by protoc-gen-es v2.16.0 with parameter "target=js+dts" // @generated from file bridge/v1/interceptor.proto (package bridge.v1, syntax proto3) /* eslint-disable */ @@ -8,7 +8,7 @@ import { enumDesc, fileDesc, messageDesc, serviceDesc, tsEnum } from "@bufbuild/ * Describes the file bridge/v1/interceptor.proto. */ export const file_bridge_v1_interceptor = /*@__PURE__*/ - fileDesc("ChticmlkZ2UvdjEvaW50ZXJjZXB0b3IucHJvdG8SCWJyaWRnZS52MSISChBHZXRTdGF0dXNSZXF1ZXN0IqABChFHZXRTdGF0dXNSZXNwb25zZRINCgVyZWFkeRgBIAEoCBIoCghsaXZlbmVzcxgCIAEoCzIWLmJyaWRnZS52MS5Qcm9iZVN0YXR1cxIpCglyZWFkaW5lc3MYAyABKAsyFi5icmlkZ2UudjEuUHJvYmVTdGF0dXMSJwoHc3RhcnR1cBgEIAEoCzIWLmJyaWRnZS52MS5Qcm9iZVN0YXR1cyKOAgoLUHJvYmVTdGF0dXMSJgoGaGVhbHRoGAEgASgOMhYuYnJpZGdlLnYxLlByb2JlSGVhbHRoEg8KB21lc3NhZ2UYAiABKAkSOAoXbGFzdF9jaGVja191bml4X3NlY29uZHMYAyABKANSF2xhc3RfY2hlY2tfdW5peF9zZWNvbmRzEjQKFWNvbnNlY3V0aXZlX3N1Y2Nlc3NlcxgEIAEoBVIVY29uc2VjdXRpdmVfc3VjY2Vzc2VzEjIKFGNvbnNlY3V0aXZlX2ZhaWx1cmVzGAUgASgFUhRjb25zZWN1dGl2ZV9mYWlsdXJlcxIiCgx0b3RhbF9jaGVja3MYBiABKANSDHRvdGFsX2NoZWNrcyIOCgxQcm9iZVJlcXVlc3QinAEKDVByb2JlUmVzcG9uc2USLQoIbGl2ZW5lc3MYASABKAsyGy5icmlkZ2UudjEuUHJvYmVDaGVja1Jlc3VsdBIuCglyZWFkaW5lc3MYAiABKAsyGy5icmlkZ2UudjEuUHJvYmVDaGVja1Jlc3VsdBIsCgdzdGFydHVwGAMgASgLMhsuYnJpZGdlLnYxLlByb2JlQ2hlY2tSZXN1bHQiawoQUHJvYmVDaGVja1Jlc3VsdBIOCgZwYXNzZWQYASABKAgSDQoFZXJyb3IYAiABKAkSOAoXY2hlY2tlZF9hdF91bml4X3NlY29uZHMYAyABKANSF2NoZWNrZWRfYXRfdW5peF9zZWNvbmRzKnsKC1Byb2JlSGVhbHRoEhwKGFBST0JFX0hFQUxUSF9VTlNQRUNJRklFRBAAEhgKFFBST0JFX0hFQUxUSF9QRU5ESU5HEAESGAoUUFJPQkVfSEVBTFRIX0hFQUxUSFkQAhIaChZQUk9CRV9IRUFMVEhfVU5IRUFMVEhZEAMymAEKEkludGVyY2VwdG9yU2VydmljZRJGCglHZXRTdGF0dXMSGy5icmlkZ2UudjEuR2V0U3RhdHVzUmVxdWVzdBocLmJyaWRnZS52MS5HZXRTdGF0dXNSZXNwb25zZRI6CgVQcm9iZRIXLmJyaWRnZS52MS5Qcm9iZVJlcXVlc3QaGC5icmlkZ2UudjEuUHJvYmVSZXNwb25zZUKaAQoNY29tLmJyaWRnZS52MUIQSW50ZXJjZXB0b3JQcm90b1ABWjJnaXRodWIuY29tL3ZlcmNlbC9icmlkZ2UvYXBpL2dvL2JyaWRnZS92MTticmlkZ2V2MaICA0JYWKoCCUJyaWRnZS5WMcoCCUJyaWRnZVxWMeICFUJyaWRnZVxWMVxHUEJNZXRhZGF0YeoCCkJyaWRnZTo6VjFiBnByb3RvMw"); + fileDesc("ChticmlkZ2UvdjEvaW50ZXJjZXB0b3IucHJvdG8SCWJyaWRnZS52MSISChBHZXRTdGF0dXNSZXF1ZXN0IskBChFHZXRTdGF0dXNSZXNwb25zZRINCgVyZWFkeRgBIAEoCBIoCghsaXZlbmVzcxgCIAEoCzIWLmJyaWRnZS52MS5Qcm9iZVN0YXR1cxIpCglyZWFkaW5lc3MYAyABKAsyFi5icmlkZ2UudjEuUHJvYmVTdGF0dXMSJwoHc3RhcnR1cBgEIAEoCzIWLmJyaWRnZS52MS5Qcm9iZVN0YXR1cxInCgZ0dW5uZWwYBSABKAsyFy5icmlkZ2UudjEuVHVubmVsU3RhdHVzIpcDCgxUdW5uZWxTdGF0dXMSJQoFc3RhdGUYASABKA4yFi5icmlkZ2UudjEuVHVubmVsU3RhdGUSHgoKbGFzdF9lcnJvchgCIAEoCVIKbGFzdF9lcnJvchI4ChdsaXZlX3NpbmNlX3VuaXhfc2Vjb25kcxgDIAEoA1IXbGl2ZV9zaW5jZV91bml4X3NlY29uZHMSEgoKcmVjb25uZWN0cxgEIAEoAxIkCg1pbmdyZXNzX2Nvbm5zGAUgASgDUg1pbmdyZXNzX2Nvbm5zEiIKDGVncmVzc19jb25ucxgGIAEoA1IMZWdyZXNzX2Nvbm5zEjAKE2luZ3Jlc3NfY29ubnNfdG90YWwYByABKANSE2luZ3Jlc3NfY29ubnNfdG90YWwSLgoSZWdyZXNzX2Nvbm5zX3RvdGFsGAggASgDUhJlZ3Jlc3NfY29ubnNfdG90YWwSHgoKYnl0ZXNfc2VudBgJIAEoA1IKYnl0ZXNfc2VudBImCg5ieXRlc19yZWNlaXZlZBgKIAEoA1IOYnl0ZXNfcmVjZWl2ZWQiJgoVU2V0VHVubmVsVG9rZW5SZXF1ZXN0Eg0KBXRva2VuGAEgASgJIhgKFlNldFR1bm5lbFRva2VuUmVzcG9uc2UijgIKC1Byb2JlU3RhdHVzEiYKBmhlYWx0aBgBIAEoDjIWLmJyaWRnZS52MS5Qcm9iZUhlYWx0aBIPCgdtZXNzYWdlGAIgASgJEjgKF2xhc3RfY2hlY2tfdW5peF9zZWNvbmRzGAMgASgDUhdsYXN0X2NoZWNrX3VuaXhfc2Vjb25kcxI0ChVjb25zZWN1dGl2ZV9zdWNjZXNzZXMYBCABKAVSFWNvbnNlY3V0aXZlX3N1Y2Nlc3NlcxIyChRjb25zZWN1dGl2ZV9mYWlsdXJlcxgFIAEoBVIUY29uc2VjdXRpdmVfZmFpbHVyZXMSIgoMdG90YWxfY2hlY2tzGAYgASgDUgx0b3RhbF9jaGVja3MiDgoMUHJvYmVSZXF1ZXN0IpwBCg1Qcm9iZVJlc3BvbnNlEi0KCGxpdmVuZXNzGAEgASgLMhsuYnJpZGdlLnYxLlByb2JlQ2hlY2tSZXN1bHQSLgoJcmVhZGluZXNzGAIgASgLMhsuYnJpZGdlLnYxLlByb2JlQ2hlY2tSZXN1bHQSLAoHc3RhcnR1cBgDIAEoCzIbLmJyaWRnZS52MS5Qcm9iZUNoZWNrUmVzdWx0ImsKEFByb2JlQ2hlY2tSZXN1bHQSDgoGcGFzc2VkGAEgASgIEg0KBWVycm9yGAIgASgJEjgKF2NoZWNrZWRfYXRfdW5peF9zZWNvbmRzGAMgASgDUhdjaGVja2VkX2F0X3VuaXhfc2Vjb25kcyquAQoLVHVubmVsU3RhdGUSHAoYVFVOTkVMX1NUQVRFX1VOU1BFQ0lGSUVEEAASGwoXVFVOTkVMX1NUQVRFX0NPTk5FQ1RJTkcQARIVChFUVU5ORUxfU1RBVEVfTElWRRACEhwKGFRVTk5FTF9TVEFURV9ORUVEU19UT0tFThADEhYKElRVTk5FTF9TVEFURV9UQUtFThAEEhcKE1RVTk5FTF9TVEFURV9GQUlMRUQQBSp7CgtQcm9iZUhlYWx0aBIcChhQUk9CRV9IRUFMVEhfVU5TUEVDSUZJRUQQABIYChRQUk9CRV9IRUFMVEhfUEVORElORxABEhgKFFBST0JFX0hFQUxUSF9IRUFMVEhZEAISGgoWUFJPQkVfSEVBTFRIX1VOSEVBTFRIWRADMu8BChJJbnRlcmNlcHRvclNlcnZpY2USRgoJR2V0U3RhdHVzEhsuYnJpZGdlLnYxLkdldFN0YXR1c1JlcXVlc3QaHC5icmlkZ2UudjEuR2V0U3RhdHVzUmVzcG9uc2USOgoFUHJvYmUSFy5icmlkZ2UudjEuUHJvYmVSZXF1ZXN0GhguYnJpZGdlLnYxLlByb2JlUmVzcG9uc2USVQoOU2V0VHVubmVsVG9rZW4SIC5icmlkZ2UudjEuU2V0VHVubmVsVG9rZW5SZXF1ZXN0GiEuYnJpZGdlLnYxLlNldFR1bm5lbFRva2VuUmVzcG9uc2VCmgEKDWNvbS5icmlkZ2UudjFCEEludGVyY2VwdG9yUHJvdG9QAVoyZ2l0aHViLmNvbS92ZXJjZWwvYnJpZGdlL2FwaS9nby9icmlkZ2UvdjE7YnJpZGdldjGiAgNCWFiqAglCcmlkZ2UuVjHKAglCcmlkZ2VcVjHiAhVCcmlkZ2VcVjFcR1BCTWV0YWRhdGHqAgpCcmlkZ2U6OlYxYgZwcm90bzM"); /** * Describes the message bridge.v1.GetStatusRequest. @@ -24,39 +24,72 @@ export const GetStatusRequestSchema = /*@__PURE__*/ export const GetStatusResponseSchema = /*@__PURE__*/ messageDesc(file_bridge_v1_interceptor, 1); +/** + * Describes the message bridge.v1.TunnelStatus. + * Use `create(TunnelStatusSchema)` to create a new message. + */ +export const TunnelStatusSchema = /*@__PURE__*/ + messageDesc(file_bridge_v1_interceptor, 2); + +/** + * Describes the message bridge.v1.SetTunnelTokenRequest. + * Use `create(SetTunnelTokenRequestSchema)` to create a new message. + */ +export const SetTunnelTokenRequestSchema = /*@__PURE__*/ + messageDesc(file_bridge_v1_interceptor, 3); + +/** + * Describes the message bridge.v1.SetTunnelTokenResponse. + * Use `create(SetTunnelTokenResponseSchema)` to create a new message. + */ +export const SetTunnelTokenResponseSchema = /*@__PURE__*/ + messageDesc(file_bridge_v1_interceptor, 4); + /** * Describes the message bridge.v1.ProbeStatus. * Use `create(ProbeStatusSchema)` to create a new message. */ export const ProbeStatusSchema = /*@__PURE__*/ - messageDesc(file_bridge_v1_interceptor, 2); + messageDesc(file_bridge_v1_interceptor, 5); /** * Describes the message bridge.v1.ProbeRequest. * Use `create(ProbeRequestSchema)` to create a new message. */ export const ProbeRequestSchema = /*@__PURE__*/ - messageDesc(file_bridge_v1_interceptor, 3); + messageDesc(file_bridge_v1_interceptor, 6); /** * Describes the message bridge.v1.ProbeResponse. * Use `create(ProbeResponseSchema)` to create a new message. */ export const ProbeResponseSchema = /*@__PURE__*/ - messageDesc(file_bridge_v1_interceptor, 4); + messageDesc(file_bridge_v1_interceptor, 7); /** * Describes the message bridge.v1.ProbeCheckResult. * Use `create(ProbeCheckResultSchema)` to create a new message. */ export const ProbeCheckResultSchema = /*@__PURE__*/ - messageDesc(file_bridge_v1_interceptor, 5); + messageDesc(file_bridge_v1_interceptor, 8); + +/** + * Describes the enum bridge.v1.TunnelState. + */ +export const TunnelStateSchema = /*@__PURE__*/ + enumDesc(file_bridge_v1_interceptor, 0); + +/** + * @generated from enum bridge.v1.TunnelState + */ +export const TunnelState = /*@__PURE__*/ + tsEnum(TunnelStateSchema); /** * Describes the enum bridge.v1.ProbeHealth. */ export const ProbeHealthSchema = /*@__PURE__*/ - enumDesc(file_bridge_v1_interceptor, 0); + enumDesc(file_bridge_v1_interceptor, 1); /** * @generated from enum bridge.v1.ProbeHealth diff --git a/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.d.ts b/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.d.ts new file mode 100644 index 0000000..0832ced --- /dev/null +++ b/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.d.ts @@ -0,0 +1,458 @@ +// Vendored from vercel/api, services/api-dev-tunnel/proto/tunnel/v1/tunnel.proto. +// Edit it there and copy it here. + +// @generated by protoc-gen-es v2.16.0 with parameter "target=js+dts" +// @generated from file vercel/api/dev_tunnel/v1/tunnel.proto (package vercel.api.dev_tunnel.v1, syntax proto3) +/* eslint-disable */ + +import type { GenEnum, GenFile, GenMessage, GenService } from "@bufbuild/protobuf/codegenv2"; +import type { Message } from "@bufbuild/protobuf"; + +/** + * Describes the file vercel/api/dev_tunnel/v1/tunnel.proto. + */ +export declare const file_vercel_api_dev_tunnel_v1_tunnel: GenFile; + +/** + * CLI to service. + * + * @generated from message vercel.api.dev_tunnel.v1.ConnectRequest + */ +export declare type ConnectRequest = Message<"vercel.api.dev_tunnel.v1.ConnectRequest"> & { + /** + * @generated from oneof vercel.api.dev_tunnel.v1.ConnectRequest.frame + */ + frame: { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Data data = 1; + */ + value: Data; + case: "data"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Close close = 2; + */ + value: Close; + case: "close"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Reset reset = 3; + */ + value: Reset; + case: "reset"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Heartbeat heartbeat = 4; + */ + value: Heartbeat; + case: "heartbeat"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Window window = 5; + */ + value: Window; + case: "window"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Open open = 6; + */ + value: Open; + case: "open"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.ConnectRequest. + * Use `create(ConnectRequestSchema)` to create a new message. + */ +export declare const ConnectRequestSchema: GenMessage; + +/** + * Service to CLI. + * + * @generated from message vercel.api.dev_tunnel.v1.ConnectResponse + */ +export declare type ConnectResponse = Message<"vercel.api.dev_tunnel.v1.ConnectResponse"> & { + /** + * @generated from oneof vercel.api.dev_tunnel.v1.ConnectResponse.frame + */ + frame: { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Open open = 1; + */ + value: Open; + case: "open"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Data data = 2; + */ + value: Data; + case: "data"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Close close = 3; + */ + value: Close; + case: "close"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Reset reset = 4; + */ + value: Reset; + case: "reset"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Heartbeat heartbeat = 5; + */ + value: Heartbeat; + case: "heartbeat"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Window window = 6; + */ + value: Window; + case: "window"; + } | { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Reconnect reconnect = 7; + */ + value: Reconnect; + case: "reconnect"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.ConnectResponse. + * Use `create(ConnectResponseSchema)` to create a new message. + */ +export declare const ConnectResponseSchema: GenMessage; + +/** + * A new conn. From the service, the CLI connects to `port` on the developer's + * machine, or resets the conn if it doesn't serve that port. From the client, + * the deployment connects to `host` and `port`, resolving `host` as its own + * code would, and the service resets the conn if it can't. The service numbers + * its conns from 1; a client numbers its own from 2^63, so the two never meet. + * + * @generated from message vercel.api.dev_tunnel.v1.Open + */ +export declare type Open = Message<"vercel.api.dev_tunnel.v1.Open"> & { + /** + * @generated from field: uint64 conn_id = 1; + */ + connId: bigint; + + /** + * @generated from field: vercel.api.dev_tunnel.v1.Protocol protocol = 2; + */ + protocol: Protocol; + + /** + * @generated from field: uint32 port = 3; + */ + port: number; + + /** + * A hostname or IP address. Only a client's Open has one. + * + * @generated from field: string host = 4; + */ + host: string; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Open. + * Use `create(OpenSchema)` to create a new message. + */ +export declare const OpenSchema: GenMessage; + +/** + * No more than the conn's window allows (see Window). + * + * @generated from message vercel.api.dev_tunnel.v1.Data + */ +export declare type Data = Message<"vercel.api.dev_tunnel.v1.Data"> & { + /** + * @generated from field: uint64 conn_id = 1; + */ + connId: bigint; + + /** + * @generated from field: bytes payload = 2; + */ + payload: Uint8Array; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Data. + * Use `create(DataSchema)` to create a new message. + */ +export declare const DataSchema: GenMessage; + +/** + * The sender has nothing more to send (TCP half-close). A conn is over once + * both sides have closed, or either has reset. + * + * @generated from message vercel.api.dev_tunnel.v1.Close + */ +export declare type Close = Message<"vercel.api.dev_tunnel.v1.Close"> & { + /** + * @generated from field: uint64 conn_id = 1; + */ + connId: bigint; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Close. + * Use `create(CloseSchema)` to create a new message. + */ +export declare const CloseSchema: GenMessage; + +/** + * Abandons the conn in both directions, e.g. connection refused. + * + * @generated from message vercel.api.dev_tunnel.v1.Reset + */ +export declare type Reset = Message<"vercel.api.dev_tunnel.v1.Reset"> & { + /** + * @generated from field: uint64 conn_id = 1; + */ + connId: bigint; + + /** + * @generated from field: string reason = 2; + */ + reason: string; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Reset. + * Use `create(ResetSchema)` to create a new message. + */ +export declare const ResetSchema: GenMessage; + +/** + * Lets the peer send `bytes` more of a conn's Data, as the sender of this + * consumes what it received. Each side starts each conn able to send 256 KiB. + * A client that sends past its window, or grants back more than it was sent, + * has the conn reset. + * + * @generated from message vercel.api.dev_tunnel.v1.Window + */ +export declare type Window = Message<"vercel.api.dev_tunnel.v1.Window"> & { + /** + * @generated from field: uint64 conn_id = 1; + */ + connId: bigint; + + /** + * @generated from field: uint32 bytes = 2; + */ + bytes: number; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Window. + * Use `create(WindowSchema)` to create a new message. + */ +export declare const WindowSchema: GenMessage; + +/** + * Sent both ways every 30s, inside the ALB's idle timeout. The service drops a + * client it hasn't heard from in three. The service also sends one as soon as + * a Connect stream holds the tunnel, so the CLI knows it's live. + * + * @generated from message vercel.api.dev_tunnel.v1.Heartbeat + */ +export declare type Heartbeat = Message<"vercel.api.dev_tunnel.v1.Heartbeat"> & { +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Heartbeat. + * Use `create(HeartbeatSchema)` to create a new message. + */ +export declare const HeartbeatSchema: GenMessage; + +/** + * This pod is shutting down. The CLI opens a new stream now, and keeps this one + * until the service ends it: it still carries the conns already open on it. + * + * @generated from message vercel.api.dev_tunnel.v1.Reconnect + */ +export declare type Reconnect = Message<"vercel.api.dev_tunnel.v1.Reconnect"> & { +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.Reconnect. + * Use `create(ReconnectSchema)` to create a new message. + */ +export declare const ReconnectSchema: GenMessage; + +/** + * @generated from message vercel.api.dev_tunnel.v1.DialRequest + */ +export declare type DialRequest = Message<"vercel.api.dev_tunnel.v1.DialRequest"> & { + /** + * @generated from oneof vercel.api.dev_tunnel.v1.DialRequest.frame + */ + frame: { + /** + * First message only. + * + * @generated from field: vercel.api.dev_tunnel.v1.DialTarget target = 1; + */ + value: DialTarget; + case: "target"; + } | { + /** + * @generated from field: bytes payload = 2; + */ + value: Uint8Array; + case: "payload"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialRequest. + * Use `create(DialRequestSchema)` to create a new message. + */ +export declare const DialRequestSchema: GenMessage; + +/** + * @generated from message vercel.api.dev_tunnel.v1.DialTarget + */ +export declare type DialTarget = Message<"vercel.api.dev_tunnel.v1.DialTarget"> & { + /** + * @generated from field: vercel.api.dev_tunnel.v1.Protocol protocol = 1; + */ + protocol: Protocol; + + /** + * @generated from field: uint32 port = 2; + */ + port: number; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialTarget. + * Use `create(DialTargetSchema)` to create a new message. + */ +export declare const DialTargetSchema: GenMessage; + +/** + * @generated from message vercel.api.dev_tunnel.v1.DialResponse + */ +export declare type DialResponse = Message<"vercel.api.dev_tunnel.v1.DialResponse"> & { + /** + * @generated from oneof vercel.api.dev_tunnel.v1.DialResponse.frame + */ + frame: { + /** + * @generated from field: bytes payload = 1; + */ + value: Uint8Array; + case: "payload"; + } | { + /** + * The developer's side has nothing more to send. Sent once; more payload + * may still go the other way. + * + * @generated from field: vercel.api.dev_tunnel.v1.DialEnd end = 2; + */ + value: DialEnd; + case: "end"; + } | { case: undefined; value?: undefined }; +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialResponse. + * Use `create(DialResponseSchema)` to create a new message. + */ +export declare const DialResponseSchema: GenMessage; + +/** + * @generated from message vercel.api.dev_tunnel.v1.DialEnd + */ +export declare type DialEnd = Message<"vercel.api.dev_tunnel.v1.DialEnd"> & { +}; + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialEnd. + * Use `create(DialEndSchema)` to create a new message. + */ +export declare const DialEndSchema: GenMessage; + +/** + * @generated from enum vercel.api.dev_tunnel.v1.Protocol + */ +export enum Protocol { + /** + * @generated from enum value: PROTOCOL_UNSPECIFIED = 0; + */ + UNSPECIFIED = 0, + + /** + * @generated from enum value: PROTOCOL_TCP = 1; + */ + TCP = 1, +} + +/** + * Describes the enum vercel.api.dev_tunnel.v1.Protocol. + */ +export declare const ProtocolSchema: GenEnum; + +/** + * Relays TCP connections between the tunnel's deployment and the developer's + * machine. Payloads are opaque bytes; nothing here interprets them. + * + * @generated from service vercel.api.dev_tunnel.v1.TunnelService + */ +export declare const TunnelService: GenService<{ + /** + * The stream `vc dev --tunnel` holds open, carrying every conn the + * deployment dials and every conn the client opens through the deployment. + * Authenticated with the client token + * (`authorization: Bearer`), which the next create-session replaces: a CLI + * refused with UNAUTHENTICATED on a reconnect asks for a new one. + * + * One client per tunnel: another stream fails with ALREADY_EXISTS, unless it + * carries the same `x-vercel-dev-tunnel-instance` (1 to 64 letters, digits + * or dashes, one per CLI process). Then it replaces the old stream, which + * ends with ABORTED, so a CLI can reconnect while its old stream is still + * open on the service. The CLI sends a Heartbeat at least every 30s; a + * stream silent for three ends with DEADLINE_EXCEEDED, and one ending with + * UNAVAILABLE means reconnect. A CLI that stops reading the stream while + * the service has frames for it has it end with RESOURCE_EXHAUSTED. + * + * A pod shutting down sends Reconnect and refuses new streams with + * UNAVAILABLE. Once the CLI's new stream holds the tunnel elsewhere, the old + * one gets no new conns and ends with ABORTED when its open ones are done. + * + * @generated from rpc vercel.api.dev_tunnel.v1.TunnelService.Connect + */ + connect: { + methodKind: "bidi_streaming"; + input: typeof ConnectRequestSchema; + output: typeof ConnectResponseSchema; + }, + /** + * One conn from the deployment: a TCP connection to a port on the + * developer's machine. The first message is the target; the rest carry + * payload. Ending the request stream half-closes the conn, and `end` says the + * developer's side has; the call ends once both have, and fails if either + * side resets. With no client connected, it waits up to 5s for one, then + * fails with UNAVAILABLE. A tunnel carries at most 256 conns at once; more + * wait up to 30s for one to end, then fail with RESOURCE_EXHAUSTED, as do + * any past 256 waiting. Authenticated with the server token + * (`authorization: Bearer`). + * + * @generated from rpc vercel.api.dev_tunnel.v1.TunnelService.Dial + */ + dial: { + methodKind: "bidi_streaming"; + input: typeof DialRequestSchema; + output: typeof DialResponseSchema; + }, +}>; + diff --git a/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js b/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js new file mode 100644 index 0000000..3ccbc1a --- /dev/null +++ b/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js @@ -0,0 +1,127 @@ +// Vendored from vercel/api, services/api-dev-tunnel/proto/tunnel/v1/tunnel.proto. +// Edit it there and copy it here. + +// @generated by protoc-gen-es v2.16.0 with parameter "target=js+dts" +// @generated from file vercel/api/dev_tunnel/v1/tunnel.proto (package vercel.api.dev_tunnel.v1, syntax proto3) +/* eslint-disable */ + +import { enumDesc, fileDesc, messageDesc, serviceDesc, tsEnum } from "@bufbuild/protobuf/codegenv2"; + +/** + * Describes the file vercel/api/dev_tunnel/v1/tunnel.proto. + */ +export const file_vercel_api_dev_tunnel_v1_tunnel = /*@__PURE__*/ + fileDesc("CiV2ZXJjZWwvYXBpL2Rldl90dW5uZWwvdjEvdHVubmVsLnByb3RvEhh2ZXJjZWwuYXBpLmRldl90dW5uZWwudjEiywIKDkNvbm5lY3RSZXF1ZXN0Ei4KBGRhdGEYASABKAsyHi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuRGF0YUgAEjAKBWNsb3NlGAIgASgLMh8udmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkNsb3NlSAASMAoFcmVzZXQYAyABKAsyHy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuUmVzZXRIABI4CgloZWFydGJlYXQYBCABKAsyIy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuSGVhcnRiZWF0SAASMgoGd2luZG93GAUgASgLMiAudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLldpbmRvd0gAEi4KBG9wZW4YBiABKAsyHi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuT3BlbkgAQgcKBWZyYW1lIoYDCg9Db25uZWN0UmVzcG9uc2USLgoEb3BlbhgBIAEoCzIeLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5PcGVuSAASLgoEZGF0YRgCIAEoCzIeLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EYXRhSAASMAoFY2xvc2UYAyABKAsyHy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ2xvc2VIABIwCgVyZXNldBgEIAEoCzIfLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5SZXNldEgAEjgKCWhlYXJ0YmVhdBgFIAEoCzIjLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5IZWFydGJlYXRIABIyCgZ3aW5kb3cYBiABKAsyIC52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuV2luZG93SAASOAoJcmVjb25uZWN0GAcgASgLMiMudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLlJlY29ubmVjdEgAQgcKBWZyYW1lImkKBE9wZW4SDwoHY29ubl9pZBgBIAEoBBI0Cghwcm90b2NvbBgCIAEoDjIiLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5Qcm90b2NvbBIMCgRwb3J0GAMgASgNEgwKBGhvc3QYBCABKAkiKAoERGF0YRIPCgdjb25uX2lkGAEgASgEEg8KB3BheWxvYWQYAiABKAwiGAoFQ2xvc2USDwoHY29ubl9pZBgBIAEoBCIoCgVSZXNldBIPCgdjb25uX2lkGAEgASgEEg4KBnJlYXNvbhgCIAEoCSIoCgZXaW5kb3cSDwoHY29ubl9pZBgBIAEoBBINCgVieXRlcxgCIAEoDSILCglIZWFydGJlYXQiCwoJUmVjb25uZWN0ImEKC0RpYWxSZXF1ZXN0EjYKBnRhcmdldBgBIAEoCzIkLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EaWFsVGFyZ2V0SAASEQoHcGF5bG9hZBgCIAEoDEgAQgcKBWZyYW1lIlAKCkRpYWxUYXJnZXQSNAoIcHJvdG9jb2wYASABKA4yIi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuUHJvdG9jb2wSDAoEcG9ydBgCIAEoDSJcCgxEaWFsUmVzcG9uc2USEQoHcGF5bG9hZBgBIAEoDEgAEjAKA2VuZBgCIAEoCzIhLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EaWFsRW5kSABCBwoFZnJhbWUiCQoHRGlhbEVuZCo2CghQcm90b2NvbBIYChRQUk9UT0NPTF9VTlNQRUNJRklFRBAAEhAKDFBST1RPQ09MX1RDUBABMs4BCg1UdW5uZWxTZXJ2aWNlEmIKB0Nvbm5lY3QSKC52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ29ubmVjdFJlcXVlc3QaKS52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ29ubmVjdFJlc3BvbnNlKAEwARJZCgREaWFsEiUudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkRpYWxSZXF1ZXN0GiYudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkRpYWxSZXNwb25zZSgBMAFC8QEKHGNvbS52ZXJjZWwuYXBpLmRldl90dW5uZWwudjFCC1R1bm5lbFByb3RvUAFaRWdpdGh1Yi5jb20vdmVyY2VsL2JyaWRnZS9hcGkvZ28vdmVyY2VsL2FwaS9kZXZfdHVubmVsL3YxO2Rldl90dW5uZWx2MaICA1ZBRKoCF1ZlcmNlbC5BcGkuRGV2VHVubmVsLlYxygIXVmVyY2VsXEFwaVxEZXZUdW5uZWxcVjHiAiNWZXJjZWxcQXBpXERldlR1bm5lbFxWMVxHUEJNZXRhZGF0YeoCGlZlcmNlbDo6QXBpOjpEZXZUdW5uZWw6OlYxYgZwcm90bzM"); + +/** + * Describes the message vercel.api.dev_tunnel.v1.ConnectRequest. + * Use `create(ConnectRequestSchema)` to create a new message. + */ +export const ConnectRequestSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 0); + +/** + * Describes the message vercel.api.dev_tunnel.v1.ConnectResponse. + * Use `create(ConnectResponseSchema)` to create a new message. + */ +export const ConnectResponseSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 1); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Open. + * Use `create(OpenSchema)` to create a new message. + */ +export const OpenSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 2); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Data. + * Use `create(DataSchema)` to create a new message. + */ +export const DataSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 3); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Close. + * Use `create(CloseSchema)` to create a new message. + */ +export const CloseSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 4); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Reset. + * Use `create(ResetSchema)` to create a new message. + */ +export const ResetSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 5); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Window. + * Use `create(WindowSchema)` to create a new message. + */ +export const WindowSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 6); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Heartbeat. + * Use `create(HeartbeatSchema)` to create a new message. + */ +export const HeartbeatSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 7); + +/** + * Describes the message vercel.api.dev_tunnel.v1.Reconnect. + * Use `create(ReconnectSchema)` to create a new message. + */ +export const ReconnectSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 8); + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialRequest. + * Use `create(DialRequestSchema)` to create a new message. + */ +export const DialRequestSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 9); + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialTarget. + * Use `create(DialTargetSchema)` to create a new message. + */ +export const DialTargetSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 10); + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialResponse. + * Use `create(DialResponseSchema)` to create a new message. + */ +export const DialResponseSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 11); + +/** + * Describes the message vercel.api.dev_tunnel.v1.DialEnd. + * Use `create(DialEndSchema)` to create a new message. + */ +export const DialEndSchema = /*@__PURE__*/ + messageDesc(file_vercel_api_dev_tunnel_v1_tunnel, 12); + +/** + * Describes the enum vercel.api.dev_tunnel.v1.Protocol. + */ +export const ProtocolSchema = /*@__PURE__*/ + enumDesc(file_vercel_api_dev_tunnel_v1_tunnel, 0); + +/** + * @generated from enum vercel.api.dev_tunnel.v1.Protocol + */ +export const Protocol = /*@__PURE__*/ + tsEnum(ProtocolSchema); + +/** + * Relays TCP connections between the tunnel's deployment and the developer's + * machine. Payloads are opaque bytes; nothing here interprets them. + * + * @generated from service vercel.api.dev_tunnel.v1.TunnelService + */ +export const TunnelService = /*@__PURE__*/ + serviceDesc(file_vercel_api_dev_tunnel_v1_tunnel, 0); + diff --git a/pkg/commands/dev_test.go b/pkg/commands/dev_test.go index 5768556..92eb992 100644 --- a/pkg/commands/dev_test.go +++ b/pkg/commands/dev_test.go @@ -37,6 +37,10 @@ func (f *fakeInterceptor) Probe(_ context.Context, _ *bridgev1.ProbeRequest, _ . return v.(*bridgev1.ProbeResponse), nil } +func (f *fakeInterceptor) SetTunnelToken(_ context.Context, _ *bridgev1.SetTunnelTokenRequest, _ ...grpc.CallOption) (*bridgev1.SetTunnelTokenResponse, error) { + return &bridgev1.SetTunnelTokenResponse{}, nil +} + func passing() *bridgev1.ProbeCheckResult { return &bridgev1.ProbeCheckResult{Passed: true} } func failing() *bridgev1.ProbeCheckResult { return &bridgev1.ProbeCheckResult{Passed: false, Error: "boom"} } diff --git a/pkg/commands/intercept.go b/pkg/commands/intercept.go index 7d44a04..b25d802 100644 --- a/pkg/commands/intercept.go +++ b/pkg/commands/intercept.go @@ -4,17 +4,20 @@ import ( "context" "fmt" "log/slog" + "net" "os" "os/exec" "os/signal" "os/user" "strings" "syscall" + "time" "github.com/hanwen/go-fuse/v2/fuse" "github.com/urfave/cli/v3" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" "github.com/vercel/bridge/pkg/conntrack" + "github.com/vercel/bridge/pkg/devtunnel" bridgedns "github.com/vercel/bridge/pkg/dns" "github.com/vercel/bridge/pkg/fsmount" "github.com/vercel/bridge/pkg/grpcutil" @@ -24,6 +27,12 @@ import ( "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/probe" "github.com/vercel/bridge/pkg/tunnel" + "google.golang.org/grpc" +) + +const ( + tunnelProtocolBridge = "bridge" + tunnelProtocolDevTunnel = "dev-tunnel" ) func Intercept() *cli.Command { @@ -34,7 +43,7 @@ func Intercept() *cli.Command { Flags: []cli.Flag{ &cli.StringFlag{ Name: "server-addr", - Usage: "Address of the bridge proxy server (e.g. localhost:9090 or k8spf:///pod.ns:9090)", + Usage: "Address of the bridge proxy server (e.g. localhost:9090 or k8spf:///pod.ns:9090), or the relay's URL with --tunnel-protocol=dev-tunnel", Sources: cli.EnvVars("BRIDGE_SERVER_ADDR"), Required: true, }, @@ -75,6 +84,22 @@ func Intercept() *cli.Command { Value: ":8080", Sources: cli.EnvVars(meta.EnvInterceptorAddr), }, + &cli.StringFlag{ + Name: "tunnel-protocol", + Usage: `What --server-addr speaks: "bridge" for a bridge proxy server, or "dev-tunnel" for api-dev-tunnel, the relay ` + "`vc dev --tunnel`" + ` uses`, + Value: tunnelProtocolBridge, + Sources: cli.EnvVars("BRIDGE_TUNNEL_PROTOCOL"), + }, + &cli.StringFlag{ + Name: "tunnel-token", + Usage: "The dev tunnel's client token, from create-session. Without one the intercept waits for InterceptorService.SetTunnelToken", + Sources: cli.EnvVars("BRIDGE_TUNNEL_TOKEN"), + }, + &cli.StringFlag{ + Name: "tunnel-instance", + Usage: "The dev tunnel client's instance, 1 to 64 letters, digits or dashes. A restarted intercept with the same one takes the tunnel back at once", + Sources: cli.EnvVars("BRIDGE_TUNNEL_INSTANCE"), + }, }, Action: runIntercept, } @@ -93,7 +118,11 @@ func doIntercept(ctx context.Context, c *cli.Command) error { proxyPort := c.Int("proxy-port") appPort := c.Int("app-port") dnsPort := c.Int("dns-port") - logger := slog.With("server_addr", serverAddr) + tunnelProtocol := c.String("tunnel-protocol") + logger := slog.With("server_addr", serverAddr, "tunnel_protocol", tunnelProtocol) + if tunnelProtocol != tunnelProtocolBridge && tunnelProtocol != tunnelProtocolDevTunnel { + return fmt.Errorf("unknown --tunnel-protocol %q: use %q or %q", tunnelProtocol, tunnelProtocolBridge, tunnelProtocolDevTunnel) + } // Parse forward-domains, handling comma-separated values from env vars var forwardDomains []string @@ -151,59 +180,49 @@ func doIntercept(ctx context.Context, c *cli.Command) error { logger.Info("No forward domains configured, DNS interception disabled") } - // Connect to the bridge proxy server via gRPC - builder := k8spf.NewBuilder(k8spf.BuilderConfig{}) - conn, err := grpcutil.NewClient(serverAddr, builder.DialOptions()...) - if err != nil { - return fmt.Errorf("failed to connect to bridge proxy: %w", err) - } - defer conn.Close() - client := bridgev1.NewBridgeProxyServiceClient(conn) - - // Fetch metadata from the proxy pod (env vars, CA cert/key). - metadata, err := client.GetMetadata(ctx, &bridgev1.GetMetadataRequest{}) - if err != nil { - slog.Warn("Failed to get metadata from proxy", "error", err) - } + // Read before /etc/resolv.conf points at the intercept's own DNS. + originalNS := readOriginalNameserver() - // Install the bridge CA certificate into the system trust store so that - // TLS connections to mocked services are trusted by the application. - if metadata != nil && len(metadata.CaCert) > 0 { - if err := installCACert(ctx, metadata.CaCert); err != nil { - slog.Warn("Failed to install CA certificate", "error", err) - } else { - slog.Info("Installed bridge CA certificate") + var ( + tun tunnel.Tunnel + dnsResolver hostResolver + metadata *bridgev1.GetMetadataResponse + mountServers []*fuseServer + devTunnel *devtunnel.Client + ) + if tunnelProtocol == tunnelProtocolDevTunnel { + if len(mountPaths) > 0 { + logger.Warn("Ignoring mount paths: a dev tunnel has no file system to mount", "paths", mountPaths) } + var err error + devTunnel, err = devtunnel.New(devtunnel.Config{ + RelayURL: serverAddr, + Token: c.String("tunnel-token"), + Instance: c.String("tunnel-instance"), + Port: appPort, + RelayDialer: relayDialer(originalNS), + Logger: logger, + }) + if err != nil { + return err + } + devTunnel.Start(ctx) + defer devTunnel.Close() + tun = devTunnel + dnsResolver = deploymentResolver{} + logger.Info("Dev tunnel starting", "app_port", appPort) + } else { + bridge, err := connectBridge(ctx, serverAddr, appPort, mountPaths) + if err != nil { + return err + } + defer bridge.conn.Close() + tun = bridge.tunnel + dnsResolver = &grpcDNSResolver{client: bridge.client} + metadata = bridge.metadata + mountServers = bridge.mounts } - // If no explicit mount paths were provided, use whatever the server - // advertises via GetMetadata. - if len(mountPaths) == 0 && metadata != nil { - mountPaths = append(mountPaths, metadata.GetMountRoots()...) - } - - // Set up FUSE mounts for the proxy pod's volume paths. Each mount uses - // the same gRPC connection that hosts the proxy service, so it doesn't - // require the tunnel. Failures are non-fatal: log and continue so the - // developer still gets network interception. - var mountServers []*fuseServer - if len(mountPaths) > 0 { - fsClient := bridgev1.NewBridgeFileSystemServiceClient(conn) - mountServers = startMounts(ctx, fsClient, mountPaths) - } - - // Open the shared tunnel stream. If app-port is set, ingress traffic from - // the server's --listen-ports will be forwarded to that local port. - stream, err := client.TunnelNetwork(ctx) - if err != nil { - return fmt.Errorf("failed to open tunnel stream: %w", err) - } - - dialer := plumbing.NewStaticPortDialer(appPort, nil) - tun := tunnel.New(dialer, stream) - tun.Start(ctx) - slog.Info("Tunnel connected", "app_port", appPort) - // Create connection tracking registry pool, err := ippool.New(proxyCIDR) if err != nil { @@ -215,9 +234,7 @@ func doIntercept(ctx context.Context, c *cli.Command) error { var dns *DNSComponent var originalResolvConf []byte if len(forwardDomains) > 0 { - // Read the original nameserver before we modify /etc/resolv.conf - originalNS := readOriginalNameserver() - exchangeClient := bridgedns.NewTunnelExchangeClient(forwardDomains, &grpcDNSResolver{client: client}, originalNS) + exchangeClient := bridgedns.NewTunnelExchangeClient(forwardDomains, dnsResolver, originalNS) dns, err = StartDNS(DNSConfig{ ListenPort: dnsPort, Client: exchangeClient, @@ -289,6 +306,9 @@ func doIntercept(ctx context.Context, c *cli.Command) error { return err } defer interceptSrv.Stop() + if devTunnel != nil { + interceptSrv.SetTunnel(devTunnel) + } // Spin up probe monitors using the source deployment's probes. Each // monitor runs in its own goroutine and is cancelled when ctx is done. @@ -329,6 +349,74 @@ func doIntercept(ctx context.Context, c *cli.Command) error { return nil } +// bridgeConn is the intercept's connection to a bridge proxy server. +type bridgeConn struct { + conn *grpc.ClientConn + client bridgev1.BridgeProxyServiceClient + metadata *bridgev1.GetMetadataResponse + mounts []*fuseServer + tunnel tunnel.Tunnel +} + +// connectBridge connects to a bridge proxy server, installs its CA, mounts +// its paths and opens the tunnel stream to it. +func connectBridge(ctx context.Context, serverAddr string, appPort int, mountPaths []string) (*bridgeConn, error) { + // Connect to the bridge proxy server via gRPC + builder := k8spf.NewBuilder(k8spf.BuilderConfig{}) + conn, err := grpcutil.NewClient(serverAddr, builder.DialOptions()...) + if err != nil { + return nil, fmt.Errorf("failed to connect to bridge proxy: %w", err) + } + client := bridgev1.NewBridgeProxyServiceClient(conn) + + // Fetch metadata from the proxy pod (env vars, CA cert/key). + metadata, err := client.GetMetadata(ctx, &bridgev1.GetMetadataRequest{}) + if err != nil { + slog.Warn("Failed to get metadata from proxy", "error", err) + } + + // Install the bridge CA certificate into the system trust store so that + // TLS connections to mocked services are trusted by the application. + if metadata != nil && len(metadata.CaCert) > 0 { + if err := installCACert(ctx, metadata.CaCert); err != nil { + slog.Warn("Failed to install CA certificate", "error", err) + } else { + slog.Info("Installed bridge CA certificate") + } + } + + // If no explicit mount paths were provided, use whatever the server + // advertises via GetMetadata. + if len(mountPaths) == 0 && metadata != nil { + mountPaths = append(mountPaths, metadata.GetMountRoots()...) + } + + // Set up FUSE mounts for the proxy pod's volume paths. Each mount uses + // the same gRPC connection that hosts the proxy service, so it doesn't + // require the tunnel. Failures are non-fatal: log and continue so the + // developer still gets network interception. + var mountServers []*fuseServer + if len(mountPaths) > 0 { + fsClient := bridgev1.NewBridgeFileSystemServiceClient(conn) + mountServers = startMounts(ctx, fsClient, mountPaths) + } + + // Open the shared tunnel stream. If app-port is set, ingress traffic from + // the server's --listen-ports will be forwarded to that local port. + stream, err := client.TunnelNetwork(ctx) + if err != nil { + _ = conn.Close() + return nil, fmt.Errorf("failed to open tunnel stream: %w", err) + } + + dialer := plumbing.NewStaticPortDialer(appPort, nil) + tun := tunnel.New(dialer, stream) + tun.Start(ctx) + slog.Info("Tunnel connected", "app_port", appPort) + + return &bridgeConn{conn: conn, client: client, metadata: metadata, mounts: mountServers, tunnel: tun}, nil +} + // installCACert writes the PEM-encoded CA certificate to the system trust // store and runs the appropriate update command. // Supports Debian/Ubuntu (update-ca-certificates) and RHEL/Alpine (update-ca-trust). @@ -432,6 +520,36 @@ func restoreResolvConf(original []byte) { } } +// hostResolver resolves the hostnames the intercept's DNS forwards. +type hostResolver interface { + ResolveDNS(ctx context.Context, hostname string) (*bridgedns.DNSResolveResult, error) +} + +// deploymentResolver answers for every forwarded hostname without resolving +// it: the deployment resolves it when it opens the conn, as its own code +// would. The address is a placeholder the conn never uses. +type deploymentResolver struct{} + +func (deploymentResolver) ResolveDNS(context.Context, string) (*bridgedns.DNSResolveResult, error) { + return &bridgedns.DNSResolveResult{Addresses: []string{"0.0.0.0"}}, nil +} + +// relayDialer reaches the relay without the intercept's own DNS, which would +// send the relay's hostname down the tunnel when every domain is forwarded. +// It asks nameserver over TCP, which the UDP :53 redirect leaves alone. +func relayDialer(nameserver string) *net.Dialer { + return &net.Dialer{ + Timeout: 10 * time.Second, + Resolver: &net.Resolver{ + PreferGo: true, + Dial: func(ctx context.Context, _, _ string) (net.Conn, error) { + var d net.Dialer + return d.DialContext(ctx, "tcp", nameserver) + }, + }, + } +} + // grpcDNSResolver wraps a BridgeProxyServiceClient to satisfy the DNS resolver // interface used by TunnelExchangeClient. type grpcDNSResolver struct { @@ -516,4 +634,3 @@ func startMounts(ctx context.Context, client bridgev1.BridgeFileSystemServiceCli } return servers } - diff --git a/pkg/commands/intercept_server.go b/pkg/commands/intercept_server.go index 2784524..bac3b55 100644 --- a/pkg/commands/intercept_server.go +++ b/pkg/commands/intercept_server.go @@ -10,10 +10,13 @@ import ( "time" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + "github.com/vercel/bridge/pkg/devtunnel" "github.com/vercel/bridge/pkg/grpcutil" "github.com/vercel/bridge/pkg/probe" "google.golang.org/grpc" + "google.golang.org/grpc/codes" healthpb "google.golang.org/grpc/health/grpc_health_v1" + "google.golang.org/grpc/status" ) // interceptServer exposes a gRPC server for the intercept process with a @@ -32,6 +35,7 @@ type interceptServer struct { ready atomic.Bool monitors atomic.Pointer[monitorSet] + tunnel atomic.Pointer[devtunnel.Client] } // monitorSet is the immutable bundle stored in interceptServer.monitors. @@ -74,6 +78,12 @@ func (s *interceptServer) SetMonitors(liveness, readiness, startup probe.Monitor }) } +// SetTunnel publishes the dev tunnel GetStatus reports on and SetTunnelToken +// hands tokens to. +func (s *interceptServer) SetTunnel(t *devtunnel.Client) { + s.tunnel.Store(t) +} + // Stop gracefully stops the gRPC server. func (s *interceptServer) Stop() { s.server.GracefulStop() @@ -107,9 +117,25 @@ func (s *interceptServer) GetStatus(_ context.Context, _ *bridgev1.GetStatusRequ resp.Startup = ms.startup.Status() } } + if t := s.tunnel.Load(); t != nil { + resp.Tunnel = t.Status() + } return resp, nil } +// SetTunnelToken hands the dev tunnel a client token. +func (s *interceptServer) SetTunnelToken(_ context.Context, req *bridgev1.SetTunnelTokenRequest) (*bridgev1.SetTunnelTokenResponse, error) { + t := s.tunnel.Load() + if t == nil { + return nil, status.Error(codes.FailedPrecondition, "this intercept isn't on a dev tunnel") + } + if req.GetToken() == "" { + return nil, status.Error(codes.InvalidArgument, "token is empty") + } + t.SetToken(req.GetToken()) + return &bridgev1.SetTunnelTokenResponse{}, nil +} + // Probe runs each configured probe synchronously and returns the per-probe // pass/fail of those single checks. Bypasses the threshold-based smoothing // surfaced by GetStatus — useful as an authoritative "is the app diff --git a/pkg/commands/intercept_server_test.go b/pkg/commands/intercept_server_test.go index 16f896a..71be281 100644 --- a/pkg/commands/intercept_server_test.go +++ b/pkg/commands/intercept_server_test.go @@ -9,7 +9,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + "github.com/vercel/bridge/pkg/devtunnel" "github.com/vercel/bridge/pkg/probe" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/status" ) // newTestInterceptServer binds the interceptServer to a free local port and @@ -128,3 +131,27 @@ func TestInterceptServer_AllThreeMonitorsRouteIndependently(t *testing.T) { time.Sleep(20 * time.Millisecond) } } + +func TestInterceptServer_ReportsAndFeedsTheDevTunnel(t *testing.T) { + srv := newTestInterceptServer(t) + ctx := context.Background() + + _, err := srv.SetTunnelToken(ctx, &bridgev1.SetTunnelTokenRequest{Token: "tok"}) + assert.Equal(t, codes.FailedPrecondition, status.Code(err)) + resp, err := srv.GetStatus(ctx, &bridgev1.GetStatusRequest{}) + require.NoError(t, err) + assert.Nil(t, resp.GetTunnel()) + + tun, err := devtunnel.New(devtunnel.Config{RelayURL: "http://127.0.0.1:1", Port: 3000}) + require.NoError(t, err) + t.Cleanup(func() { _ = tun.Close() }) + srv.SetTunnel(tun) + + resp, err = srv.GetStatus(ctx, &bridgev1.GetStatusRequest{}) + require.NoError(t, err) + assert.Equal(t, bridgev1.TunnelState_TUNNEL_STATE_CONNECTING, resp.GetTunnel().GetState()) + _, err = srv.SetTunnelToken(ctx, &bridgev1.SetTunnelTokenRequest{}) + assert.Equal(t, codes.InvalidArgument, status.Code(err)) + _, err = srv.SetTunnelToken(ctx, &bridgev1.SetTunnelTokenRequest{Token: "tok"}) + assert.NoError(t, err) +} diff --git a/pkg/devtunnel/client.go b/pkg/devtunnel/client.go new file mode 100644 index 0000000..31c8e5c --- /dev/null +++ b/pkg/devtunnel/client.go @@ -0,0 +1,543 @@ +// Package devtunnel carries an intercept's traffic over a dev tunnel: the +// Connect stream to api-dev-tunnel that `vc dev --tunnel` holds otherwise. +// The deployment's conns reach the app, and the app's outbound conns leave +// through the deployment. +package devtunnel + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net" + "net/url" + "regexp" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/segmentio/ksuid" + bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/plumbing" + "github.com/vercel/bridge/pkg/tunnel" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" +) + +const ( + // What each side of a conn may send before the other grants more. + connWindow = 256 << 10 + // Per Data frame. + maxPayload = 64 << 10 + // A client numbers its conns from here; the relay numbers its own from 1. + clientConnIDFloor = uint64(1) << 63 + + instanceHeader = "x-vercel-dev-tunnel-instance" + + retryMin = 250 * time.Millisecond + retryMax = 5 * time.Second +) + +var instancePattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + +// Codes reconnecting can't fix. +var fatal = map[codes.Code]bool{ + codes.FailedPrecondition: true, + codes.InvalidArgument: true, + codes.PermissionDenied: true, + codes.Unimplemented: true, +} + +// Config configures a Client. +type Config struct { + // RelayURL is api-dev-tunnel's URL, from create-session. + RelayURL string + // Token is the client token, from create-session. Without one the client + // waits for SetToken. + Token string + // Instance tells the relay this process's reconnects from another + // client's, so a restart can take the tunnel back while its old stream is + // still open. Random if empty. + Instance string + // Port is the only port of the app the deployment's conns may reach. + Port int + // AppDialer connects the deployment's conns to the app. Defaults to a + // net.Dialer. + AppDialer plumbing.ContextDialer + // RelayDialer connects to the relay. Defaults to a net.Dialer. + RelayDialer plumbing.ContextDialer + Logger *slog.Logger +} + +// timings are the relay's protocol intervals, shortened by tests. +type timings struct { + // The relay needs a heartbeat at least every 30s. + heartbeat time.Duration + // The relay heartbeats every 30s; three missed and the stream is dead. + silent time.Duration + // The relay holds the deployment's conns this long for a client, and an + // outbound conn waits as long for a stream. + liveWait time.Duration +} + +var defaultTimings = timings{ + heartbeat: 15 * time.Second, + silent: 90 * time.Second, + liveWait: 5 * time.Second, +} + +// Client holds a dev tunnel's stream to its relay, replacing it whenever it +// drops, and implements tunnel.Tunnel over it. +type Client struct { + cfg Config + timings timings + logger *slog.Logger + instance string + cc *grpc.ClientConn + rpc devtunnelv1.TunnelServiceClient + + mu sync.Mutex + token string + tokenSet chan struct{} + // The newest stream holding the tunnel, which new outbound conns take. + live *stream + liveCh chan struct{} + liveClosed bool + // Streams holding the tunnel, including one finishing its conns after the + // relay said to reconnect. + held int + state bridgev1.TunnelState + lastErr string + liveSince time.Time + reconnects int64 + + ingressConns, egressConns, ingressTotal, egressTotal atomic.Int64 + bytesSent, bytesReceived atomic.Int64 + nextEgress atomic.Uint64 + + ctx context.Context + cancel context.CancelFunc + started atomic.Bool + done chan struct{} + closeOnce sync.Once + err error +} + +var _ tunnel.Tunnel = (*Client)(nil) + +// New returns a Client for cfg. Nothing connects until Start. +func New(cfg Config) (*Client, error) { + target, creds, err := relayTarget(cfg.RelayURL) + if err != nil { + return nil, err + } + instance := cfg.Instance + if instance == "" { + instance = "bridge-" + ksuid.New().String() + } + if !instancePattern.MatchString(instance) { + return nil, fmt.Errorf("tunnel instance must be 1 to 64 letters, digits or dashes: %q", instance) + } + if cfg.AppDialer == nil { + cfg.AppDialer = &net.Dialer{Timeout: 10 * time.Second} + } + if cfg.RelayDialer == nil { + cfg.RelayDialer = &net.Dialer{Timeout: 10 * time.Second} + } + logger := cfg.Logger + if logger == nil { + logger = slog.Default() + } + logger = logger.With("component", "dev-tunnel", "relay", cfg.RelayURL, "instance", instance) + + relayDialer := cfg.RelayDialer + cc, err := grpc.NewClient(target, + grpc.WithTransportCredentials(creds), + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return relayDialer.DialContext(ctx, "tcp", addr) + }), + // Notices a dead connection, e.g. after sleep, sooner than the + // relay's heartbeats would. + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 10 * time.Second, + Timeout: 20 * time.Second, + PermitWithoutStream: true, + }), + ) + if err != nil { + return nil, fmt.Errorf("creating relay client: %w", err) + } + + c := &Client{ + cfg: cfg, + timings: defaultTimings, + logger: logger, + instance: instance, + cc: cc, + rpc: devtunnelv1.NewTunnelServiceClient(cc), + token: cfg.Token, + tokenSet: make(chan struct{}), + liveCh: make(chan struct{}), + state: bridgev1.TunnelState_TUNNEL_STATE_CONNECTING, + done: make(chan struct{}), + } + c.ctx, c.cancel = context.WithCancel(context.Background()) + return c, nil +} + +// relayTarget turns the relay's URL into a gRPC target. The client dials it +// with RelayDialer rather than resolving it through gRPC, so an intercept +// whose DNS forwards every domain still reaches the relay directly. +func relayTarget(relayURL string) (string, credentials.TransportCredentials, error) { + u, err := url.Parse(relayURL) + if err != nil { + return "", nil, fmt.Errorf("invalid relay URL %q: %w", relayURL, err) + } + if u.Path != "" && u.Path != "/" { + return "", nil, fmt.Errorf("relay URL %q must have no path", relayURL) + } + var creds credentials.TransportCredentials + port := u.Port() + switch u.Scheme { + case "https": + creds = credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) + if port == "" { + port = "443" + } + case "http": + creds = insecure.NewCredentials() + if port == "" { + port = "80" + } + default: + return "", nil, fmt.Errorf("relay URL %q must be http or https", relayURL) + } + if u.Hostname() == "" { + return "", nil, fmt.Errorf("relay URL %q has no host", relayURL) + } + return "passthrough:///" + net.JoinHostPort(u.Hostname(), port), creds, nil +} + +// Start holds the tunnel until ctx ends or Close. +func (c *Client) Start(ctx context.Context) { + if !c.started.CompareAndSwap(false, true) { + return + } + stop := context.AfterFunc(ctx, c.cancel) + go func() { + defer stop() + c.finish(c.run()) + }() +} + +// Close ends every stream and the conns on them. +func (c *Client) Close() error { + c.cancel() + if c.started.Load() { + <-c.done + } else { + c.finish(nil) + } + return nil +} + +func (c *Client) Done() <-chan struct{} { return c.done } + +// Err is why the client stopped: the relay refusing the tunnel for good, or +// nil after Close. Valid once Done is closed. +func (c *Client) Err() error { + <-c.done + return c.err +} + +// SetToken replaces the client token. A client waiting for one connects with +// it; a live one uses it on its next stream. +func (c *Client) SetToken(token string) { + c.mu.Lock() + defer c.mu.Unlock() + c.token = token + close(c.tokenSet) + c.tokenSet = make(chan struct{}) +} + +// Status reports the tunnel's state and traffic. +func (c *Client) Status() *bridgev1.TunnelStatus { + c.mu.Lock() + defer c.mu.Unlock() + st := &bridgev1.TunnelStatus{ + State: c.state, + LastError: c.lastErr, + Reconnects: c.reconnects, + IngressConns: c.ingressConns.Load(), + EgressConns: c.egressConns.Load(), + IngressConnsTotal: c.ingressTotal.Load(), + EgressConnsTotal: c.egressTotal.Load(), + BytesSent: c.bytesSent.Load(), + BytesReceived: c.bytesReceived.Load(), + } + if !c.liveSince.IsZero() { + st.LiveSinceUnixSeconds = c.liveSince.Unix() + } + return st +} + +// AddConn sends one of the app's outbound conns out through the deployment, +// to hostname if the intercept's DNS gave the conn one, or else to the +// address it was headed for. The deployment resolves the hostname as its own +// code would. +func (c *Client) AddConn(nc net.Conn, destOverride string, hostname string) { + dst := destOverride + if dst == "" { + dst = nc.LocalAddr().String() + } + host, portStr, err := net.SplitHostPort(dst) + port, portErr := strconv.Atoi(portStr) + if err != nil || portErr != nil { + c.logger.Warn("Dropping outbound connection with no destination", "dest", dst) + _ = nc.Close() + return + } + if hostname != "" { + host = hostname + } + logger := c.logger.With("host", host, "port", port) + + go func() { + s := c.awaitLive() + if s == nil { + logger.Warn("Dropping outbound connection: no stream holds the tunnel") + _ = nc.Close() + return + } + id := clientConnIDFloor | c.nextEgress.Add(1) + cn := s.newConn(id, egress) + if cn == nil { + logger.Warn("Dropping outbound connection: the tunnel's stream ended") + _ = nc.Close() + return + } + logger.Debug("Opening outbound connection", "conn_id", id) + // Queued ahead of any Data the conn sends. + s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Open{Open: &devtunnelv1.Open{ + ConnId: id, + Protocol: devtunnelv1.Protocol_PROTOCOL_TCP, + Host: host, + Port: uint32(port), + }}}) + cn.attach(nc) + }() +} + +// awaitLive returns the stream new outbound conns take, waiting a while for +// one, or nil. +func (c *Client) awaitLive() *stream { + timer := time.NewTimer(c.timings.liveWait) + defer timer.Stop() + for { + c.mu.Lock() + s, ch := c.live, c.liveCh + c.mu.Unlock() + if s != nil { + return s + } + select { + case <-ch: + case <-timer.C: + return nil + case <-c.ctx.Done(): + return nil + } + } +} + +// run replaces the tunnel's stream whenever it ends, for as long as it takes: +// the relay's pods roll often. A stream the relay says to move off keeps its +// open conns while the next one opens. +func (c *Client) run() error { + delay := retryMin + first := true + warnedTaken := false + for { + token, ok := c.awaitToken() + if !ok { + return nil + } + if !first { + c.mu.Lock() + c.reconnects++ + c.mu.Unlock() + } + first = false + + s := c.newStream() + ended := make(chan error, 1) + moved := make(chan struct{}) + go func() { ended <- s.run(token, sync.OnceFunc(func() { close(moved) })) }() + + var err error + select { + case <-moved: + c.logger.Info("Relay pod is shutting down, opening a new stream") + continue + case err = <-ended: + } + if c.ctx.Err() != nil { + return nil + } + if s.held.Load() { + delay = retryMin + warnedTaken = false + } + code := status.Code(err) + c.mu.Lock() + if err != nil { + c.lastErr = err.Error() + } else { + c.lastErr = "the relay ended the stream" + } + c.mu.Unlock() + if fatal[code] { + return fmt.Errorf("the relay refused the tunnel: %w", err) + } + + logger := c.logger.With("code", code.String(), "error", err) + switch code { + case codes.Unauthenticated: + // Each create-session replaces the client token. + c.refuseToken(token) + logger.Warn("The relay refused the client token, waiting for a new one") + continue + case codes.AlreadyExists: + c.setState(bridgev1.TunnelState_TUNNEL_STATE_TAKEN) + if !warnedTaken { + warnedTaken = true + logger.Warn("Another client holds the tunnel, waiting for it to let go") + } + default: + c.mu.Lock() + if c.held == 0 { + c.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING + } + c.mu.Unlock() + logger.Info("Tunnel stream ended, reconnecting", "delay", delay) + } + + select { + case <-time.After(delay): + case <-c.ctx.Done(): + return nil + } + delay = min(delay*2, retryMax) + } +} + +// awaitToken returns the client token, waiting for SetToken while there is +// none. +func (c *Client) awaitToken() (string, bool) { + for { + c.mu.Lock() + token, set := c.token, c.tokenSet + if token == "" { + c.state = bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN + } + c.mu.Unlock() + if token != "" { + return token, true + } + select { + case <-set: + case <-c.ctx.Done(): + return "", false + } + } +} + +// refuseToken forgets token, unless SetToken already replaced it. +func (c *Client) refuseToken(token string) { + c.mu.Lock() + defer c.mu.Unlock() + if c.token == token { + c.token = "" + } +} + +func (c *Client) setState(state bridgev1.TunnelState) { + c.mu.Lock() + defer c.mu.Unlock() + c.state = state +} + +// setLive makes s the stream new outbound conns take; nil for none. +func (c *Client) setLive(s *stream) { + c.live = s + if s != nil && !c.liveClosed { + close(c.liveCh) + c.liveClosed = true + } else if s == nil && c.liveClosed { + c.liveCh = make(chan struct{}) + c.liveClosed = false + } +} + +// holding: the relay routes the tunnel's conns to s. +func (c *Client) holding(s *stream) { + c.mu.Lock() + defer c.mu.Unlock() + c.held++ + c.setLive(s) + if c.state != bridgev1.TunnelState_TUNNEL_STATE_LIVE { + c.state = bridgev1.TunnelState_TUNNEL_STATE_LIVE + c.liveSince = time.Now() + c.logger.Info("Tunnel live", "reconnects", c.reconnects) + } +} + +// moving: s gets no new conns, but finishes the ones open on it. +func (c *Client) moving(s *stream) { + c.mu.Lock() + defer c.mu.Unlock() + if c.live == s { + c.setLive(nil) + } +} + +// lost: s ended. +func (c *Client) lost(s *stream) { + c.mu.Lock() + defer c.mu.Unlock() + if s.held.Load() { + c.held-- + } + if c.live == s { + c.setLive(nil) + } + if c.held == 0 && c.state == bridgev1.TunnelState_TUNNEL_STATE_LIVE { + c.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING + c.liveSince = time.Time{} + } +} + +func (c *Client) finish(err error) { + c.closeOnce.Do(func() { + c.cancel() + if err != nil { + c.mu.Lock() + c.state = bridgev1.TunnelState_TUNNEL_STATE_FAILED + c.lastErr = err.Error() + c.mu.Unlock() + c.logger.Error("Tunnel stopped", "error", err) + } + c.err = err + _ = c.cc.Close() + close(c.done) + }) +} + +var errSilent = errors.New("no heartbeat from the relay") diff --git a/pkg/devtunnel/client_test.go b/pkg/devtunnel/client_test.go new file mode 100644 index 0000000..3e528e2 --- /dev/null +++ b/pkg/devtunnel/client_test.go @@ -0,0 +1,425 @@ +package devtunnel + +import ( + "bytes" + "context" + "io" + "net" + "strconv" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" + bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" +) + +const testTimeout = 5 * time.Second + +// fakeRelay stands in for api-dev-tunnel, handing each Connect call to the +// test. +type fakeRelay struct { + devtunnelv1.UnimplementedTunnelServiceServer + url string + streams chan *relayStream + // refuse ends a call before it reaches the test, unless it returns nil. + refuse func(md metadata.MD) error +} + +type relayStream struct { + md metadata.MD + call grpc.BidiStreamingServer[devtunnelv1.ConnectRequest, devtunnelv1.ConnectResponse] + recv chan *devtunnelv1.ConnectRequest + end chan error +} + +func startRelay(t *testing.T) *fakeRelay { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + r := &fakeRelay{url: "http://" + lis.Addr().String(), streams: make(chan *relayStream, 8)} + srv := grpc.NewServer() + devtunnelv1.RegisterTunnelServiceServer(srv, r) + go func() { _ = srv.Serve(lis) }() + t.Cleanup(srv.Stop) + return r +} + +func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnelv1.ConnectRequest, devtunnelv1.ConnectResponse]) error { + md, _ := metadata.FromIncomingContext(call.Context()) + if r.refuse != nil { + if err := r.refuse(md); err != nil { + return err + } + } + s := &relayStream{md: md, call: call, recv: make(chan *devtunnelv1.ConnectRequest, 256), end: make(chan error, 1)} + go func() { + defer close(s.recv) + for { + msg, err := call.Recv() + if err != nil { + return + } + s.recv <- msg + } + }() + r.streams <- s + select { + case err := <-s.end: + return err + case <-call.Context().Done(): + return nil + } +} + +func (r *fakeRelay) next(t *testing.T) *relayStream { + t.Helper() + select { + case s := <-r.streams: + return s + case <-time.After(testTimeout): + t.Fatal("no Connect call reached the relay") + return nil + } +} + +func (s *relayStream) send(t *testing.T, msg *devtunnelv1.ConnectResponse) { + t.Helper() + require.NoError(t, s.call.Send(msg)) +} + +// hold makes the stream the tunnel's, as the relay's first heartbeat does. +func (s *relayStream) hold(t *testing.T) { + s.send(t, &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Heartbeat{Heartbeat: &devtunnelv1.Heartbeat{}}}) +} + +// expect returns the next frame from the client other than a heartbeat. +func (s *relayStream) expect(t *testing.T) *devtunnelv1.ConnectRequest { + t.Helper() + timeout := time.After(testTimeout) + for { + select { + case msg, ok := <-s.recv: + require.True(t, ok, "the client ended the stream") + if msg.GetHeartbeat() == nil { + return msg + } + case <-timeout: + t.Fatal("no frame from the client") + return nil + } + } +} + +// expectData collects the client's Data on a conn until it has n bytes. +func (s *relayStream) expectData(t *testing.T, id uint64, n int) []byte { + t.Helper() + var got []byte + for len(got) < n { + msg := s.expect(t) + if d := msg.GetData(); d != nil && d.GetConnId() == id { + got = append(got, d.GetPayload()...) + } + } + return got +} + +func open(id uint64, port int) *devtunnelv1.ConnectResponse { + return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Open{Open: &devtunnelv1.Open{ + ConnId: id, Protocol: devtunnelv1.Protocol_PROTOCOL_TCP, Port: uint32(port), + }}} +} + +func data(id uint64, payload string) *devtunnelv1.ConnectResponse { + return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Data{Data: &devtunnelv1.Data{ConnId: id, Payload: []byte(payload)}}} +} + +func closeConn(id uint64) *devtunnelv1.ConnectResponse { + return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Close{Close: &devtunnelv1.Close{ConnId: id}}} +} + +func window(id uint64, n int) *devtunnelv1.ConnectResponse { + return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Window{Window: &devtunnelv1.Window{ConnId: id, Bytes: uint32(n)}}} +} + +func newClient(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) *Client { + t.Helper() + c, err := New(Config{RelayURL: r.url, Token: token, Instance: "test-instance", Port: port}) + require.NoError(t, err) + c.timings = timings{heartbeat: 50 * time.Millisecond, silent: time.Second, liveWait: 2 * time.Second} + for _, f := range tune { + f(&c.timings) + } + c.Start(context.Background()) + t.Cleanup(func() { _ = c.Close() }) + return c +} + +// echoApp echoes each connection, then half-closes it once the peer has. +func echoApp(t *testing.T) int { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = lis.Close() }) + go func() { + for { + nc, err := lis.Accept() + if err != nil { + return + } + go func() { + _, _ = io.Copy(nc, nc) + _ = nc.(*net.TCPConn).CloseWrite() + }() + } + }() + return lis.Addr().(*net.TCPAddr).Port +} + +func waitFor(t *testing.T, cond func() bool) { + t.Helper() + require.Eventually(t, cond, testTimeout, 10*time.Millisecond) +} + +func TestIngressReachesTheApp(t *testing.T) { + r := startRelay(t) + port := echoApp(t) + c := newClient(t, r, "tok", port) + + s := r.next(t) + assert.Equal(t, []string{"Bearer tok"}, s.md.Get("authorization")) + assert.Equal(t, []string{"test-instance"}, s.md.Get(instanceHeader)) + s.hold(t) + waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_LIVE }) + + s.send(t, open(1, port)) + s.send(t, data(1, "hello")) + var echoed []byte + granted := 0 + for len(echoed) < 5 || granted < 5 { + msg := s.expect(t) + if d := msg.GetData(); d != nil { + echoed = append(echoed, d.GetPayload()...) + } + if w := msg.GetWindow(); w != nil { + granted += int(w.GetBytes()) + } + } + assert.Equal(t, "hello", string(echoed)) + + // The relay's half-close reaches the app, whose own comes back. + s.send(t, closeConn(1)) + assert.Equal(t, uint64(1), s.expect(t).GetClose().GetConnId()) + waitFor(t, func() bool { return c.Status().GetIngressConns() == 0 }) + st := c.Status() + assert.Equal(t, int64(1), st.GetIngressConnsTotal()) + assert.Equal(t, int64(5), st.GetBytesSent()) + assert.Equal(t, int64(5), st.GetBytesReceived()) +} + +func TestIngressToAnotherPortIsReset(t *testing.T) { + r := startRelay(t) + newClient(t, r, "tok", 3000) + s := r.next(t) + s.hold(t) + + s.send(t, open(1, 22)) + reset := s.expect(t).GetReset_() + require.NotNil(t, reset) + assert.Equal(t, uint64(1), reset.GetConnId()) + assert.Equal(t, "bridge serves only port 3000", reset.GetReason()) +} + +func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { + r := startRelay(t) + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = lis.Close() }) + body := bytes.Repeat([]byte("x"), connWindow+maxPayload) + go func() { + nc, err := lis.Accept() + if err == nil { + _, _ = nc.Write(body) + } + }() + port := lis.Addr().(*net.TCPAddr).Port + newClient(t, r, "tok", port) + s := r.next(t) + s.hold(t) + s.send(t, open(1, port)) + + assert.Len(t, s.expectData(t, 1, connWindow), connWindow) + select { + case msg := <-s.recv: + if msg.GetData() != nil { + t.Fatal("sent past the window") + } + case <-time.After(200 * time.Millisecond): + } + + s.send(t, window(1, maxPayload)) + assert.Len(t, s.expectData(t, 1, maxPayload), maxPayload) +} + +func TestEgressLeavesThroughTheDeployment(t *testing.T) { + r := startRelay(t) + c := newClient(t, r, "tok", 3000) + s := r.next(t) + s.hold(t) + waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_LIVE }) + + app, intercepted := tcpPair(t) + c.AddConn(intercepted, "0.0.0.0:443", "example.com") + o := s.expect(t).GetOpen() + require.NotNil(t, o) + assert.GreaterOrEqual(t, o.GetConnId(), clientConnIDFloor) + assert.Equal(t, "example.com", o.GetHost()) + assert.Equal(t, uint32(443), o.GetPort()) + assert.Equal(t, devtunnelv1.Protocol_PROTOCOL_TCP, o.GetProtocol()) + id := o.GetConnId() + + _, err := app.Write([]byte("GET")) + require.NoError(t, err) + assert.Equal(t, "GET", string(s.expectData(t, id, 3))) + + s.send(t, data(id, "200")) + buf := make([]byte, 3) + _, err = io.ReadFull(app, buf) + require.NoError(t, err) + assert.Equal(t, "200", string(buf)) + + // The deployment couldn't reach it, or it went away. + s.send(t, &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Reset_{Reset_: &devtunnelv1.Reset{ConnId: id, Reason: "refused"}}}) + _ = app.SetReadDeadline(time.Now().Add(testTimeout)) + _, err = app.Read(buf) + assert.Error(t, err) + waitFor(t, func() bool { return c.Status().GetEgressConns() == 0 }) + assert.Equal(t, int64(1), c.Status().GetEgressConnsTotal()) +} + +func TestEgressWithNoLiveStreamIsDropped(t *testing.T) { + r := startRelay(t) + c := newClient(t, r, "tok", 3000, func(tm *timings) { tm.liveWait = 50 * time.Millisecond }) + r.next(t) // never holds the tunnel + + app, intercepted := tcpPair(t) + c.AddConn(intercepted, "0.0.0.0:443", "example.com") + _ = app.SetReadDeadline(time.Now().Add(testTimeout)) + _, err := app.Read(make([]byte, 1)) + assert.ErrorIs(t, err, io.EOF) +} + +func TestReconnectKeepsTheOldStreamsConns(t *testing.T) { + r := startRelay(t) + port := echoApp(t) + c := newClient(t, r, "tok", port) + first := r.next(t) + first.hold(t) + first.send(t, open(1, port)) + first.send(t, data(1, "a")) + assert.Equal(t, "a", string(first.expectData(t, 1, 1))) + + first.send(t, &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Reconnect{Reconnect: &devtunnelv1.Reconnect{}}}) + second := r.next(t) + assert.Equal(t, []string{"test-instance"}, second.md.Get(instanceHeader)) + second.hold(t) + + first.send(t, data(1, "b")) + assert.Equal(t, "b", string(first.expectData(t, 1, 1))) + st := c.Status() + assert.Equal(t, bridgev1.TunnelState_TUNNEL_STATE_LIVE, st.GetState()) + assert.Equal(t, int64(1), st.GetReconnects()) +} + +func TestRefusedTokenWaitsForANewOne(t *testing.T) { + r := startRelay(t) + r.refuse = func(md metadata.MD) error { + if md.Get("authorization")[0] == "Bearer old" { + return status.Error(codes.Unauthenticated, "missing or invalid dev tunnel token") + } + return nil + } + c := newClient(t, r, "old", 3000) + waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN }) + + c.SetToken("new") + s := r.next(t) + assert.Equal(t, []string{"Bearer new"}, s.md.Get("authorization")) +} + +func TestNoTokenWaitsForOne(t *testing.T) { + r := startRelay(t) + c := newClient(t, r, "", 3000) + waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN }) + select { + case <-r.streams: + t.Fatal("connected without a token") + case <-time.After(100 * time.Millisecond): + } + + c.SetToken("tok") + assert.Equal(t, []string{"Bearer tok"}, r.next(t).md.Get("authorization")) +} + +func TestFatalRefusalStops(t *testing.T) { + r := startRelay(t) + r.refuse = func(metadata.MD) error { + return status.Error(codes.PermissionDenied, "not allowed") + } + c := newClient(t, r, "tok", 3000) + + select { + case <-c.Done(): + case <-time.After(testTimeout): + t.Fatal("kept retrying") + } + assert.ErrorContains(t, c.Err(), "not allowed") + assert.Equal(t, bridgev1.TunnelState_TUNNEL_STATE_FAILED, c.Status().GetState()) +} + +func TestSilentRelayIsReplaced(t *testing.T) { + r := startRelay(t) + c := newClient(t, r, "tok", 3000, func(tm *timings) { tm.silent = 200 * time.Millisecond }) + r.next(t) // never heard from again + + r.next(t) + assert.Contains(t, c.Status().GetLastError(), errSilent.Error()) +} + +func TestRelayTarget(t *testing.T) { + target, _, err := relayTarget("https://api-dev-tunnel.example.com") + require.NoError(t, err) + assert.Equal(t, "passthrough:///api-dev-tunnel.example.com:443", target) + + target, _, err = relayTarget("http://127.0.0.1:" + strconv.Itoa(8080)) + require.NoError(t, err) + assert.Equal(t, "passthrough:///127.0.0.1:8080", target) + + for _, bad := range []string{"ftp://relay", "https://relay/v1", "https://"} { + _, _, err := relayTarget(bad) + assert.Error(t, err, bad) + } +} + +// tcpPair returns both ends of a TCP connection: the app's, and the one the +// transparent proxy would have accepted. +func tcpPair(t *testing.T) (net.Conn, net.Conn) { + t.Helper() + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer lis.Close() + accepted := make(chan net.Conn, 1) + go func() { + nc, _ := lis.Accept() + accepted <- nc + }() + app, err := net.Dial("tcp", lis.Addr().String()) + require.NoError(t, err) + t.Cleanup(func() { _ = app.Close() }) + return app, <-accepted +} diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go new file mode 100644 index 0000000..3906099 --- /dev/null +++ b/pkg/devtunnel/conn.go @@ -0,0 +1,234 @@ +package devtunnel + +import ( + "bytes" + "errors" + "io" + "log/slog" + "net" + "sync" + + devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" +) + +type direction int + +const ( + // From the deployment to the app. + ingress direction = iota + // From the app out through the deployment. + egress +) + +func (d direction) String() string { + if d == egress { + return "egress" + } + return "ingress" +} + +// conn joins one of the stream's conns to a local TCP connection. Each +// direction moves only as fast as its reader takes it: what the local side +// sends waits on the relay's window, and the relay gets window back as what it +// sent is written locally. Either side may half-close; the conn is over once +// both have, or either resets. +type conn struct { + id uint64 + s *stream + dir direction + logger *slog.Logger + + mu sync.Mutex + cond *sync.Cond + nc net.Conn + // What this side may still send the relay. + sendWindow int + // From the relay, not yet written locally; a nil entry half-closes. + pending [][]byte + closeSent, closeQueued bool + writeClosed, over bool +} + +// attach starts moving bytes once the local connection is up. +func (c *conn) attach(nc net.Conn) { + c.mu.Lock() + if c.over { + c.mu.Unlock() + _ = nc.Close() + return + } + c.nc = nc + c.mu.Unlock() + go c.readLocal() + go c.writeLocal() +} + +// readLocal sends what the local side sends, as the window allows. +func (c *conn) readLocal() { + buf := make([]byte, maxPayload) + for { + c.mu.Lock() + for c.sendWindow == 0 && !c.over { + c.cond.Wait() + } + if c.over { + c.mu.Unlock() + return + } + size := min(c.sendWindow, maxPayload) + c.mu.Unlock() + + n, err := c.nc.Read(buf[:size]) + if n > 0 { + c.mu.Lock() + if c.over { + c.mu.Unlock() + return + } + c.sendWindow -= n + c.mu.Unlock() + c.s.c.bytesSent.Add(int64(n)) + c.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Data{Data: &devtunnelv1.Data{ + ConnId: c.id, + Payload: bytes.Clone(buf[:n]), + }}}) + } + if errors.Is(err, io.EOF) { + c.sendClose() + return + } + if err != nil { + c.reset(err.Error()) + return + } + } +} + +// writeLocal writes what the relay sent, granting it back as it goes. +func (c *conn) writeLocal() { + for { + c.mu.Lock() + for len(c.pending) == 0 && !c.over { + c.cond.Wait() + } + if c.over { + c.mu.Unlock() + return + } + payload := c.pending[0] + c.pending[0] = nil + c.pending = c.pending[1:] + c.mu.Unlock() + + if payload == nil { + if cw, ok := c.nc.(interface{ CloseWrite() error }); ok { + _ = cw.CloseWrite() + } + c.mu.Lock() + c.writeClosed = true + done := c.closeSent + c.mu.Unlock() + if done { + c.end() + } + return + } + if _, err := c.nc.Write(payload); err != nil { + c.reset(err.Error()) + return + } + c.mu.Lock() + over := c.over + c.mu.Unlock() + if !over { + c.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Window{Window: &devtunnelv1.Window{ + ConnId: c.id, + Bytes: uint32(len(payload)), + }}}) + } + } +} + +// received queues payload from the relay. +func (c *conn) received(payload []byte) { + if len(payload) == 0 { + return + } + c.mu.Lock() + defer c.mu.Unlock() + if c.over || c.closeQueued { + return + } + c.pending = append(c.pending, payload) + c.cond.Broadcast() +} + +// closeReceived: the relay's side has nothing more to send. +func (c *conn) closeReceived() { + c.mu.Lock() + defer c.mu.Unlock() + if c.over || c.closeQueued { + return + } + c.closeQueued = true + c.pending = append(c.pending, nil) + c.cond.Broadcast() +} + +// grant: the relay may take this many more bytes. +func (c *conn) grant(bytes int) { + c.mu.Lock() + defer c.mu.Unlock() + c.sendWindow += bytes + c.cond.Broadcast() +} + +// sendClose: the local side has nothing more to send. +func (c *conn) sendClose() { + c.mu.Lock() + if c.over || c.closeSent { + c.mu.Unlock() + return + } + c.closeSent = true + done := c.writeClosed + c.mu.Unlock() + c.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Close{Close: &devtunnelv1.Close{ConnId: c.id}}}) + if done { + c.end() + } +} + +// reset abandons the conn and tells the relay why. +func (c *conn) reset(reason string) { + c.mu.Lock() + over := c.over + c.mu.Unlock() + if over { + return + } + c.s.send(resetFrame(c.id, reason)) + c.abandon() +} + +// abandon drops the conn in both directions without telling the relay, which +// already knows. +func (c *conn) abandon() { + c.end() +} + +func (c *conn) end() { + c.mu.Lock() + if c.over { + c.mu.Unlock() + return + } + c.over = true + nc := c.nc + c.cond.Broadcast() + c.mu.Unlock() + if nc != nil { + _ = nc.Close() + } + c.s.forget(c) +} diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go new file mode 100644 index 0000000..902fd32 --- /dev/null +++ b/pkg/devtunnel/stream.go @@ -0,0 +1,275 @@ +package devtunnel + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "net" + "strconv" + "sync" + "sync/atomic" + "time" + + devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "google.golang.org/grpc/metadata" +) + +var heartbeat = &devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Heartbeat{Heartbeat: &devtunnelv1.Heartbeat{}}} + +// stream is one Connect call and the conns it carries. The relay ends every +// conn with the stream. +type stream struct { + c *Client + logger *slog.Logger + ctx context.Context + cancel context.CancelCauseFunc + out outbox + + // Set once the relay routes the tunnel's conns to this stream. + held atomic.Bool + lastHeard atomic.Int64 + + mu sync.Mutex + conns map[uint64]*conn + closed bool +} + +func (c *Client) newStream() *stream { + ctx, cancel := context.WithCancelCause(c.ctx) + return &stream{ + c: c, + logger: c.logger, + ctx: ctx, + cancel: cancel, + out: outbox{wake: make(chan struct{}, 1)}, + conns: map[uint64]*conn{}, + } +} + +// run holds the stream until the relay ends it. moving is called if the +// relay says to reconnect elsewhere; the stream keeps its conns until then. +func (s *stream) run(token string, moving func()) error { + defer s.cancel(nil) + ctx := metadata.AppendToOutgoingContext(s.ctx, + "authorization", "Bearer "+token, + instanceHeader, s.c.instance, + ) + call, err := s.c.rpc.Connect(ctx) + if err != nil { + return err + } + defer s.c.lost(s) + defer s.abandonAll() + + s.lastHeard.Store(time.Now().UnixNano()) + // The call starts with its first message. + s.send(heartbeat) + go func() { + for { + frame, ok := s.out.pop(s.ctx) + if !ok { + return + } + // A failed send ends the call, which Recv reports. + if err := call.Send(frame); err != nil { + return + } + } + }() + go s.heartbeats() + + for { + msg, err := call.Recv() + if err != nil { + if cause := context.Cause(s.ctx); errors.Is(cause, errSilent) { + return cause + } + if errors.Is(err, io.EOF) { + return nil + } + return err + } + s.lastHeard.Store(time.Now().UnixNano()) + s.handle(msg, moving) + } +} + +func (s *stream) heartbeats() { + ticker := time.NewTicker(s.c.timings.heartbeat) + defer ticker.Stop() + for { + select { + case <-s.ctx.Done(): + return + case <-ticker.C: + if time.Since(time.Unix(0, s.lastHeard.Load())) > s.c.timings.silent { + s.logger.Info("Relay went silent, reconnecting") + s.cancel(errSilent) + return + } + s.send(heartbeat) + } + } +} + +func (s *stream) handle(msg *devtunnelv1.ConnectResponse, moving func()) { + switch f := msg.GetFrame().(type) { + case *devtunnelv1.ConnectResponse_Heartbeat: + if s.held.CompareAndSwap(false, true) { + s.c.holding(s) + } + case *devtunnelv1.ConnectResponse_Reconnect: + s.c.moving(s) + moving() + case *devtunnelv1.ConnectResponse_Open: + s.openIngress(f.Open) + case *devtunnelv1.ConnectResponse_Data: + if cn := s.conn(f.Data.GetConnId()); cn != nil { + s.c.bytesReceived.Add(int64(len(f.Data.GetPayload()))) + cn.received(f.Data.GetPayload()) + } + case *devtunnelv1.ConnectResponse_Close: + if cn := s.conn(f.Close.GetConnId()); cn != nil { + cn.closeReceived() + } + case *devtunnelv1.ConnectResponse_Reset_: + if cn := s.conn(f.Reset_.GetConnId()); cn != nil { + cn.logger.Debug("Relay reset the connection", "reason", f.Reset_.GetReason()) + cn.abandon() + } + case *devtunnelv1.ConnectResponse_Window: + if cn := s.conn(f.Window.GetConnId()); cn != nil { + cn.grant(int(f.Window.GetBytes())) + } + } +} + +// openIngress connects one of the deployment's conns to the app. +func (s *stream) openIngress(open *devtunnelv1.Open) { + id := open.GetConnId() + port := s.c.cfg.Port + if open.GetProtocol() != devtunnelv1.Protocol_PROTOCOL_TCP || int(open.GetPort()) != port { + s.send(resetFrame(id, fmt.Sprintf("bridge serves only port %d", port))) + return + } + cn := s.newConn(id, ingress) + if cn == nil { + return + } + go func() { + nc, err := s.c.cfg.AppDialer.DialContext(s.ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) + if err != nil { + cn.logger.Debug("App refused the connection", "error", err) + cn.reset(err.Error()) + return + } + cn.attach(nc) + }() +} + +// newConn registers a conn, or returns nil once the stream has ended. +func (s *stream) newConn(id uint64, dir direction) *conn { + s.mu.Lock() + defer s.mu.Unlock() + if s.closed { + return nil + } + cn := &conn{ + id: id, + s: s, + dir: dir, + logger: s.logger.With("conn_id", id, "direction", dir.String()), + sendWindow: connWindow, + } + cn.cond = sync.NewCond(&cn.mu) + s.conns[id] = cn + s.c.counters(dir, 1) + return cn +} + +func (s *stream) conn(id uint64) *conn { + s.mu.Lock() + defer s.mu.Unlock() + return s.conns[id] +} + +func (s *stream) forget(cn *conn) { + s.mu.Lock() + defer s.mu.Unlock() + if s.conns[cn.id] == cn { + delete(s.conns, cn.id) + s.c.counters(cn.dir, -1) + } +} + +func (s *stream) abandonAll() { + s.mu.Lock() + s.closed = true + conns := make([]*conn, 0, len(s.conns)) + for _, cn := range s.conns { + conns = append(conns, cn) + } + s.mu.Unlock() + for _, cn := range conns { + cn.abandon() + } +} + +func (s *stream) send(frame *devtunnelv1.ConnectRequest) { + s.out.push(frame) +} + +// counters tracks conns opening (delta 1) and ending (delta -1). +func (c *Client) counters(dir direction, delta int64) { + open, total := &c.ingressConns, &c.ingressTotal + if dir == egress { + open, total = &c.egressConns, &c.egressTotal + } + open.Add(delta) + if delta > 0 { + total.Add(delta) + } +} + +// outbox queues frames for the relay in order. Data is bounded by each conn's +// window; the rest is small. +type outbox struct { + mu sync.Mutex + frames []*devtunnelv1.ConnectRequest + wake chan struct{} +} + +func (o *outbox) push(frame *devtunnelv1.ConnectRequest) { + o.mu.Lock() + o.frames = append(o.frames, frame) + o.mu.Unlock() + select { + case o.wake <- struct{}{}: + default: + } +} + +func (o *outbox) pop(ctx context.Context) (*devtunnelv1.ConnectRequest, bool) { + for { + o.mu.Lock() + if len(o.frames) > 0 { + frame := o.frames[0] + o.frames[0] = nil + o.frames = o.frames[1:] + o.mu.Unlock() + return frame, true + } + o.mu.Unlock() + select { + case <-o.wake: + case <-ctx.Done(): + return nil, false + } + } +} + +func resetFrame(id uint64, reason string) *devtunnelv1.ConnectRequest { + return &devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Reset_{Reset_: &devtunnelv1.Reset{ConnId: id, Reason: reason}}} +} diff --git a/proto/bridge/v1/interceptor.proto b/proto/bridge/v1/interceptor.proto index bccc13c..92c0a99 100644 --- a/proto/bridge/v1/interceptor.proto +++ b/proto/bridge/v1/interceptor.proto @@ -19,6 +19,12 @@ service InterceptorService { // immediate check passed. Useful as a "is the app healthy right now?" // primitive that ignores any stale state from a previous dev session. rpc Probe(ProbeRequest) returns (ProbeResponse); + + // SetTunnelToken gives an intercept on a dev tunnel + // (--tunnel-protocol=dev-tunnel) its client token, from create-session. One + // started without a token, or whose token the relay refused + // (TUNNEL_STATE_NEEDS_TOKEN), connects with it. + rpc SetTunnelToken(SetTunnelTokenRequest) returns (SetTunnelTokenResponse); } message GetStatusRequest {} @@ -39,8 +45,63 @@ message GetStatusResponse { // Startup probe result against the local app. Unset if no startup // probe was configured on the source. ProbeStatus startup = 4; + + // The dev tunnel's stream to its relay. Unset unless + // --tunnel-protocol=dev-tunnel. + TunnelStatus tunnel = 5; } +// TunnelStatus is the state of a dev tunnel's stream to api-dev-tunnel, and +// the conns it carries. +message TunnelStatus { + TunnelState state = 1; + + // Why the last stream ended, or the relay refused one. Empty before any + // has. + string last_error = 2 [json_name = "last_error"]; + + // Unix seconds since a stream has held the tunnel. Zero unless LIVE. + int64 live_since_unix_seconds = 3 [json_name = "live_since_unix_seconds"]; + + // Streams opened after the first. + int64 reconnects = 4; + + // Conns open now from the deployment to the app (ingress), and from the + // app out through the deployment (egress). + int64 ingress_conns = 5 [json_name = "ingress_conns"]; + int64 egress_conns = 6 [json_name = "egress_conns"]; + + // Conns opened since the intercept started. + int64 ingress_conns_total = 7 [json_name = "ingress_conns_total"]; + int64 egress_conns_total = 8 [json_name = "egress_conns_total"]; + + // Payload bytes sent to and received from the relay. + int64 bytes_sent = 9 [json_name = "bytes_sent"]; + int64 bytes_received = 10 [json_name = "bytes_received"]; +} + +enum TunnelState { + TUNNEL_STATE_UNSPECIFIED = 0; + // No stream holds the tunnel yet, or the last one ended and the next is + // opening. + TUNNEL_STATE_CONNECTING = 1; + // A stream holds the tunnel: the deployment's conns reach the app. + TUNNEL_STATE_LIVE = 2; + // The relay refused the client token, or there is none yet. Waiting for + // SetTunnelToken. + TUNNEL_STATE_NEEDS_TOKEN = 3; + // Another client holds the tunnel. Retrying until it lets go. + TUNNEL_STATE_TAKEN = 4; + // The relay refused the tunnel for good; last_error says why. + TUNNEL_STATE_FAILED = 5; +} + +message SetTunnelTokenRequest { + string token = 1; +} + +message SetTunnelTokenResponse {} + // ProbeStatus reflects the current health view from a single configured // probe — what the source deployment's probe would have reported if it had // been run against the developer's local app. diff --git a/proto/vercel/api/dev_tunnel/v1/tunnel.proto b/proto/vercel/api/dev_tunnel/v1/tunnel.proto new file mode 100644 index 0000000..67252cd --- /dev/null +++ b/proto/vercel/api/dev_tunnel/v1/tunnel.proto @@ -0,0 +1,144 @@ +// Vendored from vercel/api, services/api-dev-tunnel/proto/tunnel/v1/tunnel.proto. +// Edit it there and copy it here. + +syntax = "proto3"; + +package vercel.api.dev_tunnel.v1; + +// Relays TCP connections between the tunnel's deployment and the developer's +// machine. Payloads are opaque bytes; nothing here interprets them. +service TunnelService { + // The stream `vc dev --tunnel` holds open, carrying every conn the + // deployment dials and every conn the client opens through the deployment. + // Authenticated with the client token + // (`authorization: Bearer`), which the next create-session replaces: a CLI + // refused with UNAUTHENTICATED on a reconnect asks for a new one. + // + // One client per tunnel: another stream fails with ALREADY_EXISTS, unless it + // carries the same `x-vercel-dev-tunnel-instance` (1 to 64 letters, digits + // or dashes, one per CLI process). Then it replaces the old stream, which + // ends with ABORTED, so a CLI can reconnect while its old stream is still + // open on the service. The CLI sends a Heartbeat at least every 30s; a + // stream silent for three ends with DEADLINE_EXCEEDED, and one ending with + // UNAVAILABLE means reconnect. A CLI that stops reading the stream while + // the service has frames for it has it end with RESOURCE_EXHAUSTED. + // + // A pod shutting down sends Reconnect and refuses new streams with + // UNAVAILABLE. Once the CLI's new stream holds the tunnel elsewhere, the old + // one gets no new conns and ends with ABORTED when its open ones are done. + rpc Connect(stream ConnectRequest) returns (stream ConnectResponse); + + // One conn from the deployment: a TCP connection to a port on the + // developer's machine. The first message is the target; the rest carry + // payload. Ending the request stream half-closes the conn, and `end` says the + // developer's side has; the call ends once both have, and fails if either + // side resets. With no client connected, it waits up to 5s for one, then + // fails with UNAVAILABLE. A tunnel carries at most 256 conns at once; more + // wait up to 30s for one to end, then fail with RESOURCE_EXHAUSTED, as do + // any past 256 waiting. Authenticated with the server token + // (`authorization: Bearer`). + rpc Dial(stream DialRequest) returns (stream DialResponse); +} + +enum Protocol { + PROTOCOL_UNSPECIFIED = 0; + PROTOCOL_TCP = 1; +} + +// CLI to service. +message ConnectRequest { + oneof frame { + Data data = 1; + Close close = 2; + Reset reset = 3; + Heartbeat heartbeat = 4; + Window window = 5; + Open open = 6; + } +} + +// Service to CLI. +message ConnectResponse { + oneof frame { + Open open = 1; + Data data = 2; + Close close = 3; + Reset reset = 4; + Heartbeat heartbeat = 5; + Window window = 6; + Reconnect reconnect = 7; + } +} + +// A new conn. From the service, the CLI connects to `port` on the developer's +// machine, or resets the conn if it doesn't serve that port. From the client, +// the deployment connects to `host` and `port`, resolving `host` as its own +// code would, and the service resets the conn if it can't. The service numbers +// its conns from 1; a client numbers its own from 2^63, so the two never meet. +message Open { + uint64 conn_id = 1; + Protocol protocol = 2; + uint32 port = 3; + // A hostname or IP address. Only a client's Open has one. + string host = 4; +} + +// No more than the conn's window allows (see Window). +message Data { + uint64 conn_id = 1; + bytes payload = 2; +} + +// The sender has nothing more to send (TCP half-close). A conn is over once +// both sides have closed, or either has reset. +message Close { + uint64 conn_id = 1; +} + +// Abandons the conn in both directions, e.g. connection refused. +message Reset { + uint64 conn_id = 1; + string reason = 2; +} + +// Lets the peer send `bytes` more of a conn's Data, as the sender of this +// consumes what it received. Each side starts each conn able to send 256 KiB. +// A client that sends past its window, or grants back more than it was sent, +// has the conn reset. +message Window { + uint64 conn_id = 1; + uint32 bytes = 2; +} + +// Sent both ways every 30s, inside the ALB's idle timeout. The service drops a +// client it hasn't heard from in three. The service also sends one as soon as +// a Connect stream holds the tunnel, so the CLI knows it's live. +message Heartbeat {} + +// This pod is shutting down. The CLI opens a new stream now, and keeps this one +// until the service ends it: it still carries the conns already open on it. +message Reconnect {} + +message DialRequest { + oneof frame { + // First message only. + DialTarget target = 1; + bytes payload = 2; + } +} + +message DialTarget { + Protocol protocol = 1; + uint32 port = 2; +} + +message DialResponse { + oneof frame { + bytes payload = 1; + // The developer's side has nothing more to send. Sent once; more payload + // may still go the other way. + DialEnd end = 2; + } +} + +message DialEnd {} From 5da93b2963b56ade5b4d6837da4e48f7796ad1f6 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 12:51:56 -0700 Subject: [PATCH 02/11] refactor(devtunnel): keep a stream's conns in an xsync map Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/devtunnel/stream.go | 56 +++++++++++++++++++++++------------------ 1 file changed, 32 insertions(+), 24 deletions(-) diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index 902fd32..e97021e 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -12,6 +12,7 @@ import ( "sync/atomic" "time" + "github.com/puzpuzpuz/xsync/v3" devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "google.golang.org/grpc/metadata" ) @@ -31,9 +32,8 @@ type stream struct { held atomic.Bool lastHeard atomic.Int64 - mu sync.Mutex - conns map[uint64]*conn - closed bool + conns *xsync.MapOf[uint64, *conn] + closed atomic.Bool } func (c *Client) newStream() *stream { @@ -44,7 +44,7 @@ func (c *Client) newStream() *stream { ctx: ctx, cancel: cancel, out: outbox{wake: make(chan struct{}, 1)}, - conns: map[uint64]*conn{}, + conns: xsync.NewMapOf[uint64, *conn](), } } @@ -171,9 +171,7 @@ func (s *stream) openIngress(open *devtunnelv1.Open) { // newConn registers a conn, or returns nil once the stream has ended. func (s *stream) newConn(id uint64, dir direction) *conn { - s.mu.Lock() - defer s.mu.Unlock() - if s.closed { + if s.closed.Load() { return nil } cn := &conn{ @@ -184,37 +182,47 @@ func (s *stream) newConn(id uint64, dir direction) *conn { sendWindow: connWindow, } cn.cond = sync.NewCond(&cn.mu) - s.conns[id] = cn + // Counted before it's stored, so whichever of forget's callers removes it + // uncounts it. s.c.counters(dir, 1) + s.conns.Store(id, cn) + // abandonAll may have swept the map before the Store. + if s.closed.Load() { + s.forget(cn) + return nil + } return cn } func (s *stream) conn(id uint64) *conn { - s.mu.Lock() - defer s.mu.Unlock() - return s.conns[id] + cn, _ := s.conns.Load(id) + return cn } +// forget removes cn, unless another conn has since taken its id. func (s *stream) forget(cn *conn) { - s.mu.Lock() - defer s.mu.Unlock() - if s.conns[cn.id] == cn { - delete(s.conns, cn.id) + removed := false + s.conns.Compute(cn.id, func(cur *conn, loaded bool) (*conn, bool) { + if !loaded { + return nil, true + } + if cur != cn { + return cur, false + } + removed = true + return nil, true + }) + if removed { s.c.counters(cn.dir, -1) } } func (s *stream) abandonAll() { - s.mu.Lock() - s.closed = true - conns := make([]*conn, 0, len(s.conns)) - for _, cn := range s.conns { - conns = append(conns, cn) - } - s.mu.Unlock() - for _, cn := range conns { + s.closed.Store(true) + s.conns.Range(func(_ uint64, cn *conn) bool { cn.abandon() - } + return true + }) } func (s *stream) send(frame *devtunnelv1.ConnectRequest) { From 6bc42ba4cee6750775cc7d5ff549378bec2388d6 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 13:50:57 -0700 Subject: [PATCH 03/11] refactor(tunnel): run a tunnel over a backend, and make the dev tunnel one pkg/tunnel keeps the conns and pumps each one: its side on the transport is a net.Conn, copied to and from the local conn through two buffers from a pool that go back when the conn ends. A Backend carries the frames: the bridge TunnelNetwork stream, or a dev tunnel's Connect stream. Half-close and per-conn windows apply where the backend has them. pkg/devtunnel shrinks to that backend: frames, heartbeats, reconnects, the client token and status. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/commands/intercept.go | 8 +- pkg/commands/intercept_server.go | 4 +- pkg/commands/intercept_server_test.go | 1 - pkg/devtunnel/backend.go | 565 ++++++++++++++++++ .../{client_test.go => backend_test.go} | 54 +- pkg/devtunnel/client.go | 543 ----------------- pkg/devtunnel/conn.go | 234 -------- pkg/devtunnel/stream.go | 178 +++--- pkg/tunnel/backend.go | 74 +++ pkg/tunnel/conn.go | 270 +++++++++ pkg/tunnel/pump_test.go | 144 +++++ pkg/tunnel/stream.go | 136 +++++ pkg/tunnel/tunnel.go | 377 ++++++------ 13 files changed, 1471 insertions(+), 1117 deletions(-) create mode 100644 pkg/devtunnel/backend.go rename pkg/devtunnel/{client_test.go => backend_test.go} (89%) delete mode 100644 pkg/devtunnel/client.go delete mode 100644 pkg/devtunnel/conn.go create mode 100644 pkg/tunnel/backend.go create mode 100644 pkg/tunnel/conn.go create mode 100644 pkg/tunnel/pump_test.go create mode 100644 pkg/tunnel/stream.go diff --git a/pkg/commands/intercept.go b/pkg/commands/intercept.go index b25d802..55a6c94 100644 --- a/pkg/commands/intercept.go +++ b/pkg/commands/intercept.go @@ -188,7 +188,7 @@ func doIntercept(ctx context.Context, c *cli.Command) error { dnsResolver hostResolver metadata *bridgev1.GetMetadataResponse mountServers []*fuseServer - devTunnel *devtunnel.Client + devTunnel *devtunnel.Backend ) if tunnelProtocol == tunnelProtocolDevTunnel { if len(mountPaths) > 0 { @@ -206,9 +206,9 @@ func doIntercept(ctx context.Context, c *cli.Command) error { if err != nil { return err } - devTunnel.Start(ctx) - defer devTunnel.Close() - tun = devTunnel + tun = tunnel.NewWithBackend(plumbing.NewStaticPortDialer(appPort, nil), devTunnel) + tun.Start(ctx) + defer tun.Close() dnsResolver = deploymentResolver{} logger.Info("Dev tunnel starting", "app_port", appPort) } else { diff --git a/pkg/commands/intercept_server.go b/pkg/commands/intercept_server.go index bac3b55..fca9a66 100644 --- a/pkg/commands/intercept_server.go +++ b/pkg/commands/intercept_server.go @@ -35,7 +35,7 @@ type interceptServer struct { ready atomic.Bool monitors atomic.Pointer[monitorSet] - tunnel atomic.Pointer[devtunnel.Client] + tunnel atomic.Pointer[devtunnel.Backend] } // monitorSet is the immutable bundle stored in interceptServer.monitors. @@ -80,7 +80,7 @@ func (s *interceptServer) SetMonitors(liveness, readiness, startup probe.Monitor // SetTunnel publishes the dev tunnel GetStatus reports on and SetTunnelToken // hands tokens to. -func (s *interceptServer) SetTunnel(t *devtunnel.Client) { +func (s *interceptServer) SetTunnel(t *devtunnel.Backend) { s.tunnel.Store(t) } diff --git a/pkg/commands/intercept_server_test.go b/pkg/commands/intercept_server_test.go index 71be281..c4f4c91 100644 --- a/pkg/commands/intercept_server_test.go +++ b/pkg/commands/intercept_server_test.go @@ -144,7 +144,6 @@ func TestInterceptServer_ReportsAndFeedsTheDevTunnel(t *testing.T) { tun, err := devtunnel.New(devtunnel.Config{RelayURL: "http://127.0.0.1:1", Port: 3000}) require.NoError(t, err) - t.Cleanup(func() { _ = tun.Close() }) srv.SetTunnel(tun) resp, err = srv.GetStatus(ctx, &bridgev1.GetStatusRequest{}) diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go new file mode 100644 index 0000000..669703b --- /dev/null +++ b/pkg/devtunnel/backend.go @@ -0,0 +1,565 @@ +// Package devtunnel is a tunnel.Backend over a dev tunnel: the Connect stream +// to api-dev-tunnel that `vc dev --tunnel` holds otherwise. The deployment's +// conns reach the app, and the app's outbound conns leave through the +// deployment. +package devtunnel + +import ( + "context" + "crypto/tls" + "errors" + "fmt" + "log/slog" + "net" + "net/url" + "regexp" + "strconv" + "strings" + "sync" + "sync/atomic" + "time" + + "github.com/puzpuzpuz/xsync/v3" + "github.com/segmentio/ksuid" + bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/plumbing" + "github.com/vercel/bridge/pkg/tunnel" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" +) + +const ( + // What each side of a conn may send before the other grants more. + connWindow = 256 << 10 + // A client numbers its conns from here; the relay numbers its own from 1. + clientConnIDFloor = uint64(1) << 63 + + instanceHeader = "x-vercel-dev-tunnel-instance" + + retryMin = 250 * time.Millisecond + retryMax = 5 * time.Second +) + +var instancePattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + +// Codes reconnecting can't fix. +var fatal = map[codes.Code]bool{ + codes.FailedPrecondition: true, + codes.InvalidArgument: true, + codes.PermissionDenied: true, + codes.Unimplemented: true, +} + +// Config configures a Backend. +type Config struct { + // RelayURL is api-dev-tunnel's URL, from create-session. + RelayURL string + // Token is the client token, from create-session. Without one the + // backend waits for SetToken. + Token string + // Instance tells the relay this process's reconnects from another + // client's, so a restart can take the tunnel back while its old stream is + // still open. Random if empty. + Instance string + // Port is the only port of the app the deployment's conns may reach. + Port int + // RelayDialer connects to the relay. Defaults to a net.Dialer. + RelayDialer plumbing.ContextDialer + Logger *slog.Logger +} + +// timings are the relay's protocol intervals, shortened by tests. +type timings struct { + // The relay needs a heartbeat at least every 30s. + heartbeat time.Duration + // The relay heartbeats every 30s; three missed and the stream is dead. + silent time.Duration + // The relay holds the deployment's conns this long for a client, and an + // outbound conn waits as long for a stream. + liveWait time.Duration +} + +var defaultTimings = timings{ + heartbeat: 15 * time.Second, + silent: 90 * time.Second, + liveWait: 5 * time.Second, +} + +// Backend holds a dev tunnel's stream to its relay, replacing it whenever it +// drops. +type Backend struct { + cfg Config + timings timings + logger *slog.Logger + instance string + target string + creds credentials.TransportCredentials + + // Set by Run, before any stream. + rpc devtunnelv1.TunnelServiceClient + deliver func(tunnel.Frame) + ctx context.Context + + // Each open conn's stream, by the tunnel's id for it. + routes *xsync.MapOf[string, route] + + mu sync.Mutex + token string + tokenSet chan struct{} + // The newest stream holding the tunnel, which new outbound conns take. + live *stream + liveCh chan struct{} + liveClosed bool + // Streams holding the tunnel, including one finishing its conns after the + // relay said to reconnect. + held int + state bridgev1.TunnelState + lastErr string + liveSince time.Time + reconnects int64 + + ingressConns, egressConns, ingressTotal, egressTotal atomic.Int64 + bytesSent, bytesReceived atomic.Int64 + nextEgress, nextStream atomic.Uint64 +} + +// route is where one of the tunnel's conns lives on the relay's protocol. +type route struct { + s *stream + // The conn's id on s. + wire uint64 + dir direction +} + +type direction int + +const ( + // From the deployment to the app. + ingress direction = iota + // From the app out through the deployment. + egress +) + +var _ tunnel.Backend = (*Backend)(nil) + +// New returns a Backend for cfg. Nothing connects until Run. +func New(cfg Config) (*Backend, error) { + target, creds, err := relayTarget(cfg.RelayURL) + if err != nil { + return nil, err + } + instance := cfg.Instance + if instance == "" { + instance = "bridge-" + ksuid.New().String() + } + if !instancePattern.MatchString(instance) { + return nil, fmt.Errorf("tunnel instance must be 1 to 64 letters, digits or dashes: %q", instance) + } + if cfg.RelayDialer == nil { + cfg.RelayDialer = &net.Dialer{Timeout: 10 * time.Second} + } + logger := cfg.Logger + if logger == nil { + logger = slog.Default() + } + return &Backend{ + cfg: cfg, + timings: defaultTimings, + logger: logger.With("component", "dev-tunnel", "relay", cfg.RelayURL, "instance", instance), + instance: instance, + target: target, + creds: creds, + routes: xsync.NewMapOf[string, route](), + token: cfg.Token, + tokenSet: make(chan struct{}), + liveCh: make(chan struct{}), + state: bridgev1.TunnelState_TUNNEL_STATE_CONNECTING, + }, nil +} + +// relayTarget turns the relay's URL into a gRPC target. Run dials it with +// RelayDialer rather than resolving it through gRPC, so an intercept whose DNS +// forwards every domain still reaches the relay directly. +func relayTarget(relayURL string) (string, credentials.TransportCredentials, error) { + u, err := url.Parse(relayURL) + if err != nil { + return "", nil, fmt.Errorf("invalid relay URL %q: %w", relayURL, err) + } + if u.Path != "" && u.Path != "/" { + return "", nil, fmt.Errorf("relay URL %q must have no path", relayURL) + } + var creds credentials.TransportCredentials + port := u.Port() + switch u.Scheme { + case "https": + creds = credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) + if port == "" { + port = "443" + } + case "http": + creds = insecure.NewCredentials() + if port == "" { + port = "80" + } + default: + return "", nil, fmt.Errorf("relay URL %q must be http or https", relayURL) + } + if u.Hostname() == "" { + return "", nil, fmt.Errorf("relay URL %q has no host", relayURL) + } + return "passthrough:///" + net.JoinHostPort(u.Hostname(), port), creds, nil +} + +// Run holds the tunnel until ctx ends, replacing its stream whenever it +// drops. It returns an error only if the relay refuses the tunnel for good. +func (b *Backend) Run(ctx context.Context, deliver func(tunnel.Frame)) error { + relayDialer := b.cfg.RelayDialer + cc, err := grpc.NewClient(b.target, + grpc.WithTransportCredentials(b.creds), + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return relayDialer.DialContext(ctx, "tcp", addr) + }), + // Notices a dead connection, e.g. after sleep, sooner than the + // relay's heartbeats would. + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 10 * time.Second, + Timeout: 20 * time.Second, + PermitWithoutStream: true, + }), + ) + if err != nil { + return fmt.Errorf("creating relay client: %w", err) + } + defer cc.Close() + b.rpc = devtunnelv1.NewTunnelServiceClient(cc) + b.deliver = deliver + b.ctx = ctx + + err = b.run() + if err != nil { + b.mu.Lock() + b.state = bridgev1.TunnelState_TUNNEL_STATE_FAILED + b.lastErr = err.Error() + b.mu.Unlock() + b.logger.Error("Tunnel stopped", "error", err) + } + return err +} + +// ConnID numbers an outbound conn. Open gives it its id on the stream. +func (b *Backend) ConnID(*tunnel.ConnInfo) string { + return "e" + strconv.FormatUint(b.nextEgress.Add(1), 10) +} + +// Open sends an outbound conn out through the deployment, to its hostname if +// the intercept's DNS gave it one, or else to the address it was headed for. +// The deployment resolves the hostname as its own code would. It waits a +// while for a stream to hold the tunnel. +func (b *Backend) Open(ctx context.Context, id string, conn *tunnel.ConnInfo) error { + n, err := strconv.ParseUint(strings.TrimPrefix(id, "e"), 10, 64) + if err != nil { + return fmt.Errorf("not an outbound conn id: %q", id) + } + s := b.awaitLive(ctx) + if s == nil { + return errors.New("no stream holds the tunnel") + } + wire := clientConnIDFloor | n + if !s.track(id, route{s: s, wire: wire, dir: egress}) { + return errors.New("the tunnel's stream ended") + } + host := conn.Hostname + if host == "" { + host = conn.Dest.Host + } + s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Open{Open: &devtunnelv1.Open{ + ConnId: wire, + Protocol: devtunnelv1.Protocol_PROTOCOL_TCP, + Host: host, + Port: uint32(conn.Dest.Port), + }}}) + return nil +} + +func (b *Backend) Send(f tunnel.Frame) { + r, ok := b.routes.Load(f.ConnID) + if !ok { + return + } + switch f.Kind { + case tunnel.FrameData: + b.bytesSent.Add(int64(len(f.Payload))) + r.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Data{Data: &devtunnelv1.Data{ConnId: r.wire, Payload: f.Payload}}}) + case tunnel.FrameCloseWrite: + r.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Close{Close: &devtunnelv1.Close{ConnId: r.wire}}}) + case tunnel.FrameReset: + r.s.send(resetFrame(r.wire, f.Reason)) + b.untrack(f.ConnID, r.s) + case tunnel.FrameWindow: + r.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Window{Window: &devtunnelv1.Window{ConnId: r.wire, Bytes: uint32(f.Bytes)}}}) + } +} + +func (b *Backend) Release(id string) { b.untrack(id, nil) } +func (b *Backend) Window() int { return connWindow } +func (b *Backend) HalfClose() bool { return true } + +// untrack forgets id's route if it's on s, or on any stream for nil. Reports +// whether it did. +func (b *Backend) untrack(id string, s *stream) bool { + var gone route + removed := false + b.routes.Compute(id, func(cur route, loaded bool) (route, bool) { + if !loaded { + return cur, true + } + if s != nil && cur.s != s { + return cur, false + } + gone, removed = cur, true + return cur, true + }) + if removed { + b.counters(gone.dir, -1) + } + return removed +} + +// counters tracks conns opening (delta 1) and ending (delta -1). +func (b *Backend) counters(dir direction, delta int64) { + open, total := &b.ingressConns, &b.ingressTotal + if dir == egress { + open, total = &b.egressConns, &b.egressTotal + } + open.Add(delta) + if delta > 0 { + total.Add(delta) + } +} + +// SetToken replaces the client token. A backend waiting for one connects with +// it; a live one uses it on its next stream. +func (b *Backend) SetToken(token string) { + b.mu.Lock() + defer b.mu.Unlock() + b.token = token + close(b.tokenSet) + b.tokenSet = make(chan struct{}) +} + +// Status reports the tunnel's state and traffic. +func (b *Backend) Status() *bridgev1.TunnelStatus { + b.mu.Lock() + defer b.mu.Unlock() + st := &bridgev1.TunnelStatus{ + State: b.state, + LastError: b.lastErr, + Reconnects: b.reconnects, + IngressConns: b.ingressConns.Load(), + EgressConns: b.egressConns.Load(), + IngressConnsTotal: b.ingressTotal.Load(), + EgressConnsTotal: b.egressTotal.Load(), + BytesSent: b.bytesSent.Load(), + BytesReceived: b.bytesReceived.Load(), + } + if !b.liveSince.IsZero() { + st.LiveSinceUnixSeconds = b.liveSince.Unix() + } + return st +} + +// awaitLive returns the stream new outbound conns take, waiting a while for +// one, or nil. +func (b *Backend) awaitLive(ctx context.Context) *stream { + timer := time.NewTimer(b.timings.liveWait) + defer timer.Stop() + for { + b.mu.Lock() + s, ch := b.live, b.liveCh + b.mu.Unlock() + if s != nil { + return s + } + select { + case <-ch: + case <-timer.C: + return nil + case <-ctx.Done(): + return nil + } + } +} + +// run replaces the tunnel's stream whenever it ends, for as long as it takes: +// the relay's pods roll often. A stream the relay says to move off keeps its +// open conns while the next one opens. +func (b *Backend) run() error { + delay := retryMin + first := true + warnedTaken := false + for { + token, ok := b.awaitToken() + if !ok { + return nil + } + if !first { + b.mu.Lock() + b.reconnects++ + b.mu.Unlock() + } + first = false + + s := b.newStream() + ended := make(chan error, 1) + moved := make(chan struct{}) + go func() { ended <- s.run(token, sync.OnceFunc(func() { close(moved) })) }() + + var err error + select { + case <-moved: + b.logger.Info("Relay pod is shutting down, opening a new stream") + continue + case err = <-ended: + } + if b.ctx.Err() != nil { + return nil + } + if s.held.Load() { + delay = retryMin + warnedTaken = false + } + code := status.Code(err) + b.mu.Lock() + if err != nil { + b.lastErr = err.Error() + } else { + b.lastErr = "the relay ended the stream" + } + b.mu.Unlock() + if fatal[code] { + return fmt.Errorf("the relay refused the tunnel: %w", err) + } + + logger := b.logger.With("code", code.String(), "error", err) + switch code { + case codes.Unauthenticated: + // Each create-session replaces the client token. + b.refuseToken(token) + logger.Warn("The relay refused the client token, waiting for a new one") + continue + case codes.AlreadyExists: + b.setState(bridgev1.TunnelState_TUNNEL_STATE_TAKEN) + if !warnedTaken { + warnedTaken = true + logger.Warn("Another client holds the tunnel, waiting for it to let go") + } + default: + b.mu.Lock() + if b.held == 0 { + b.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING + } + b.mu.Unlock() + logger.Info("Tunnel stream ended, reconnecting", "delay", delay) + } + + select { + case <-time.After(delay): + case <-b.ctx.Done(): + return nil + } + delay = min(delay*2, retryMax) + } +} + +// awaitToken returns the client token, waiting for SetToken while there is +// none. +func (b *Backend) awaitToken() (string, bool) { + for { + b.mu.Lock() + token, set := b.token, b.tokenSet + if token == "" { + b.state = bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN + } + b.mu.Unlock() + if token != "" { + return token, true + } + select { + case <-set: + case <-b.ctx.Done(): + return "", false + } + } +} + +// refuseToken forgets token, unless SetToken already replaced it. +func (b *Backend) refuseToken(token string) { + b.mu.Lock() + defer b.mu.Unlock() + if b.token == token { + b.token = "" + } +} + +func (b *Backend) setState(state bridgev1.TunnelState) { + b.mu.Lock() + defer b.mu.Unlock() + b.state = state +} + +// setLive makes s the stream new outbound conns take; nil for none. +func (b *Backend) setLive(s *stream) { + b.live = s + if s != nil && !b.liveClosed { + close(b.liveCh) + b.liveClosed = true + } else if s == nil && b.liveClosed { + b.liveCh = make(chan struct{}) + b.liveClosed = false + } +} + +// holding: the relay routes the tunnel's conns to s. +func (b *Backend) holding(s *stream) { + b.mu.Lock() + defer b.mu.Unlock() + b.held++ + b.setLive(s) + if b.state != bridgev1.TunnelState_TUNNEL_STATE_LIVE { + b.state = bridgev1.TunnelState_TUNNEL_STATE_LIVE + b.liveSince = time.Now() + b.logger.Info("Tunnel live", "reconnects", b.reconnects) + } +} + +// moving: s gets no new conns, but finishes the ones open on it. +func (b *Backend) moving(s *stream) { + b.mu.Lock() + defer b.mu.Unlock() + if b.live == s { + b.setLive(nil) + } +} + +// lost: s ended. +func (b *Backend) lost(s *stream) { + b.mu.Lock() + defer b.mu.Unlock() + if s.held.Load() { + b.held-- + } + if b.live == s { + b.setLive(nil) + } + if b.held == 0 && b.state == bridgev1.TunnelState_TUNNEL_STATE_LIVE { + b.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING + b.liveSince = time.Time{} + } +} + +var errSilent = errors.New("no heartbeat from the relay") diff --git a/pkg/devtunnel/client_test.go b/pkg/devtunnel/backend_test.go similarity index 89% rename from pkg/devtunnel/client_test.go rename to pkg/devtunnel/backend_test.go index 3e528e2..2443f11 100644 --- a/pkg/devtunnel/client_test.go +++ b/pkg/devtunnel/backend_test.go @@ -13,6 +13,7 @@ import ( "github.com/stretchr/testify/require" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/tunnel" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" @@ -21,6 +22,9 @@ import ( const testTimeout = 5 * time.Second +// More than one Data frame's worth. +const frameBytes = 64 << 10 + // fakeRelay stands in for api-dev-tunnel, handing each Connect call to the // test. type fakeRelay struct { @@ -147,17 +151,19 @@ func window(id uint64, n int) *devtunnelv1.ConnectResponse { return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Window{Window: &devtunnelv1.Window{ConnId: id, Bytes: uint32(n)}}} } -func newClient(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) *Client { +// newTunnel runs a tunnel over a Backend for r. +func newTunnel(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) (*Backend, tunnel.Tunnel) { t.Helper() - c, err := New(Config{RelayURL: r.url, Token: token, Instance: "test-instance", Port: port}) + b, err := New(Config{RelayURL: r.url, Token: token, Instance: "test-instance", Port: port}) require.NoError(t, err) - c.timings = timings{heartbeat: 50 * time.Millisecond, silent: time.Second, liveWait: 2 * time.Second} + b.timings = timings{heartbeat: 50 * time.Millisecond, silent: time.Second, liveWait: 2 * time.Second} for _, f := range tune { - f(&c.timings) + f(&b.timings) } - c.Start(context.Background()) - t.Cleanup(func() { _ = c.Close() }) - return c + tun := tunnel.NewWithBackend(&net.Dialer{Timeout: time.Second}, b) + tun.Start(context.Background()) + t.Cleanup(func() { _ = tun.Close() }) + return b, tun } // echoApp echoes each connection, then half-closes it once the peer has. @@ -189,7 +195,7 @@ func waitFor(t *testing.T, cond func() bool) { func TestIngressReachesTheApp(t *testing.T) { r := startRelay(t) port := echoApp(t) - c := newClient(t, r, "tok", port) + c, _ := newTunnel(t, r, "tok", port) s := r.next(t) assert.Equal(t, []string{"Bearer tok"}, s.md.Get("authorization")) @@ -224,7 +230,7 @@ func TestIngressReachesTheApp(t *testing.T) { func TestIngressToAnotherPortIsReset(t *testing.T) { r := startRelay(t) - newClient(t, r, "tok", 3000) + newTunnel(t, r, "tok", 3000) s := r.next(t) s.hold(t) @@ -240,7 +246,7 @@ func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { _ = lis.Close() }) - body := bytes.Repeat([]byte("x"), connWindow+maxPayload) + body := bytes.Repeat([]byte("x"), connWindow+frameBytes) go func() { nc, err := lis.Accept() if err == nil { @@ -248,7 +254,7 @@ func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { } }() port := lis.Addr().(*net.TCPAddr).Port - newClient(t, r, "tok", port) + newTunnel(t, r, "tok", port) s := r.next(t) s.hold(t) s.send(t, open(1, port)) @@ -262,19 +268,19 @@ func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { case <-time.After(200 * time.Millisecond): } - s.send(t, window(1, maxPayload)) - assert.Len(t, s.expectData(t, 1, maxPayload), maxPayload) + s.send(t, window(1, frameBytes)) + assert.Len(t, s.expectData(t, 1, frameBytes), frameBytes) } func TestEgressLeavesThroughTheDeployment(t *testing.T) { r := startRelay(t) - c := newClient(t, r, "tok", 3000) + c, tun := newTunnel(t, r, "tok", 3000) s := r.next(t) s.hold(t) waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_LIVE }) app, intercepted := tcpPair(t) - c.AddConn(intercepted, "0.0.0.0:443", "example.com") + tun.AddConn(intercepted, "0.0.0.0:443", "example.com") o := s.expect(t).GetOpen() require.NotNil(t, o) assert.GreaterOrEqual(t, o.GetConnId(), clientConnIDFloor) @@ -304,11 +310,11 @@ func TestEgressLeavesThroughTheDeployment(t *testing.T) { func TestEgressWithNoLiveStreamIsDropped(t *testing.T) { r := startRelay(t) - c := newClient(t, r, "tok", 3000, func(tm *timings) { tm.liveWait = 50 * time.Millisecond }) + _, tun := newTunnel(t, r, "tok", 3000, func(tm *timings) { tm.liveWait = 50 * time.Millisecond }) r.next(t) // never holds the tunnel app, intercepted := tcpPair(t) - c.AddConn(intercepted, "0.0.0.0:443", "example.com") + tun.AddConn(intercepted, "0.0.0.0:443", "example.com") _ = app.SetReadDeadline(time.Now().Add(testTimeout)) _, err := app.Read(make([]byte, 1)) assert.ErrorIs(t, err, io.EOF) @@ -317,7 +323,7 @@ func TestEgressWithNoLiveStreamIsDropped(t *testing.T) { func TestReconnectKeepsTheOldStreamsConns(t *testing.T) { r := startRelay(t) port := echoApp(t) - c := newClient(t, r, "tok", port) + c, _ := newTunnel(t, r, "tok", port) first := r.next(t) first.hold(t) first.send(t, open(1, port)) @@ -344,7 +350,7 @@ func TestRefusedTokenWaitsForANewOne(t *testing.T) { } return nil } - c := newClient(t, r, "old", 3000) + c, _ := newTunnel(t, r, "old", 3000) waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN }) c.SetToken("new") @@ -354,7 +360,7 @@ func TestRefusedTokenWaitsForANewOne(t *testing.T) { func TestNoTokenWaitsForOne(t *testing.T) { r := startRelay(t) - c := newClient(t, r, "", 3000) + c, _ := newTunnel(t, r, "", 3000) waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN }) select { case <-r.streams: @@ -371,20 +377,20 @@ func TestFatalRefusalStops(t *testing.T) { r.refuse = func(metadata.MD) error { return status.Error(codes.PermissionDenied, "not allowed") } - c := newClient(t, r, "tok", 3000) + c, tun := newTunnel(t, r, "tok", 3000) select { - case <-c.Done(): + case <-tun.Done(): case <-time.After(testTimeout): t.Fatal("kept retrying") } - assert.ErrorContains(t, c.Err(), "not allowed") + assert.Contains(t, c.Status().GetLastError(), "not allowed") assert.Equal(t, bridgev1.TunnelState_TUNNEL_STATE_FAILED, c.Status().GetState()) } func TestSilentRelayIsReplaced(t *testing.T) { r := startRelay(t) - c := newClient(t, r, "tok", 3000, func(tm *timings) { tm.silent = 200 * time.Millisecond }) + c, _ := newTunnel(t, r, "tok", 3000, func(tm *timings) { tm.silent = 200 * time.Millisecond }) r.next(t) // never heard from again r.next(t) diff --git a/pkg/devtunnel/client.go b/pkg/devtunnel/client.go deleted file mode 100644 index 31c8e5c..0000000 --- a/pkg/devtunnel/client.go +++ /dev/null @@ -1,543 +0,0 @@ -// Package devtunnel carries an intercept's traffic over a dev tunnel: the -// Connect stream to api-dev-tunnel that `vc dev --tunnel` holds otherwise. -// The deployment's conns reach the app, and the app's outbound conns leave -// through the deployment. -package devtunnel - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "log/slog" - "net" - "net/url" - "regexp" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/segmentio/ksuid" - bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" - devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" - "github.com/vercel/bridge/pkg/plumbing" - "github.com/vercel/bridge/pkg/tunnel" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/status" -) - -const ( - // What each side of a conn may send before the other grants more. - connWindow = 256 << 10 - // Per Data frame. - maxPayload = 64 << 10 - // A client numbers its conns from here; the relay numbers its own from 1. - clientConnIDFloor = uint64(1) << 63 - - instanceHeader = "x-vercel-dev-tunnel-instance" - - retryMin = 250 * time.Millisecond - retryMax = 5 * time.Second -) - -var instancePattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) - -// Codes reconnecting can't fix. -var fatal = map[codes.Code]bool{ - codes.FailedPrecondition: true, - codes.InvalidArgument: true, - codes.PermissionDenied: true, - codes.Unimplemented: true, -} - -// Config configures a Client. -type Config struct { - // RelayURL is api-dev-tunnel's URL, from create-session. - RelayURL string - // Token is the client token, from create-session. Without one the client - // waits for SetToken. - Token string - // Instance tells the relay this process's reconnects from another - // client's, so a restart can take the tunnel back while its old stream is - // still open. Random if empty. - Instance string - // Port is the only port of the app the deployment's conns may reach. - Port int - // AppDialer connects the deployment's conns to the app. Defaults to a - // net.Dialer. - AppDialer plumbing.ContextDialer - // RelayDialer connects to the relay. Defaults to a net.Dialer. - RelayDialer plumbing.ContextDialer - Logger *slog.Logger -} - -// timings are the relay's protocol intervals, shortened by tests. -type timings struct { - // The relay needs a heartbeat at least every 30s. - heartbeat time.Duration - // The relay heartbeats every 30s; three missed and the stream is dead. - silent time.Duration - // The relay holds the deployment's conns this long for a client, and an - // outbound conn waits as long for a stream. - liveWait time.Duration -} - -var defaultTimings = timings{ - heartbeat: 15 * time.Second, - silent: 90 * time.Second, - liveWait: 5 * time.Second, -} - -// Client holds a dev tunnel's stream to its relay, replacing it whenever it -// drops, and implements tunnel.Tunnel over it. -type Client struct { - cfg Config - timings timings - logger *slog.Logger - instance string - cc *grpc.ClientConn - rpc devtunnelv1.TunnelServiceClient - - mu sync.Mutex - token string - tokenSet chan struct{} - // The newest stream holding the tunnel, which new outbound conns take. - live *stream - liveCh chan struct{} - liveClosed bool - // Streams holding the tunnel, including one finishing its conns after the - // relay said to reconnect. - held int - state bridgev1.TunnelState - lastErr string - liveSince time.Time - reconnects int64 - - ingressConns, egressConns, ingressTotal, egressTotal atomic.Int64 - bytesSent, bytesReceived atomic.Int64 - nextEgress atomic.Uint64 - - ctx context.Context - cancel context.CancelFunc - started atomic.Bool - done chan struct{} - closeOnce sync.Once - err error -} - -var _ tunnel.Tunnel = (*Client)(nil) - -// New returns a Client for cfg. Nothing connects until Start. -func New(cfg Config) (*Client, error) { - target, creds, err := relayTarget(cfg.RelayURL) - if err != nil { - return nil, err - } - instance := cfg.Instance - if instance == "" { - instance = "bridge-" + ksuid.New().String() - } - if !instancePattern.MatchString(instance) { - return nil, fmt.Errorf("tunnel instance must be 1 to 64 letters, digits or dashes: %q", instance) - } - if cfg.AppDialer == nil { - cfg.AppDialer = &net.Dialer{Timeout: 10 * time.Second} - } - if cfg.RelayDialer == nil { - cfg.RelayDialer = &net.Dialer{Timeout: 10 * time.Second} - } - logger := cfg.Logger - if logger == nil { - logger = slog.Default() - } - logger = logger.With("component", "dev-tunnel", "relay", cfg.RelayURL, "instance", instance) - - relayDialer := cfg.RelayDialer - cc, err := grpc.NewClient(target, - grpc.WithTransportCredentials(creds), - grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { - return relayDialer.DialContext(ctx, "tcp", addr) - }), - // Notices a dead connection, e.g. after sleep, sooner than the - // relay's heartbeats would. - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 10 * time.Second, - Timeout: 20 * time.Second, - PermitWithoutStream: true, - }), - ) - if err != nil { - return nil, fmt.Errorf("creating relay client: %w", err) - } - - c := &Client{ - cfg: cfg, - timings: defaultTimings, - logger: logger, - instance: instance, - cc: cc, - rpc: devtunnelv1.NewTunnelServiceClient(cc), - token: cfg.Token, - tokenSet: make(chan struct{}), - liveCh: make(chan struct{}), - state: bridgev1.TunnelState_TUNNEL_STATE_CONNECTING, - done: make(chan struct{}), - } - c.ctx, c.cancel = context.WithCancel(context.Background()) - return c, nil -} - -// relayTarget turns the relay's URL into a gRPC target. The client dials it -// with RelayDialer rather than resolving it through gRPC, so an intercept -// whose DNS forwards every domain still reaches the relay directly. -func relayTarget(relayURL string) (string, credentials.TransportCredentials, error) { - u, err := url.Parse(relayURL) - if err != nil { - return "", nil, fmt.Errorf("invalid relay URL %q: %w", relayURL, err) - } - if u.Path != "" && u.Path != "/" { - return "", nil, fmt.Errorf("relay URL %q must have no path", relayURL) - } - var creds credentials.TransportCredentials - port := u.Port() - switch u.Scheme { - case "https": - creds = credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) - if port == "" { - port = "443" - } - case "http": - creds = insecure.NewCredentials() - if port == "" { - port = "80" - } - default: - return "", nil, fmt.Errorf("relay URL %q must be http or https", relayURL) - } - if u.Hostname() == "" { - return "", nil, fmt.Errorf("relay URL %q has no host", relayURL) - } - return "passthrough:///" + net.JoinHostPort(u.Hostname(), port), creds, nil -} - -// Start holds the tunnel until ctx ends or Close. -func (c *Client) Start(ctx context.Context) { - if !c.started.CompareAndSwap(false, true) { - return - } - stop := context.AfterFunc(ctx, c.cancel) - go func() { - defer stop() - c.finish(c.run()) - }() -} - -// Close ends every stream and the conns on them. -func (c *Client) Close() error { - c.cancel() - if c.started.Load() { - <-c.done - } else { - c.finish(nil) - } - return nil -} - -func (c *Client) Done() <-chan struct{} { return c.done } - -// Err is why the client stopped: the relay refusing the tunnel for good, or -// nil after Close. Valid once Done is closed. -func (c *Client) Err() error { - <-c.done - return c.err -} - -// SetToken replaces the client token. A client waiting for one connects with -// it; a live one uses it on its next stream. -func (c *Client) SetToken(token string) { - c.mu.Lock() - defer c.mu.Unlock() - c.token = token - close(c.tokenSet) - c.tokenSet = make(chan struct{}) -} - -// Status reports the tunnel's state and traffic. -func (c *Client) Status() *bridgev1.TunnelStatus { - c.mu.Lock() - defer c.mu.Unlock() - st := &bridgev1.TunnelStatus{ - State: c.state, - LastError: c.lastErr, - Reconnects: c.reconnects, - IngressConns: c.ingressConns.Load(), - EgressConns: c.egressConns.Load(), - IngressConnsTotal: c.ingressTotal.Load(), - EgressConnsTotal: c.egressTotal.Load(), - BytesSent: c.bytesSent.Load(), - BytesReceived: c.bytesReceived.Load(), - } - if !c.liveSince.IsZero() { - st.LiveSinceUnixSeconds = c.liveSince.Unix() - } - return st -} - -// AddConn sends one of the app's outbound conns out through the deployment, -// to hostname if the intercept's DNS gave the conn one, or else to the -// address it was headed for. The deployment resolves the hostname as its own -// code would. -func (c *Client) AddConn(nc net.Conn, destOverride string, hostname string) { - dst := destOverride - if dst == "" { - dst = nc.LocalAddr().String() - } - host, portStr, err := net.SplitHostPort(dst) - port, portErr := strconv.Atoi(portStr) - if err != nil || portErr != nil { - c.logger.Warn("Dropping outbound connection with no destination", "dest", dst) - _ = nc.Close() - return - } - if hostname != "" { - host = hostname - } - logger := c.logger.With("host", host, "port", port) - - go func() { - s := c.awaitLive() - if s == nil { - logger.Warn("Dropping outbound connection: no stream holds the tunnel") - _ = nc.Close() - return - } - id := clientConnIDFloor | c.nextEgress.Add(1) - cn := s.newConn(id, egress) - if cn == nil { - logger.Warn("Dropping outbound connection: the tunnel's stream ended") - _ = nc.Close() - return - } - logger.Debug("Opening outbound connection", "conn_id", id) - // Queued ahead of any Data the conn sends. - s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Open{Open: &devtunnelv1.Open{ - ConnId: id, - Protocol: devtunnelv1.Protocol_PROTOCOL_TCP, - Host: host, - Port: uint32(port), - }}}) - cn.attach(nc) - }() -} - -// awaitLive returns the stream new outbound conns take, waiting a while for -// one, or nil. -func (c *Client) awaitLive() *stream { - timer := time.NewTimer(c.timings.liveWait) - defer timer.Stop() - for { - c.mu.Lock() - s, ch := c.live, c.liveCh - c.mu.Unlock() - if s != nil { - return s - } - select { - case <-ch: - case <-timer.C: - return nil - case <-c.ctx.Done(): - return nil - } - } -} - -// run replaces the tunnel's stream whenever it ends, for as long as it takes: -// the relay's pods roll often. A stream the relay says to move off keeps its -// open conns while the next one opens. -func (c *Client) run() error { - delay := retryMin - first := true - warnedTaken := false - for { - token, ok := c.awaitToken() - if !ok { - return nil - } - if !first { - c.mu.Lock() - c.reconnects++ - c.mu.Unlock() - } - first = false - - s := c.newStream() - ended := make(chan error, 1) - moved := make(chan struct{}) - go func() { ended <- s.run(token, sync.OnceFunc(func() { close(moved) })) }() - - var err error - select { - case <-moved: - c.logger.Info("Relay pod is shutting down, opening a new stream") - continue - case err = <-ended: - } - if c.ctx.Err() != nil { - return nil - } - if s.held.Load() { - delay = retryMin - warnedTaken = false - } - code := status.Code(err) - c.mu.Lock() - if err != nil { - c.lastErr = err.Error() - } else { - c.lastErr = "the relay ended the stream" - } - c.mu.Unlock() - if fatal[code] { - return fmt.Errorf("the relay refused the tunnel: %w", err) - } - - logger := c.logger.With("code", code.String(), "error", err) - switch code { - case codes.Unauthenticated: - // Each create-session replaces the client token. - c.refuseToken(token) - logger.Warn("The relay refused the client token, waiting for a new one") - continue - case codes.AlreadyExists: - c.setState(bridgev1.TunnelState_TUNNEL_STATE_TAKEN) - if !warnedTaken { - warnedTaken = true - logger.Warn("Another client holds the tunnel, waiting for it to let go") - } - default: - c.mu.Lock() - if c.held == 0 { - c.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING - } - c.mu.Unlock() - logger.Info("Tunnel stream ended, reconnecting", "delay", delay) - } - - select { - case <-time.After(delay): - case <-c.ctx.Done(): - return nil - } - delay = min(delay*2, retryMax) - } -} - -// awaitToken returns the client token, waiting for SetToken while there is -// none. -func (c *Client) awaitToken() (string, bool) { - for { - c.mu.Lock() - token, set := c.token, c.tokenSet - if token == "" { - c.state = bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN - } - c.mu.Unlock() - if token != "" { - return token, true - } - select { - case <-set: - case <-c.ctx.Done(): - return "", false - } - } -} - -// refuseToken forgets token, unless SetToken already replaced it. -func (c *Client) refuseToken(token string) { - c.mu.Lock() - defer c.mu.Unlock() - if c.token == token { - c.token = "" - } -} - -func (c *Client) setState(state bridgev1.TunnelState) { - c.mu.Lock() - defer c.mu.Unlock() - c.state = state -} - -// setLive makes s the stream new outbound conns take; nil for none. -func (c *Client) setLive(s *stream) { - c.live = s - if s != nil && !c.liveClosed { - close(c.liveCh) - c.liveClosed = true - } else if s == nil && c.liveClosed { - c.liveCh = make(chan struct{}) - c.liveClosed = false - } -} - -// holding: the relay routes the tunnel's conns to s. -func (c *Client) holding(s *stream) { - c.mu.Lock() - defer c.mu.Unlock() - c.held++ - c.setLive(s) - if c.state != bridgev1.TunnelState_TUNNEL_STATE_LIVE { - c.state = bridgev1.TunnelState_TUNNEL_STATE_LIVE - c.liveSince = time.Now() - c.logger.Info("Tunnel live", "reconnects", c.reconnects) - } -} - -// moving: s gets no new conns, but finishes the ones open on it. -func (c *Client) moving(s *stream) { - c.mu.Lock() - defer c.mu.Unlock() - if c.live == s { - c.setLive(nil) - } -} - -// lost: s ended. -func (c *Client) lost(s *stream) { - c.mu.Lock() - defer c.mu.Unlock() - if s.held.Load() { - c.held-- - } - if c.live == s { - c.setLive(nil) - } - if c.held == 0 && c.state == bridgev1.TunnelState_TUNNEL_STATE_LIVE { - c.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING - c.liveSince = time.Time{} - } -} - -func (c *Client) finish(err error) { - c.closeOnce.Do(func() { - c.cancel() - if err != nil { - c.mu.Lock() - c.state = bridgev1.TunnelState_TUNNEL_STATE_FAILED - c.lastErr = err.Error() - c.mu.Unlock() - c.logger.Error("Tunnel stopped", "error", err) - } - c.err = err - _ = c.cc.Close() - close(c.done) - }) -} - -var errSilent = errors.New("no heartbeat from the relay") diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go deleted file mode 100644 index 3906099..0000000 --- a/pkg/devtunnel/conn.go +++ /dev/null @@ -1,234 +0,0 @@ -package devtunnel - -import ( - "bytes" - "errors" - "io" - "log/slog" - "net" - "sync" - - devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" -) - -type direction int - -const ( - // From the deployment to the app. - ingress direction = iota - // From the app out through the deployment. - egress -) - -func (d direction) String() string { - if d == egress { - return "egress" - } - return "ingress" -} - -// conn joins one of the stream's conns to a local TCP connection. Each -// direction moves only as fast as its reader takes it: what the local side -// sends waits on the relay's window, and the relay gets window back as what it -// sent is written locally. Either side may half-close; the conn is over once -// both have, or either resets. -type conn struct { - id uint64 - s *stream - dir direction - logger *slog.Logger - - mu sync.Mutex - cond *sync.Cond - nc net.Conn - // What this side may still send the relay. - sendWindow int - // From the relay, not yet written locally; a nil entry half-closes. - pending [][]byte - closeSent, closeQueued bool - writeClosed, over bool -} - -// attach starts moving bytes once the local connection is up. -func (c *conn) attach(nc net.Conn) { - c.mu.Lock() - if c.over { - c.mu.Unlock() - _ = nc.Close() - return - } - c.nc = nc - c.mu.Unlock() - go c.readLocal() - go c.writeLocal() -} - -// readLocal sends what the local side sends, as the window allows. -func (c *conn) readLocal() { - buf := make([]byte, maxPayload) - for { - c.mu.Lock() - for c.sendWindow == 0 && !c.over { - c.cond.Wait() - } - if c.over { - c.mu.Unlock() - return - } - size := min(c.sendWindow, maxPayload) - c.mu.Unlock() - - n, err := c.nc.Read(buf[:size]) - if n > 0 { - c.mu.Lock() - if c.over { - c.mu.Unlock() - return - } - c.sendWindow -= n - c.mu.Unlock() - c.s.c.bytesSent.Add(int64(n)) - c.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Data{Data: &devtunnelv1.Data{ - ConnId: c.id, - Payload: bytes.Clone(buf[:n]), - }}}) - } - if errors.Is(err, io.EOF) { - c.sendClose() - return - } - if err != nil { - c.reset(err.Error()) - return - } - } -} - -// writeLocal writes what the relay sent, granting it back as it goes. -func (c *conn) writeLocal() { - for { - c.mu.Lock() - for len(c.pending) == 0 && !c.over { - c.cond.Wait() - } - if c.over { - c.mu.Unlock() - return - } - payload := c.pending[0] - c.pending[0] = nil - c.pending = c.pending[1:] - c.mu.Unlock() - - if payload == nil { - if cw, ok := c.nc.(interface{ CloseWrite() error }); ok { - _ = cw.CloseWrite() - } - c.mu.Lock() - c.writeClosed = true - done := c.closeSent - c.mu.Unlock() - if done { - c.end() - } - return - } - if _, err := c.nc.Write(payload); err != nil { - c.reset(err.Error()) - return - } - c.mu.Lock() - over := c.over - c.mu.Unlock() - if !over { - c.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Window{Window: &devtunnelv1.Window{ - ConnId: c.id, - Bytes: uint32(len(payload)), - }}}) - } - } -} - -// received queues payload from the relay. -func (c *conn) received(payload []byte) { - if len(payload) == 0 { - return - } - c.mu.Lock() - defer c.mu.Unlock() - if c.over || c.closeQueued { - return - } - c.pending = append(c.pending, payload) - c.cond.Broadcast() -} - -// closeReceived: the relay's side has nothing more to send. -func (c *conn) closeReceived() { - c.mu.Lock() - defer c.mu.Unlock() - if c.over || c.closeQueued { - return - } - c.closeQueued = true - c.pending = append(c.pending, nil) - c.cond.Broadcast() -} - -// grant: the relay may take this many more bytes. -func (c *conn) grant(bytes int) { - c.mu.Lock() - defer c.mu.Unlock() - c.sendWindow += bytes - c.cond.Broadcast() -} - -// sendClose: the local side has nothing more to send. -func (c *conn) sendClose() { - c.mu.Lock() - if c.over || c.closeSent { - c.mu.Unlock() - return - } - c.closeSent = true - done := c.writeClosed - c.mu.Unlock() - c.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Close{Close: &devtunnelv1.Close{ConnId: c.id}}}) - if done { - c.end() - } -} - -// reset abandons the conn and tells the relay why. -func (c *conn) reset(reason string) { - c.mu.Lock() - over := c.over - c.mu.Unlock() - if over { - return - } - c.s.send(resetFrame(c.id, reason)) - c.abandon() -} - -// abandon drops the conn in both directions without telling the relay, which -// already knows. -func (c *conn) abandon() { - c.end() -} - -func (c *conn) end() { - c.mu.Lock() - if c.over { - c.mu.Unlock() - return - } - c.over = true - nc := c.nc - c.cond.Broadcast() - c.mu.Unlock() - if nc != nil { - _ = nc.Close() - } - c.s.forget(c) -} diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index e97021e..ee4fcd6 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -6,24 +6,25 @@ import ( "fmt" "io" "log/slog" - "net" "strconv" "sync" "sync/atomic" "time" - "github.com/puzpuzpuz/xsync/v3" devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/tunnel" "google.golang.org/grpc/metadata" ) var heartbeat = &devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Heartbeat{Heartbeat: &devtunnelv1.Heartbeat{}}} -// stream is one Connect call and the conns it carries. The relay ends every -// conn with the stream. +// stream is one Connect call. The relay ends every conn on it with it. type stream struct { - c *Client + b *Backend logger *slog.Logger + // The relay numbers each stream's conns from 1, so the tunnel's id for one + // starts with this. + prefix string ctx context.Context cancel context.CancelCauseFunc out outbox @@ -31,20 +32,18 @@ type stream struct { // Set once the relay routes the tunnel's conns to this stream. held atomic.Bool lastHeard atomic.Int64 - - conns *xsync.MapOf[uint64, *conn] - closed atomic.Bool + closed atomic.Bool } -func (c *Client) newStream() *stream { - ctx, cancel := context.WithCancelCause(c.ctx) +func (b *Backend) newStream() *stream { + ctx, cancel := context.WithCancelCause(b.ctx) return &stream{ - c: c, - logger: c.logger, + b: b, + logger: b.logger, + prefix: strconv.FormatUint(b.nextStream.Add(1), 10) + ".", ctx: ctx, cancel: cancel, out: outbox{wake: make(chan struct{}, 1)}, - conns: xsync.NewMapOf[uint64, *conn](), } } @@ -54,14 +53,14 @@ func (s *stream) run(token string, moving func()) error { defer s.cancel(nil) ctx := metadata.AppendToOutgoingContext(s.ctx, "authorization", "Bearer "+token, - instanceHeader, s.c.instance, + instanceHeader, s.b.instance, ) - call, err := s.c.rpc.Connect(ctx) + call, err := s.b.rpc.Connect(ctx) if err != nil { return err } - defer s.c.lost(s) - defer s.abandonAll() + defer s.b.lost(s) + defer s.endConns() s.lastHeard.Store(time.Now().UnixNano()) // The call starts with its first message. @@ -97,14 +96,14 @@ func (s *stream) run(token string, moving func()) error { } func (s *stream) heartbeats() { - ticker := time.NewTicker(s.c.timings.heartbeat) + ticker := time.NewTicker(s.b.timings.heartbeat) defer ticker.Stop() for { select { case <-s.ctx.Done(): return case <-ticker.C: - if time.Since(time.Unix(0, s.lastHeard.Load())) > s.c.timings.silent { + if time.Since(time.Unix(0, s.lastHeard.Load())) > s.b.timings.silent { s.logger.Info("Relay went silent, reconnecting") s.cancel(errSilent) return @@ -114,113 +113,98 @@ func (s *stream) heartbeats() { } } +// handle passes one of the relay's frames to the tunnel. func (s *stream) handle(msg *devtunnelv1.ConnectResponse, moving func()) { switch f := msg.GetFrame().(type) { case *devtunnelv1.ConnectResponse_Heartbeat: if s.held.CompareAndSwap(false, true) { - s.c.holding(s) + s.b.holding(s) } case *devtunnelv1.ConnectResponse_Reconnect: - s.c.moving(s) + s.b.moving(s) moving() case *devtunnelv1.ConnectResponse_Open: - s.openIngress(f.Open) + s.open(f.Open) case *devtunnelv1.ConnectResponse_Data: - if cn := s.conn(f.Data.GetConnId()); cn != nil { - s.c.bytesReceived.Add(int64(len(f.Data.GetPayload()))) - cn.received(f.Data.GetPayload()) + if id, ok := s.routed(f.Data.GetConnId()); ok { + s.b.bytesReceived.Add(int64(len(f.Data.GetPayload()))) + s.b.deliver(tunnel.Frame{Kind: tunnel.FrameData, ConnID: id, Payload: f.Data.GetPayload()}) } case *devtunnelv1.ConnectResponse_Close: - if cn := s.conn(f.Close.GetConnId()); cn != nil { - cn.closeReceived() + if id, ok := s.routed(f.Close.GetConnId()); ok { + s.b.deliver(tunnel.Frame{Kind: tunnel.FrameCloseWrite, ConnID: id}) } case *devtunnelv1.ConnectResponse_Reset_: - if cn := s.conn(f.Reset_.GetConnId()); cn != nil { - cn.logger.Debug("Relay reset the connection", "reason", f.Reset_.GetReason()) - cn.abandon() + if id, ok := s.routed(f.Reset_.GetConnId()); ok && s.b.untrack(id, s) { + s.logger.Debug("Relay reset the connection", "conn_id", id, "reason", f.Reset_.GetReason()) + s.b.deliver(tunnel.Frame{Kind: tunnel.FrameReset, ConnID: id, Reason: f.Reset_.GetReason()}) } case *devtunnelv1.ConnectResponse_Window: - if cn := s.conn(f.Window.GetConnId()); cn != nil { - cn.grant(int(f.Window.GetBytes())) + if id, ok := s.routed(f.Window.GetConnId()); ok { + s.b.deliver(tunnel.Frame{Kind: tunnel.FrameWindow, ConnID: id, Bytes: int(f.Window.GetBytes())}) } } } -// openIngress connects one of the deployment's conns to the app. -func (s *stream) openIngress(open *devtunnelv1.Open) { - id := open.GetConnId() - port := s.c.cfg.Port +// open hands one of the deployment's conns to the tunnel, which connects it +// to the app. +func (s *stream) open(open *devtunnelv1.Open) { + wire := open.GetConnId() + port := s.b.cfg.Port if open.GetProtocol() != devtunnelv1.Protocol_PROTOCOL_TCP || int(open.GetPort()) != port { - s.send(resetFrame(id, fmt.Sprintf("bridge serves only port %d", port))) + s.send(resetFrame(wire, fmt.Sprintf("bridge serves only port %d", port))) return } - cn := s.newConn(id, ingress) - if cn == nil { + id := s.idFor(wire) + if !s.track(id, route{s: s, wire: wire, dir: ingress}) { return } - go func() { - nc, err := s.c.cfg.AppDialer.DialContext(s.ctx, "tcp", net.JoinHostPort("127.0.0.1", strconv.Itoa(port))) - if err != nil { - cn.logger.Debug("App refused the connection", "error", err) - cn.reset(err.Error()) - return - } - cn.attach(nc) - }() + s.b.deliver(tunnel.Frame{ + Kind: tunnel.FrameOpen, + ConnID: id, + Conn: &tunnel.ConnInfo{Dest: tunnel.Addr{Host: "127.0.0.1", Port: port}}, + }) } -// newConn registers a conn, or returns nil once the stream has ended. -func (s *stream) newConn(id uint64, dir direction) *conn { - if s.closed.Load() { - return nil - } - cn := &conn{ - id: id, - s: s, - dir: dir, - logger: s.logger.With("conn_id", id, "direction", dir.String()), - sendWindow: connWindow, - } - cn.cond = sync.NewCond(&cn.mu) - // Counted before it's stored, so whichever of forget's callers removes it - // uncounts it. - s.c.counters(dir, 1) - s.conns.Store(id, cn) - // abandonAll may have swept the map before the Store. - if s.closed.Load() { - s.forget(cn) - return nil +// idFor is the tunnel's id for the conn the relay calls wire on this stream. +// Outbound conns keep the id the tunnel gave them. +func (s *stream) idFor(wire uint64) string { + if wire >= clientConnIDFloor { + return "e" + strconv.FormatUint(wire&^clientConnIDFloor, 10) } - return cn + return s.prefix + strconv.FormatUint(wire, 10) } -func (s *stream) conn(id uint64) *conn { - cn, _ := s.conns.Load(id) - return cn +// routed returns the tunnel's id for wire, if it's a conn open on this stream. +func (s *stream) routed(wire uint64) (string, bool) { + id := s.idFor(wire) + r, ok := s.b.routes.Load(id) + return id, ok && r.s == s } -// forget removes cn, unless another conn has since taken its id. -func (s *stream) forget(cn *conn) { - removed := false - s.conns.Compute(cn.id, func(cur *conn, loaded bool) (*conn, bool) { - if !loaded { - return nil, true - } - if cur != cn { - return cur, false - } - removed = true - return nil, true - }) - if removed { - s.c.counters(cn.dir, -1) +// track routes id to this stream, or reports false once the stream has ended. +func (s *stream) track(id string, r route) bool { + if s.closed.Load() { + return false + } + // Counted before it's stored, so whichever untrack removes it uncounts it. + s.b.counters(r.dir, 1) + s.b.routes.Store(id, r) + // endConns may have swept the routes before the Store. + if s.closed.Load() { + s.b.untrack(id, s) + return false } + return true } -func (s *stream) abandonAll() { +// endConns resets every conn on the stream, which the relay ended with it. +func (s *stream) endConns() { s.closed.Store(true) - s.conns.Range(func(_ uint64, cn *conn) bool { - cn.abandon() + s.b.routes.Range(func(id string, r route) bool { + if r.s == s && s.b.untrack(id, s) { + s.b.deliver(tunnel.Frame{Kind: tunnel.FrameReset, ConnID: id, Reason: "the tunnel's stream ended"}) + } return true }) } @@ -229,18 +213,6 @@ func (s *stream) send(frame *devtunnelv1.ConnectRequest) { s.out.push(frame) } -// counters tracks conns opening (delta 1) and ending (delta -1). -func (c *Client) counters(dir direction, delta int64) { - open, total := &c.ingressConns, &c.ingressTotal - if dir == egress { - open, total = &c.egressConns, &c.egressTotal - } - open.Add(delta) - if delta > 0 { - total.Add(delta) - } -} - // outbox queues frames for the relay in order. Data is bounded by each conn's // window; the rest is small. type outbox struct { diff --git a/pkg/tunnel/backend.go b/pkg/tunnel/backend.go new file mode 100644 index 0000000..fdd93d7 --- /dev/null +++ b/pkg/tunnel/backend.go @@ -0,0 +1,74 @@ +package tunnel + +import ( + "context" + "net" + "strconv" +) + +// Backend carries a tunnel's conns over one multiplexed transport: a bridge +// server's TunnelNetwork stream (NewStreamBackend), or a dev tunnel's Connect +// stream (devtunnel.Backend). +type Backend interface { + // Run carries frames until ctx ends or the transport does, passing each of + // the peer's to deliver in order. deliver blocks only while a conn has too + // much queued, and only for a backend without flow control. + Run(ctx context.Context, deliver func(Frame)) error + // ConnID numbers a conn this side opens. + ConnID(conn *ConnInfo) string + // Open tells the peer about a conn this side opens, ahead of its Data. + Open(ctx context.Context, id string, conn *ConnInfo) error + // Send queues a frame for the peer. + Send(f Frame) + // Release forgets a conn that's over. + Release(id string) + // Window is what each side of a conn may send before the other grants + // more, or 0 if the transport has no flow control. + Window() int + // HalfClose reports whether the transport carries a conn's half-close. + // Without it, a conn ends at either side's EOF. + HalfClose() bool +} + +type FrameKind uint8 + +const ( + // The peer opened a conn to Conn.Dest. + FrameOpen FrameKind = iota + 1 + // Payload for a conn. From a peer that sends no Open, Data for an unknown + // conn opens it when it has a Conn. + FrameData + // The sender has nothing more to send on the conn. + FrameCloseWrite + // The conn is abandoned in both directions, for Reason. + FrameReset + // The receiver may send Bytes more of the conn's Data. + FrameWindow +) + +// Frame is one message about one conn. +type Frame struct { + Kind FrameKind + ConnID string + Payload []byte + Bytes int + Reason string + // Where the conn goes. Set on Open, and on every frame of a backend that + // addresses each one. + Conn *ConnInfo +} + +// Addr is a host (an IP address or a hostname) and a port. +type Addr struct { + Host string + Port int +} + +func (a Addr) String() string { return net.JoinHostPort(a.Host, strconv.Itoa(a.Port)) } + +// ConnInfo addresses a conn, as the side sending on it sees it. +type ConnInfo struct { + Source, Dest Addr + // The DNS name Dest was looked up as, if any. + Hostname string +} diff --git a/pkg/tunnel/conn.go b/pkg/tunnel/conn.go new file mode 100644 index 0000000..df3b9e1 --- /dev/null +++ b/pkg/tunnel/conn.go @@ -0,0 +1,270 @@ +package tunnel + +import ( + "bytes" + "io" + "net" + "sync" + "sync/atomic" + "time" +) + +const ( + // Per Data frame. + maxFrame = 64 << 10 + // What a conn may have queued from a peer without flow control before the + // backend waits for it to be read. + maxQueued = 256 << 10 +) + +// streamConn is one conn's side on the tunnel's transport, as a net.Conn: Read +// takes what the peer sent, Write sends to it as the conn's window allows, and +// CloseWrite half-closes it. +type streamConn struct { + t *tunnelImpl + id string + info *ConnInfo + // 0 when the backend has no flow control. + window int + halfClose bool + + // From the peer, not yet read. + inMu sync.Mutex + in [][]byte + inBytes int + inEOF bool + inWake chan struct{} + inRoom chan struct{} + // The rest of the chunk Read is on; only Read touches it. + pending []byte + + // What this side may still send the peer. + sendWindow atomic.Int64 + winWake chan struct{} + closeSent atomic.Bool + + local atomic.Pointer[net.Conn] + done chan struct{} + doneOnce sync.Once +} + +var _ net.Conn = (*streamConn)(nil) + +func (t *tunnelImpl) newConn(id string, info *ConnInfo) *streamConn { + c := &streamConn{ + t: t, + id: id, + info: info, + window: t.backend.Window(), + halfClose: t.backend.HalfClose(), + inWake: make(chan struct{}, 1), + inRoom: make(chan struct{}, 1), + winWake: make(chan struct{}, 1), + done: make(chan struct{}), + } + c.sendWindow.Store(int64(c.window)) + return c +} + +func (c *streamConn) Read(p []byte) (int, error) { + for len(c.pending) == 0 { + if c.isDone() { + return 0, net.ErrClosed + } + c.inMu.Lock() + if len(c.in) > 0 { + c.pending = c.in[0] + c.in[0] = nil + c.in = c.in[1:] + c.inMu.Unlock() + break + } + eof := c.inEOF + c.inMu.Unlock() + if eof { + return 0, io.EOF + } + select { + case <-c.inWake: + case <-c.done: + } + } + n := copy(p, c.pending) + c.pending = c.pending[n:] + c.consumed(n) + return n, nil +} + +// consumed gives the peer back what Read took. +func (c *streamConn) consumed(n int) { + if c.window > 0 { + c.send(Frame{Kind: FrameWindow, Bytes: n}) + return + } + c.inMu.Lock() + c.inBytes -= n + c.inMu.Unlock() + wake(c.inRoom) +} + +func (c *streamConn) Write(p []byte) (int, error) { + written := 0 + for written < len(p) { + size := min(len(p)-written, maxFrame) + if c.window > 0 { + w, err := c.awaitWindow() + if err != nil { + return written, err + } + size = min(size, w) + c.sendWindow.Add(-int64(size)) + } else if c.isDone() { + return written, net.ErrClosed + } + // The backend may send it after Write returns. + c.send(Frame{Kind: FrameData, Payload: bytes.Clone(p[written : written+size])}) + written += size + } + return written, nil +} + +func (c *streamConn) awaitWindow() (int, error) { + for { + if c.isDone() { + return 0, net.ErrClosed + } + if w := c.sendWindow.Load(); w > 0 { + return int(w), nil + } + select { + case <-c.winWake: + case <-c.done: + } + } +} + +// CloseWrite tells the peer this side has nothing more to send. Over a +// transport without half-close, it ends the conn. +func (c *streamConn) CloseWrite() error { + if !c.halfClose { + c.finish() + return nil + } + if !c.isDone() && c.closeSent.CompareAndSwap(false, true) { + c.send(Frame{Kind: FrameCloseWrite}) + } + return nil +} + +// Close resets the conn. +func (c *streamConn) Close() error { + c.reset("closed") + return nil +} + +func (c *streamConn) LocalAddr() net.Addr { return addr(c.info.Source.String()) } +func (c *streamConn) RemoteAddr() net.Addr { return addr(c.info.Dest.String()) } +func (c *streamConn) SetDeadline(time.Time) error { return nil } +func (c *streamConn) SetReadDeadline(time.Time) error { return nil } +func (c *streamConn) SetWriteDeadline(time.Time) error { return nil } + +// deliver queues payload from the peer. +func (c *streamConn) deliver(p []byte) { + if len(p) == 0 { + return + } + for { + c.inMu.Lock() + if c.inEOF || c.isDone() { + c.inMu.Unlock() + return + } + // A window already bounds what the peer can send. + if c.window > 0 || c.inBytes < maxQueued { + c.in = append(c.in, p) + c.inBytes += len(p) + c.inMu.Unlock() + wake(c.inWake) + return + } + c.inMu.Unlock() + select { + case <-c.inRoom: + case <-c.done: + return + } + } +} + +// peerClosed: the peer has nothing more to send. Read returns io.EOF once it +// has returned the rest. +func (c *streamConn) peerClosed() { + c.inMu.Lock() + c.inEOF = true + c.inMu.Unlock() + wake(c.inWake) +} + +func (c *streamConn) grant(n int) { + c.sendWindow.Add(int64(n)) + wake(c.winWake) +} + +// attach hands the conn its local end, which it closes when it's over. +// Reports false if the conn is already over. +func (c *streamConn) attach(local net.Conn) bool { + c.local.Store(&local) + if c.isDone() { + _ = local.Close() + return false + } + return true +} + +// reset ends the conn and tells the peer why. +func (c *streamConn) reset(reason string) { c.end(reason, true) } + +// finish ends the conn without telling the peer: it's done, or already knows. +func (c *streamConn) finish() { c.end("", false) } + +func (c *streamConn) end(reason string, tell bool) { + c.doneOnce.Do(func() { + if tell { + c.send(Frame{Kind: FrameReset, Reason: reason}) + } + close(c.done) + if local := c.local.Load(); local != nil { + _ = (*local).Close() + } + c.t.forget(c) + c.t.backend.Release(c.id) + }) +} + +func (c *streamConn) isDone() bool { + select { + case <-c.done: + return true + default: + return false + } +} + +func (c *streamConn) send(f Frame) { + f.ConnID = c.id + f.Conn = c.info + c.t.backend.Send(f) +} + +// wake signals ch without blocking; ch holds one signal. +func wake(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} + +type addr string + +func (a addr) Network() string { return "tunnel" } +func (a addr) String() string { return string(a) } diff --git a/pkg/tunnel/pump_test.go b/pkg/tunnel/pump_test.go new file mode 100644 index 0000000..d00c706 --- /dev/null +++ b/pkg/tunnel/pump_test.go @@ -0,0 +1,144 @@ +package tunnel + +import ( + "context" + "io" + "net" + "testing" + "time" +) + +// fakeBackend has the flow control and half-close a dev tunnel has. +type fakeBackend struct { + window int + frames chan Frame + sent chan Frame + released chan string +} + +func newFakeBackend(window int) *fakeBackend { + return &fakeBackend{ + window: window, + frames: make(chan Frame, 16), + sent: make(chan Frame, 1024), + released: make(chan string, 16), + } +} + +func (b *fakeBackend) Run(ctx context.Context, deliver func(Frame)) error { + for { + select { + case f := <-b.frames: + deliver(f) + case <-ctx.Done(): + return nil + } + } +} + +func (b *fakeBackend) ConnID(*ConnInfo) string { return "out" } +func (b *fakeBackend) Open(context.Context, string, *ConnInfo) error { return nil } +func (b *fakeBackend) Send(f Frame) { b.sent <- f } +func (b *fakeBackend) Release(id string) { b.released <- id } +func (b *fakeBackend) Window() int { return b.window } +func (b *fakeBackend) HalfClose() bool { return true } + +// next returns the next frame the tunnel sent other than a window grant. +func (b *fakeBackend) next(t *testing.T) Frame { + t.Helper() + for { + select { + case f := <-b.sent: + if f.Kind != FrameWindow { + return f + } + case <-time.After(testTimeout): + t.Fatal("the tunnel sent nothing") + } + } +} + +func startBackendTunnel(t *testing.T, b *fakeBackend) Tunnel { + t.Helper() + tun := NewWithBackend(&net.Dialer{Timeout: time.Second}, b) + tun.Start(context.Background()) + t.Cleanup(func() { tun.Close() }) + return tun +} + +func TestHalfCloseCrossesThePump(t *testing.T) { + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer lis.Close() + got := make(chan string, 1) + go func() { + nc, err := lis.Accept() + if err != nil { + return + } + // Reads to the peer's half-close, then answers and closes. + req, _ := io.ReadAll(nc) + got <- string(req) + _, _ = nc.Write([]byte("bye")) + _ = nc.Close() + }() + b := newFakeBackend(1 << 20) + startBackendTunnel(t, b) + dest := splitAddr(lis.Addr().String()) + + b.frames <- Frame{Kind: FrameOpen, ConnID: "1", Conn: &ConnInfo{Dest: dest}} + b.frames <- Frame{Kind: FrameData, ConnID: "1", Payload: []byte("hi")} + b.frames <- Frame{Kind: FrameCloseWrite, ConnID: "1"} + + select { + case req := <-got: + if req != "hi" { + t.Fatalf("the app read %q", req) + } + case <-time.After(testTimeout): + t.Fatal("the app never saw the peer's half-close") + } + if f := b.next(t); f.Kind != FrameData || string(f.Payload) != "bye" { + t.Fatalf("expected the app's answer, got %+v", f) + } + if f := b.next(t); f.Kind != FrameCloseWrite { + t.Fatalf("expected the app's half-close, got %+v", f) + } + select { + case id := <-b.released: + if id != "1" { + t.Fatalf("released %q", id) + } + case <-time.After(testTimeout): + t.Fatal("the conn was never released") + } +} + +func TestWritesWaitForWindow(t *testing.T) { + b := newFakeBackend(10) + tun := startBackendTunnel(t, b) + app, intercepted := net.Pipe() + defer app.Close() + tun.AddConn(intercepted, "10.0.0.1:443", "example.com") + + go func() { _, _ = app.Write([]byte("0123456789abcdefghijklmno")) }() + sent := "" + for len(sent) < 10 { + sent += string(b.next(t).Payload) + } + select { + case f := <-b.sent: + t.Fatalf("sent past the window: %+v", f) + case <-time.After(100 * time.Millisecond): + } + + b.frames <- Frame{Kind: FrameWindow, ConnID: "out", Bytes: 15} + for len(sent) < 25 { + sent += string(b.next(t).Payload) + } + if sent != "0123456789abcdefghijklmno" { + t.Fatalf("sent %q", sent) + } +} diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go new file mode 100644 index 0000000..5fc6754 --- /dev/null +++ b/pkg/tunnel/stream.go @@ -0,0 +1,136 @@ +package tunnel + +import ( + "context" + "io" + "log/slog" + + bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + "github.com/vercel/bridge/pkg/plumbing" +) + +// Stream abstracts a bidirectional gRPC stream. Both the server-side +// BidiStreamingServer and client-side TunnelNetworkClient satisfy this +// interface directly since both directions use TunnelNetworkMessage. +type Stream interface { + Send(*bridgev1.TunnelNetworkMessage) error + Recv() (*bridgev1.TunnelNetworkMessage, error) +} + +// NewStreamBackend carries a tunnel over a bridge TunnelNetwork stream. Every +// message addresses its conn, the first one to an unknown conn opens it, and +// an error ends it. The stream has no half-close and no flow control. +func NewStreamBackend(stream Stream) Backend { + return &streamBackend{ + stream: stream, + sendCh: make(chan *bridgev1.TunnelNetworkMessage, 64), + stop: make(chan struct{}), + } +} + +type streamBackend struct { + stream Stream + sendCh chan *bridgev1.TunnelNetworkMessage + // Closed once Run returns, so a Send waiting for room gives up. + stop chan struct{} +} + +func (b *streamBackend) Run(ctx context.Context, deliver func(Frame)) error { + defer close(b.stop) + + // Single goroutine drains the send channel onto the stream. + go func() { + for { + select { + case msg := <-b.sendCh: + if err := b.stream.Send(msg); err != nil { + slog.Info("Tunnel: stream send error", "conn_id", msg.GetConnectionId(), "error", err) + return + } + slog.Debug("Tunnel: sent", "conn_id", msg.GetConnectionId(), "bytes", len(msg.GetData())) + case <-ctx.Done(): + return + } + } + }() + + recvCh := make(chan *bridgev1.TunnelNetworkMessage) + recvErr := make(chan error, 1) + go func() { + for { + msg, err := b.stream.Recv() + if err != nil { + recvErr <- err + return + } + select { + case recvCh <- msg: + case <-ctx.Done(): + return + } + } + }() + + for { + select { + case msg := <-recvCh: + deliver(frameFor(msg)) + case err := <-recvErr: + if err == io.EOF { + slog.Info("Tunnel: stream closed (EOF)") + return nil + } + slog.Info("Tunnel: stream recv error", "error", err) + return err + case <-ctx.Done(): + return nil + } + } +} + +func frameFor(msg *bridgev1.TunnelNetworkMessage) Frame { + if msg.GetError() != "" { + return Frame{Kind: FrameReset, ConnID: msg.GetConnectionId(), Reason: msg.GetError()} + } + f := Frame{Kind: FrameData, ConnID: msg.GetConnectionId(), Payload: msg.GetData()} + if dest := msg.GetDest(); dest != nil { + f.Conn = &ConnInfo{ + Source: Addr{Host: msg.GetSource().GetIp(), Port: int(msg.GetSource().GetPort())}, + Dest: Addr{Host: dest.GetIp(), Port: int(dest.GetPort())}, + Hostname: msg.GetHostname(), + } + } + return f +} + +func (b *streamBackend) ConnID(conn *ConnInfo) string { + return plumbing.ConnectionID(conn.Source.Host, conn.Source.Port, conn.Dest.Host, conn.Dest.Port) +} + +// Open sends nothing: the conn's first Data opens it. +func (b *streamBackend) Open(context.Context, string, *ConnInfo) error { return nil } + +func (b *streamBackend) Send(f Frame) { + msg := &bridgev1.TunnelNetworkMessage{ConnectionId: f.ConnID} + switch f.Kind { + case FrameData: + msg.Data = f.Payload + if f.Conn != nil { + msg.Source = tunnelAddress(f.Conn.Source) + msg.Dest = tunnelAddress(f.Conn.Dest) + msg.Hostname = f.Conn.Hostname + } + case FrameReset: + msg.Error = f.Reason + default: + return + } + select { + case b.sendCh <- msg: + case <-b.stop: + } +} + +func (b *streamBackend) Release(string) {} +func (b *streamBackend) Window() int { return 0 } +func (b *streamBackend) HalfClose() bool { return false } diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index 4755f22..9ebcdf7 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -2,7 +2,6 @@ package tunnel import ( "context" - "fmt" "io" "log/slog" "net" @@ -11,31 +10,29 @@ import ( "github.com/puzpuzpuz/xsync/v3" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" - "github.com/vercel/bridge/pkg/ioutil" "github.com/vercel/bridge/pkg/mitm" "github.com/vercel/bridge/pkg/plumbing" ) -// Stream abstracts a bidirectional gRPC stream. Both the server-side -// BidiStreamingServer and client-side TunnelNetworkClient satisfy this -// interface directly since both directions use TunnelNetworkMessage. -type Stream interface { - Send(*bridgev1.TunnelNetworkMessage) error - Recv() (*bridgev1.TunnelNetworkMessage, error) -} +// Each direction of a conn copies through one buffer of this size. +const copyBufferSize = 32 << 10 -// Tunnel is a wrapper around a Tunnel gRPC stream that is responsible for managing the connections it is currently multiplexing. It is intended to be used by both client and server-side +// Tunnel multiplexes conns over one transport, its Backend. It is used on both +// ends: the intercept and the bridge server, or the intercept and a dev +// tunnel's relay. type Tunnel interface { - // AddConn Adds a connection to the map of connections that are tracked under this Tunnel. When a conn is added, it is placed into a sync map and a goroutine is spawned that reads from the conn. Whenever data is received from the conn, its data is forwarded to the underlying gRPC stream. A destination override may be provided to override the call for original destination from the Conn. The hostname is the DNS name associated with this connection (may be empty). + // AddConn sends conn through the tunnel, to destOverride if given or else + // where conn was headed. hostname is the DNS name the destination was + // looked up as (may be empty). AddConn(conn net.Conn, destOverride string, hostname string) - // Start starts the pump for the Tunnel. It has a loop for receiving messages which are routed based on their connection ID. + // Start runs the backend, opening a local conn for each the peer opens. Start(ctx context.Context) // Close cancels the tunnel context, closing all connections and stopping all goroutines. Close() error - // Done returns a channel that is closed when the recv pump exits. + // Done returns a channel that is closed when the backend stops. Done() <-chan struct{} } @@ -48,15 +45,28 @@ func WithHijacker(h mitm.Hijacker) Option { return func(t *tunnelImpl) { t.hijacker = h } } -// New creates a Tunnel. When a message is received down the stream and a corresponding connection ID cannot be found, the dialer is called to instantiate that connection. The client-side uses the StaticPortDialer while the server-side uses a standard ContextDialer. +// New creates a Tunnel over a bridge TunnelNetwork stream. A conn the peer +// opens is dialed with dialer: the client side uses the StaticPortDialer while +// the server side uses a standard ContextDialer. func New(dialer plumbing.ContextDialer, stream Stream, opts ...Option) Tunnel { + return NewWithBackend(dialer, NewStreamBackend(stream), opts...) +} + +// NewWithBackend creates a Tunnel over backend, dialing the conns the peer +// opens with dialer. +func NewWithBackend(dialer plumbing.ContextDialer, backend Backend, opts ...Option) Tunnel { t := &tunnelImpl{ - dialer: dialer, - stream: stream, - sendCh: make(chan *bridgev1.TunnelNetworkMessage, 64), - conns: xsync.NewMapOf[string, io.ReadWriteCloser](), - done: make(chan struct{}), + dialer: dialer, + backend: backend, + conns: xsync.NewMapOf[string, *streamConn](), + done: make(chan struct{}), + logger: slog.Default(), } + t.bufs.New = func() any { + buf := make([]byte, copyBufferSize) + return &buf + } + t.ctx, t.cancel = context.WithCancel(context.Background()) for _, o := range opts { o(t) } @@ -66,9 +76,10 @@ func New(dialer plumbing.ContextDialer, stream Stream, opts ...Option) Tunnel { type tunnelImpl struct { dialer plumbing.ContextDialer hijacker mitm.Hijacker - stream Stream - sendCh chan *bridgev1.TunnelNetworkMessage - conns *xsync.MapOf[string, io.ReadWriteCloser] + backend Backend + conns *xsync.MapOf[string, *streamConn] + bufs sync.Pool + logger *slog.Logger done chan struct{} ctx context.Context cancel context.CancelFunc @@ -77,229 +88,183 @@ type tunnelImpl struct { func (t *tunnelImpl) Done() <-chan struct{} { return t.done } -func (t *tunnelImpl) AddConn(conn net.Conn, destOverride string, hostname string) { - srcAddr := conn.RemoteAddr().String() - dstAddr := conn.LocalAddr().String() - if destOverride != "" { - dstAddr = destOverride - } - - srcHost, srcPortStr, _ := net.SplitHostPort(srcAddr) - srcPort, _ := strconv.Atoi(srcPortStr) - - dstHost, dstPortStr, _ := net.SplitHostPort(dstAddr) - dstPort, _ := strconv.Atoi(dstPortStr) - - connID := plumbing.ConnectionID(srcHost, srcPort, dstHost, dstPort) - src := &bridgev1.TunnelAddress{Ip: srcHost, Port: int32(srcPort)} - dst := &bridgev1.TunnelAddress{Ip: dstHost, Port: int32(dstPort)} - - slog.Debug("Tunnel: adding connection", "conn_id", connID, "src", srcAddr, "dst", dstAddr, "hostname", hostname) - - t.conns.Store(connID, conn) - go t.readFromConn(conn, connID, src, dst, hostname) -} - -// readFromConn reads from a connection and forwards data to the stream via sendCh. -func (t *tunnelImpl) readFromConn(conn io.ReadCloser, connID string, src, dst *bridgev1.TunnelAddress, hostname string) { - defer func() { - conn.Close() - t.conns.Delete(connID) - }() - - buf := make([]byte, 32*1024) - for { - n, err := conn.Read(buf) - if n > 0 { - data := make([]byte, n) - copy(data, buf[:n]) - select { - case t.sendCh <- &bridgev1.TunnelNetworkMessage{ - ConnectionId: connID, - Source: src, - Dest: dst, - Data: data, - Hostname: hostname, - }: - case <-t.ctx.Done(): - return - } - } - if err != nil { - return +func (t *tunnelImpl) Start(ctx context.Context) { + stop := context.AfterFunc(ctx, t.cancel) + go func() { + defer stop() + defer t.Close() + if err := t.backend.Run(t.ctx, t.deliver); err != nil { + t.logger.Info("Tunnel: backend stopped", "error", err) } - } + }() } func (t *tunnelImpl) Close() error { t.closeOnce.Do(func() { t.cancel() - t.closeAll() + t.conns.Range(func(_ string, c *streamConn) bool { + c.finish() + return true + }) close(t.done) }) return nil } -func (t *tunnelImpl) Start(ctx context.Context) { - t.ctx, t.cancel = context.WithCancel(ctx) +func (t *tunnelImpl) AddConn(conn net.Conn, destOverride string, hostname string) { + srcAddr := conn.RemoteAddr().String() + dstAddr := conn.LocalAddr().String() + if destOverride != "" { + dstAddr = destOverride + } + info := &ConnInfo{Source: splitAddr(srcAddr), Dest: splitAddr(dstAddr), Hostname: hostname} + id := t.backend.ConnID(info) + logger := t.logger.With("conn_id", id, "src", srcAddr, "dst", dstAddr, "hostname", hostname) + logger.Debug("Tunnel: adding connection") - // Single goroutine drains the send channel onto the stream. + c := t.newConn(id, info) + t.conns.Store(id, c) go func() { - for { - select { - case msg, ok := <-t.sendCh: - if !ok { - return - } - if err := t.stream.Send(msg); err != nil { - slog.Info("Tunnel: stream send error", "conn_id", msg.GetConnectionId(), "error", err) - return - } - slog.Debug("Tunnel: sent", "conn_id", msg.GetConnectionId(), "bytes", len(msg.GetData())) - case <-t.ctx.Done(): - return - } + if err := t.backend.Open(t.ctx, id, info); err != nil { + logger.Info("Tunnel: open failed", "error", err) + c.finish() + _ = conn.Close() + return } + t.pump(conn, c) }() +} - // Recv pump runs until the stream closes or context is cancelled. - go func() { - defer t.Close() - - recvCh := make(chan *bridgev1.TunnelNetworkMessage) - recvErr := make(chan error, 1) - go func() { - for { - msg, err := t.stream.Recv() - if err != nil { - recvErr <- err - return - } - recvCh <- msg - } - }() - - for { - select { - case msg := <-recvCh: - connID := msg.GetConnectionId() - - // Route to existing connection. - if conn, ok := t.conns.Load(connID); ok { - if msg.GetError() != "" { - conn.Close() - t.conns.Delete(connID) - continue - } - if data := msg.GetData(); len(data) > 0 { - if _, err := conn.Write(data); err != nil { - slog.Debug("Write to conn failed", "connection_id", connID, "error", err) - conn.Close() - t.conns.Delete(connID) - } - } - continue - } - - // Unknown connection ID → this message opens it. - t.openConn(msg) - - case err := <-recvErr: - if err != io.EOF { - slog.Info("Tunnel: stream recv error", "error", err) - } else { - slog.Info("Tunnel: stream closed (EOF)") - } - return - - case <-t.ctx.Done(): - return - } +// deliver routes one of the peer's frames to its conn. +func (t *tunnelImpl) deliver(f Frame) { + c, ok := t.conns.Load(f.ConnID) + switch f.Kind { + case FrameOpen: + t.open(f) + case FrameData: + if ok { + c.deliver(f.Payload) + } else { + t.open(f) } - }() + case FrameCloseWrite: + if ok { + c.peerClosed() + } + case FrameReset: + if ok { + c.finish() + } + case FrameWindow: + if ok { + c.grant(f.Bytes) + } + } } -// openConn registers a buffered connection for msg's ID before dialing. The -// peer has no explicit open message: the first chunk of a connection is what -// opens it, and the next chunks can arrive before the dial completes. Storing -// the buffer synchronously on the recv pump makes those chunks queue behind -// the first one instead of each dialing a second connection under the same ID -// and splitting the stream between them. -func (t *tunnelImpl) openConn(msg *bridgev1.TunnelNetworkMessage) { - connID := msg.GetConnectionId() - if msg.GetDest() == nil { - slog.Info("Tunnel: ignoring message with no dest", "conn_id", connID) +// open registers a conn the peer opened before dialing it, so the frames that +// arrive during the dial queue on it instead of each opening another. +func (t *tunnelImpl) open(f Frame) { + if f.Conn == nil { + t.logger.Info("Tunnel: ignoring message with no dest", "conn_id", f.ConnID) return } - - buffered := ioutil.NewBufferedReadWriteCloser() - t.conns.Store(connID, buffered) - if data := msg.GetData(); len(data) > 0 { - _, _ = buffered.Write(data) + // This side sends from where the peer's conn was headed, back to it. + reply := &ConnInfo{Source: f.Conn.Dest, Dest: f.Conn.Source, Hostname: f.Conn.Hostname} + c, loaded := t.conns.LoadOrStore(f.ConnID, t.newConn(f.ConnID, reply)) + c.deliver(f.Payload) + if loaded { + return } - go t.dial(msg, buffered) + go func() { + local, err := t.dial(f) + if err != nil { + t.logger.Info("Tunnel: connect failed", "conn_id", f.ConnID, "hostname", f.Conn.Hostname, "error", err) + c.reset(err.Error()) + return + } + t.pump(local, c) + }() } -func (t *tunnelImpl) dial(msg *bridgev1.TunnelNetworkMessage, buffered ioutil.BufferedReadWriteCloser) { - connID := msg.GetConnectionId() - hostname := msg.GetHostname() - dest := msg.GetDest() - - // Resolve the connection: hijacker gets first shot, then fall through to the dialer. - var conn net.Conn - var err error - - if t.hijacker != nil && t.hijacker.ShouldHijack(msg) { - conn, err = t.hijacker.Hijack(t.ctx, msg) - if err != nil { - slog.Info("Tunnel: hijack failed, falling back to dialer", "conn_id", connID, "hostname", hostname, "error", err) +// dial gives the hijacker first shot at a conn the peer opened, then falls +// through to the dialer. +func (t *tunnelImpl) dial(f Frame) (net.Conn, error) { + if t.hijacker != nil { + msg := &bridgev1.TunnelNetworkMessage{ + ConnectionId: f.ConnID, + Source: tunnelAddress(f.Conn.Source), + Dest: tunnelAddress(f.Conn.Dest), + Hostname: f.Conn.Hostname, + } + if t.hijacker.ShouldHijack(msg) { + conn, err := t.hijacker.Hijack(t.ctx, msg) + if err == nil { + return conn, nil + } + t.logger.Info("Tunnel: hijack failed, falling back to dialer", "conn_id", f.ConnID, "hostname", f.Conn.Hostname, "error", err) } } + return t.dialer.DialContext(t.ctx, "tcp", f.Conn.Dest.String()) +} - if conn == nil { - addr := fmt.Sprintf("%s:%d", dest.GetIp(), dest.GetPort()) - conn, err = t.dialer.DialContext(t.ctx, "tcp", addr) +// pump copies between local and c in both directions, each through a buffer +// from the pool, until the conn is over. +func (t *tunnelImpl) pump(local net.Conn, c *streamConn) { + if !c.attach(local) { + return } + toPeer := t.bufs.Get().(*[]byte) + fromPeer := t.bufs.Get().(*[]byte) + defer func() { + t.bufs.Put(toPeer) + t.bufs.Put(fromPeer) + }() + + var wg sync.WaitGroup + wg.Add(2) + go func() { + defer wg.Done() + copyHalf(c, local, *toPeer, c) + }() + go func() { + defer wg.Done() + copyHalf(local, c, *fromPeer, c) + }() + wg.Wait() + c.finish() +} +// copyHalf copies src to dst until src ends, then half-closes dst. A failure +// either way abandons the conn. +func copyHalf(dst net.Conn, src net.Conn, buf []byte, c *streamConn) { + // Hides ReadFrom and WriteTo, which would copy through buffers of their own. + _, err := io.CopyBuffer(struct{ io.Writer }{dst}, struct{ io.Reader }{src}, buf) if err != nil { - slog.Info("Tunnel: connect failed", "conn_id", connID, "hostname", hostname, "error", err) - t.deleteIfSame(connID, buffered) - buffered.Close() - select { - case t.sendCh <- &bridgev1.TunnelNetworkMessage{ - ConnectionId: connID, - Error: err.Error(), - }: - case <-t.ctx.Done(): - } + c.reset(err.Error()) return } - - if err := buffered.Set(conn); err != nil { - slog.Debug("Tunnel: dropping dialed connection", "connection_id", connID, "error", err) - t.deleteIfSame(connID, buffered) - return + if cw, ok := dst.(interface{ CloseWrite() error }); ok { + _ = cw.CloseWrite() } - - go t.readFromConn(buffered, connID, dest, msg.GetSource(), hostname) } -// deleteIfSame removes connID only while it still maps to conn, so a connection -// the peer has since reopened under the same ID is left alone. -func (t *tunnelImpl) deleteIfSame(connID string, conn ioutil.BufferedReadWriteCloser) { - t.conns.Compute(connID, func(cur io.ReadWriteCloser, loaded bool) (io.ReadWriteCloser, bool) { +// forget removes c, unless another conn has since taken its id. +func (t *tunnelImpl) forget(c *streamConn) { + t.conns.Compute(c.id, func(cur *streamConn, loaded bool) (*streamConn, bool) { if !loaded { return nil, true } - b, ok := cur.(ioutil.BufferedReadWriteCloser) - return cur, ok && b == conn + return cur, cur == c }) } -func (t *tunnelImpl) closeAll() { - t.conns.Range(func(key string, conn io.ReadWriteCloser) bool { - conn.Close() - t.conns.Delete(key) - return true - }) +func splitAddr(s string) Addr { + host, portStr, _ := net.SplitHostPort(s) + port, _ := strconv.Atoi(portStr) + return Addr{Host: host, Port: port} +} + +func tunnelAddress(a Addr) *bridgev1.TunnelAddress { + return &bridgev1.TunnelAddress{Ip: a.Host, Port: int32(a.Port)} } From acfd89ec86ba63d5ff4fe3b0fa22fee2b580311f Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 14:01:35 -0700 Subject: [PATCH 04/11] refactor(tunnel): move per-conn flow control into plumbing.MeteredConn MeteredConn wraps any net.Conn: Write holds to the credit Grant gives, Read reports what it returns so the caller can grant it back, and Stats counts bytes and the time writes waited for credit. The tunnel wraps each conn's stream side in one and logs its stats when the conn ends. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/plumbing/metered.go | 160 +++++++++++++++++++++++++++++++++++ pkg/plumbing/metered_test.go | 130 ++++++++++++++++++++++++++++ pkg/tunnel/conn.go | 66 +++++---------- pkg/tunnel/tunnel.go | 19 +++-- 4 files changed, 325 insertions(+), 50 deletions(-) create mode 100644 pkg/plumbing/metered.go create mode 100644 pkg/plumbing/metered_test.go diff --git a/pkg/plumbing/metered.go b/pkg/plumbing/metered.go new file mode 100644 index 0000000..274dfcb --- /dev/null +++ b/pkg/plumbing/metered.go @@ -0,0 +1,160 @@ +package plumbing + +import ( + "errors" + "net" + "sync" + "sync/atomic" + "time" +) + +// MeteredConn wraps a net.Conn and counts what passes through it. Given a +// window, Write sends no more than the peer has granted with Grant, and Read +// reports what it returns to OnRead so the caller can grant it back: the flow +// control a conn multiplexed with others needs, since the transport's own +// can only slow all of them. +type MeteredConn struct { + net.Conn + cfg MeteredConfig + + credit atomic.Int64 + wake chan struct{} + closed chan struct{} + closeOnce sync.Once + // The wrapped conn's, if it has a Done method, so a write waiting for + // credit returns once that conn is over. + innerDone <-chan struct{} + + bytesRead, bytesWritten atomic.Int64 + writeWait, writeWaits atomic.Int64 +} + +// MeteredConfig configures a MeteredConn. +type MeteredConfig struct { + // Window is what Write may send before Grant gives more; 0 for no limit. + Window int + // OnRead is called with what each Read returned. + OnRead func(n int) +} + +// MeteredStats is what has passed through a MeteredConn. +type MeteredStats struct { + BytesRead int64 + BytesWritten int64 + // What Write may still send. Zero without a window. + Credit int64 + // How long writes waited for credit, over how many waits. + WriteWait time.Duration + WriteWaits int64 +} + +var _ net.Conn = (*MeteredConn)(nil) + +func NewMeteredConn(conn net.Conn, cfg MeteredConfig) *MeteredConn { + m := &MeteredConn{ + Conn: conn, + cfg: cfg, + wake: make(chan struct{}, 1), + closed: make(chan struct{}), + } + m.credit.Store(int64(cfg.Window)) + if d, ok := conn.(interface{ Done() <-chan struct{} }); ok { + m.innerDone = d.Done() + } + return m +} + +func (m *MeteredConn) Read(p []byte) (int, error) { + n, err := m.Conn.Read(p) + if n > 0 { + m.bytesRead.Add(int64(n)) + if m.cfg.OnRead != nil { + m.cfg.OnRead(n) + } + } + return n, err +} + +func (m *MeteredConn) Write(p []byte) (int, error) { + if m.cfg.Window <= 0 { + n, err := m.Conn.Write(p) + m.bytesWritten.Add(int64(n)) + return n, err + } + written := 0 + for written < len(p) { + credit, err := m.awaitCredit() + if err != nil { + return written, err + } + size := min(len(p)-written, int(credit)) + n, err := m.Conn.Write(p[written : written+size]) + m.credit.Add(-int64(n)) + m.bytesWritten.Add(int64(n)) + written += n + if err != nil { + return written, err + } + } + return written, nil +} + +// Grant lets Write send n more bytes. +func (m *MeteredConn) Grant(n int) { + m.credit.Add(int64(n)) + select { + case m.wake <- struct{}{}: + default: + } +} + +// CloseWrite half-closes the wrapped conn, if it can. +func (m *MeteredConn) CloseWrite() error { + if cw, ok := m.Conn.(interface{ CloseWrite() error }); ok { + return cw.CloseWrite() + } + return errors.ErrUnsupported +} + +func (m *MeteredConn) Close() error { + m.closeOnce.Do(func() { close(m.closed) }) + return m.Conn.Close() +} + +func (m *MeteredConn) Stats() MeteredStats { + st := MeteredStats{ + BytesRead: m.bytesRead.Load(), + BytesWritten: m.bytesWritten.Load(), + WriteWait: time.Duration(m.writeWait.Load()), + WriteWaits: m.writeWaits.Load(), + } + if m.cfg.Window > 0 { + st.Credit = m.credit.Load() + } + return st +} + +func (m *MeteredConn) awaitCredit() (int64, error) { + var start time.Time + defer func() { + if !start.IsZero() { + m.writeWait.Add(int64(time.Since(start))) + } + }() + for { + if credit := m.credit.Load(); credit > 0 { + return credit, nil + } + if start.IsZero() { + start = time.Now() + m.writeWaits.Add(1) + } + select { + case <-m.wake: + case <-m.closed: + return 0, net.ErrClosed + case <-m.innerDone: + return 0, net.ErrClosed + } + } +} diff --git a/pkg/plumbing/metered_test.go b/pkg/plumbing/metered_test.go new file mode 100644 index 0000000..115199b --- /dev/null +++ b/pkg/plumbing/metered_test.go @@ -0,0 +1,130 @@ +package plumbing + +import ( + "bytes" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordingConn keeps what's written to it and reads from r. +type recordingConn struct { + net.Conn + r io.Reader + mu sync.Mutex + w bytes.Buffer + done chan struct{} +} + +func (c *recordingConn) Read(p []byte) (int, error) { return c.r.Read(p) } + +func (c *recordingConn) Write(p []byte) (int, error) { + c.mu.Lock() + defer c.mu.Unlock() + return c.w.Write(p) +} + +func (c *recordingConn) written() string { + c.mu.Lock() + defer c.mu.Unlock() + return c.w.String() +} + +func (c *recordingConn) Close() error { return nil } + +func (c *recordingConn) Done() <-chan struct{} { return c.done } + +func newRecordingConn(r string) *recordingConn { + return &recordingConn{r: bytes.NewReader([]byte(r)), done: make(chan struct{})} +} + +// writeAsync starts a Write and returns its result. +func writeAsync(m *MeteredConn, p string) <-chan error { + errc := make(chan error, 1) + go func() { + _, err := m.Write([]byte(p)) + errc <- err + }() + return errc +} + +func TestMeteredConnCountsWithoutAWindow(t *testing.T) { + inner := newRecordingConn("hello") + var reads []int + m := NewMeteredConn(inner, MeteredConfig{OnRead: func(n int) { reads = append(reads, n) }}) + + _, err := m.Write([]byte("abc")) + require.NoError(t, err) + buf := make([]byte, 16) + n, err := m.Read(buf) + require.NoError(t, err) + + assert.Equal(t, "abc", inner.written()) + assert.Equal(t, []int{n}, reads) + assert.Equal(t, MeteredStats{BytesRead: 5, BytesWritten: 3}, m.Stats()) +} + +func TestMeteredConnHoldsWritesToTheWindow(t *testing.T) { + inner := newRecordingConn("") + m := NewMeteredConn(inner, MeteredConfig{Window: 4}) + + errc := writeAsync(m, "abcdefgh") + require.Eventually(t, func() bool { return inner.written() == "abcd" }, time.Second, time.Millisecond) + select { + case <-errc: + t.Fatal("wrote past the window") + case <-time.After(50 * time.Millisecond): + } + + m.Grant(4) + require.NoError(t, <-errc) + assert.Equal(t, "abcdefgh", inner.written()) + st := m.Stats() + assert.Equal(t, int64(0), st.Credit) + assert.Equal(t, int64(1), st.WriteWaits) + assert.Greater(t, st.WriteWait, time.Duration(0)) +} + +func TestMeteredConnWriteWaitingEndsWithTheConn(t *testing.T) { + m := NewMeteredConn(newRecordingConn(""), MeteredConfig{Window: 1}) + errc := writeAsync(m, "ab") + time.Sleep(10 * time.Millisecond) + require.NoError(t, m.Close()) + assert.ErrorIs(t, <-errc, net.ErrClosed) + + inner := newRecordingConn("") + m = NewMeteredConn(inner, MeteredConfig{Window: 1}) + errc = writeAsync(m, "ab") + time.Sleep(10 * time.Millisecond) + close(inner.done) + assert.ErrorIs(t, <-errc, net.ErrClosed) +} + +func TestMeteredConnCloseWrite(t *testing.T) { + assert.True(t, errors.Is(NewMeteredConn(newRecordingConn(""), MeteredConfig{}).CloseWrite(), errors.ErrUnsupported)) + + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + defer lis.Close() + go func() { + nc, err := lis.Accept() + if err == nil { + _, _ = io.Copy(io.Discard, nc) + _ = nc.Close() + } + }() + nc, err := net.Dial("tcp", lis.Addr().String()) + require.NoError(t, err) + defer nc.Close() + m := NewMeteredConn(nc, MeteredConfig{}) + require.NoError(t, m.CloseWrite()) + // The peer saw EOF and closed its side. + _, err = nc.Read(make([]byte, 1)) + assert.ErrorIs(t, err, io.EOF) +} diff --git a/pkg/tunnel/conn.go b/pkg/tunnel/conn.go index df3b9e1..853ef31 100644 --- a/pkg/tunnel/conn.go +++ b/pkg/tunnel/conn.go @@ -7,6 +7,8 @@ import ( "sync" "sync/atomic" "time" + + "github.com/vercel/bridge/pkg/plumbing" ) const ( @@ -18,12 +20,13 @@ const ( ) // streamConn is one conn's side on the tunnel's transport, as a net.Conn: Read -// takes what the peer sent, Write sends to it as the conn's window allows, and -// CloseWrite half-closes it. +// takes what the peer sent, Write sends to it, and CloseWrite half-closes it. +// The tunnel uses it through meter, which holds writes to the conn's window. type streamConn struct { - t *tunnelImpl - id string - info *ConnInfo + t *tunnelImpl + id string + info *ConnInfo + meter *plumbing.MeteredConn // 0 when the backend has no flow control. window int halfClose bool @@ -38,10 +41,7 @@ type streamConn struct { // The rest of the chunk Read is on; only Read touches it. pending []byte - // What this side may still send the peer. - sendWindow atomic.Int64 - winWake chan struct{} - closeSent atomic.Bool + closeSent atomic.Bool local atomic.Pointer[net.Conn] done chan struct{} @@ -59,10 +59,14 @@ func (t *tunnelImpl) newConn(id string, info *ConnInfo) *streamConn { halfClose: t.backend.HalfClose(), inWake: make(chan struct{}, 1), inRoom: make(chan struct{}, 1), - winWake: make(chan struct{}, 1), done: make(chan struct{}), } - c.sendWindow.Store(int64(c.window)) + cfg := plumbing.MeteredConfig{Window: c.window} + if c.window > 0 { + // Gives the peer back what was read. + cfg.OnRead = func(n int) { c.send(Frame{Kind: FrameWindow, Bytes: n}) } + } + c.meter = plumbing.NewMeteredConn(c, cfg) return c } @@ -95,12 +99,8 @@ func (c *streamConn) Read(p []byte) (int, error) { return n, nil } -// consumed gives the peer back what Read took. +// consumed makes room for what a peer without flow control sends next. func (c *streamConn) consumed(n int) { - if c.window > 0 { - c.send(Frame{Kind: FrameWindow, Bytes: n}) - return - } c.inMu.Lock() c.inBytes -= n c.inMu.Unlock() @@ -110,17 +110,10 @@ func (c *streamConn) consumed(n int) { func (c *streamConn) Write(p []byte) (int, error) { written := 0 for written < len(p) { - size := min(len(p)-written, maxFrame) - if c.window > 0 { - w, err := c.awaitWindow() - if err != nil { - return written, err - } - size = min(size, w) - c.sendWindow.Add(-int64(size)) - } else if c.isDone() { + if c.isDone() { return written, net.ErrClosed } + size := min(len(p)-written, maxFrame) // The backend may send it after Write returns. c.send(Frame{Kind: FrameData, Payload: bytes.Clone(p[written : written+size])}) written += size @@ -128,21 +121,6 @@ func (c *streamConn) Write(p []byte) (int, error) { return written, nil } -func (c *streamConn) awaitWindow() (int, error) { - for { - if c.isDone() { - return 0, net.ErrClosed - } - if w := c.sendWindow.Load(); w > 0 { - return int(w), nil - } - select { - case <-c.winWake: - case <-c.done: - } - } -} - // CloseWrite tells the peer this side has nothing more to send. Over a // transport without half-close, it ends the conn. func (c *streamConn) CloseWrite() error { @@ -156,6 +134,9 @@ func (c *streamConn) CloseWrite() error { return nil } +// Done is closed once the conn is over. +func (c *streamConn) Done() <-chan struct{} { return c.done } + // Close resets the conn. func (c *streamConn) Close() error { c.reset("closed") @@ -205,11 +186,6 @@ func (c *streamConn) peerClosed() { wake(c.inWake) } -func (c *streamConn) grant(n int) { - c.sendWindow.Add(int64(n)) - wake(c.winWake) -} - // attach hands the conn its local end, which it closes when it's over. // Reports false if the conn is already over. func (c *streamConn) attach(local net.Conn) bool { diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index 9ebcdf7..be1641f 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -157,7 +157,7 @@ func (t *tunnelImpl) deliver(f Frame) { } case FrameWindow: if ok { - c.grant(f.Bytes) + c.meter.Grant(f.Bytes) } } } @@ -208,8 +208,8 @@ func (t *tunnelImpl) dial(f Frame) (net.Conn, error) { return t.dialer.DialContext(t.ctx, "tcp", f.Conn.Dest.String()) } -// pump copies between local and c in both directions, each through a buffer -// from the pool, until the conn is over. +// pump copies between local and c, through its meter, in both directions, +// each through a buffer from the pool, until the conn is over. func (t *tunnelImpl) pump(local net.Conn, c *streamConn) { if !c.attach(local) { return @@ -225,14 +225,23 @@ func (t *tunnelImpl) pump(local net.Conn, c *streamConn) { wg.Add(2) go func() { defer wg.Done() - copyHalf(c, local, *toPeer, c) + copyHalf(c.meter, local, *toPeer, c) }() go func() { defer wg.Done() - copyHalf(local, c, *fromPeer, c) + copyHalf(local, c.meter, *fromPeer, c) }() wg.Wait() c.finish() + + st := c.meter.Stats() + t.logger.Debug("Tunnel: connection closed", + "conn_id", c.id, + "bytes_sent", st.BytesWritten, + "bytes_received", st.BytesRead, + "window_wait", st.WriteWait, + "window_waits", st.WriteWaits, + ) } // copyHalf copies src to dst until src ends, then half-closes dst. A failure From 7746dd5f6ce3cf0bcb4443c54e6cc60723ab6d63 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 15:33:53 -0700 Subject: [PATCH 05/11] refactor(plumbing): make MeteredConn an interface without a read callback Granting back what was read is the transport's protocol, so the tunnel's stream conn sends its Window frames again. NewMeteredConn takes the window and returns the interface; Stats returns a snapshot by pointer. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/plumbing/metered.go | 72 +++++++++++++++++------------------- pkg/plumbing/metered_test.go | 21 +++++------ pkg/tunnel/conn.go | 16 ++++---- 3 files changed, 50 insertions(+), 59 deletions(-) diff --git a/pkg/plumbing/metered.go b/pkg/plumbing/metered.go index 274dfcb..6d87a92 100644 --- a/pkg/plumbing/metered.go +++ b/pkg/plumbing/metered.go @@ -8,14 +8,23 @@ import ( "time" ) -// MeteredConn wraps a net.Conn and counts what passes through it. Given a -// window, Write sends no more than the peer has granted with Grant, and Read -// reports what it returns to OnRead so the caller can grant it back: the flow -// control a conn multiplexed with others needs, since the transport's own -// can only slow all of them. -type MeteredConn struct { +// MeteredConn is a net.Conn that counts what passes through it. Given a +// window, Write sends no more than the peer has granted with Grant: the flow +// control a conn multiplexed with others needs, since the transport's own can +// only slow all of them. +type MeteredConn interface { net.Conn - cfg MeteredConfig + // Grant lets Write send n more bytes. + Grant(n int) + // CloseWrite half-closes the wrapped conn, if it can. + CloseWrite() error + // Stats reports what has passed through so far. + Stats() *MeteredStats +} + +type meteredConn struct { + net.Conn + window int credit atomic.Int64 wake chan struct{} @@ -29,14 +38,6 @@ type MeteredConn struct { writeWait, writeWaits atomic.Int64 } -// MeteredConfig configures a MeteredConn. -type MeteredConfig struct { - // Window is what Write may send before Grant gives more; 0 for no limit. - Window int - // OnRead is called with what each Read returned. - OnRead func(n int) -} - // MeteredStats is what has passed through a MeteredConn. type MeteredStats struct { BytesRead int64 @@ -48,35 +49,30 @@ type MeteredStats struct { WriteWaits int64 } -var _ net.Conn = (*MeteredConn)(nil) - -func NewMeteredConn(conn net.Conn, cfg MeteredConfig) *MeteredConn { - m := &MeteredConn{ +// NewMeteredConn wraps conn. Write may send window bytes before Grant gives +// more; 0 for no limit. +func NewMeteredConn(conn net.Conn, window int) MeteredConn { + m := &meteredConn{ Conn: conn, - cfg: cfg, + window: window, wake: make(chan struct{}, 1), closed: make(chan struct{}), } - m.credit.Store(int64(cfg.Window)) + m.credit.Store(int64(window)) if d, ok := conn.(interface{ Done() <-chan struct{} }); ok { m.innerDone = d.Done() } return m } -func (m *MeteredConn) Read(p []byte) (int, error) { +func (m *meteredConn) Read(p []byte) (int, error) { n, err := m.Conn.Read(p) - if n > 0 { - m.bytesRead.Add(int64(n)) - if m.cfg.OnRead != nil { - m.cfg.OnRead(n) - } - } + m.bytesRead.Add(int64(n)) return n, err } -func (m *MeteredConn) Write(p []byte) (int, error) { - if m.cfg.Window <= 0 { +func (m *meteredConn) Write(p []byte) (int, error) { + if m.window <= 0 { n, err := m.Conn.Write(p) m.bytesWritten.Add(int64(n)) return n, err @@ -99,8 +95,7 @@ func (m *MeteredConn) Write(p []byte) (int, error) { return written, nil } -// Grant lets Write send n more bytes. -func (m *MeteredConn) Grant(n int) { +func (m *meteredConn) Grant(n int) { m.credit.Add(int64(n)) select { case m.wake <- struct{}{}: @@ -108,33 +103,32 @@ func (m *MeteredConn) Grant(n int) { } } -// CloseWrite half-closes the wrapped conn, if it can. -func (m *MeteredConn) CloseWrite() error { +func (m *meteredConn) CloseWrite() error { if cw, ok := m.Conn.(interface{ CloseWrite() error }); ok { return cw.CloseWrite() } return errors.ErrUnsupported } -func (m *MeteredConn) Close() error { +func (m *meteredConn) Close() error { m.closeOnce.Do(func() { close(m.closed) }) return m.Conn.Close() } -func (m *MeteredConn) Stats() MeteredStats { - st := MeteredStats{ +func (m *meteredConn) Stats() *MeteredStats { + st := &MeteredStats{ BytesRead: m.bytesRead.Load(), BytesWritten: m.bytesWritten.Load(), WriteWait: time.Duration(m.writeWait.Load()), WriteWaits: m.writeWaits.Load(), } - if m.cfg.Window > 0 { + if m.window > 0 { st.Credit = m.credit.Load() } return st } -func (m *MeteredConn) awaitCredit() (int64, error) { +func (m *meteredConn) awaitCredit() (int64, error) { var start time.Time defer func() { if !start.IsZero() { diff --git a/pkg/plumbing/metered_test.go b/pkg/plumbing/metered_test.go index 115199b..2b0831b 100644 --- a/pkg/plumbing/metered_test.go +++ b/pkg/plumbing/metered_test.go @@ -45,7 +45,7 @@ func newRecordingConn(r string) *recordingConn { } // writeAsync starts a Write and returns its result. -func writeAsync(m *MeteredConn, p string) <-chan error { +func writeAsync(m MeteredConn, p string) <-chan error { errc := make(chan error, 1) go func() { _, err := m.Write([]byte(p)) @@ -56,23 +56,20 @@ func writeAsync(m *MeteredConn, p string) <-chan error { func TestMeteredConnCountsWithoutAWindow(t *testing.T) { inner := newRecordingConn("hello") - var reads []int - m := NewMeteredConn(inner, MeteredConfig{OnRead: func(n int) { reads = append(reads, n) }}) + m := NewMeteredConn(inner, 0) _, err := m.Write([]byte("abc")) require.NoError(t, err) - buf := make([]byte, 16) - n, err := m.Read(buf) + _, err = m.Read(make([]byte, 16)) require.NoError(t, err) assert.Equal(t, "abc", inner.written()) - assert.Equal(t, []int{n}, reads) - assert.Equal(t, MeteredStats{BytesRead: 5, BytesWritten: 3}, m.Stats()) + assert.Equal(t, &MeteredStats{BytesRead: 5, BytesWritten: 3}, m.Stats()) } func TestMeteredConnHoldsWritesToTheWindow(t *testing.T) { inner := newRecordingConn("") - m := NewMeteredConn(inner, MeteredConfig{Window: 4}) + m := NewMeteredConn(inner, 4) errc := writeAsync(m, "abcdefgh") require.Eventually(t, func() bool { return inner.written() == "abcd" }, time.Second, time.Millisecond) @@ -92,14 +89,14 @@ func TestMeteredConnHoldsWritesToTheWindow(t *testing.T) { } func TestMeteredConnWriteWaitingEndsWithTheConn(t *testing.T) { - m := NewMeteredConn(newRecordingConn(""), MeteredConfig{Window: 1}) + m := NewMeteredConn(newRecordingConn(""), 1) errc := writeAsync(m, "ab") time.Sleep(10 * time.Millisecond) require.NoError(t, m.Close()) assert.ErrorIs(t, <-errc, net.ErrClosed) inner := newRecordingConn("") - m = NewMeteredConn(inner, MeteredConfig{Window: 1}) + m = NewMeteredConn(inner, 1) errc = writeAsync(m, "ab") time.Sleep(10 * time.Millisecond) close(inner.done) @@ -107,7 +104,7 @@ func TestMeteredConnWriteWaitingEndsWithTheConn(t *testing.T) { } func TestMeteredConnCloseWrite(t *testing.T) { - assert.True(t, errors.Is(NewMeteredConn(newRecordingConn(""), MeteredConfig{}).CloseWrite(), errors.ErrUnsupported)) + assert.True(t, errors.Is(NewMeteredConn(newRecordingConn(""), 0).CloseWrite(), errors.ErrUnsupported)) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -122,7 +119,7 @@ func TestMeteredConnCloseWrite(t *testing.T) { nc, err := net.Dial("tcp", lis.Addr().String()) require.NoError(t, err) defer nc.Close() - m := NewMeteredConn(nc, MeteredConfig{}) + m := NewMeteredConn(nc, 0) require.NoError(t, m.CloseWrite()) // The peer saw EOF and closed its side. _, err = nc.Read(make([]byte, 1)) diff --git a/pkg/tunnel/conn.go b/pkg/tunnel/conn.go index 853ef31..82fb7f1 100644 --- a/pkg/tunnel/conn.go +++ b/pkg/tunnel/conn.go @@ -26,7 +26,7 @@ type streamConn struct { t *tunnelImpl id string info *ConnInfo - meter *plumbing.MeteredConn + meter plumbing.MeteredConn // 0 when the backend has no flow control. window int halfClose bool @@ -61,12 +61,7 @@ func (t *tunnelImpl) newConn(id string, info *ConnInfo) *streamConn { inRoom: make(chan struct{}, 1), done: make(chan struct{}), } - cfg := plumbing.MeteredConfig{Window: c.window} - if c.window > 0 { - // Gives the peer back what was read. - cfg.OnRead = func(n int) { c.send(Frame{Kind: FrameWindow, Bytes: n}) } - } - c.meter = plumbing.NewMeteredConn(c, cfg) + c.meter = plumbing.NewMeteredConn(c, c.window) return c } @@ -99,8 +94,13 @@ func (c *streamConn) Read(p []byte) (int, error) { return n, nil } -// consumed makes room for what a peer without flow control sends next. +// consumed gives back what Read took: as window to a peer with flow control, +// and as room in the queue to one without. func (c *streamConn) consumed(n int) { + if c.window > 0 { + c.send(Frame{Kind: FrameWindow, Bytes: n}) + return + } c.inMu.Lock() c.inBytes -= n c.inMu.Unlock() From 50e1d1801fd3bd7807886290b972593accac5a79 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 15:53:29 -0700 Subject: [PATCH 06/11] refactor: turn any plumbing.Stream into a net.Conn; name the dev tunnel protos devtunnel plumbing.Stream is one conn's bytes each way (Send, Recv, CloseSend, Close), and NewStreamConn makes a net.Conn of one. It replaces the unused TunnelConn. The tunnel's side of each conn is now a Stream, wrapped in a StreamConn and a MeteredConn, instead of a net.Conn of its own. The vendored dev tunnel proto keeps vercel/api's file as is; a go_package override in buf.gen.yaml names its Go package devtunnel. Co-Authored-By: Claude Opus 5.5 (1M context) --- api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go | 6 +- .../api/dev_tunnel/v1/tunnel_grpc.pb.go | 2 +- api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js | 2 +- buf.gen.yaml | 4 + pkg/devtunnel/backend.go | 16 +- pkg/devtunnel/backend_test.go | 44 +++--- pkg/devtunnel/stream.go | 36 ++--- pkg/plumbing/conn.go | 139 +++++++----------- pkg/plumbing/conn_test.go | 71 +++++++++ pkg/tunnel/conn.go | 111 +++++++------- pkg/tunnel/tunnel.go | 26 ++-- 11 files changed, 244 insertions(+), 213 deletions(-) create mode 100644 pkg/plumbing/conn_test.go diff --git a/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go b/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go index 16c6f12..0da3bb4 100644 --- a/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go +++ b/api/go/vercel/api/dev_tunnel/v1/tunnel.pb.go @@ -7,7 +7,7 @@ // protoc (unknown) // source: vercel/api/dev_tunnel/v1/tunnel.proto -package dev_tunnelv1 +package devtunnel import ( protoreflect "google.golang.org/protobuf/reflect/protoreflect" @@ -1052,8 +1052,8 @@ const file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDesc = "" + "\fPROTOCOL_TCP\x10\x012\xce\x01\n" + "\rTunnelService\x12b\n" + "\aConnect\x12(.vercel.api.dev_tunnel.v1.ConnectRequest\x1a).vercel.api.dev_tunnel.v1.ConnectResponse(\x010\x01\x12Y\n" + - "\x04Dial\x12%.vercel.api.dev_tunnel.v1.DialRequest\x1a&.vercel.api.dev_tunnel.v1.DialResponse(\x010\x01B\xf1\x01\n" + - "\x1ccom.vercel.api.dev_tunnel.v1B\vTunnelProtoP\x01ZEgithub.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1;dev_tunnelv1\xa2\x02\x03VAD\xaa\x02\x17Vercel.Api.DevTunnel.V1\xca\x02\x17Vercel\\Api\\DevTunnel\\V1\xe2\x02#Vercel\\Api\\DevTunnel\\V1\\GPBMetadata\xea\x02\x1aVercel::Api::DevTunnel::V1b\x06proto3" + "\x04Dial\x12%.vercel.api.dev_tunnel.v1.DialRequest\x1a&.vercel.api.dev_tunnel.v1.DialResponse(\x010\x01B\xee\x01\n" + + "\x1ccom.vercel.api.dev_tunnel.v1B\vTunnelProtoP\x01ZBgithub.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1;devtunnel\xa2\x02\x03VAD\xaa\x02\x17Vercel.Api.DevTunnel.V1\xca\x02\x17Vercel\\Api\\DevTunnel\\V1\xe2\x02#Vercel\\Api\\DevTunnel\\V1\\GPBMetadata\xea\x02\x1aVercel::Api::DevTunnel::V1b\x06proto3" var ( file_vercel_api_dev_tunnel_v1_tunnel_proto_rawDescOnce sync.Once diff --git a/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go b/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go index ac0728a..8b661e7 100644 --- a/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go +++ b/api/go/vercel/api/dev_tunnel/v1/tunnel_grpc.pb.go @@ -7,7 +7,7 @@ // - protoc (unknown) // source: vercel/api/dev_tunnel/v1/tunnel.proto -package dev_tunnelv1 +package devtunnel import ( context "context" diff --git a/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js b/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js index 3ccbc1a..385350b 100644 --- a/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js +++ b/api/ts/vercel/api/dev_tunnel/v1/tunnel_pb.js @@ -11,7 +11,7 @@ import { enumDesc, fileDesc, messageDesc, serviceDesc, tsEnum } from "@bufbuild/ * Describes the file vercel/api/dev_tunnel/v1/tunnel.proto. */ export const file_vercel_api_dev_tunnel_v1_tunnel = /*@__PURE__*/ - fileDesc("CiV2ZXJjZWwvYXBpL2Rldl90dW5uZWwvdjEvdHVubmVsLnByb3RvEhh2ZXJjZWwuYXBpLmRldl90dW5uZWwudjEiywIKDkNvbm5lY3RSZXF1ZXN0Ei4KBGRhdGEYASABKAsyHi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuRGF0YUgAEjAKBWNsb3NlGAIgASgLMh8udmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkNsb3NlSAASMAoFcmVzZXQYAyABKAsyHy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuUmVzZXRIABI4CgloZWFydGJlYXQYBCABKAsyIy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuSGVhcnRiZWF0SAASMgoGd2luZG93GAUgASgLMiAudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLldpbmRvd0gAEi4KBG9wZW4YBiABKAsyHi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuT3BlbkgAQgcKBWZyYW1lIoYDCg9Db25uZWN0UmVzcG9uc2USLgoEb3BlbhgBIAEoCzIeLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5PcGVuSAASLgoEZGF0YRgCIAEoCzIeLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EYXRhSAASMAoFY2xvc2UYAyABKAsyHy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ2xvc2VIABIwCgVyZXNldBgEIAEoCzIfLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5SZXNldEgAEjgKCWhlYXJ0YmVhdBgFIAEoCzIjLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5IZWFydGJlYXRIABIyCgZ3aW5kb3cYBiABKAsyIC52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuV2luZG93SAASOAoJcmVjb25uZWN0GAcgASgLMiMudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLlJlY29ubmVjdEgAQgcKBWZyYW1lImkKBE9wZW4SDwoHY29ubl9pZBgBIAEoBBI0Cghwcm90b2NvbBgCIAEoDjIiLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5Qcm90b2NvbBIMCgRwb3J0GAMgASgNEgwKBGhvc3QYBCABKAkiKAoERGF0YRIPCgdjb25uX2lkGAEgASgEEg8KB3BheWxvYWQYAiABKAwiGAoFQ2xvc2USDwoHY29ubl9pZBgBIAEoBCIoCgVSZXNldBIPCgdjb25uX2lkGAEgASgEEg4KBnJlYXNvbhgCIAEoCSIoCgZXaW5kb3cSDwoHY29ubl9pZBgBIAEoBBINCgVieXRlcxgCIAEoDSILCglIZWFydGJlYXQiCwoJUmVjb25uZWN0ImEKC0RpYWxSZXF1ZXN0EjYKBnRhcmdldBgBIAEoCzIkLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EaWFsVGFyZ2V0SAASEQoHcGF5bG9hZBgCIAEoDEgAQgcKBWZyYW1lIlAKCkRpYWxUYXJnZXQSNAoIcHJvdG9jb2wYASABKA4yIi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuUHJvdG9jb2wSDAoEcG9ydBgCIAEoDSJcCgxEaWFsUmVzcG9uc2USEQoHcGF5bG9hZBgBIAEoDEgAEjAKA2VuZBgCIAEoCzIhLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EaWFsRW5kSABCBwoFZnJhbWUiCQoHRGlhbEVuZCo2CghQcm90b2NvbBIYChRQUk9UT0NPTF9VTlNQRUNJRklFRBAAEhAKDFBST1RPQ09MX1RDUBABMs4BCg1UdW5uZWxTZXJ2aWNlEmIKB0Nvbm5lY3QSKC52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ29ubmVjdFJlcXVlc3QaKS52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ29ubmVjdFJlc3BvbnNlKAEwARJZCgREaWFsEiUudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkRpYWxSZXF1ZXN0GiYudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkRpYWxSZXNwb25zZSgBMAFC8QEKHGNvbS52ZXJjZWwuYXBpLmRldl90dW5uZWwudjFCC1R1bm5lbFByb3RvUAFaRWdpdGh1Yi5jb20vdmVyY2VsL2JyaWRnZS9hcGkvZ28vdmVyY2VsL2FwaS9kZXZfdHVubmVsL3YxO2Rldl90dW5uZWx2MaICA1ZBRKoCF1ZlcmNlbC5BcGkuRGV2VHVubmVsLlYxygIXVmVyY2VsXEFwaVxEZXZUdW5uZWxcVjHiAiNWZXJjZWxcQXBpXERldlR1bm5lbFxWMVxHUEJNZXRhZGF0YeoCGlZlcmNlbDo6QXBpOjpEZXZUdW5uZWw6OlYxYgZwcm90bzM"); + fileDesc("CiV2ZXJjZWwvYXBpL2Rldl90dW5uZWwvdjEvdHVubmVsLnByb3RvEhh2ZXJjZWwuYXBpLmRldl90dW5uZWwudjEiywIKDkNvbm5lY3RSZXF1ZXN0Ei4KBGRhdGEYASABKAsyHi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuRGF0YUgAEjAKBWNsb3NlGAIgASgLMh8udmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkNsb3NlSAASMAoFcmVzZXQYAyABKAsyHy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuUmVzZXRIABI4CgloZWFydGJlYXQYBCABKAsyIy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuSGVhcnRiZWF0SAASMgoGd2luZG93GAUgASgLMiAudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLldpbmRvd0gAEi4KBG9wZW4YBiABKAsyHi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuT3BlbkgAQgcKBWZyYW1lIoYDCg9Db25uZWN0UmVzcG9uc2USLgoEb3BlbhgBIAEoCzIeLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5PcGVuSAASLgoEZGF0YRgCIAEoCzIeLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EYXRhSAASMAoFY2xvc2UYAyABKAsyHy52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ2xvc2VIABIwCgVyZXNldBgEIAEoCzIfLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5SZXNldEgAEjgKCWhlYXJ0YmVhdBgFIAEoCzIjLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5IZWFydGJlYXRIABIyCgZ3aW5kb3cYBiABKAsyIC52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuV2luZG93SAASOAoJcmVjb25uZWN0GAcgASgLMiMudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLlJlY29ubmVjdEgAQgcKBWZyYW1lImkKBE9wZW4SDwoHY29ubl9pZBgBIAEoBBI0Cghwcm90b2NvbBgCIAEoDjIiLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5Qcm90b2NvbBIMCgRwb3J0GAMgASgNEgwKBGhvc3QYBCABKAkiKAoERGF0YRIPCgdjb25uX2lkGAEgASgEEg8KB3BheWxvYWQYAiABKAwiGAoFQ2xvc2USDwoHY29ubl9pZBgBIAEoBCIoCgVSZXNldBIPCgdjb25uX2lkGAEgASgEEg4KBnJlYXNvbhgCIAEoCSIoCgZXaW5kb3cSDwoHY29ubl9pZBgBIAEoBBINCgVieXRlcxgCIAEoDSILCglIZWFydGJlYXQiCwoJUmVjb25uZWN0ImEKC0RpYWxSZXF1ZXN0EjYKBnRhcmdldBgBIAEoCzIkLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EaWFsVGFyZ2V0SAASEQoHcGF5bG9hZBgCIAEoDEgAQgcKBWZyYW1lIlAKCkRpYWxUYXJnZXQSNAoIcHJvdG9jb2wYASABKA4yIi52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuUHJvdG9jb2wSDAoEcG9ydBgCIAEoDSJcCgxEaWFsUmVzcG9uc2USEQoHcGF5bG9hZBgBIAEoDEgAEjAKA2VuZBgCIAEoCzIhLnZlcmNlbC5hcGkuZGV2X3R1bm5lbC52MS5EaWFsRW5kSABCBwoFZnJhbWUiCQoHRGlhbEVuZCo2CghQcm90b2NvbBIYChRQUk9UT0NPTF9VTlNQRUNJRklFRBAAEhAKDFBST1RPQ09MX1RDUBABMs4BCg1UdW5uZWxTZXJ2aWNlEmIKB0Nvbm5lY3QSKC52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ29ubmVjdFJlcXVlc3QaKS52ZXJjZWwuYXBpLmRldl90dW5uZWwudjEuQ29ubmVjdFJlc3BvbnNlKAEwARJZCgREaWFsEiUudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkRpYWxSZXF1ZXN0GiYudmVyY2VsLmFwaS5kZXZfdHVubmVsLnYxLkRpYWxSZXNwb25zZSgBMAFC7gEKHGNvbS52ZXJjZWwuYXBpLmRldl90dW5uZWwudjFCC1R1bm5lbFByb3RvUAFaQmdpdGh1Yi5jb20vdmVyY2VsL2JyaWRnZS9hcGkvZ28vdmVyY2VsL2FwaS9kZXZfdHVubmVsL3YxO2RldnR1bm5lbKICA1ZBRKoCF1ZlcmNlbC5BcGkuRGV2VHVubmVsLlYxygIXVmVyY2VsXEFwaVxEZXZUdW5uZWxcVjHiAiNWZXJjZWxcQXBpXERldlR1bm5lbFxWMVxHUEJNZXRhZGF0YeoCGlZlcmNlbDo6QXBpOjpEZXZUdW5uZWw6OlYxYgZwcm90bzM"); /** * Describes the message vercel.api.dev_tunnel.v1.ConnectRequest. diff --git a/buf.gen.yaml b/buf.gen.yaml index f6dc66f..0f9730d 100644 --- a/buf.gen.yaml +++ b/buf.gen.yaml @@ -4,6 +4,10 @@ managed: override: - file_option: go_package_prefix value: github.com/vercel/bridge/api/go + # Vendored from vercel/api, which names it vercel.api.dev_tunnel.v1. + - file_option: go_package + path: vercel/api/dev_tunnel/v1/tunnel.proto + value: github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1;devtunnel disable: - file_option: go_package_prefix module: buf.build/bufbuild/protovalidate diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go index 669703b..47c397a 100644 --- a/pkg/devtunnel/backend.go +++ b/pkg/devtunnel/backend.go @@ -22,7 +22,7 @@ import ( "github.com/puzpuzpuz/xsync/v3" "github.com/segmentio/ksuid" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" - devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/tunnel" "google.golang.org/grpc" @@ -101,7 +101,7 @@ type Backend struct { creds credentials.TransportCredentials // Set by Run, before any stream. - rpc devtunnelv1.TunnelServiceClient + rpc devtunnel.TunnelServiceClient deliver func(tunnel.Frame) ctx context.Context @@ -236,7 +236,7 @@ func (b *Backend) Run(ctx context.Context, deliver func(tunnel.Frame)) error { return fmt.Errorf("creating relay client: %w", err) } defer cc.Close() - b.rpc = devtunnelv1.NewTunnelServiceClient(cc) + b.rpc = devtunnel.NewTunnelServiceClient(cc) b.deliver = deliver b.ctx = ctx @@ -277,9 +277,9 @@ func (b *Backend) Open(ctx context.Context, id string, conn *tunnel.ConnInfo) er if host == "" { host = conn.Dest.Host } - s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Open{Open: &devtunnelv1.Open{ + s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Open{Open: &devtunnel.Open{ ConnId: wire, - Protocol: devtunnelv1.Protocol_PROTOCOL_TCP, + Protocol: devtunnel.Protocol_PROTOCOL_TCP, Host: host, Port: uint32(conn.Dest.Port), }}}) @@ -294,14 +294,14 @@ func (b *Backend) Send(f tunnel.Frame) { switch f.Kind { case tunnel.FrameData: b.bytesSent.Add(int64(len(f.Payload))) - r.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Data{Data: &devtunnelv1.Data{ConnId: r.wire, Payload: f.Payload}}}) + r.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: r.wire, Payload: f.Payload}}}) case tunnel.FrameCloseWrite: - r.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Close{Close: &devtunnelv1.Close{ConnId: r.wire}}}) + r.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: r.wire}}}) case tunnel.FrameReset: r.s.send(resetFrame(r.wire, f.Reason)) b.untrack(f.ConnID, r.s) case tunnel.FrameWindow: - r.s.send(&devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Window{Window: &devtunnelv1.Window{ConnId: r.wire, Bytes: uint32(f.Bytes)}}}) + r.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: r.wire, Bytes: uint32(f.Bytes)}}}) } } diff --git a/pkg/devtunnel/backend_test.go b/pkg/devtunnel/backend_test.go index 2443f11..9c9f84b 100644 --- a/pkg/devtunnel/backend_test.go +++ b/pkg/devtunnel/backend_test.go @@ -12,7 +12,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" - devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "github.com/vercel/bridge/pkg/tunnel" "google.golang.org/grpc" "google.golang.org/grpc/codes" @@ -28,7 +28,7 @@ const frameBytes = 64 << 10 // fakeRelay stands in for api-dev-tunnel, handing each Connect call to the // test. type fakeRelay struct { - devtunnelv1.UnimplementedTunnelServiceServer + devtunnel.UnimplementedTunnelServiceServer url string streams chan *relayStream // refuse ends a call before it reaches the test, unless it returns nil. @@ -37,8 +37,8 @@ type fakeRelay struct { type relayStream struct { md metadata.MD - call grpc.BidiStreamingServer[devtunnelv1.ConnectRequest, devtunnelv1.ConnectResponse] - recv chan *devtunnelv1.ConnectRequest + call grpc.BidiStreamingServer[devtunnel.ConnectRequest, devtunnel.ConnectResponse] + recv chan *devtunnel.ConnectRequest end chan error } @@ -48,20 +48,20 @@ func startRelay(t *testing.T) *fakeRelay { require.NoError(t, err) r := &fakeRelay{url: "http://" + lis.Addr().String(), streams: make(chan *relayStream, 8)} srv := grpc.NewServer() - devtunnelv1.RegisterTunnelServiceServer(srv, r) + devtunnel.RegisterTunnelServiceServer(srv, r) go func() { _ = srv.Serve(lis) }() t.Cleanup(srv.Stop) return r } -func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnelv1.ConnectRequest, devtunnelv1.ConnectResponse]) error { +func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnel.ConnectRequest, devtunnel.ConnectResponse]) error { md, _ := metadata.FromIncomingContext(call.Context()) if r.refuse != nil { if err := r.refuse(md); err != nil { return err } } - s := &relayStream{md: md, call: call, recv: make(chan *devtunnelv1.ConnectRequest, 256), end: make(chan error, 1)} + s := &relayStream{md: md, call: call, recv: make(chan *devtunnel.ConnectRequest, 256), end: make(chan error, 1)} go func() { defer close(s.recv) for { @@ -92,18 +92,18 @@ func (r *fakeRelay) next(t *testing.T) *relayStream { } } -func (s *relayStream) send(t *testing.T, msg *devtunnelv1.ConnectResponse) { +func (s *relayStream) send(t *testing.T, msg *devtunnel.ConnectResponse) { t.Helper() require.NoError(t, s.call.Send(msg)) } // hold makes the stream the tunnel's, as the relay's first heartbeat does. func (s *relayStream) hold(t *testing.T) { - s.send(t, &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Heartbeat{Heartbeat: &devtunnelv1.Heartbeat{}}}) + s.send(t, &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Heartbeat{Heartbeat: &devtunnel.Heartbeat{}}}) } // expect returns the next frame from the client other than a heartbeat. -func (s *relayStream) expect(t *testing.T) *devtunnelv1.ConnectRequest { +func (s *relayStream) expect(t *testing.T) *devtunnel.ConnectRequest { t.Helper() timeout := time.After(testTimeout) for { @@ -133,22 +133,22 @@ func (s *relayStream) expectData(t *testing.T, id uint64, n int) []byte { return got } -func open(id uint64, port int) *devtunnelv1.ConnectResponse { - return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Open{Open: &devtunnelv1.Open{ - ConnId: id, Protocol: devtunnelv1.Protocol_PROTOCOL_TCP, Port: uint32(port), +func open(id uint64, port int) *devtunnel.ConnectResponse { + return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Open{Open: &devtunnel.Open{ + ConnId: id, Protocol: devtunnel.Protocol_PROTOCOL_TCP, Port: uint32(port), }}} } -func data(id uint64, payload string) *devtunnelv1.ConnectResponse { - return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Data{Data: &devtunnelv1.Data{ConnId: id, Payload: []byte(payload)}}} +func data(id uint64, payload string) *devtunnel.ConnectResponse { + return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Data{Data: &devtunnel.Data{ConnId: id, Payload: []byte(payload)}}} } -func closeConn(id uint64) *devtunnelv1.ConnectResponse { - return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Close{Close: &devtunnelv1.Close{ConnId: id}}} +func closeConn(id uint64) *devtunnel.ConnectResponse { + return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Close{Close: &devtunnel.Close{ConnId: id}}} } -func window(id uint64, n int) *devtunnelv1.ConnectResponse { - return &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Window{Window: &devtunnelv1.Window{ConnId: id, Bytes: uint32(n)}}} +func window(id uint64, n int) *devtunnel.ConnectResponse { + return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Window{Window: &devtunnel.Window{ConnId: id, Bytes: uint32(n)}}} } // newTunnel runs a tunnel over a Backend for r. @@ -286,7 +286,7 @@ func TestEgressLeavesThroughTheDeployment(t *testing.T) { assert.GreaterOrEqual(t, o.GetConnId(), clientConnIDFloor) assert.Equal(t, "example.com", o.GetHost()) assert.Equal(t, uint32(443), o.GetPort()) - assert.Equal(t, devtunnelv1.Protocol_PROTOCOL_TCP, o.GetProtocol()) + assert.Equal(t, devtunnel.Protocol_PROTOCOL_TCP, o.GetProtocol()) id := o.GetConnId() _, err := app.Write([]byte("GET")) @@ -300,7 +300,7 @@ func TestEgressLeavesThroughTheDeployment(t *testing.T) { assert.Equal(t, "200", string(buf)) // The deployment couldn't reach it, or it went away. - s.send(t, &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Reset_{Reset_: &devtunnelv1.Reset{ConnId: id, Reason: "refused"}}}) + s.send(t, &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Reset_{Reset_: &devtunnel.Reset{ConnId: id, Reason: "refused"}}}) _ = app.SetReadDeadline(time.Now().Add(testTimeout)) _, err = app.Read(buf) assert.Error(t, err) @@ -330,7 +330,7 @@ func TestReconnectKeepsTheOldStreamsConns(t *testing.T) { first.send(t, data(1, "a")) assert.Equal(t, "a", string(first.expectData(t, 1, 1))) - first.send(t, &devtunnelv1.ConnectResponse{Frame: &devtunnelv1.ConnectResponse_Reconnect{Reconnect: &devtunnelv1.Reconnect{}}}) + first.send(t, &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Reconnect{Reconnect: &devtunnel.Reconnect{}}}) second := r.next(t) assert.Equal(t, []string{"test-instance"}, second.md.Get(instanceHeader)) second.hold(t) diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index ee4fcd6..5129f1e 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -11,12 +11,12 @@ import ( "sync/atomic" "time" - devtunnelv1 "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "github.com/vercel/bridge/pkg/tunnel" "google.golang.org/grpc/metadata" ) -var heartbeat = &devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Heartbeat{Heartbeat: &devtunnelv1.Heartbeat{}}} +var heartbeat = &devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Heartbeat{Heartbeat: &devtunnel.Heartbeat{}}} // stream is one Connect call. The relay ends every conn on it with it. type stream struct { @@ -114,32 +114,32 @@ func (s *stream) heartbeats() { } // handle passes one of the relay's frames to the tunnel. -func (s *stream) handle(msg *devtunnelv1.ConnectResponse, moving func()) { +func (s *stream) handle(msg *devtunnel.ConnectResponse, moving func()) { switch f := msg.GetFrame().(type) { - case *devtunnelv1.ConnectResponse_Heartbeat: + case *devtunnel.ConnectResponse_Heartbeat: if s.held.CompareAndSwap(false, true) { s.b.holding(s) } - case *devtunnelv1.ConnectResponse_Reconnect: + case *devtunnel.ConnectResponse_Reconnect: s.b.moving(s) moving() - case *devtunnelv1.ConnectResponse_Open: + case *devtunnel.ConnectResponse_Open: s.open(f.Open) - case *devtunnelv1.ConnectResponse_Data: + case *devtunnel.ConnectResponse_Data: if id, ok := s.routed(f.Data.GetConnId()); ok { s.b.bytesReceived.Add(int64(len(f.Data.GetPayload()))) s.b.deliver(tunnel.Frame{Kind: tunnel.FrameData, ConnID: id, Payload: f.Data.GetPayload()}) } - case *devtunnelv1.ConnectResponse_Close: + case *devtunnel.ConnectResponse_Close: if id, ok := s.routed(f.Close.GetConnId()); ok { s.b.deliver(tunnel.Frame{Kind: tunnel.FrameCloseWrite, ConnID: id}) } - case *devtunnelv1.ConnectResponse_Reset_: + case *devtunnel.ConnectResponse_Reset_: if id, ok := s.routed(f.Reset_.GetConnId()); ok && s.b.untrack(id, s) { s.logger.Debug("Relay reset the connection", "conn_id", id, "reason", f.Reset_.GetReason()) s.b.deliver(tunnel.Frame{Kind: tunnel.FrameReset, ConnID: id, Reason: f.Reset_.GetReason()}) } - case *devtunnelv1.ConnectResponse_Window: + case *devtunnel.ConnectResponse_Window: if id, ok := s.routed(f.Window.GetConnId()); ok { s.b.deliver(tunnel.Frame{Kind: tunnel.FrameWindow, ConnID: id, Bytes: int(f.Window.GetBytes())}) } @@ -148,10 +148,10 @@ func (s *stream) handle(msg *devtunnelv1.ConnectResponse, moving func()) { // open hands one of the deployment's conns to the tunnel, which connects it // to the app. -func (s *stream) open(open *devtunnelv1.Open) { +func (s *stream) open(open *devtunnel.Open) { wire := open.GetConnId() port := s.b.cfg.Port - if open.GetProtocol() != devtunnelv1.Protocol_PROTOCOL_TCP || int(open.GetPort()) != port { + if open.GetProtocol() != devtunnel.Protocol_PROTOCOL_TCP || int(open.GetPort()) != port { s.send(resetFrame(wire, fmt.Sprintf("bridge serves only port %d", port))) return } @@ -209,7 +209,7 @@ func (s *stream) endConns() { }) } -func (s *stream) send(frame *devtunnelv1.ConnectRequest) { +func (s *stream) send(frame *devtunnel.ConnectRequest) { s.out.push(frame) } @@ -217,11 +217,11 @@ func (s *stream) send(frame *devtunnelv1.ConnectRequest) { // window; the rest is small. type outbox struct { mu sync.Mutex - frames []*devtunnelv1.ConnectRequest + frames []*devtunnel.ConnectRequest wake chan struct{} } -func (o *outbox) push(frame *devtunnelv1.ConnectRequest) { +func (o *outbox) push(frame *devtunnel.ConnectRequest) { o.mu.Lock() o.frames = append(o.frames, frame) o.mu.Unlock() @@ -231,7 +231,7 @@ func (o *outbox) push(frame *devtunnelv1.ConnectRequest) { } } -func (o *outbox) pop(ctx context.Context) (*devtunnelv1.ConnectRequest, bool) { +func (o *outbox) pop(ctx context.Context) (*devtunnel.ConnectRequest, bool) { for { o.mu.Lock() if len(o.frames) > 0 { @@ -250,6 +250,6 @@ func (o *outbox) pop(ctx context.Context) (*devtunnelv1.ConnectRequest, bool) { } } -func resetFrame(id uint64, reason string) *devtunnelv1.ConnectRequest { - return &devtunnelv1.ConnectRequest{Frame: &devtunnelv1.ConnectRequest_Reset_{Reset_: &devtunnelv1.Reset{ConnId: id, Reason: reason}}} +func resetFrame(id uint64, reason string) *devtunnel.ConnectRequest { + return &devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Reset_{Reset_: &devtunnel.Reset{ConnId: id, Reason: reason}}} } diff --git a/pkg/plumbing/conn.go b/pkg/plumbing/conn.go index bfff5f0..09b0a18 100644 --- a/pkg/plumbing/conn.go +++ b/pkg/plumbing/conn.go @@ -1,112 +1,79 @@ package plumbing import ( - "io" "net" "sync" "time" ) -// TunnelConn wraps one side of a multiplexed tunnel connection as a net.Conn. -// -// Reads come from a channel fed by a central read loop (demuxed by connection -// ID). Writes go through a caller-provided send function that handles stream -// serialization and any mutex. -type TunnelConn struct { - readCh chan []byte - closeCh chan struct{} - closeOnce sync.Once - sendFn func([]byte) error - onClose func() - - localAddr net.Addr - remoteAddr net.Addr - - buf []byte // leftover from a partial Read -} - -// TunnelConnConfig configures a TunnelConn. -type TunnelConnConfig struct { - // SendFn writes data onto the tunnel stream. The caller is responsible for - // any mutex and message framing. - SendFn func([]byte) error - - // OnClose is called once when Close is invoked (e.g. to remove the conn - // from a tracking map). May be nil. - OnClose func() - - LocalAddr net.Addr - RemoteAddr net.Addr -} - -// NewTunnelConn creates a TunnelConn ready for use with bidi.New. -func NewTunnelConn(cfg TunnelConnConfig) *TunnelConn { - return &TunnelConn{ - readCh: make(chan []byte, 64), - closeCh: make(chan struct{}), - sendFn: cfg.SendFn, - onClose: cfg.OnClose, - localAddr: cfg.LocalAddr, - remoteAddr: cfg.RemoteAddr, - } +// Stream carries one conn's bytes each way: a per-conn gRPC stream, or one +// conn's frames on a multiplexed one. +type Stream interface { + // Send sends p to the peer. It doesn't keep p after returning. + Send(p []byte) error + // Recv returns the peer's next bytes, or io.EOF once the peer has + // nothing more to send. + Recv() ([]byte, error) + // CloseSend tells the peer this side has nothing more to send. + CloseSend() error + // Close abandons the stream in both directions. + Close() error } -// Deliver pushes data into the read buffer. Called by the central read loop -// when a message arrives for this connection's ID. -func (tc *TunnelConn) Deliver(data []byte) { - select { - case tc.readCh <- data: - case <-tc.closeCh: +// NewStreamConn turns s into a net.Conn, whose CloseWrite is s.CloseSend. +// Deadlines aren't supported; Close unblocks a pending Read or Write. +func NewStreamConn(s Stream, local, remote net.Addr) net.Conn { + c := &streamConn{stream: s, local: local, remote: remote, closed: make(chan struct{})} + c.done = c.closed + if d, ok := s.(interface{ Done() <-chan struct{} }); ok { + c.done = d.Done() } + return c } -// SignalClose tears down the connection from the remote side. -func (tc *TunnelConn) SignalClose() { - tc.closeOnce.Do(func() { close(tc.closeCh) }) +type streamConn struct { + stream Stream + local, remote net.Addr + // The rest of the last Recv; only Read touches it. + pending []byte + closed chan struct{} + closeOnce sync.Once + // The stream's, if it has one, else closed. + done <-chan struct{} } -// --- net.Conn implementation --- - -func (tc *TunnelConn) Read(b []byte) (int, error) { - if len(tc.buf) > 0 { - n := copy(b, tc.buf) - tc.buf = tc.buf[n:] - return n, nil - } - select { - case data, ok := <-tc.readCh: - if !ok { - return 0, io.EOF - } - n := copy(b, data) - if n < len(data) { - tc.buf = data[n:] +func (c *streamConn) Read(p []byte) (int, error) { + for len(c.pending) == 0 { + chunk, err := c.stream.Recv() + if err != nil { + return 0, err } - return n, nil - case <-tc.closeCh: - return 0, io.EOF + c.pending = chunk } + n := copy(p, c.pending) + c.pending = c.pending[n:] + return n, nil } -func (tc *TunnelConn) Write(b []byte) (int, error) { - if err := tc.sendFn(b); err != nil { +func (c *streamConn) Write(p []byte) (int, error) { + if err := c.stream.Send(p); err != nil { return 0, err } - return len(b), nil + return len(p), nil } -func (tc *TunnelConn) Close() error { - tc.closeOnce.Do(func() { close(tc.closeCh) }) - if tc.onClose != nil { - tc.onClose() - } - return nil +func (c *streamConn) CloseWrite() error { return c.stream.CloseSend() } + +func (c *streamConn) Close() error { + c.closeOnce.Do(func() { close(c.closed) }) + return c.stream.Close() } -func (tc *TunnelConn) LocalAddr() net.Addr { return tc.localAddr } -func (tc *TunnelConn) RemoteAddr() net.Addr { return tc.remoteAddr } -func (tc *TunnelConn) SetDeadline(_ time.Time) error { return nil } -func (tc *TunnelConn) SetReadDeadline(_ time.Time) error { return nil } -func (tc *TunnelConn) SetWriteDeadline(_ time.Time) error { return nil } +// Done is closed once the conn is over. +func (c *streamConn) Done() <-chan struct{} { return c.done } -var _ net.Conn = (*TunnelConn)(nil) +func (c *streamConn) LocalAddr() net.Addr { return c.local } +func (c *streamConn) RemoteAddr() net.Addr { return c.remote } +func (c *streamConn) SetDeadline(time.Time) error { return nil } +func (c *streamConn) SetReadDeadline(time.Time) error { return nil } +func (c *streamConn) SetWriteDeadline(time.Time) error { return nil } diff --git a/pkg/plumbing/conn_test.go b/pkg/plumbing/conn_test.go new file mode 100644 index 0000000..ee04939 --- /dev/null +++ b/pkg/plumbing/conn_test.go @@ -0,0 +1,71 @@ +package plumbing + +import ( + "io" + "net" + "testing" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// chunkStream replays chunks, then EOF, and records what's done to it. +type chunkStream struct { + chunks [][]byte + sent []string + sendClosed bool + closed bool + done chan struct{} +} + +func (s *chunkStream) Send(p []byte) error { + s.sent = append(s.sent, string(p)) + return nil +} + +func (s *chunkStream) Recv() ([]byte, error) { + if len(s.chunks) == 0 { + return nil, io.EOF + } + chunk := s.chunks[0] + s.chunks = s.chunks[1:] + return chunk, nil +} + +func (s *chunkStream) CloseSend() error { + s.sendClosed = true + return nil +} + +func (s *chunkStream) Close() error { + s.closed = true + return nil +} + +func (s *chunkStream) Done() <-chan struct{} { return s.done } + +func TestStreamConn(t *testing.T) { + s := &chunkStream{chunks: [][]byte{[]byte("hello "), []byte("world")}, done: make(chan struct{})} + conn := NewStreamConn(s, nil, nil) + + // A Read smaller than a chunk takes the rest of it next. + buf := make([]byte, 4) + n, err := conn.Read(buf) + require.NoError(t, err) + assert.Equal(t, "hell", string(buf[:n])) + rest, err := io.ReadAll(conn) + require.NoError(t, err) + assert.Equal(t, "o world", string(rest)) + + _, err = conn.Write([]byte("hi")) + require.NoError(t, err) + assert.Equal(t, []string{"hi"}, s.sent) + + require.NoError(t, conn.(interface{ CloseWrite() error }).CloseWrite()) + assert.True(t, s.sendClosed) + assert.Equal(t, (<-chan struct{})(s.done), conn.(interface{ Done() <-chan struct{} }).Done()) + require.NoError(t, conn.Close()) + assert.True(t, s.closed) + + var _ net.Conn = conn +} diff --git a/pkg/tunnel/conn.go b/pkg/tunnel/conn.go index 82fb7f1..ee825c4 100644 --- a/pkg/tunnel/conn.go +++ b/pkg/tunnel/conn.go @@ -6,7 +6,6 @@ import ( "net" "sync" "sync/atomic" - "time" "github.com/vercel/bridge/pkg/plumbing" ) @@ -19,39 +18,36 @@ const ( maxQueued = 256 << 10 ) -// streamConn is one conn's side on the tunnel's transport, as a net.Conn: Read -// takes what the peer sent, Write sends to it, and CloseWrite half-closes it. -// The tunnel uses it through meter, which holds writes to the conn's window. -type streamConn struct { - t *tunnelImpl - id string - info *ConnInfo - meter plumbing.MeteredConn +// connStream is one conn's frames on the tunnel's transport, as a +// plumbing.Stream. The tunnel uses it as conn: a net.Conn over it, metered +// to the conn's window. +type connStream struct { + t *tunnelImpl + id string + info *ConnInfo + conn plumbing.MeteredConn // 0 when the backend has no flow control. window int halfClose bool - // From the peer, not yet read. + // From the peer, not yet received. inMu sync.Mutex in [][]byte inBytes int inEOF bool inWake chan struct{} inRoom chan struct{} - // The rest of the chunk Read is on; only Read touches it. - pending []byte closeSent atomic.Bool - - local atomic.Pointer[net.Conn] - done chan struct{} - doneOnce sync.Once + local atomic.Pointer[net.Conn] + done chan struct{} + doneOnce sync.Once } -var _ net.Conn = (*streamConn)(nil) +var _ plumbing.Stream = (*connStream)(nil) -func (t *tunnelImpl) newConn(id string, info *ConnInfo) *streamConn { - c := &streamConn{ +func (t *tunnelImpl) newConn(id string, info *ConnInfo) *connStream { + c := &connStream{ t: t, id: id, info: info, @@ -61,42 +57,42 @@ func (t *tunnelImpl) newConn(id string, info *ConnInfo) *streamConn { inRoom: make(chan struct{}, 1), done: make(chan struct{}), } - c.meter = plumbing.NewMeteredConn(c, c.window) + c.conn = plumbing.NewMeteredConn( + plumbing.NewStreamConn(c, addr(info.Source.String()), addr(info.Dest.String())), + c.window, + ) return c } -func (c *streamConn) Read(p []byte) (int, error) { - for len(c.pending) == 0 { +func (c *connStream) Recv() ([]byte, error) { + for { if c.isDone() { - return 0, net.ErrClosed + return nil, net.ErrClosed } c.inMu.Lock() if len(c.in) > 0 { - c.pending = c.in[0] + chunk := c.in[0] c.in[0] = nil c.in = c.in[1:] c.inMu.Unlock() - break + c.consumed(len(chunk)) + return chunk, nil } eof := c.inEOF c.inMu.Unlock() if eof { - return 0, io.EOF + return nil, io.EOF } select { case <-c.inWake: case <-c.done: } } - n := copy(p, c.pending) - c.pending = c.pending[n:] - c.consumed(n) - return n, nil } -// consumed gives back what Read took: as window to a peer with flow control, +// consumed gives back what Recv took: as window to a peer with flow control, // and as room in the queue to one without. -func (c *streamConn) consumed(n int) { +func (c *connStream) consumed(n int) { if c.window > 0 { c.send(Frame{Kind: FrameWindow, Bytes: n}) return @@ -107,23 +103,22 @@ func (c *streamConn) consumed(n int) { wake(c.inRoom) } -func (c *streamConn) Write(p []byte) (int, error) { - written := 0 - for written < len(p) { +func (c *connStream) Send(p []byte) error { + for len(p) > 0 { if c.isDone() { - return written, net.ErrClosed + return net.ErrClosed } - size := min(len(p)-written, maxFrame) - // The backend may send it after Write returns. - c.send(Frame{Kind: FrameData, Payload: bytes.Clone(p[written : written+size])}) - written += size + size := min(len(p), maxFrame) + // The backend may send it after Send returns. + c.send(Frame{Kind: FrameData, Payload: bytes.Clone(p[:size])}) + p = p[size:] } - return written, nil + return nil } -// CloseWrite tells the peer this side has nothing more to send. Over a +// CloseSend tells the peer this side has nothing more to send. Over a // transport without half-close, it ends the conn. -func (c *streamConn) CloseWrite() error { +func (c *connStream) CloseSend() error { if !c.halfClose { c.finish() return nil @@ -134,23 +129,17 @@ func (c *streamConn) CloseWrite() error { return nil } -// Done is closed once the conn is over. -func (c *streamConn) Done() <-chan struct{} { return c.done } - // Close resets the conn. -func (c *streamConn) Close() error { +func (c *connStream) Close() error { c.reset("closed") return nil } -func (c *streamConn) LocalAddr() net.Addr { return addr(c.info.Source.String()) } -func (c *streamConn) RemoteAddr() net.Addr { return addr(c.info.Dest.String()) } -func (c *streamConn) SetDeadline(time.Time) error { return nil } -func (c *streamConn) SetReadDeadline(time.Time) error { return nil } -func (c *streamConn) SetWriteDeadline(time.Time) error { return nil } +// Done is closed once the conn is over. +func (c *connStream) Done() <-chan struct{} { return c.done } // deliver queues payload from the peer. -func (c *streamConn) deliver(p []byte) { +func (c *connStream) deliver(p []byte) { if len(p) == 0 { return } @@ -177,9 +166,9 @@ func (c *streamConn) deliver(p []byte) { } } -// peerClosed: the peer has nothing more to send. Read returns io.EOF once it +// peerClosed: the peer has nothing more to send. Recv returns io.EOF once it // has returned the rest. -func (c *streamConn) peerClosed() { +func (c *connStream) peerClosed() { c.inMu.Lock() c.inEOF = true c.inMu.Unlock() @@ -188,7 +177,7 @@ func (c *streamConn) peerClosed() { // attach hands the conn its local end, which it closes when it's over. // Reports false if the conn is already over. -func (c *streamConn) attach(local net.Conn) bool { +func (c *connStream) attach(local net.Conn) bool { c.local.Store(&local) if c.isDone() { _ = local.Close() @@ -198,12 +187,12 @@ func (c *streamConn) attach(local net.Conn) bool { } // reset ends the conn and tells the peer why. -func (c *streamConn) reset(reason string) { c.end(reason, true) } +func (c *connStream) reset(reason string) { c.end(reason, true) } // finish ends the conn without telling the peer: it's done, or already knows. -func (c *streamConn) finish() { c.end("", false) } +func (c *connStream) finish() { c.end("", false) } -func (c *streamConn) end(reason string, tell bool) { +func (c *connStream) end(reason string, tell bool) { c.doneOnce.Do(func() { if tell { c.send(Frame{Kind: FrameReset, Reason: reason}) @@ -217,7 +206,7 @@ func (c *streamConn) end(reason string, tell bool) { }) } -func (c *streamConn) isDone() bool { +func (c *connStream) isDone() bool { select { case <-c.done: return true @@ -226,7 +215,7 @@ func (c *streamConn) isDone() bool { } } -func (c *streamConn) send(f Frame) { +func (c *connStream) send(f Frame) { f.ConnID = c.id f.Conn = c.info c.t.backend.Send(f) diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index be1641f..80c85ec 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -58,7 +58,7 @@ func NewWithBackend(dialer plumbing.ContextDialer, backend Backend, opts ...Opti t := &tunnelImpl{ dialer: dialer, backend: backend, - conns: xsync.NewMapOf[string, *streamConn](), + conns: xsync.NewMapOf[string, *connStream](), done: make(chan struct{}), logger: slog.Default(), } @@ -77,7 +77,7 @@ type tunnelImpl struct { dialer plumbing.ContextDialer hijacker mitm.Hijacker backend Backend - conns *xsync.MapOf[string, *streamConn] + conns *xsync.MapOf[string, *connStream] bufs sync.Pool logger *slog.Logger done chan struct{} @@ -102,7 +102,7 @@ func (t *tunnelImpl) Start(ctx context.Context) { func (t *tunnelImpl) Close() error { t.closeOnce.Do(func() { t.cancel() - t.conns.Range(func(_ string, c *streamConn) bool { + t.conns.Range(func(_ string, c *connStream) bool { c.finish() return true }) @@ -157,7 +157,7 @@ func (t *tunnelImpl) deliver(f Frame) { } case FrameWindow: if ok { - c.meter.Grant(f.Bytes) + c.conn.Grant(f.Bytes) } } } @@ -208,9 +208,9 @@ func (t *tunnelImpl) dial(f Frame) (net.Conn, error) { return t.dialer.DialContext(t.ctx, "tcp", f.Conn.Dest.String()) } -// pump copies between local and c, through its meter, in both directions, -// each through a buffer from the pool, until the conn is over. -func (t *tunnelImpl) pump(local net.Conn, c *streamConn) { +// pump copies between local and c.conn in both directions, each through a +// buffer from the pool, until the conn is over. +func (t *tunnelImpl) pump(local net.Conn, c *connStream) { if !c.attach(local) { return } @@ -225,16 +225,16 @@ func (t *tunnelImpl) pump(local net.Conn, c *streamConn) { wg.Add(2) go func() { defer wg.Done() - copyHalf(c.meter, local, *toPeer, c) + copyHalf(c.conn, local, *toPeer, c) }() go func() { defer wg.Done() - copyHalf(local, c.meter, *fromPeer, c) + copyHalf(local, c.conn, *fromPeer, c) }() wg.Wait() c.finish() - st := c.meter.Stats() + st := c.conn.Stats() t.logger.Debug("Tunnel: connection closed", "conn_id", c.id, "bytes_sent", st.BytesWritten, @@ -246,7 +246,7 @@ func (t *tunnelImpl) pump(local net.Conn, c *streamConn) { // copyHalf copies src to dst until src ends, then half-closes dst. A failure // either way abandons the conn. -func copyHalf(dst net.Conn, src net.Conn, buf []byte, c *streamConn) { +func copyHalf(dst net.Conn, src net.Conn, buf []byte, c *connStream) { // Hides ReadFrom and WriteTo, which would copy through buffers of their own. _, err := io.CopyBuffer(struct{ io.Writer }{dst}, struct{ io.Reader }{src}, buf) if err != nil { @@ -259,8 +259,8 @@ func copyHalf(dst net.Conn, src net.Conn, buf []byte, c *streamConn) { } // forget removes c, unless another conn has since taken its id. -func (t *tunnelImpl) forget(c *streamConn) { - t.conns.Compute(c.id, func(cur *streamConn, loaded bool) (*streamConn, bool) { +func (t *tunnelImpl) forget(c *connStream) { + t.conns.Compute(c.id, func(cur *connStream, loaded bool) (*connStream, bool) { if !loaded { return nil, true } From 9801f5953228541284fce55ce374247d9c258905 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 16:12:31 -0700 Subject: [PATCH 07/11] refactor(tunnel): backends hand the tunnel net.Conns; drop connStream A Backend now accepts and opens conns as net.Conns: a plumbing.StreamConn over its own per-conn Stream, in a MeteredConn that holds writes to the window. The tunnel only dials and pumps; it has no frames, windows or per-conn state. The bridge backend's Stream is a bounded queue of its messages. The dev tunnel backend's grants back what Recv returns and sends Data, Close and Reset. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/devtunnel/backend.go | 113 ++++------------ pkg/devtunnel/backend_test.go | 18 +-- pkg/devtunnel/conn.go | 197 ++++++++++++++++++++++++++++ pkg/devtunnel/stream.go | 68 ++++------ pkg/tunnel/backend.go | 62 ++------- pkg/tunnel/conn.go | 235 ---------------------------------- pkg/tunnel/pump_test.go | 146 ++++++++------------- pkg/tunnel/stream.go | 178 ++++++++++++++++++++----- pkg/tunnel/tunnel.go | 180 +++++++++----------------- pkg/tunnel/tunnel_test.go | 2 +- 10 files changed, 528 insertions(+), 671 deletions(-) create mode 100644 pkg/devtunnel/conn.go delete mode 100644 pkg/tunnel/conn.go diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go index 47c397a..9db6dfc 100644 --- a/pkg/devtunnel/backend.go +++ b/pkg/devtunnel/backend.go @@ -14,7 +14,6 @@ import ( "net/url" "regexp" "strconv" - "strings" "sync" "sync/atomic" "time" @@ -101,12 +100,12 @@ type Backend struct { creds credentials.TransportCredentials // Set by Run, before any stream. - rpc devtunnel.TunnelServiceClient - deliver func(tunnel.Frame) - ctx context.Context + rpc devtunnel.TunnelServiceClient + accept func(net.Conn, *tunnel.ConnInfo) + ctx context.Context - // Each open conn's stream, by the tunnel's id for it. - routes *xsync.MapOf[string, route] + // Each open conn, by an id unique across streams. + conns *xsync.MapOf[string, *relayStream] mu sync.Mutex token string @@ -128,23 +127,6 @@ type Backend struct { nextEgress, nextStream atomic.Uint64 } -// route is where one of the tunnel's conns lives on the relay's protocol. -type route struct { - s *stream - // The conn's id on s. - wire uint64 - dir direction -} - -type direction int - -const ( - // From the deployment to the app. - ingress direction = iota - // From the app out through the deployment. - egress -) - var _ tunnel.Backend = (*Backend)(nil) // New returns a Backend for cfg. Nothing connects until Run. @@ -174,7 +156,7 @@ func New(cfg Config) (*Backend, error) { instance: instance, target: target, creds: creds, - routes: xsync.NewMapOf[string, route](), + conns: xsync.NewMapOf[string, *relayStream](), token: cfg.Token, tokenSet: make(chan struct{}), liveCh: make(chan struct{}), @@ -217,7 +199,7 @@ func relayTarget(relayURL string) (string, credentials.TransportCredentials, err // Run holds the tunnel until ctx ends, replacing its stream whenever it // drops. It returns an error only if the relay refuses the tunnel for good. -func (b *Backend) Run(ctx context.Context, deliver func(tunnel.Frame)) error { +func (b *Backend) Run(ctx context.Context, accept func(net.Conn, *tunnel.ConnInfo)) error { relayDialer := b.cfg.RelayDialer cc, err := grpc.NewClient(b.target, grpc.WithTransportCredentials(b.creds), @@ -237,8 +219,13 @@ func (b *Backend) Run(ctx context.Context, deliver func(tunnel.Frame)) error { } defer cc.Close() b.rpc = devtunnel.NewTunnelServiceClient(cc) - b.deliver = deliver + b.accept = accept b.ctx = ctx + // A stream the relay moved off may still be finishing. + defer b.conns.Range(func(_ string, rs *relayStream) bool { + rs.end() + return true + }) err = b.run() if err != nil { @@ -251,83 +238,31 @@ func (b *Backend) Run(ctx context.Context, deliver func(tunnel.Frame)) error { return err } -// ConnID numbers an outbound conn. Open gives it its id on the stream. -func (b *Backend) ConnID(*tunnel.ConnInfo) string { - return "e" + strconv.FormatUint(b.nextEgress.Add(1), 10) -} - // Open sends an outbound conn out through the deployment, to its hostname if // the intercept's DNS gave it one, or else to the address it was headed for. // The deployment resolves the hostname as its own code would. It waits a // while for a stream to hold the tunnel. -func (b *Backend) Open(ctx context.Context, id string, conn *tunnel.ConnInfo) error { - n, err := strconv.ParseUint(strings.TrimPrefix(id, "e"), 10, 64) - if err != nil { - return fmt.Errorf("not an outbound conn id: %q", id) - } +func (b *Backend) Open(ctx context.Context, info *tunnel.ConnInfo) (net.Conn, error) { s := b.awaitLive(ctx) if s == nil { - return errors.New("no stream holds the tunnel") + return nil, errors.New("no stream holds the tunnel") } - wire := clientConnIDFloor | n - if !s.track(id, route{s: s, wire: wire, dir: egress}) { - return errors.New("the tunnel's stream ended") + n := b.nextEgress.Add(1) + rs := s.newConn("e"+strconv.FormatUint(n, 10), clientConnIDFloor|n, egress, info.Dest) + if rs == nil { + return nil, errors.New("the tunnel's stream ended") } - host := conn.Hostname + host := info.Hostname if host == "" { - host = conn.Dest.Host + host = info.Dest.Host } s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Open{Open: &devtunnel.Open{ - ConnId: wire, + ConnId: rs.wire, Protocol: devtunnel.Protocol_PROTOCOL_TCP, Host: host, - Port: uint32(conn.Dest.Port), + Port: uint32(info.Dest.Port), }}}) - return nil -} - -func (b *Backend) Send(f tunnel.Frame) { - r, ok := b.routes.Load(f.ConnID) - if !ok { - return - } - switch f.Kind { - case tunnel.FrameData: - b.bytesSent.Add(int64(len(f.Payload))) - r.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: r.wire, Payload: f.Payload}}}) - case tunnel.FrameCloseWrite: - r.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: r.wire}}}) - case tunnel.FrameReset: - r.s.send(resetFrame(r.wire, f.Reason)) - b.untrack(f.ConnID, r.s) - case tunnel.FrameWindow: - r.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: r.wire, Bytes: uint32(f.Bytes)}}}) - } -} - -func (b *Backend) Release(id string) { b.untrack(id, nil) } -func (b *Backend) Window() int { return connWindow } -func (b *Backend) HalfClose() bool { return true } - -// untrack forgets id's route if it's on s, or on any stream for nil. Reports -// whether it did. -func (b *Backend) untrack(id string, s *stream) bool { - var gone route - removed := false - b.routes.Compute(id, func(cur route, loaded bool) (route, bool) { - if !loaded { - return cur, true - } - if s != nil && cur.s != s { - return cur, false - } - gone, removed = cur, true - return cur, true - }) - if removed { - b.counters(gone.dir, -1) - } - return removed + return rs.conn, nil } // counters tracks conns opening (delta 1) and ending (delta -1). diff --git a/pkg/devtunnel/backend_test.go b/pkg/devtunnel/backend_test.go index 9c9f84b..c8fba0a 100644 --- a/pkg/devtunnel/backend_test.go +++ b/pkg/devtunnel/backend_test.go @@ -30,12 +30,12 @@ const frameBytes = 64 << 10 type fakeRelay struct { devtunnel.UnimplementedTunnelServiceServer url string - streams chan *relayStream + streams chan *relayCall // refuse ends a call before it reaches the test, unless it returns nil. refuse func(md metadata.MD) error } -type relayStream struct { +type relayCall struct { md metadata.MD call grpc.BidiStreamingServer[devtunnel.ConnectRequest, devtunnel.ConnectResponse] recv chan *devtunnel.ConnectRequest @@ -46,7 +46,7 @@ func startRelay(t *testing.T) *fakeRelay { t.Helper() lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) - r := &fakeRelay{url: "http://" + lis.Addr().String(), streams: make(chan *relayStream, 8)} + r := &fakeRelay{url: "http://" + lis.Addr().String(), streams: make(chan *relayCall, 8)} srv := grpc.NewServer() devtunnel.RegisterTunnelServiceServer(srv, r) go func() { _ = srv.Serve(lis) }() @@ -61,7 +61,7 @@ func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnel.ConnectReque return err } } - s := &relayStream{md: md, call: call, recv: make(chan *devtunnel.ConnectRequest, 256), end: make(chan error, 1)} + s := &relayCall{md: md, call: call, recv: make(chan *devtunnel.ConnectRequest, 256), end: make(chan error, 1)} go func() { defer close(s.recv) for { @@ -81,7 +81,7 @@ func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnel.ConnectReque } } -func (r *fakeRelay) next(t *testing.T) *relayStream { +func (r *fakeRelay) next(t *testing.T) *relayCall { t.Helper() select { case s := <-r.streams: @@ -92,18 +92,18 @@ func (r *fakeRelay) next(t *testing.T) *relayStream { } } -func (s *relayStream) send(t *testing.T, msg *devtunnel.ConnectResponse) { +func (s *relayCall) send(t *testing.T, msg *devtunnel.ConnectResponse) { t.Helper() require.NoError(t, s.call.Send(msg)) } // hold makes the stream the tunnel's, as the relay's first heartbeat does. -func (s *relayStream) hold(t *testing.T) { +func (s *relayCall) hold(t *testing.T) { s.send(t, &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Heartbeat{Heartbeat: &devtunnel.Heartbeat{}}}) } // expect returns the next frame from the client other than a heartbeat. -func (s *relayStream) expect(t *testing.T) *devtunnel.ConnectRequest { +func (s *relayCall) expect(t *testing.T) *devtunnel.ConnectRequest { t.Helper() timeout := time.After(testTimeout) for { @@ -121,7 +121,7 @@ func (s *relayStream) expect(t *testing.T) *devtunnel.ConnectRequest { } // expectData collects the client's Data on a conn until it has n bytes. -func (s *relayStream) expectData(t *testing.T, id uint64, n int) []byte { +func (s *relayCall) expectData(t *testing.T, id uint64, n int) []byte { t.Helper() var got []byte for len(got) < n { diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go new file mode 100644 index 0000000..aa932ee --- /dev/null +++ b/pkg/devtunnel/conn.go @@ -0,0 +1,197 @@ +package devtunnel + +import ( + "bytes" + "io" + "net" + "sync" + "sync/atomic" + + "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/plumbing" + "github.com/vercel/bridge/pkg/tunnel" +) + +// Per Data frame. +const maxPayload = 64 << 10 + +type direction int + +const ( + // From the deployment to the app. + ingress direction = iota + // From the app out through the deployment. + egress +) + +// relayStream is one conn's frames on a Connect stream, as a plumbing.Stream: +// Recv returns the relay's Data and grants it back, Send sends Data, +// CloseSend sends Close and Close sends Reset. The tunnel gets conn, which +// holds writes to the relay's window. +type relayStream struct { + s *stream + id string + wire uint64 + dir direction + conn plumbing.MeteredConn + + // From the relay, not yet received. The window bounds it. + mu sync.Mutex + in [][]byte + eof bool + wake chan struct{} + + // Both sides have half-closed once both are set. + closeSent, eofRead atomic.Bool + done chan struct{} + once sync.Once +} + +var _ plumbing.Stream = (*relayStream)(nil) + +// newConn opens id on the stream as the relay's conn wire, or returns nil +// once the stream has ended. +func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Addr) *relayStream { + if s.closed.Load() { + return nil + } + rs := &relayStream{s: s, id: id, wire: wire, dir: dir, wake: make(chan struct{}, 1), done: make(chan struct{})} + rs.conn = plumbing.NewMeteredConn(plumbing.NewStreamConn(rs, addr("bridge"), addr(remote.String())), connWindow) + s.b.counters(dir, 1) + s.b.conns.Store(id, rs) + // endConns may have swept the conns before the Store. + if s.closed.Load() { + rs.end() + return nil + } + return rs +} + +func (rs *relayStream) Recv() ([]byte, error) { + for { + if rs.isDone() { + return nil, net.ErrClosed + } + rs.mu.Lock() + if len(rs.in) > 0 { + chunk := rs.in[0] + rs.in[0] = nil + rs.in = rs.in[1:] + rs.mu.Unlock() + rs.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ + ConnId: rs.wire, + Bytes: uint32(len(chunk)), + }}}) + return chunk, nil + } + eof := rs.eof + rs.mu.Unlock() + if eof { + rs.eofRead.Store(true) + if rs.closeSent.Load() { + rs.end() + } + return nil, io.EOF + } + select { + case <-rs.wake: + case <-rs.done: + } + } +} + +func (rs *relayStream) Send(p []byte) error { + for len(p) > 0 { + if rs.isDone() { + return net.ErrClosed + } + size := min(len(p), maxPayload) + rs.s.b.bytesSent.Add(int64(size)) + // Queued, so it can't share p. + rs.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ + ConnId: rs.wire, + Payload: bytes.Clone(p[:size]), + }}}) + p = p[size:] + } + return nil +} + +func (rs *relayStream) CloseSend() error { + if rs.isDone() || !rs.closeSent.CompareAndSwap(false, true) { + return nil + } + rs.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: rs.wire}}}) + if rs.eofRead.Load() { + rs.end() + } + return nil +} + +// Close resets the conn. +func (rs *relayStream) Close() error { + if rs.isDone() { + return nil + } + rs.s.send(resetFrame(rs.wire, "connection closed")) + rs.end() + return nil +} + +func (rs *relayStream) Done() <-chan struct{} { return rs.done } + +// push queues payload from the relay. +func (rs *relayStream) push(payload []byte) { + if len(payload) == 0 { + return + } + rs.mu.Lock() + if !rs.eof && !rs.isDone() { + rs.in = append(rs.in, payload) + } + rs.mu.Unlock() + rs.signal() +} + +// peerClosed: the relay's side has nothing more to send. +func (rs *relayStream) peerClosed() { + rs.mu.Lock() + rs.eof = true + rs.mu.Unlock() + rs.signal() +} + +// end forgets the conn, without telling the relay. +func (rs *relayStream) end() { + rs.once.Do(func() { + close(rs.done) + rs.s.b.conns.Compute(rs.id, func(cur *relayStream, loaded bool) (*relayStream, bool) { + if !loaded { + return nil, true + } + return cur, cur == rs + }) + rs.s.b.counters(rs.dir, -1) + }) +} + +func (rs *relayStream) signal() { + select { + case rs.wake <- struct{}{}: + default: + } +} + +func (rs *relayStream) isDone() bool { + select { + case <-rs.done: + return true + default: + return false + } +} + +type addr string + +func (a addr) Network() string { return "dev-tunnel" } +func (a addr) String() string { return string(a) } diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index 5129f1e..1c4c6b3 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -126,22 +126,22 @@ func (s *stream) handle(msg *devtunnel.ConnectResponse, moving func()) { case *devtunnel.ConnectResponse_Open: s.open(f.Open) case *devtunnel.ConnectResponse_Data: - if id, ok := s.routed(f.Data.GetConnId()); ok { + if rs := s.conn(f.Data.GetConnId()); rs != nil { s.b.bytesReceived.Add(int64(len(f.Data.GetPayload()))) - s.b.deliver(tunnel.Frame{Kind: tunnel.FrameData, ConnID: id, Payload: f.Data.GetPayload()}) + rs.push(f.Data.GetPayload()) } case *devtunnel.ConnectResponse_Close: - if id, ok := s.routed(f.Close.GetConnId()); ok { - s.b.deliver(tunnel.Frame{Kind: tunnel.FrameCloseWrite, ConnID: id}) + if rs := s.conn(f.Close.GetConnId()); rs != nil { + rs.peerClosed() } case *devtunnel.ConnectResponse_Reset_: - if id, ok := s.routed(f.Reset_.GetConnId()); ok && s.b.untrack(id, s) { - s.logger.Debug("Relay reset the connection", "conn_id", id, "reason", f.Reset_.GetReason()) - s.b.deliver(tunnel.Frame{Kind: tunnel.FrameReset, ConnID: id, Reason: f.Reset_.GetReason()}) + if rs := s.conn(f.Reset_.GetConnId()); rs != nil { + s.logger.Debug("Relay reset the connection", "conn_id", rs.id, "reason", f.Reset_.GetReason()) + rs.end() } case *devtunnel.ConnectResponse_Window: - if id, ok := s.routed(f.Window.GetConnId()); ok { - s.b.deliver(tunnel.Frame{Kind: tunnel.FrameWindow, ConnID: id, Bytes: int(f.Window.GetBytes())}) + if rs := s.conn(f.Window.GetConnId()); rs != nil { + rs.conn.Grant(int(f.Window.GetBytes())) } } } @@ -155,19 +155,15 @@ func (s *stream) open(open *devtunnel.Open) { s.send(resetFrame(wire, fmt.Sprintf("bridge serves only port %d", port))) return } - id := s.idFor(wire) - if !s.track(id, route{s: s, wire: wire, dir: ingress}) { - return + dest := tunnel.Addr{Host: "127.0.0.1", Port: port} + if rs := s.newConn(s.idFor(wire), wire, ingress, dest); rs != nil { + s.b.accept(rs.conn, &tunnel.ConnInfo{Dest: dest}) } - s.b.deliver(tunnel.Frame{ - Kind: tunnel.FrameOpen, - ConnID: id, - Conn: &tunnel.ConnInfo{Dest: tunnel.Addr{Host: "127.0.0.1", Port: port}}, - }) } -// idFor is the tunnel's id for the conn the relay calls wire on this stream. -// Outbound conns keep the id the tunnel gave them. +// idFor is the backend's id for the conn the relay calls wire on this stream. +// The relay numbers each stream's conns from 1; outbound conns are numbered +// across streams. func (s *stream) idFor(wire uint64) string { if wire >= clientConnIDFloor { return "e" + strconv.FormatUint(wire&^clientConnIDFloor, 10) @@ -175,35 +171,21 @@ func (s *stream) idFor(wire uint64) string { return s.prefix + strconv.FormatUint(wire, 10) } -// routed returns the tunnel's id for wire, if it's a conn open on this stream. -func (s *stream) routed(wire uint64) (string, bool) { - id := s.idFor(wire) - r, ok := s.b.routes.Load(id) - return id, ok && r.s == s -} - -// track routes id to this stream, or reports false once the stream has ended. -func (s *stream) track(id string, r route) bool { - if s.closed.Load() { - return false - } - // Counted before it's stored, so whichever untrack removes it uncounts it. - s.b.counters(r.dir, 1) - s.b.routes.Store(id, r) - // endConns may have swept the routes before the Store. - if s.closed.Load() { - s.b.untrack(id, s) - return false +// conn returns the conn the relay calls wire, if it's open on this stream. +func (s *stream) conn(wire uint64) *relayStream { + rs, ok := s.b.conns.Load(s.idFor(wire)) + if !ok || rs.s != s { + return nil } - return true + return rs } -// endConns resets every conn on the stream, which the relay ended with it. +// endConns ends every conn on the stream, which the relay ended with it. func (s *stream) endConns() { s.closed.Store(true) - s.b.routes.Range(func(id string, r route) bool { - if r.s == s && s.b.untrack(id, s) { - s.b.deliver(tunnel.Frame{Kind: tunnel.FrameReset, ConnID: id, Reason: "the tunnel's stream ended"}) + s.b.conns.Range(func(_ string, rs *relayStream) bool { + if rs.s == s { + rs.end() } return true }) diff --git a/pkg/tunnel/backend.go b/pkg/tunnel/backend.go index fdd93d7..dcb75c7 100644 --- a/pkg/tunnel/backend.go +++ b/pkg/tunnel/backend.go @@ -8,54 +8,15 @@ import ( // Backend carries a tunnel's conns over one multiplexed transport: a bridge // server's TunnelNetwork stream (NewStreamBackend), or a dev tunnel's Connect -// stream (devtunnel.Backend). +// stream (devtunnel.Backend). Each conn is a net.Conn over one of its streams; +// its CloseWrite half-closes it and Close resets it. type Backend interface { - // Run carries frames until ctx ends or the transport does, passing each of - // the peer's to deliver in order. deliver blocks only while a conn has too - // much queued, and only for a backend without flow control. - Run(ctx context.Context, deliver func(Frame)) error - // ConnID numbers a conn this side opens. - ConnID(conn *ConnInfo) string - // Open tells the peer about a conn this side opens, ahead of its Data. - Open(ctx context.Context, id string, conn *ConnInfo) error - // Send queues a frame for the peer. - Send(f Frame) - // Release forgets a conn that's over. - Release(id string) - // Window is what each side of a conn may send before the other grants - // more, or 0 if the transport has no flow control. - Window() int - // HalfClose reports whether the transport carries a conn's half-close. - // Without it, a conn ends at either side's EOF. - HalfClose() bool -} - -type FrameKind uint8 - -const ( - // The peer opened a conn to Conn.Dest. - FrameOpen FrameKind = iota + 1 - // Payload for a conn. From a peer that sends no Open, Data for an unknown - // conn opens it when it has a Conn. - FrameData - // The sender has nothing more to send on the conn. - FrameCloseWrite - // The conn is abandoned in both directions, for Reason. - FrameReset - // The receiver may send Bytes more of the conn's Data. - FrameWindow -) - -// Frame is one message about one conn. -type Frame struct { - Kind FrameKind - ConnID string - Payload []byte - Bytes int - Reason string - // Where the conn goes. Set on Open, and on every frame of a backend that - // addresses each one. - Conn *ConnInfo + // Run carries the transport until ctx ends or the transport does, handing + // accept each conn the peer opens, with where it goes. It ends every conn + // before returning. + Run(ctx context.Context, accept func(conn net.Conn, info *ConnInfo)) error + // Open opens a conn through the transport to info.Dest. + Open(ctx context.Context, info *ConnInfo) (net.Conn, error) } // Addr is a host (an IP address or a hostname) and a port. @@ -66,9 +27,14 @@ type Addr struct { func (a Addr) String() string { return net.JoinHostPort(a.Host, strconv.Itoa(a.Port)) } -// ConnInfo addresses a conn, as the side sending on it sees it. +// ConnInfo addresses a conn. type ConnInfo struct { Source, Dest Addr // The DNS name Dest was looked up as, if any. Hostname string } + +type addr string + +func (a addr) Network() string { return "tunnel" } +func (a addr) String() string { return string(a) } diff --git a/pkg/tunnel/conn.go b/pkg/tunnel/conn.go deleted file mode 100644 index ee825c4..0000000 --- a/pkg/tunnel/conn.go +++ /dev/null @@ -1,235 +0,0 @@ -package tunnel - -import ( - "bytes" - "io" - "net" - "sync" - "sync/atomic" - - "github.com/vercel/bridge/pkg/plumbing" -) - -const ( - // Per Data frame. - maxFrame = 64 << 10 - // What a conn may have queued from a peer without flow control before the - // backend waits for it to be read. - maxQueued = 256 << 10 -) - -// connStream is one conn's frames on the tunnel's transport, as a -// plumbing.Stream. The tunnel uses it as conn: a net.Conn over it, metered -// to the conn's window. -type connStream struct { - t *tunnelImpl - id string - info *ConnInfo - conn plumbing.MeteredConn - // 0 when the backend has no flow control. - window int - halfClose bool - - // From the peer, not yet received. - inMu sync.Mutex - in [][]byte - inBytes int - inEOF bool - inWake chan struct{} - inRoom chan struct{} - - closeSent atomic.Bool - local atomic.Pointer[net.Conn] - done chan struct{} - doneOnce sync.Once -} - -var _ plumbing.Stream = (*connStream)(nil) - -func (t *tunnelImpl) newConn(id string, info *ConnInfo) *connStream { - c := &connStream{ - t: t, - id: id, - info: info, - window: t.backend.Window(), - halfClose: t.backend.HalfClose(), - inWake: make(chan struct{}, 1), - inRoom: make(chan struct{}, 1), - done: make(chan struct{}), - } - c.conn = plumbing.NewMeteredConn( - plumbing.NewStreamConn(c, addr(info.Source.String()), addr(info.Dest.String())), - c.window, - ) - return c -} - -func (c *connStream) Recv() ([]byte, error) { - for { - if c.isDone() { - return nil, net.ErrClosed - } - c.inMu.Lock() - if len(c.in) > 0 { - chunk := c.in[0] - c.in[0] = nil - c.in = c.in[1:] - c.inMu.Unlock() - c.consumed(len(chunk)) - return chunk, nil - } - eof := c.inEOF - c.inMu.Unlock() - if eof { - return nil, io.EOF - } - select { - case <-c.inWake: - case <-c.done: - } - } -} - -// consumed gives back what Recv took: as window to a peer with flow control, -// and as room in the queue to one without. -func (c *connStream) consumed(n int) { - if c.window > 0 { - c.send(Frame{Kind: FrameWindow, Bytes: n}) - return - } - c.inMu.Lock() - c.inBytes -= n - c.inMu.Unlock() - wake(c.inRoom) -} - -func (c *connStream) Send(p []byte) error { - for len(p) > 0 { - if c.isDone() { - return net.ErrClosed - } - size := min(len(p), maxFrame) - // The backend may send it after Send returns. - c.send(Frame{Kind: FrameData, Payload: bytes.Clone(p[:size])}) - p = p[size:] - } - return nil -} - -// CloseSend tells the peer this side has nothing more to send. Over a -// transport without half-close, it ends the conn. -func (c *connStream) CloseSend() error { - if !c.halfClose { - c.finish() - return nil - } - if !c.isDone() && c.closeSent.CompareAndSwap(false, true) { - c.send(Frame{Kind: FrameCloseWrite}) - } - return nil -} - -// Close resets the conn. -func (c *connStream) Close() error { - c.reset("closed") - return nil -} - -// Done is closed once the conn is over. -func (c *connStream) Done() <-chan struct{} { return c.done } - -// deliver queues payload from the peer. -func (c *connStream) deliver(p []byte) { - if len(p) == 0 { - return - } - for { - c.inMu.Lock() - if c.inEOF || c.isDone() { - c.inMu.Unlock() - return - } - // A window already bounds what the peer can send. - if c.window > 0 || c.inBytes < maxQueued { - c.in = append(c.in, p) - c.inBytes += len(p) - c.inMu.Unlock() - wake(c.inWake) - return - } - c.inMu.Unlock() - select { - case <-c.inRoom: - case <-c.done: - return - } - } -} - -// peerClosed: the peer has nothing more to send. Recv returns io.EOF once it -// has returned the rest. -func (c *connStream) peerClosed() { - c.inMu.Lock() - c.inEOF = true - c.inMu.Unlock() - wake(c.inWake) -} - -// attach hands the conn its local end, which it closes when it's over. -// Reports false if the conn is already over. -func (c *connStream) attach(local net.Conn) bool { - c.local.Store(&local) - if c.isDone() { - _ = local.Close() - return false - } - return true -} - -// reset ends the conn and tells the peer why. -func (c *connStream) reset(reason string) { c.end(reason, true) } - -// finish ends the conn without telling the peer: it's done, or already knows. -func (c *connStream) finish() { c.end("", false) } - -func (c *connStream) end(reason string, tell bool) { - c.doneOnce.Do(func() { - if tell { - c.send(Frame{Kind: FrameReset, Reason: reason}) - } - close(c.done) - if local := c.local.Load(); local != nil { - _ = (*local).Close() - } - c.t.forget(c) - c.t.backend.Release(c.id) - }) -} - -func (c *connStream) isDone() bool { - select { - case <-c.done: - return true - default: - return false - } -} - -func (c *connStream) send(f Frame) { - f.ConnID = c.id - f.Conn = c.info - c.t.backend.Send(f) -} - -// wake signals ch without blocking; ch holds one signal. -func wake(ch chan struct{}) { - select { - case ch <- struct{}{}: - default: - } -} - -type addr string - -func (a addr) Network() string { return "tunnel" } -func (a addr) String() string { return string(a) } diff --git a/pkg/tunnel/pump_test.go b/pkg/tunnel/pump_test.go index d00c706..6fca973 100644 --- a/pkg/tunnel/pump_test.go +++ b/pkg/tunnel/pump_test.go @@ -8,89 +8,80 @@ import ( "time" ) -// fakeBackend has the flow control and half-close a dev tunnel has. -type fakeBackend struct { - window int - frames chan Frame - sent chan Frame - released chan string +// acceptingBackend hands the tunnel the conns a test gives it. +type acceptingBackend struct { + conns chan net.Conn + dest Addr } -func newFakeBackend(window int) *fakeBackend { - return &fakeBackend{ - window: window, - frames: make(chan Frame, 16), - sent: make(chan Frame, 1024), - released: make(chan string, 16), - } -} - -func (b *fakeBackend) Run(ctx context.Context, deliver func(Frame)) error { +func (b *acceptingBackend) Run(ctx context.Context, accept func(net.Conn, *ConnInfo)) error { for { select { - case f := <-b.frames: - deliver(f) + case c := <-b.conns: + accept(c, &ConnInfo{Dest: b.dest}) case <-ctx.Done(): return nil } } } -func (b *fakeBackend) ConnID(*ConnInfo) string { return "out" } -func (b *fakeBackend) Open(context.Context, string, *ConnInfo) error { return nil } -func (b *fakeBackend) Send(f Frame) { b.sent <- f } -func (b *fakeBackend) Release(id string) { b.released <- id } -func (b *fakeBackend) Window() int { return b.window } -func (b *fakeBackend) HalfClose() bool { return true } - -// next returns the next frame the tunnel sent other than a window grant. -func (b *fakeBackend) next(t *testing.T) Frame { - t.Helper() - for { - select { - case f := <-b.sent: - if f.Kind != FrameWindow { - return f - } - case <-time.After(testTimeout): - t.Fatal("the tunnel sent nothing") - } - } +func (b *acceptingBackend) Open(context.Context, *ConnInfo) (net.Conn, error) { + return nil, net.ErrClosed } -func startBackendTunnel(t *testing.T, b *fakeBackend) Tunnel { +// tcpPair returns both ends of a TCP connection. +func tcpPair(t *testing.T) (net.Conn, net.Conn) { t.Helper() - tun := NewWithBackend(&net.Dialer{Timeout: time.Second}, b) - tun.Start(context.Background()) - t.Cleanup(func() { tun.Close() }) - return tun + lis, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + defer lis.Close() + accepted := make(chan net.Conn, 1) + go func() { + c, _ := lis.Accept() + accepted <- c + }() + a, err := net.Dial("tcp", lis.Addr().String()) + if err != nil { + t.Fatal(err) + } + b := <-accepted + t.Cleanup(func() { + a.Close() + b.Close() + }) + return a, b } func TestHalfCloseCrossesThePump(t *testing.T) { - lis, err := net.Listen("tcp", "127.0.0.1:0") + app, err := net.Listen("tcp", "127.0.0.1:0") if err != nil { t.Fatal(err) } - defer lis.Close() + defer app.Close() got := make(chan string, 1) go func() { - nc, err := lis.Accept() + c, err := app.Accept() if err != nil { return } // Reads to the peer's half-close, then answers and closes. - req, _ := io.ReadAll(nc) + req, _ := io.ReadAll(c) got <- string(req) - _, _ = nc.Write([]byte("bye")) - _ = nc.Close() + _, _ = c.Write([]byte("bye")) + _ = c.Close() }() - b := newFakeBackend(1 << 20) - startBackendTunnel(t, b) - dest := splitAddr(lis.Addr().String()) - b.frames <- Frame{Kind: FrameOpen, ConnID: "1", Conn: &ConnInfo{Dest: dest}} - b.frames <- Frame{Kind: FrameData, ConnID: "1", Payload: []byte("hi")} - b.frames <- Frame{Kind: FrameCloseWrite, ConnID: "1"} + b := &acceptingBackend{conns: make(chan net.Conn, 1), dest: splitAddr(app.Addr().String())} + tun := NewWithBackend(&net.Dialer{Timeout: time.Second}, b) + tun.Start(context.Background()) + t.Cleanup(func() { tun.Close() }) + + peer, remote := tcpPair(t) + b.conns <- remote + _, _ = peer.Write([]byte("hi")) + _ = peer.(*net.TCPConn).CloseWrite() select { case req := <-got: @@ -100,45 +91,12 @@ func TestHalfCloseCrossesThePump(t *testing.T) { case <-time.After(testTimeout): t.Fatal("the app never saw the peer's half-close") } - if f := b.next(t); f.Kind != FrameData || string(f.Payload) != "bye" { - t.Fatalf("expected the app's answer, got %+v", f) - } - if f := b.next(t); f.Kind != FrameCloseWrite { - t.Fatalf("expected the app's half-close, got %+v", f) - } - select { - case id := <-b.released: - if id != "1" { - t.Fatalf("released %q", id) - } - case <-time.After(testTimeout): - t.Fatal("the conn was never released") - } -} - -func TestWritesWaitForWindow(t *testing.T) { - b := newFakeBackend(10) - tun := startBackendTunnel(t, b) - app, intercepted := net.Pipe() - defer app.Close() - tun.AddConn(intercepted, "10.0.0.1:443", "example.com") - - go func() { _, _ = app.Write([]byte("0123456789abcdefghijklmno")) }() - sent := "" - for len(sent) < 10 { - sent += string(b.next(t).Payload) - } - select { - case f := <-b.sent: - t.Fatalf("sent past the window: %+v", f) - case <-time.After(100 * time.Millisecond): - } - - b.frames <- Frame{Kind: FrameWindow, ConnID: "out", Bytes: 15} - for len(sent) < 25 { - sent += string(b.next(t).Payload) + _ = peer.SetReadDeadline(time.Now().Add(testTimeout)) + resp, err := io.ReadAll(peer) + if err != nil { + t.Fatal(err) } - if sent != "0123456789abcdefghijklmno" { - t.Fatalf("sent %q", sent) + if string(resp) != "bye" { + t.Fatalf("the peer read %q", resp) } } diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go index 5fc6754..4ec54bb 100644 --- a/pkg/tunnel/stream.go +++ b/pkg/tunnel/stream.go @@ -1,14 +1,22 @@ package tunnel import ( + "bytes" "context" "io" "log/slog" + "net" + "sync" + "github.com/puzpuzpuz/xsync/v3" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" "github.com/vercel/bridge/pkg/plumbing" ) +// What a conn may have queued from the peer before the stream waits for it +// to be read. The stream has no flow control of its own. +const queuedMessages = 64 + // Stream abstracts a bidirectional gRPC stream. Both the server-side // BidiStreamingServer and client-side TunnelNetworkClient satisfy this // interface directly since both directions use TunnelNetworkMessage. @@ -25,6 +33,7 @@ func NewStreamBackend(stream Stream) Backend { stream: stream, sendCh: make(chan *bridgev1.TunnelNetworkMessage, 64), stop: make(chan struct{}), + conns: xsync.NewMapOf[string, *messageStream](), } } @@ -32,11 +41,16 @@ type streamBackend struct { stream Stream sendCh chan *bridgev1.TunnelNetworkMessage // Closed once Run returns, so a Send waiting for room gives up. - stop chan struct{} + stop chan struct{} + conns *xsync.MapOf[string, *messageStream] } -func (b *streamBackend) Run(ctx context.Context, deliver func(Frame)) error { +func (b *streamBackend) Run(ctx context.Context, accept func(net.Conn, *ConnInfo)) error { defer close(b.stop) + defer b.conns.Range(func(_ string, s *messageStream) bool { + s.end() + return true + }) // Single goroutine drains the send channel onto the stream. go func() { @@ -74,7 +88,7 @@ func (b *streamBackend) Run(ctx context.Context, deliver func(Frame)) error { for { select { case msg := <-recvCh: - deliver(frameFor(msg)) + b.handle(ctx, msg, accept) case err := <-recvErr: if err == io.EOF { slog.Info("Tunnel: stream closed (EOF)") @@ -88,49 +102,151 @@ func (b *streamBackend) Run(ctx context.Context, deliver func(Frame)) error { } } -func frameFor(msg *bridgev1.TunnelNetworkMessage) Frame { +// handle routes one of the peer's messages to its conn. The first message of +// an unknown conn opens it, before the dial, so the chunks that arrive during +// the dial queue on it instead of each opening another. +func (b *streamBackend) handle(ctx context.Context, msg *bridgev1.TunnelNetworkMessage, accept func(net.Conn, *ConnInfo)) { + id := msg.GetConnectionId() + s, ok := b.conns.Load(id) if msg.GetError() != "" { - return Frame{Kind: FrameReset, ConnID: msg.GetConnectionId(), Reason: msg.GetError()} + if ok { + s.end() + } + return } - f := Frame{Kind: FrameData, ConnID: msg.GetConnectionId(), Payload: msg.GetData()} - if dest := msg.GetDest(); dest != nil { - f.Conn = &ConnInfo{ + if !ok { + dest := msg.GetDest() + if dest == nil { + slog.Info("Tunnel: ignoring message with no dest", "conn_id", id) + return + } + info := &ConnInfo{ Source: Addr{Host: msg.GetSource().GetIp(), Port: int(msg.GetSource().GetPort())}, Dest: Addr{Host: dest.GetIp(), Port: int(dest.GetPort())}, Hostname: msg.GetHostname(), } + // This side sends from where the peer's conn was headed, back to it. + s = b.newStream(id, &ConnInfo{Source: info.Dest, Dest: info.Source, Hostname: info.Hostname}) + b.conns.Store(id, s) + accept(s.conn(), info) } - return f + s.push(ctx, msg.GetData()) } -func (b *streamBackend) ConnID(conn *ConnInfo) string { - return plumbing.ConnectionID(conn.Source.Host, conn.Source.Port, conn.Dest.Host, conn.Dest.Port) +func (b *streamBackend) Open(_ context.Context, info *ConnInfo) (net.Conn, error) { + id := plumbing.ConnectionID(info.Source.Host, info.Source.Port, info.Dest.Host, info.Dest.Port) + s := b.newStream(id, info) + b.conns.Store(id, s) + // Its first Data opens it. + return s.conn(), nil } -// Open sends nothing: the conn's first Data opens it. -func (b *streamBackend) Open(context.Context, string, *ConnInfo) error { return nil } - -func (b *streamBackend) Send(f Frame) { - msg := &bridgev1.TunnelNetworkMessage{ConnectionId: f.ConnID} - switch f.Kind { - case FrameData: - msg.Data = f.Payload - if f.Conn != nil { - msg.Source = tunnelAddress(f.Conn.Source) - msg.Dest = tunnelAddress(f.Conn.Dest) - msg.Hostname = f.Conn.Hostname - } - case FrameReset: - msg.Error = f.Reason - default: - return +func (b *streamBackend) newStream(id string, info *ConnInfo) *messageStream { + return &messageStream{ + b: b, + id: id, + info: info, + in: make(chan []byte, queuedMessages), + done: make(chan struct{}), } +} + +func (b *streamBackend) send(msg *bridgev1.TunnelNetworkMessage) error { select { case b.sendCh <- msg: + return nil case <-b.stop: + return net.ErrClosed + } +} + +// messageStream is one conn's messages on the stream, as a plumbing.Stream. +type messageStream struct { + b *streamBackend + id string + // As this side sends it. + info *ConnInfo + in chan []byte + done chan struct{} + once sync.Once +} + +var _ plumbing.Stream = (*messageStream)(nil) + +func (s *messageStream) conn() net.Conn { + return plumbing.NewMeteredConn(plumbing.NewStreamConn(s, addr(s.info.Source.String()), addr(s.info.Dest.String())), 0) +} + +// push queues payload for Recv, waiting while the conn has a full queue. +func (s *messageStream) push(ctx context.Context, payload []byte) { + if len(payload) == 0 { + return + } + select { + case s.in <- payload: + case <-s.done: + case <-ctx.Done(): } } -func (b *streamBackend) Release(string) {} -func (b *streamBackend) Window() int { return 0 } -func (b *streamBackend) HalfClose() bool { return false } +func (s *messageStream) Recv() ([]byte, error) { + // A conn the peer ended drops what it hadn't read. + select { + case <-s.done: + return nil, net.ErrClosed + default: + } + select { + case p := <-s.in: + return p, nil + case <-s.done: + return nil, net.ErrClosed + } +} + +func (s *messageStream) Send(p []byte) error { + select { + case <-s.done: + return net.ErrClosed + default: + } + return s.b.send(&bridgev1.TunnelNetworkMessage{ + ConnectionId: s.id, + Source: tunnelAddress(s.info.Source), + Dest: tunnelAddress(s.info.Dest), + Hostname: s.info.Hostname, + Data: bytes.Clone(p), + }) +} + +// CloseSend ends the conn: the stream can't carry a half-close. +func (s *messageStream) CloseSend() error { + s.end() + return nil +} + +// Close ends the conn and tells the peer. +func (s *messageStream) Close() error { + select { + case <-s.done: + return nil + default: + } + s.end() + return s.b.send(&bridgev1.TunnelNetworkMessage{ConnectionId: s.id, Error: "connection closed"}) +} + +func (s *messageStream) Done() <-chan struct{} { return s.done } + +// end forgets the conn, unless another has since taken its id. +func (s *messageStream) end() { + s.once.Do(func() { + close(s.done) + s.b.conns.Compute(s.id, func(cur *messageStream, loaded bool) (*messageStream, bool) { + if !loaded { + return nil, true + } + return cur, cur == s + }) + }) +} diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index 80c85ec..df1ece5 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -8,7 +8,6 @@ import ( "strconv" "sync" - "github.com/puzpuzpuz/xsync/v3" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" "github.com/vercel/bridge/pkg/mitm" "github.com/vercel/bridge/pkg/plumbing" @@ -26,10 +25,10 @@ type Tunnel interface { // looked up as (may be empty). AddConn(conn net.Conn, destOverride string, hostname string) - // Start runs the backend, opening a local conn for each the peer opens. + // Start runs the backend, dialing a local conn for each the peer opens. Start(ctx context.Context) - // Close cancels the tunnel context, closing all connections and stopping all goroutines. + // Close stops the backend, which ends every conn. Close() error // Done returns a channel that is closed when the backend stops. @@ -58,7 +57,6 @@ func NewWithBackend(dialer plumbing.ContextDialer, backend Backend, opts ...Opti t := &tunnelImpl{ dialer: dialer, backend: backend, - conns: xsync.NewMapOf[string, *connStream](), done: make(chan struct{}), logger: slog.Default(), } @@ -74,16 +72,14 @@ func NewWithBackend(dialer plumbing.ContextDialer, backend Backend, opts ...Opti } type tunnelImpl struct { - dialer plumbing.ContextDialer - hijacker mitm.Hijacker - backend Backend - conns *xsync.MapOf[string, *connStream] - bufs sync.Pool - logger *slog.Logger - done chan struct{} - ctx context.Context - cancel context.CancelFunc - closeOnce sync.Once + dialer plumbing.ContextDialer + hijacker mitm.Hijacker + backend Backend + bufs sync.Pool + logger *slog.Logger + done chan struct{} + ctx context.Context + cancel context.CancelFunc } func (t *tunnelImpl) Done() <-chan struct{} { return t.done } @@ -92,22 +88,16 @@ func (t *tunnelImpl) Start(ctx context.Context) { stop := context.AfterFunc(ctx, t.cancel) go func() { defer stop() - defer t.Close() - if err := t.backend.Run(t.ctx, t.deliver); err != nil { + defer close(t.done) + defer t.cancel() + if err := t.backend.Run(t.ctx, t.accept); err != nil { t.logger.Info("Tunnel: backend stopped", "error", err) } }() } func (t *tunnelImpl) Close() error { - t.closeOnce.Do(func() { - t.cancel() - t.conns.Range(func(_ string, c *connStream) bool { - c.finish() - return true - }) - close(t.done) - }) + t.cancel() return nil } @@ -118,139 +108,97 @@ func (t *tunnelImpl) AddConn(conn net.Conn, destOverride string, hostname string dstAddr = destOverride } info := &ConnInfo{Source: splitAddr(srcAddr), Dest: splitAddr(dstAddr), Hostname: hostname} - id := t.backend.ConnID(info) - logger := t.logger.With("conn_id", id, "src", srcAddr, "dst", dstAddr, "hostname", hostname) + logger := t.logger.With("src", srcAddr, "dst", dstAddr, "hostname", hostname) logger.Debug("Tunnel: adding connection") - c := t.newConn(id, info) - t.conns.Store(id, c) go func() { - if err := t.backend.Open(t.ctx, id, info); err != nil { + remote, err := t.backend.Open(t.ctx, info) + if err != nil { logger.Info("Tunnel: open failed", "error", err) - c.finish() _ = conn.Close() return } - t.pump(conn, c) + t.pump(conn, remote) }() } -// deliver routes one of the peer's frames to its conn. -func (t *tunnelImpl) deliver(f Frame) { - c, ok := t.conns.Load(f.ConnID) - switch f.Kind { - case FrameOpen: - t.open(f) - case FrameData: - if ok { - c.deliver(f.Payload) - } else { - t.open(f) - } - case FrameCloseWrite: - if ok { - c.peerClosed() - } - case FrameReset: - if ok { - c.finish() - } - case FrameWindow: - if ok { - c.conn.Grant(f.Bytes) - } - } -} - -// open registers a conn the peer opened before dialing it, so the frames that -// arrive during the dial queue on it instead of each opening another. -func (t *tunnelImpl) open(f Frame) { - if f.Conn == nil { - t.logger.Info("Tunnel: ignoring message with no dest", "conn_id", f.ConnID) - return - } - // This side sends from where the peer's conn was headed, back to it. - reply := &ConnInfo{Source: f.Conn.Dest, Dest: f.Conn.Source, Hostname: f.Conn.Hostname} - c, loaded := t.conns.LoadOrStore(f.ConnID, t.newConn(f.ConnID, reply)) - c.deliver(f.Payload) - if loaded { - return - } +// accept dials a conn the peer opened, giving the hijacker first shot at it. +func (t *tunnelImpl) accept(remote net.Conn, info *ConnInfo) { go func() { - local, err := t.dial(f) + local, err := t.dial(info) if err != nil { - t.logger.Info("Tunnel: connect failed", "conn_id", f.ConnID, "hostname", f.Conn.Hostname, "error", err) - c.reset(err.Error()) + t.logger.Info("Tunnel: connect failed", "dest", info.Dest.String(), "hostname", info.Hostname, "error", err) + _ = remote.Close() return } - t.pump(local, c) + t.pump(local, remote) }() } -// dial gives the hijacker first shot at a conn the peer opened, then falls -// through to the dialer. -func (t *tunnelImpl) dial(f Frame) (net.Conn, error) { +func (t *tunnelImpl) dial(info *ConnInfo) (net.Conn, error) { if t.hijacker != nil { msg := &bridgev1.TunnelNetworkMessage{ - ConnectionId: f.ConnID, - Source: tunnelAddress(f.Conn.Source), - Dest: tunnelAddress(f.Conn.Dest), - Hostname: f.Conn.Hostname, + Source: tunnelAddress(info.Source), + Dest: tunnelAddress(info.Dest), + Hostname: info.Hostname, } if t.hijacker.ShouldHijack(msg) { conn, err := t.hijacker.Hijack(t.ctx, msg) if err == nil { return conn, nil } - t.logger.Info("Tunnel: hijack failed, falling back to dialer", "conn_id", f.ConnID, "hostname", f.Conn.Hostname, "error", err) + t.logger.Info("Tunnel: hijack failed, falling back to dialer", "hostname", info.Hostname, "error", err) } } - return t.dialer.DialContext(t.ctx, "tcp", f.Conn.Dest.String()) + return t.dialer.DialContext(t.ctx, "tcp", info.Dest.String()) } -// pump copies between local and c.conn in both directions, each through a -// buffer from the pool, until the conn is over. -func (t *tunnelImpl) pump(local net.Conn, c *connStream) { - if !c.attach(local) { - return - } - toPeer := t.bufs.Get().(*[]byte) - fromPeer := t.bufs.Get().(*[]byte) +// pump copies between local and remote in both directions, each through a +// buffer from the pool, then closes both. Either side's EOF half-closes the +// other; a failure either way closes both, which resets remote. +func (t *tunnelImpl) pump(local, remote net.Conn) { + toRemote := t.bufs.Get().(*[]byte) + fromRemote := t.bufs.Get().(*[]byte) defer func() { - t.bufs.Put(toPeer) - t.bufs.Put(fromPeer) + t.bufs.Put(toRemote) + t.bufs.Put(fromRemote) }() + abort := sync.OnceFunc(func() { + _ = remote.Close() + _ = local.Close() + }) var wg sync.WaitGroup wg.Add(2) go func() { defer wg.Done() - copyHalf(c.conn, local, *toPeer, c) + copyHalf(remote, local, *toRemote, abort) }() go func() { defer wg.Done() - copyHalf(local, c.conn, *fromPeer, c) + copyHalf(local, remote, *fromRemote, abort) }() wg.Wait() - c.finish() - - st := c.conn.Stats() - t.logger.Debug("Tunnel: connection closed", - "conn_id", c.id, - "bytes_sent", st.BytesWritten, - "bytes_received", st.BytesRead, - "window_wait", st.WriteWait, - "window_waits", st.WriteWaits, - ) + abort() + + if m, ok := remote.(plumbing.MeteredConn); ok { + st := m.Stats() + t.logger.Debug("Tunnel: connection closed", + "remote", remote.RemoteAddr().String(), + "bytes_sent", st.BytesWritten, + "bytes_received", st.BytesRead, + "window_wait", st.WriteWait, + "window_waits", st.WriteWaits, + ) + } } -// copyHalf copies src to dst until src ends, then half-closes dst. A failure -// either way abandons the conn. -func copyHalf(dst net.Conn, src net.Conn, buf []byte, c *connStream) { +// copyHalf copies src to dst until src ends, then half-closes dst. +func copyHalf(dst, src net.Conn, buf []byte, abort func()) { // Hides ReadFrom and WriteTo, which would copy through buffers of their own. _, err := io.CopyBuffer(struct{ io.Writer }{dst}, struct{ io.Reader }{src}, buf) if err != nil { - c.reset(err.Error()) + abort() return } if cw, ok := dst.(interface{ CloseWrite() error }); ok { @@ -258,16 +206,6 @@ func copyHalf(dst net.Conn, src net.Conn, buf []byte, c *connStream) { } } -// forget removes c, unless another conn has since taken its id. -func (t *tunnelImpl) forget(c *connStream) { - t.conns.Compute(c.id, func(cur *connStream, loaded bool) (*connStream, bool) { - if !loaded { - return nil, true - } - return cur, cur == c - }) -} - func splitAddr(s string) Addr { host, portStr, _ := net.SplitHostPort(s) port, _ := strconv.Atoi(portStr) diff --git a/pkg/tunnel/tunnel_test.go b/pkg/tunnel/tunnel_test.go index 6b51fab..f612139 100644 --- a/pkg/tunnel/tunnel_test.go +++ b/pkg/tunnel/tunnel_test.go @@ -221,7 +221,7 @@ func TestPeerErrorDuringDialClosesTheDialedConnection(t *testing.T) { waitFor(t, "first dial", func() bool { return dialer.dials.Load() == 1 }) stream.in <- &bridgev1.TunnelNetworkMessage{ConnectionId: connID, Error: "peer closed"} waitFor(t, "the peer error to drop the buffered connection", func() bool { - _, loaded := tun.(*tunnelImpl).conns.Load(connID) + _, loaded := tun.(*tunnelImpl).backend.(*streamBackend).conns.Load(connID) return !loaded }) From 20030a3233c7696c9595d0eb6a957611c0fcdecb Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 16:36:25 -0700 Subject: [PATCH 08/11] refactor(plumbing): share one MuxStream between the bridge and dev tunnel backends MuxStream is one conn's Stream on any multiplexed transport: it queues what the transport delivers, ends the conn once both sides half-close or either resets, and builds the metered net.Conn over itself. A backend gives it Frames, the four frames its protocol writes and a Release, and feeds it from its demux. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/devtunnel/backend.go | 16 +-- pkg/devtunnel/conn.go | 182 +++++++------------------------- pkg/devtunnel/stream.go | 36 +++---- pkg/plumbing/mux.go | 218 +++++++++++++++++++++++++++++++++++++++ pkg/plumbing/mux_test.go | 163 +++++++++++++++++++++++++++++ pkg/tunnel/stream.go | 145 +++++++++----------------- 6 files changed, 490 insertions(+), 270 deletions(-) create mode 100644 pkg/plumbing/mux.go create mode 100644 pkg/plumbing/mux_test.go diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go index 9db6dfc..38e6bbf 100644 --- a/pkg/devtunnel/backend.go +++ b/pkg/devtunnel/backend.go @@ -105,7 +105,7 @@ type Backend struct { ctx context.Context // Each open conn, by an id unique across streams. - conns *xsync.MapOf[string, *relayStream] + conns *xsync.MapOf[string, *relayFrames] mu sync.Mutex token string @@ -156,7 +156,7 @@ func New(cfg Config) (*Backend, error) { instance: instance, target: target, creds: creds, - conns: xsync.NewMapOf[string, *relayStream](), + conns: xsync.NewMapOf[string, *relayFrames](), token: cfg.Token, tokenSet: make(chan struct{}), liveCh: make(chan struct{}), @@ -222,8 +222,8 @@ func (b *Backend) Run(ctx context.Context, accept func(net.Conn, *tunnel.ConnInf b.accept = accept b.ctx = ctx // A stream the relay moved off may still be finishing. - defer b.conns.Range(func(_ string, rs *relayStream) bool { - rs.end() + defer b.conns.Range(func(_ string, f *relayFrames) bool { + f.stream.PeerReset() return true }) @@ -248,8 +248,8 @@ func (b *Backend) Open(ctx context.Context, info *tunnel.ConnInfo) (net.Conn, er return nil, errors.New("no stream holds the tunnel") } n := b.nextEgress.Add(1) - rs := s.newConn("e"+strconv.FormatUint(n, 10), clientConnIDFloor|n, egress, info.Dest) - if rs == nil { + f := s.newConn("e"+strconv.FormatUint(n, 10), clientConnIDFloor|n, egress, info.Dest) + if f == nil { return nil, errors.New("the tunnel's stream ended") } host := info.Hostname @@ -257,12 +257,12 @@ func (b *Backend) Open(ctx context.Context, info *tunnel.ConnInfo) (net.Conn, er host = info.Dest.Host } s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Open{Open: &devtunnel.Open{ - ConnId: rs.wire, + ConnId: f.wire, Protocol: devtunnel.Protocol_PROTOCOL_TCP, Host: host, Port: uint32(info.Dest.Port), }}}) - return rs.conn, nil + return f.stream.Conn(), nil } // counters tracks conns opening (delta 1) and ending (delta -1). diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go index aa932ee..edffed3 100644 --- a/pkg/devtunnel/conn.go +++ b/pkg/devtunnel/conn.go @@ -1,12 +1,6 @@ package devtunnel import ( - "bytes" - "io" - "net" - "sync" - "sync/atomic" - "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/tunnel" @@ -24,171 +18,69 @@ const ( egress ) -// relayStream is one conn's frames on a Connect stream, as a plumbing.Stream: -// Recv returns the relay's Data and grants it back, Send sends Data, -// CloseSend sends Close and Close sends Reset. The tunnel gets conn, which -// holds writes to the relay's window. -type relayStream struct { - s *stream - id string - wire uint64 - dir direction - conn plumbing.MeteredConn - - // From the relay, not yet received. The window bounds it. - mu sync.Mutex - in [][]byte - eof bool - wake chan struct{} - - // Both sides have half-closed once both are set. - closeSent, eofRead atomic.Bool - done chan struct{} - once sync.Once +// relayFrames writes one conn's frames on a Connect stream: Data, Close, +// Reset, and a Window for each payload received. +type relayFrames struct { + s *stream + id string + wire uint64 + dir direction + stream *plumbing.MuxStream } -var _ plumbing.Stream = (*relayStream)(nil) - // newConn opens id on the stream as the relay's conn wire, or returns nil // once the stream has ended. -func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Addr) *relayStream { +func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Addr) *relayFrames { if s.closed.Load() { return nil } - rs := &relayStream{s: s, id: id, wire: wire, dir: dir, wake: make(chan struct{}, 1), done: make(chan struct{})} - rs.conn = plumbing.NewMeteredConn(plumbing.NewStreamConn(rs, addr("bridge"), addr(remote.String())), connWindow) + f := &relayFrames{s: s, id: id, wire: wire, dir: dir} + f.stream = plumbing.NewMuxStream(f, plumbing.MuxConfig{ + MaxFrame: maxPayload, + Window: connWindow, + Local: addr("bridge"), + Remote: addr(remote.String()), + }) s.b.counters(dir, 1) - s.b.conns.Store(id, rs) + s.b.conns.Store(id, f) // endConns may have swept the conns before the Store. if s.closed.Load() { - rs.end() + f.stream.PeerReset() return nil } - return rs -} - -func (rs *relayStream) Recv() ([]byte, error) { - for { - if rs.isDone() { - return nil, net.ErrClosed - } - rs.mu.Lock() - if len(rs.in) > 0 { - chunk := rs.in[0] - rs.in[0] = nil - rs.in = rs.in[1:] - rs.mu.Unlock() - rs.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ - ConnId: rs.wire, - Bytes: uint32(len(chunk)), - }}}) - return chunk, nil - } - eof := rs.eof - rs.mu.Unlock() - if eof { - rs.eofRead.Store(true) - if rs.closeSent.Load() { - rs.end() - } - return nil, io.EOF - } - select { - case <-rs.wake: - case <-rs.done: - } - } + return f } -func (rs *relayStream) Send(p []byte) error { - for len(p) > 0 { - if rs.isDone() { - return net.ErrClosed - } - size := min(len(p), maxPayload) - rs.s.b.bytesSent.Add(int64(size)) - // Queued, so it can't share p. - rs.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ - ConnId: rs.wire, - Payload: bytes.Clone(p[:size]), - }}}) - p = p[size:] - } +func (f *relayFrames) Data(payload []byte) error { + f.s.b.bytesSent.Add(int64(len(payload))) + f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: f.wire, Payload: payload}}}) return nil } -func (rs *relayStream) CloseSend() error { - if rs.isDone() || !rs.closeSent.CompareAndSwap(false, true) { - return nil - } - rs.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: rs.wire}}}) - if rs.eofRead.Load() { - rs.end() - } +func (f *relayFrames) CloseWrite() error { + f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: f.wire}}}) return nil } -// Close resets the conn. -func (rs *relayStream) Close() error { - if rs.isDone() { - return nil - } - rs.s.send(resetFrame(rs.wire, "connection closed")) - rs.end() +func (f *relayFrames) Reset(reason string) error { + f.s.send(resetFrame(f.wire, reason)) return nil } -func (rs *relayStream) Done() <-chan struct{} { return rs.done } - -// push queues payload from the relay. -func (rs *relayStream) push(payload []byte) { - if len(payload) == 0 { - return - } - rs.mu.Lock() - if !rs.eof && !rs.isDone() { - rs.in = append(rs.in, payload) - } - rs.mu.Unlock() - rs.signal() -} - -// peerClosed: the relay's side has nothing more to send. -func (rs *relayStream) peerClosed() { - rs.mu.Lock() - rs.eof = true - rs.mu.Unlock() - rs.signal() +func (f *relayFrames) Window(n int) error { + f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: f.wire, Bytes: uint32(n)}}}) + return nil } -// end forgets the conn, without telling the relay. -func (rs *relayStream) end() { - rs.once.Do(func() { - close(rs.done) - rs.s.b.conns.Compute(rs.id, func(cur *relayStream, loaded bool) (*relayStream, bool) { - if !loaded { - return nil, true - } - return cur, cur == rs - }) - rs.s.b.counters(rs.dir, -1) +// Release forgets the conn. +func (f *relayFrames) Release() { + f.s.b.conns.Compute(f.id, func(cur *relayFrames, loaded bool) (*relayFrames, bool) { + if !loaded { + return nil, true + } + return cur, cur == f }) -} - -func (rs *relayStream) signal() { - select { - case rs.wake <- struct{}{}: - default: - } -} - -func (rs *relayStream) isDone() bool { - select { - case <-rs.done: - return true - default: - return false - } + f.s.b.counters(f.dir, -1) } type addr string diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index 1c4c6b3..6e6c3a2 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -126,22 +126,22 @@ func (s *stream) handle(msg *devtunnel.ConnectResponse, moving func()) { case *devtunnel.ConnectResponse_Open: s.open(f.Open) case *devtunnel.ConnectResponse_Data: - if rs := s.conn(f.Data.GetConnId()); rs != nil { + if c := s.conn(f.Data.GetConnId()); c != nil { s.b.bytesReceived.Add(int64(len(f.Data.GetPayload()))) - rs.push(f.Data.GetPayload()) + c.stream.Deliver(s.ctx, f.Data.GetPayload()) } case *devtunnel.ConnectResponse_Close: - if rs := s.conn(f.Close.GetConnId()); rs != nil { - rs.peerClosed() + if c := s.conn(f.Close.GetConnId()); c != nil { + c.stream.PeerClosed() } case *devtunnel.ConnectResponse_Reset_: - if rs := s.conn(f.Reset_.GetConnId()); rs != nil { - s.logger.Debug("Relay reset the connection", "conn_id", rs.id, "reason", f.Reset_.GetReason()) - rs.end() + if c := s.conn(f.Reset_.GetConnId()); c != nil { + s.logger.Debug("Relay reset the connection", "conn_id", c.id, "reason", f.Reset_.GetReason()) + c.stream.PeerReset() } case *devtunnel.ConnectResponse_Window: - if rs := s.conn(f.Window.GetConnId()); rs != nil { - rs.conn.Grant(int(f.Window.GetBytes())) + if c := s.conn(f.Window.GetConnId()); c != nil { + c.stream.Conn().Grant(int(f.Window.GetBytes())) } } } @@ -156,8 +156,8 @@ func (s *stream) open(open *devtunnel.Open) { return } dest := tunnel.Addr{Host: "127.0.0.1", Port: port} - if rs := s.newConn(s.idFor(wire), wire, ingress, dest); rs != nil { - s.b.accept(rs.conn, &tunnel.ConnInfo{Dest: dest}) + if f := s.newConn(s.idFor(wire), wire, ingress, dest); f != nil { + s.b.accept(f.stream.Conn(), &tunnel.ConnInfo{Dest: dest}) } } @@ -172,20 +172,20 @@ func (s *stream) idFor(wire uint64) string { } // conn returns the conn the relay calls wire, if it's open on this stream. -func (s *stream) conn(wire uint64) *relayStream { - rs, ok := s.b.conns.Load(s.idFor(wire)) - if !ok || rs.s != s { +func (s *stream) conn(wire uint64) *relayFrames { + f, ok := s.b.conns.Load(s.idFor(wire)) + if !ok || f.s != s { return nil } - return rs + return f } // endConns ends every conn on the stream, which the relay ended with it. func (s *stream) endConns() { s.closed.Store(true) - s.b.conns.Range(func(_ string, rs *relayStream) bool { - if rs.s == s { - rs.end() + s.b.conns.Range(func(_ string, f *relayFrames) bool { + if f.s == s { + f.stream.PeerReset() } return true }) diff --git a/pkg/plumbing/mux.go b/pkg/plumbing/mux.go new file mode 100644 index 0000000..daee405 --- /dev/null +++ b/pkg/plumbing/mux.go @@ -0,0 +1,218 @@ +package plumbing + +import ( + "bytes" + "context" + "errors" + "io" + "net" + "sync" + "sync/atomic" +) + +// Frames writes one conn's frames onto a multiplexed transport. +type Frames interface { + // Data sends payload, which Frames may keep. + Data(payload []byte) error + // CloseWrite tells the peer this side has nothing more to send. A + // transport that can't returns errors.ErrUnsupported, and the conn ends + // instead. + CloseWrite() error + // Reset abandons the conn on the peer, for reason. + Reset(reason string) error + // Window grants the peer n more bytes, what Recv returned. A no-op + // without flow control. + Window(n int) error + // Release forgets the conn once it's over. + Release() +} + +// MuxConfig configures a MuxStream. +type MuxConfig struct { + // MaxFrame caps each Data's payload; 0 for no cap. + MaxFrame int + // Queue is how many payloads from the peer may wait unread before Deliver + // waits; 0 for no limit, for a transport whose window bounds them. + Queue int + // Window is what the conn may send before the peer grants more; 0 for no + // limit. + Window int + Local, Remote net.Addr +} + +// MuxStream is one conn's Stream on a multiplexed transport: Recv returns what +// the transport Delivers for it, and Send, CloseSend and Close write its +// frames. Conn is a net.Conn over it, metered to its window. +type MuxStream struct { + frames Frames + cfg MuxConfig + conn MeteredConn + + mu sync.Mutex + in [][]byte + eof bool + wake chan struct{} + room chan struct{} + + // Both sides have half-closed once both are set. + closeSent, eofRead atomic.Bool + done chan struct{} + once sync.Once +} + +var _ Stream = (*MuxStream)(nil) + +func NewMuxStream(frames Frames, cfg MuxConfig) *MuxStream { + s := &MuxStream{ + frames: frames, + cfg: cfg, + wake: make(chan struct{}, 1), + room: make(chan struct{}, 1), + done: make(chan struct{}), + } + s.conn = NewMeteredConn(NewStreamConn(s, cfg.Local, cfg.Remote), cfg.Window) + return s +} + +// Conn is the net.Conn over the stream. Grant it the peer's window. +func (s *MuxStream) Conn() MeteredConn { return s.conn } + +// Deliver queues payload from the peer, waiting while a bounded queue is +// full, until ctx ends. +func (s *MuxStream) Deliver(ctx context.Context, payload []byte) { + if len(payload) == 0 { + return + } + for { + s.mu.Lock() + if s.eof || s.isDone() { + s.mu.Unlock() + return + } + if s.cfg.Queue == 0 || len(s.in) < s.cfg.Queue { + s.in = append(s.in, payload) + s.mu.Unlock() + signal(s.wake) + return + } + s.mu.Unlock() + select { + case <-s.room: + case <-s.done: + return + case <-ctx.Done(): + return + } + } +} + +// PeerClosed: the peer has nothing more to send. Recv returns io.EOF once it +// has returned the rest. +func (s *MuxStream) PeerClosed() { + s.mu.Lock() + s.eof = true + s.mu.Unlock() + signal(s.wake) +} + +// PeerReset ends the conn: the peer abandoned it, so what it sent unread is +// dropped. +func (s *MuxStream) PeerReset() { s.end() } + +func (s *MuxStream) Recv() ([]byte, error) { + for { + if s.isDone() { + return nil, net.ErrClosed + } + s.mu.Lock() + if len(s.in) > 0 { + payload := s.in[0] + s.in[0] = nil + s.in = s.in[1:] + s.mu.Unlock() + signal(s.room) + if err := s.frames.Window(len(payload)); err != nil { + return nil, err + } + return payload, nil + } + eof := s.eof + s.mu.Unlock() + if eof { + s.eofRead.Store(true) + if s.closeSent.Load() { + s.end() + } + return nil, io.EOF + } + select { + case <-s.wake: + case <-s.done: + } + } +} + +func (s *MuxStream) Send(p []byte) error { + for len(p) > 0 { + if s.isDone() { + return net.ErrClosed + } + size := len(p) + if s.cfg.MaxFrame > 0 { + size = min(size, s.cfg.MaxFrame) + } + if err := s.frames.Data(bytes.Clone(p[:size])); err != nil { + return err + } + p = p[size:] + } + return nil +} + +func (s *MuxStream) CloseSend() error { + if s.isDone() || !s.closeSent.CompareAndSwap(false, true) { + return nil + } + err := s.frames.CloseWrite() + if errors.Is(err, errors.ErrUnsupported) || s.eofRead.Load() { + s.end() + return nil + } + return err +} + +// Close resets the conn. +func (s *MuxStream) Close() error { + if s.isDone() { + return nil + } + err := s.frames.Reset("connection closed") + s.end() + return err +} + +// Done is closed once the conn is over. +func (s *MuxStream) Done() <-chan struct{} { return s.done } + +func (s *MuxStream) end() { + s.once.Do(func() { + close(s.done) + s.frames.Release() + }) +} + +func (s *MuxStream) isDone() bool { + select { + case <-s.done: + return true + default: + return false + } +} + +func signal(ch chan struct{}) { + select { + case ch <- struct{}{}: + default: + } +} diff --git a/pkg/plumbing/mux_test.go b/pkg/plumbing/mux_test.go new file mode 100644 index 0000000..4d9a72d --- /dev/null +++ b/pkg/plumbing/mux_test.go @@ -0,0 +1,163 @@ +package plumbing + +import ( + "context" + "errors" + "io" + "net" + "sync" + "testing" + "time" + + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +// recordingFrames keeps the frames a MuxStream writes. +type recordingFrames struct { + mu sync.Mutex + data []string + windows []int + closes int + resets []string + releases int + halfClose bool +} + +func (f *recordingFrames) Data(p []byte) error { + f.mu.Lock() + defer f.mu.Unlock() + f.data = append(f.data, string(p)) + return nil +} + +func (f *recordingFrames) CloseWrite() error { + if !f.halfClose { + return errors.ErrUnsupported + } + f.mu.Lock() + defer f.mu.Unlock() + f.closes++ + return nil +} + +func (f *recordingFrames) Reset(reason string) error { + f.mu.Lock() + defer f.mu.Unlock() + f.resets = append(f.resets, reason) + return nil +} + +func (f *recordingFrames) Window(n int) error { + f.mu.Lock() + defer f.mu.Unlock() + f.windows = append(f.windows, n) + return nil +} + +func (f *recordingFrames) Release() { + f.mu.Lock() + defer f.mu.Unlock() + f.releases++ +} + +func TestMuxStreamReceivesInOrderAndGrantsWhatItReturns(t *testing.T) { + f := &recordingFrames{halfClose: true} + s := NewMuxStream(f, MuxConfig{}) + ctx := context.Background() + s.Deliver(ctx, []byte("ab")) + s.Deliver(ctx, []byte("cde")) + s.PeerClosed() + + got, err := io.ReadAll(s.Conn()) + require.NoError(t, err) + assert.Equal(t, "abcde", string(got)) + assert.Equal(t, []int{2, 3}, f.windows) +} + +func TestMuxStreamBoundedQueueWaitsForRecv(t *testing.T) { + s := NewMuxStream(&recordingFrames{}, MuxConfig{Queue: 1}) + ctx := context.Background() + s.Deliver(ctx, []byte("a")) + + delivered := make(chan struct{}) + go func() { + s.Deliver(ctx, []byte("b")) + close(delivered) + }() + select { + case <-delivered: + t.Fatal("queued past the bound") + case <-time.After(50 * time.Millisecond): + } + _, err := s.Recv() + require.NoError(t, err) + select { + case <-delivered: + case <-time.After(time.Second): + t.Fatal("Recv made no room") + } + + // A cancelled ctx gives up a wait. + waiting, cancel := context.WithCancel(ctx) + done := make(chan struct{}) + go func() { + s.Deliver(waiting, []byte("c")) + close(done) + }() + cancel() + select { + case <-done: + case <-time.After(time.Second): + t.Fatal("Deliver kept waiting after ctx ended") + } +} + +func TestMuxStreamEndsOnceBothSidesHalfClose(t *testing.T) { + f := &recordingFrames{halfClose: true} + s := NewMuxStream(f, MuxConfig{}) + require.NoError(t, s.CloseSend()) + assert.Equal(t, 1, f.closes) + assert.Equal(t, 0, f.releases) + + s.PeerClosed() + _, err := s.Recv() + assert.ErrorIs(t, err, io.EOF) + assert.Equal(t, 1, f.releases) + require.NoError(t, s.Close()) + assert.Empty(t, f.resets, "a conn both sides closed isn't reset") +} + +func TestMuxStreamWithoutHalfCloseEndsAtCloseSend(t *testing.T) { + f := &recordingFrames{} + s := NewMuxStream(f, MuxConfig{}) + require.NoError(t, s.CloseSend()) + assert.Equal(t, 1, f.releases) + _, err := s.Recv() + assert.ErrorIs(t, err, net.ErrClosed) +} + +func TestMuxStreamSendsInFramesAndResets(t *testing.T) { + f := &recordingFrames{halfClose: true} + s := NewMuxStream(f, MuxConfig{MaxFrame: 2}) + require.NoError(t, s.Send([]byte("abcde"))) + assert.Equal(t, []string{"ab", "cd", "e"}, f.data) + + require.NoError(t, s.Close()) + assert.Equal(t, []string{"connection closed"}, f.resets) + assert.ErrorIs(t, s.Send([]byte("x")), net.ErrClosed) + require.NoError(t, s.Close()) + assert.Equal(t, 1, f.releases) +} + +func TestMuxStreamPeerResetDropsWhatWasUnread(t *testing.T) { + f := &recordingFrames{halfClose: true} + s := NewMuxStream(f, MuxConfig{}) + s.Deliver(context.Background(), []byte("unread")) + s.PeerReset() + + _, err := s.Recv() + assert.ErrorIs(t, err, net.ErrClosed) + assert.Empty(t, f.resets, "the peer already knows") + assert.Equal(t, 1, f.releases) +} diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go index 4ec54bb..33eaf5d 100644 --- a/pkg/tunnel/stream.go +++ b/pkg/tunnel/stream.go @@ -1,12 +1,11 @@ package tunnel import ( - "bytes" "context" + "errors" "io" "log/slog" "net" - "sync" "github.com/puzpuzpuz/xsync/v3" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" @@ -33,7 +32,7 @@ func NewStreamBackend(stream Stream) Backend { stream: stream, sendCh: make(chan *bridgev1.TunnelNetworkMessage, 64), stop: make(chan struct{}), - conns: xsync.NewMapOf[string, *messageStream](), + conns: xsync.NewMapOf[string, *messageFrames](), } } @@ -42,13 +41,13 @@ type streamBackend struct { sendCh chan *bridgev1.TunnelNetworkMessage // Closed once Run returns, so a Send waiting for room gives up. stop chan struct{} - conns *xsync.MapOf[string, *messageStream] + conns *xsync.MapOf[string, *messageFrames] } func (b *streamBackend) Run(ctx context.Context, accept func(net.Conn, *ConnInfo)) error { defer close(b.stop) - defer b.conns.Range(func(_ string, s *messageStream) bool { - s.end() + defer b.conns.Range(func(_ string, f *messageFrames) bool { + f.stream.PeerReset() return true }) @@ -107,10 +106,10 @@ func (b *streamBackend) Run(ctx context.Context, accept func(net.Conn, *ConnInfo // the dial queue on it instead of each opening another. func (b *streamBackend) handle(ctx context.Context, msg *bridgev1.TunnelNetworkMessage, accept func(net.Conn, *ConnInfo)) { id := msg.GetConnectionId() - s, ok := b.conns.Load(id) + f, ok := b.conns.Load(id) if msg.GetError() != "" { if ok { - s.end() + f.stream.PeerReset() } return } @@ -126,29 +125,28 @@ func (b *streamBackend) handle(ctx context.Context, msg *bridgev1.TunnelNetworkM Hostname: msg.GetHostname(), } // This side sends from where the peer's conn was headed, back to it. - s = b.newStream(id, &ConnInfo{Source: info.Dest, Dest: info.Source, Hostname: info.Hostname}) - b.conns.Store(id, s) - accept(s.conn(), info) + f = b.newConn(id, &ConnInfo{Source: info.Dest, Dest: info.Source, Hostname: info.Hostname}) + accept(f.stream.Conn(), info) } - s.push(ctx, msg.GetData()) + f.stream.Deliver(ctx, msg.GetData()) } func (b *streamBackend) Open(_ context.Context, info *ConnInfo) (net.Conn, error) { id := plumbing.ConnectionID(info.Source.Host, info.Source.Port, info.Dest.Host, info.Dest.Port) - s := b.newStream(id, info) - b.conns.Store(id, s) // Its first Data opens it. - return s.conn(), nil + return b.newConn(id, info).stream.Conn(), nil } -func (b *streamBackend) newStream(id string, info *ConnInfo) *messageStream { - return &messageStream{ - b: b, - id: id, - info: info, - in: make(chan []byte, queuedMessages), - done: make(chan struct{}), - } +// newConn registers a conn that sends as info says. +func (b *streamBackend) newConn(id string, info *ConnInfo) *messageFrames { + f := &messageFrames{b: b, id: id, info: info} + f.stream = plumbing.NewMuxStream(f, plumbing.MuxConfig{ + Queue: queuedMessages, + Local: addr(info.Source.String()), + Remote: addr(info.Dest.String()), + }) + b.conns.Store(id, f) + return f } func (b *streamBackend) send(msg *bridgev1.TunnelNetworkMessage) error { @@ -160,93 +158,42 @@ func (b *streamBackend) send(msg *bridgev1.TunnelNetworkMessage) error { } } -// messageStream is one conn's messages on the stream, as a plumbing.Stream. -type messageStream struct { +// messageFrames writes one conn's messages: every one addresses the conn, and +// an error ends it. +type messageFrames struct { b *streamBackend id string // As this side sends it. - info *ConnInfo - in chan []byte - done chan struct{} - once sync.Once -} - -var _ plumbing.Stream = (*messageStream)(nil) - -func (s *messageStream) conn() net.Conn { - return plumbing.NewMeteredConn(plumbing.NewStreamConn(s, addr(s.info.Source.String()), addr(s.info.Dest.String())), 0) -} - -// push queues payload for Recv, waiting while the conn has a full queue. -func (s *messageStream) push(ctx context.Context, payload []byte) { - if len(payload) == 0 { - return - } - select { - case s.in <- payload: - case <-s.done: - case <-ctx.Done(): - } -} - -func (s *messageStream) Recv() ([]byte, error) { - // A conn the peer ended drops what it hadn't read. - select { - case <-s.done: - return nil, net.ErrClosed - default: - } - select { - case p := <-s.in: - return p, nil - case <-s.done: - return nil, net.ErrClosed - } + info *ConnInfo + stream *plumbing.MuxStream } -func (s *messageStream) Send(p []byte) error { - select { - case <-s.done: - return net.ErrClosed - default: - } - return s.b.send(&bridgev1.TunnelNetworkMessage{ - ConnectionId: s.id, - Source: tunnelAddress(s.info.Source), - Dest: tunnelAddress(s.info.Dest), - Hostname: s.info.Hostname, - Data: bytes.Clone(p), +func (f *messageFrames) Data(payload []byte) error { + return f.b.send(&bridgev1.TunnelNetworkMessage{ + ConnectionId: f.id, + Source: tunnelAddress(f.info.Source), + Dest: tunnelAddress(f.info.Dest), + Hostname: f.info.Hostname, + Data: payload, }) } -// CloseSend ends the conn: the stream can't carry a half-close. -func (s *messageStream) CloseSend() error { - s.end() - return nil -} +// CloseWrite: the stream can't carry a half-close. +func (f *messageFrames) CloseWrite() error { return errors.ErrUnsupported } -// Close ends the conn and tells the peer. -func (s *messageStream) Close() error { - select { - case <-s.done: - return nil - default: - } - s.end() - return s.b.send(&bridgev1.TunnelNetworkMessage{ConnectionId: s.id, Error: "connection closed"}) +func (f *messageFrames) Reset(reason string) error { + return f.b.send(&bridgev1.TunnelNetworkMessage{ConnectionId: f.id, Error: reason}) } -func (s *messageStream) Done() <-chan struct{} { return s.done } +// Window: the stream has no flow control; the queue holds the peer back. +func (f *messageFrames) Window(int) error { return nil } -// end forgets the conn, unless another has since taken its id. -func (s *messageStream) end() { - s.once.Do(func() { - close(s.done) - s.b.conns.Compute(s.id, func(cur *messageStream, loaded bool) (*messageStream, bool) { - if !loaded { - return nil, true - } - return cur, cur == s - }) +// Release forgets the conn, unless another has since taken its id. +func (f *messageFrames) Release() { + f.b.conns.Compute(f.id, func(cur *messageFrames, loaded bool) (*messageFrames, bool) { + if !loaded { + return nil, true + } + return cur, cur == f }) } From 9489e762d20130da6cf4fe82d7b4fea403ef7685 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 18:29:49 -0700 Subject: [PATCH 09/11] refactor(devtunnel): pace the dev tunnel in bridge alone The relay no longer holds a client to a window (vercel/api), so bridge sends no Window frames and ignores the relay's: what the relay sends is taken as fast as it comes. A conn with 1 MiB queued for the relay but not yet on the stream waits to send more, which holds back a dev server that outruns the stream while the other conns keep flowing. The window leaves plumbing: MeteredConn only counts, and MuxStream and Frames have no Window. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/devtunnel/backend.go | 2 - pkg/devtunnel/backend_test.go | 117 ++++++++++++------ pkg/devtunnel/conn.go | 49 ++++++-- pkg/devtunnel/stream.go | 51 ++++---- pkg/plumbing/metered.go | 125 +++----------------- pkg/plumbing/metered_test.go | 68 ++--------- pkg/plumbing/mux.go | 19 +-- pkg/plumbing/mux_test.go | 11 +- pkg/tunnel/stream.go | 3 - pkg/tunnel/tunnel.go | 3 +- proto/vercel/api/dev_tunnel/v1/tunnel.proto | 10 +- 11 files changed, 189 insertions(+), 269 deletions(-) diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go index 38e6bbf..08900c1 100644 --- a/pkg/devtunnel/backend.go +++ b/pkg/devtunnel/backend.go @@ -33,8 +33,6 @@ import ( ) const ( - // What each side of a conn may send before the other grants more. - connWindow = 256 << 10 // A client numbers its conns from here; the relay numbers its own from 1. clientConnIDFloor = uint64(1) << 63 diff --git a/pkg/devtunnel/backend_test.go b/pkg/devtunnel/backend_test.go index c8fba0a..22a5ac1 100644 --- a/pkg/devtunnel/backend_test.go +++ b/pkg/devtunnel/backend_test.go @@ -33,6 +33,8 @@ type fakeRelay struct { streams chan *relayCall // refuse ends a call before it reaches the test, unless it returns nil. refuse func(md metadata.MD) error + // gate, if set, holds off reading each call until it's closed. + gate chan struct{} } type relayCall struct { @@ -42,12 +44,16 @@ type relayCall struct { end chan error } -func startRelay(t *testing.T) *fakeRelay { +func startRelay(t *testing.T, opts ...func(*fakeRelay)) *fakeRelay { t.Helper() lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) r := &fakeRelay{url: "http://" + lis.Addr().String(), streams: make(chan *relayCall, 8)} - srv := grpc.NewServer() + for _, o := range opts { + o(r) + } + // Fixed windows, so a relay that stops reading backs the client up soon. + srv := grpc.NewServer(grpc.InitialWindowSize(64<<10), grpc.InitialConnWindowSize(64<<10)) devtunnel.RegisterTunnelServiceServer(srv, r) go func() { _ = srv.Serve(lis) }() t.Cleanup(srv.Stop) @@ -64,6 +70,9 @@ func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnel.ConnectReque s := &relayCall{md: md, call: call, recv: make(chan *devtunnel.ConnectRequest, 256), end: make(chan error, 1)} go func() { defer close(s.recv) + if r.gate != nil { + <-r.gate + } for { msg, err := call.Recv() if err != nil { @@ -147,10 +156,6 @@ func closeConn(id uint64) *devtunnel.ConnectResponse { return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Close{Close: &devtunnel.Close{ConnId: id}}} } -func window(id uint64, n int) *devtunnel.ConnectResponse { - return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Window{Window: &devtunnel.Window{ConnId: id, Bytes: uint32(n)}}} -} - // newTunnel runs a tunnel over a Backend for r. func newTunnel(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) (*Backend, tunnel.Tunnel) { t.Helper() @@ -206,15 +211,10 @@ func TestIngressReachesTheApp(t *testing.T) { s.send(t, open(1, port)) s.send(t, data(1, "hello")) var echoed []byte - granted := 0 - for len(echoed) < 5 || granted < 5 { + for len(echoed) < 5 { msg := s.expect(t) - if d := msg.GetData(); d != nil { - echoed = append(echoed, d.GetPayload()...) - } - if w := msg.GetWindow(); w != nil { - granted += int(w.GetBytes()) - } + require.Nil(t, msg.GetWindow(), "pacing is the client's own") + echoed = append(echoed, msg.GetData().GetPayload()...) } assert.Equal(t, "hello", string(echoed)) @@ -241,16 +241,17 @@ func TestIngressToAnotherPortIsReset(t *testing.T) { assert.Equal(t, "bridge serves only port 3000", reset.GetReason()) } -func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { +func TestRelayDataIsTakenAsFastAsItComes(t *testing.T) { r := startRelay(t) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { _ = lis.Close() }) - body := bytes.Repeat([]byte("x"), connWindow+frameBytes) + got := make(chan int, 1) go func() { nc, err := lis.Accept() if err == nil { - _, _ = nc.Write(body) + n, _ := io.Copy(io.Discard, nc) + got <- int(n) } }() port := lis.Addr().(*net.TCPAddr).Port @@ -259,17 +260,63 @@ func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { s.hold(t) s.send(t, open(1, port)) - assert.Len(t, s.expectData(t, 1, connWindow), connWindow) + // Far past what any window would allow, with nothing granted. + const total = 4 << 20 + chunk := string(bytes.Repeat([]byte("x"), frameBytes)) + for sent := 0; sent < total; sent += frameBytes { + s.send(t, data(1, chunk)) + } + s.send(t, closeConn(1)) select { - case msg := <-s.recv: - if msg.GetData() != nil { - t.Fatal("sent past the window") + case n := <-got: + assert.Equal(t, total, n) + case <-time.After(testTimeout): + t.Fatal("the app didn't get it all") + } +} + +func TestADevServerOutrunningTheStreamIsHeldBack(t *testing.T) { + gate := make(chan struct{}) + r := startRelay(t, func(r *fakeRelay) { r.gate = gate }) + lis, err := net.Listen("tcp", "127.0.0.1:0") + require.NoError(t, err) + t.Cleanup(func() { _ = lis.Close() }) + const total = 8 << 20 + written := make(chan struct{}) + go func() { + nc, err := lis.Accept() + if err == nil { + _, _ = nc.Write(bytes.Repeat([]byte("x"), total)) + close(written) } - case <-time.After(200 * time.Millisecond): + }() + port := lis.Addr().(*net.TCPAddr).Port + b, _ := newTunnel(t, r, "tok", port, func(tm *timings) { tm.silent = time.Minute }) + s := r.next(t) + s.hold(t) + s.send(t, open(1, port)) + + // The relay isn't reading, so the conn's sends wait once it has a + // megabyte queued, and the dev server's writes back up. + unsent := func() (n int64) { + b.conns.Range(func(_ string, f *relayFrames) bool { + n = f.unsent.Load() + return true + }) + return n + } + waitFor(t, func() bool { return unsent() >= maxUnsentBytes }) + time.Sleep(200 * time.Millisecond) + assert.LessOrEqual(t, unsent(), int64(maxUnsentBytes+maxPayload)) + select { + case <-written: + t.Fatal("the dev server wasn't held back") + default: } - s.send(t, window(1, frameBytes)) - assert.Len(t, s.expectData(t, 1, frameBytes), frameBytes) + close(gate) + assert.Len(t, s.expectData(t, 1, total), total) + <-written } func TestEgressLeavesThroughTheDeployment(t *testing.T) { @@ -343,13 +390,14 @@ func TestReconnectKeepsTheOldStreamsConns(t *testing.T) { } func TestRefusedTokenWaitsForANewOne(t *testing.T) { - r := startRelay(t) - r.refuse = func(md metadata.MD) error { - if md.Get("authorization")[0] == "Bearer old" { - return status.Error(codes.Unauthenticated, "missing or invalid dev tunnel token") + r := startRelay(t, func(r *fakeRelay) { + r.refuse = func(md metadata.MD) error { + if md.Get("authorization")[0] == "Bearer old" { + return status.Error(codes.Unauthenticated, "missing or invalid dev tunnel token") + } + return nil } - return nil - } + }) c, _ := newTunnel(t, r, "old", 3000) waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN }) @@ -373,10 +421,11 @@ func TestNoTokenWaitsForOne(t *testing.T) { } func TestFatalRefusalStops(t *testing.T) { - r := startRelay(t) - r.refuse = func(metadata.MD) error { - return status.Error(codes.PermissionDenied, "not allowed") - } + r := startRelay(t, func(r *fakeRelay) { + r.refuse = func(metadata.MD) error { + return status.Error(codes.PermissionDenied, "not allowed") + } + }) c, tun := newTunnel(t, r, "tok", 3000) select { diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go index edffed3..9eb28f8 100644 --- a/pkg/devtunnel/conn.go +++ b/pkg/devtunnel/conn.go @@ -1,13 +1,22 @@ package devtunnel import ( + "net" + "sync/atomic" + "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/tunnel" ) -// Per Data frame. -const maxPayload = 64 << 10 +const ( + // Per Data frame. + maxPayload = 64 << 10 + // What a conn may have queued for the relay, not yet on the stream, before + // its sends wait. A dev server writing faster than the stream drains is + // held back by TCP, while the other conns keep flowing. + maxUnsentBytes = 1 << 20 +) type direction int @@ -18,14 +27,18 @@ const ( egress ) -// relayFrames writes one conn's frames on a Connect stream: Data, Close, -// Reset, and a Window for each payload received. +// relayFrames writes one conn's frames on a Connect stream: Data, Close and +// Reset. The relay's Window frames are ignored: what the relay sends is taken +// as fast as it comes. type relayFrames struct { s *stream id string wire uint64 dir direction stream *plumbing.MuxStream + + unsent atomic.Int64 + drained chan struct{} } // newConn opens id on the stream as the relay's conn wire, or returns nil @@ -34,10 +47,9 @@ func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Ad if s.closed.Load() { return nil } - f := &relayFrames{s: s, id: id, wire: wire, dir: dir} + f := &relayFrames{s: s, id: id, wire: wire, dir: dir, drained: make(chan struct{}, 1)} f.stream = plumbing.NewMuxStream(f, plumbing.MuxConfig{ MaxFrame: maxPayload, - Window: connWindow, Local: addr("bridge"), Remote: addr(remote.String()), }) @@ -51,9 +63,25 @@ func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Ad return f } +// Data waits while the conn has maxUnsentBytes queued for the relay. func (f *relayFrames) Data(payload []byte) error { - f.s.b.bytesSent.Add(int64(len(payload))) - f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: f.wire, Payload: payload}}}) + for f.unsent.Load() >= maxUnsentBytes { + select { + case <-f.drained: + case <-f.stream.Done(): + return net.ErrClosed + } + } + n := int64(len(payload)) + f.unsent.Add(n) + f.s.b.bytesSent.Add(n) + f.s.sendThen(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: f.wire, Payload: payload}}}, func() { + f.unsent.Add(-n) + select { + case f.drained <- struct{}{}: + default: + } + }) return nil } @@ -67,11 +95,6 @@ func (f *relayFrames) Reset(reason string) error { return nil } -func (f *relayFrames) Window(n int) error { - f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: f.wire, Bytes: uint32(n)}}}) - return nil -} - // Release forgets the conn. func (f *relayFrames) Release() { f.s.b.conns.Compute(f.id, func(cur *relayFrames, loaded bool) (*relayFrames, bool) { diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index 6e6c3a2..f753c3c 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -67,14 +67,17 @@ func (s *stream) run(token string, moving func()) error { s.send(heartbeat) go func() { for { - frame, ok := s.out.pop(s.ctx) + entry, ok := s.out.pop(s.ctx) if !ok { return } // A failed send ends the call, which Recv reports. - if err := call.Send(frame); err != nil { + if err := call.Send(entry.frame); err != nil { return } + if entry.sent != nil { + entry.sent() + } } }() go s.heartbeats() @@ -139,10 +142,6 @@ func (s *stream) handle(msg *devtunnel.ConnectResponse, moving func()) { s.logger.Debug("Relay reset the connection", "conn_id", c.id, "reason", f.Reset_.GetReason()) c.stream.PeerReset() } - case *devtunnel.ConnectResponse_Window: - if c := s.conn(f.Window.GetConnId()); c != nil { - c.stream.Conn().Grant(int(f.Window.GetBytes())) - } } } @@ -192,20 +191,30 @@ func (s *stream) endConns() { } func (s *stream) send(frame *devtunnel.ConnectRequest) { - s.out.push(frame) + s.out.push(outboxEntry{frame: frame}) +} + +// sendThen queues frame, calling sent once it's on the stream. +func (s *stream) sendThen(frame *devtunnel.ConnectRequest, sent func()) { + s.out.push(outboxEntry{frame: frame, sent: sent}) } -// outbox queues frames for the relay in order. Data is bounded by each conn's -// window; the rest is small. +// outbox queues frames for the relay in order. Each conn has at most +// maxUnsentBytes of Data in it; the rest is small. type outbox struct { - mu sync.Mutex - frames []*devtunnel.ConnectRequest - wake chan struct{} + mu sync.Mutex + entries []outboxEntry + wake chan struct{} +} + +type outboxEntry struct { + frame *devtunnel.ConnectRequest + sent func() } -func (o *outbox) push(frame *devtunnel.ConnectRequest) { +func (o *outbox) push(e outboxEntry) { o.mu.Lock() - o.frames = append(o.frames, frame) + o.entries = append(o.entries, e) o.mu.Unlock() select { case o.wake <- struct{}{}: @@ -213,21 +222,21 @@ func (o *outbox) push(frame *devtunnel.ConnectRequest) { } } -func (o *outbox) pop(ctx context.Context) (*devtunnel.ConnectRequest, bool) { +func (o *outbox) pop(ctx context.Context) (outboxEntry, bool) { for { o.mu.Lock() - if len(o.frames) > 0 { - frame := o.frames[0] - o.frames[0] = nil - o.frames = o.frames[1:] + if len(o.entries) > 0 { + e := o.entries[0] + o.entries[0] = outboxEntry{} + o.entries = o.entries[1:] o.mu.Unlock() - return frame, true + return e, true } o.mu.Unlock() select { case <-o.wake: case <-ctx.Done(): - return nil, false + return outboxEntry{}, false } } } diff --git a/pkg/plumbing/metered.go b/pkg/plumbing/metered.go index 6d87a92..56d5570 100644 --- a/pkg/plumbing/metered.go +++ b/pkg/plumbing/metered.go @@ -3,66 +3,34 @@ package plumbing import ( "errors" "net" - "sync" "sync/atomic" "time" ) -// MeteredConn is a net.Conn that counts what passes through it. Given a -// window, Write sends no more than the peer has granted with Grant: the flow -// control a conn multiplexed with others needs, since the transport's own can -// only slow all of them. +// MeteredConn is a net.Conn that counts what passes through it. type MeteredConn interface { net.Conn - // Grant lets Write send n more bytes. - Grant(n int) // CloseWrite half-closes the wrapped conn, if it can. CloseWrite() error // Stats reports what has passed through so far. Stats() *MeteredStats } -type meteredConn struct { - net.Conn - window int - - credit atomic.Int64 - wake chan struct{} - closed chan struct{} - closeOnce sync.Once - // The wrapped conn's, if it has a Done method, so a write waiting for - // credit returns once that conn is over. - innerDone <-chan struct{} - - bytesRead, bytesWritten atomic.Int64 - writeWait, writeWaits atomic.Int64 -} - // MeteredStats is what has passed through a MeteredConn. type MeteredStats struct { BytesRead int64 BytesWritten int64 - // What Write may still send. Zero without a window. - Credit int64 - // How long writes waited for credit, over how many waits. - WriteWait time.Duration - WriteWaits int64 + // How long writes took, which grows when the transport holds them back. + WriteTime time.Duration } -// NewMeteredConn wraps conn. Write may send window bytes before Grant gives -// more; 0 for no limit. -func NewMeteredConn(conn net.Conn, window int) MeteredConn { - m := &meteredConn{ - Conn: conn, - window: window, - wake: make(chan struct{}, 1), - closed: make(chan struct{}), - } - m.credit.Store(int64(window)) - if d, ok := conn.(interface{ Done() <-chan struct{} }); ok { - m.innerDone = d.Done() - } - return m +type meteredConn struct { + net.Conn + bytesRead, bytesWritten, writeTime atomic.Int64 +} + +func NewMeteredConn(conn net.Conn) MeteredConn { + return &meteredConn{Conn: conn} } func (m *meteredConn) Read(p []byte) (int, error) { @@ -72,35 +40,11 @@ func (m *meteredConn) Read(p []byte) (int, error) { } func (m *meteredConn) Write(p []byte) (int, error) { - if m.window <= 0 { - n, err := m.Conn.Write(p) - m.bytesWritten.Add(int64(n)) - return n, err - } - written := 0 - for written < len(p) { - credit, err := m.awaitCredit() - if err != nil { - return written, err - } - size := min(len(p)-written, int(credit)) - n, err := m.Conn.Write(p[written : written+size]) - m.credit.Add(-int64(n)) - m.bytesWritten.Add(int64(n)) - written += n - if err != nil { - return written, err - } - } - return written, nil -} - -func (m *meteredConn) Grant(n int) { - m.credit.Add(int64(n)) - select { - case m.wake <- struct{}{}: - default: - } + start := time.Now() + n, err := m.Conn.Write(p) + m.writeTime.Add(int64(time.Since(start))) + m.bytesWritten.Add(int64(n)) + return n, err } func (m *meteredConn) CloseWrite() error { @@ -110,45 +54,10 @@ func (m *meteredConn) CloseWrite() error { return errors.ErrUnsupported } -func (m *meteredConn) Close() error { - m.closeOnce.Do(func() { close(m.closed) }) - return m.Conn.Close() -} - func (m *meteredConn) Stats() *MeteredStats { - st := &MeteredStats{ + return &MeteredStats{ BytesRead: m.bytesRead.Load(), BytesWritten: m.bytesWritten.Load(), - WriteWait: time.Duration(m.writeWait.Load()), - WriteWaits: m.writeWaits.Load(), - } - if m.window > 0 { - st.Credit = m.credit.Load() - } - return st -} - -func (m *meteredConn) awaitCredit() (int64, error) { - var start time.Time - defer func() { - if !start.IsZero() { - m.writeWait.Add(int64(time.Since(start))) - } - }() - for { - if credit := m.credit.Load(); credit > 0 { - return credit, nil - } - if start.IsZero() { - start = time.Now() - m.writeWaits.Add(1) - } - select { - case <-m.wake: - case <-m.closed: - return 0, net.ErrClosed - case <-m.innerDone: - return 0, net.ErrClosed - } + WriteTime: time.Duration(m.writeTime.Load()), } } diff --git a/pkg/plumbing/metered_test.go b/pkg/plumbing/metered_test.go index 2b0831b..7b7fe0e 100644 --- a/pkg/plumbing/metered_test.go +++ b/pkg/plumbing/metered_test.go @@ -16,10 +16,9 @@ import ( // recordingConn keeps what's written to it and reads from r. type recordingConn struct { net.Conn - r io.Reader - mu sync.Mutex - w bytes.Buffer - done chan struct{} + r io.Reader + mu sync.Mutex + w bytes.Buffer } func (c *recordingConn) Read(p []byte) (int, error) { return c.r.Read(p) } @@ -38,25 +37,13 @@ func (c *recordingConn) written() string { func (c *recordingConn) Close() error { return nil } -func (c *recordingConn) Done() <-chan struct{} { return c.done } - func newRecordingConn(r string) *recordingConn { - return &recordingConn{r: bytes.NewReader([]byte(r)), done: make(chan struct{})} -} - -// writeAsync starts a Write and returns its result. -func writeAsync(m MeteredConn, p string) <-chan error { - errc := make(chan error, 1) - go func() { - _, err := m.Write([]byte(p)) - errc <- err - }() - return errc + return &recordingConn{r: bytes.NewReader([]byte(r))} } -func TestMeteredConnCountsWithoutAWindow(t *testing.T) { +func TestMeteredConnCounts(t *testing.T) { inner := newRecordingConn("hello") - m := NewMeteredConn(inner, 0) + m := NewMeteredConn(inner) _, err := m.Write([]byte("abc")) require.NoError(t, err) @@ -64,47 +51,14 @@ func TestMeteredConnCountsWithoutAWindow(t *testing.T) { require.NoError(t, err) assert.Equal(t, "abc", inner.written()) - assert.Equal(t, &MeteredStats{BytesRead: 5, BytesWritten: 3}, m.Stats()) -} - -func TestMeteredConnHoldsWritesToTheWindow(t *testing.T) { - inner := newRecordingConn("") - m := NewMeteredConn(inner, 4) - - errc := writeAsync(m, "abcdefgh") - require.Eventually(t, func() bool { return inner.written() == "abcd" }, time.Second, time.Millisecond) - select { - case <-errc: - t.Fatal("wrote past the window") - case <-time.After(50 * time.Millisecond): - } - - m.Grant(4) - require.NoError(t, <-errc) - assert.Equal(t, "abcdefgh", inner.written()) st := m.Stats() - assert.Equal(t, int64(0), st.Credit) - assert.Equal(t, int64(1), st.WriteWaits) - assert.Greater(t, st.WriteWait, time.Duration(0)) -} - -func TestMeteredConnWriteWaitingEndsWithTheConn(t *testing.T) { - m := NewMeteredConn(newRecordingConn(""), 1) - errc := writeAsync(m, "ab") - time.Sleep(10 * time.Millisecond) - require.NoError(t, m.Close()) - assert.ErrorIs(t, <-errc, net.ErrClosed) - - inner := newRecordingConn("") - m = NewMeteredConn(inner, 1) - errc = writeAsync(m, "ab") - time.Sleep(10 * time.Millisecond) - close(inner.done) - assert.ErrorIs(t, <-errc, net.ErrClosed) + assert.Equal(t, int64(5), st.BytesRead) + assert.Equal(t, int64(3), st.BytesWritten) + assert.Greater(t, st.WriteTime, time.Duration(0)) } func TestMeteredConnCloseWrite(t *testing.T) { - assert.True(t, errors.Is(NewMeteredConn(newRecordingConn(""), 0).CloseWrite(), errors.ErrUnsupported)) + assert.True(t, errors.Is(NewMeteredConn(newRecordingConn("")).CloseWrite(), errors.ErrUnsupported)) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -119,7 +73,7 @@ func TestMeteredConnCloseWrite(t *testing.T) { nc, err := net.Dial("tcp", lis.Addr().String()) require.NoError(t, err) defer nc.Close() - m := NewMeteredConn(nc, 0) + m := NewMeteredConn(nc) require.NoError(t, m.CloseWrite()) // The peer saw EOF and closed its side. _, err = nc.Read(make([]byte, 1)) diff --git a/pkg/plumbing/mux.go b/pkg/plumbing/mux.go index daee405..32fe3a7 100644 --- a/pkg/plumbing/mux.go +++ b/pkg/plumbing/mux.go @@ -20,9 +20,6 @@ type Frames interface { CloseWrite() error // Reset abandons the conn on the peer, for reason. Reset(reason string) error - // Window grants the peer n more bytes, what Recv returned. A no-op - // without flow control. - Window(n int) error // Release forgets the conn once it's over. Release() } @@ -32,17 +29,14 @@ type MuxConfig struct { // MaxFrame caps each Data's payload; 0 for no cap. MaxFrame int // Queue is how many payloads from the peer may wait unread before Deliver - // waits; 0 for no limit, for a transport whose window bounds them. - Queue int - // Window is what the conn may send before the peer grants more; 0 for no - // limit. - Window int + // waits; 0 for no limit. + Queue int Local, Remote net.Addr } // MuxStream is one conn's Stream on a multiplexed transport: Recv returns what // the transport Delivers for it, and Send, CloseSend and Close write its -// frames. Conn is a net.Conn over it, metered to its window. +// frames. Conn is a metered net.Conn over it. type MuxStream struct { frames Frames cfg MuxConfig @@ -70,11 +64,11 @@ func NewMuxStream(frames Frames, cfg MuxConfig) *MuxStream { room: make(chan struct{}, 1), done: make(chan struct{}), } - s.conn = NewMeteredConn(NewStreamConn(s, cfg.Local, cfg.Remote), cfg.Window) + s.conn = NewMeteredConn(NewStreamConn(s, cfg.Local, cfg.Remote)) return s } -// Conn is the net.Conn over the stream. Grant it the peer's window. +// Conn is the net.Conn over the stream. func (s *MuxStream) Conn() MeteredConn { return s.conn } // Deliver queues payload from the peer, waiting while a bounded queue is @@ -131,9 +125,6 @@ func (s *MuxStream) Recv() ([]byte, error) { s.in = s.in[1:] s.mu.Unlock() signal(s.room) - if err := s.frames.Window(len(payload)); err != nil { - return nil, err - } return payload, nil } eof := s.eof diff --git a/pkg/plumbing/mux_test.go b/pkg/plumbing/mux_test.go index 4d9a72d..42d534f 100644 --- a/pkg/plumbing/mux_test.go +++ b/pkg/plumbing/mux_test.go @@ -17,7 +17,6 @@ import ( type recordingFrames struct { mu sync.Mutex data []string - windows []int closes int resets []string releases int @@ -48,20 +47,13 @@ func (f *recordingFrames) Reset(reason string) error { return nil } -func (f *recordingFrames) Window(n int) error { - f.mu.Lock() - defer f.mu.Unlock() - f.windows = append(f.windows, n) - return nil -} - func (f *recordingFrames) Release() { f.mu.Lock() defer f.mu.Unlock() f.releases++ } -func TestMuxStreamReceivesInOrderAndGrantsWhatItReturns(t *testing.T) { +func TestMuxStreamReceivesInOrder(t *testing.T) { f := &recordingFrames{halfClose: true} s := NewMuxStream(f, MuxConfig{}) ctx := context.Background() @@ -72,7 +64,6 @@ func TestMuxStreamReceivesInOrderAndGrantsWhatItReturns(t *testing.T) { got, err := io.ReadAll(s.Conn()) require.NoError(t, err) assert.Equal(t, "abcde", string(got)) - assert.Equal(t, []int{2, 3}, f.windows) } func TestMuxStreamBoundedQueueWaitsForRecv(t *testing.T) { diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go index 33eaf5d..c1b6285 100644 --- a/pkg/tunnel/stream.go +++ b/pkg/tunnel/stream.go @@ -185,9 +185,6 @@ func (f *messageFrames) Reset(reason string) error { return f.b.send(&bridgev1.TunnelNetworkMessage{ConnectionId: f.id, Error: reason}) } -// Window: the stream has no flow control; the queue holds the peer back. -func (f *messageFrames) Window(int) error { return nil } - // Release forgets the conn, unless another has since taken its id. func (f *messageFrames) Release() { f.b.conns.Compute(f.id, func(cur *messageFrames, loaded bool) (*messageFrames, bool) { diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index df1ece5..7063444 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -187,8 +187,7 @@ func (t *tunnelImpl) pump(local, remote net.Conn) { "remote", remote.RemoteAddr().String(), "bytes_sent", st.BytesWritten, "bytes_received", st.BytesRead, - "window_wait", st.WriteWait, - "window_waits", st.WriteWaits, + "write_time", st.WriteTime, ) } } diff --git a/proto/vercel/api/dev_tunnel/v1/tunnel.proto b/proto/vercel/api/dev_tunnel/v1/tunnel.proto index 67252cd..8009fd3 100644 --- a/proto/vercel/api/dev_tunnel/v1/tunnel.proto +++ b/proto/vercel/api/dev_tunnel/v1/tunnel.proto @@ -83,7 +83,6 @@ message Open { string host = 4; } -// No more than the conn's window allows (see Window). message Data { uint64 conn_id = 1; bytes payload = 2; @@ -101,10 +100,11 @@ message Reset { string reason = 2; } -// Lets the peer send `bytes` more of a conn's Data, as the sender of this -// consumes what it received. Each side starts each conn able to send 256 KiB. -// A client that sends past its window, or grants back more than it was sent, -// has the conn reset. +// The deployment has read `bytes` more of the Data the client sent on a conn. +// A client may pace its sends on these, as `vc dev` has: each conn starting +// able to send 256 KiB, and waiting for the service to grant more. Pacing is +// the client's; the service holds it to no window, sends Data as fast as the +// stream takes it, and ignores a client's Window. message Window { uint64 conn_id = 1; uint32 bytes = 2; From c5983cf41a98f2a8866dbf0f9a50234e5a793047 Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 18:37:43 -0700 Subject: [PATCH 10/11] Revert "refactor(devtunnel): pace the dev tunnel in bridge alone" The relay keeps its windows, so bridge keeps speaking them. This reverts commit 9489e76. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/devtunnel/backend.go | 2 + pkg/devtunnel/backend_test.go | 117 ++++++------------ pkg/devtunnel/conn.go | 49 ++------ pkg/devtunnel/stream.go | 51 ++++---- pkg/plumbing/metered.go | 125 +++++++++++++++++--- pkg/plumbing/metered_test.go | 68 +++++++++-- pkg/plumbing/mux.go | 19 ++- pkg/plumbing/mux_test.go | 11 +- pkg/tunnel/stream.go | 3 + pkg/tunnel/tunnel.go | 3 +- proto/vercel/api/dev_tunnel/v1/tunnel.proto | 10 +- 11 files changed, 269 insertions(+), 189 deletions(-) diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go index 08900c1..38e6bbf 100644 --- a/pkg/devtunnel/backend.go +++ b/pkg/devtunnel/backend.go @@ -33,6 +33,8 @@ import ( ) const ( + // What each side of a conn may send before the other grants more. + connWindow = 256 << 10 // A client numbers its conns from here; the relay numbers its own from 1. clientConnIDFloor = uint64(1) << 63 diff --git a/pkg/devtunnel/backend_test.go b/pkg/devtunnel/backend_test.go index 22a5ac1..c8fba0a 100644 --- a/pkg/devtunnel/backend_test.go +++ b/pkg/devtunnel/backend_test.go @@ -33,8 +33,6 @@ type fakeRelay struct { streams chan *relayCall // refuse ends a call before it reaches the test, unless it returns nil. refuse func(md metadata.MD) error - // gate, if set, holds off reading each call until it's closed. - gate chan struct{} } type relayCall struct { @@ -44,16 +42,12 @@ type relayCall struct { end chan error } -func startRelay(t *testing.T, opts ...func(*fakeRelay)) *fakeRelay { +func startRelay(t *testing.T) *fakeRelay { t.Helper() lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) r := &fakeRelay{url: "http://" + lis.Addr().String(), streams: make(chan *relayCall, 8)} - for _, o := range opts { - o(r) - } - // Fixed windows, so a relay that stops reading backs the client up soon. - srv := grpc.NewServer(grpc.InitialWindowSize(64<<10), grpc.InitialConnWindowSize(64<<10)) + srv := grpc.NewServer() devtunnel.RegisterTunnelServiceServer(srv, r) go func() { _ = srv.Serve(lis) }() t.Cleanup(srv.Stop) @@ -70,9 +64,6 @@ func (r *fakeRelay) Connect(call grpc.BidiStreamingServer[devtunnel.ConnectReque s := &relayCall{md: md, call: call, recv: make(chan *devtunnel.ConnectRequest, 256), end: make(chan error, 1)} go func() { defer close(s.recv) - if r.gate != nil { - <-r.gate - } for { msg, err := call.Recv() if err != nil { @@ -156,6 +147,10 @@ func closeConn(id uint64) *devtunnel.ConnectResponse { return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Close{Close: &devtunnel.Close{ConnId: id}}} } +func window(id uint64, n int) *devtunnel.ConnectResponse { + return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Window{Window: &devtunnel.Window{ConnId: id, Bytes: uint32(n)}}} +} + // newTunnel runs a tunnel over a Backend for r. func newTunnel(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) (*Backend, tunnel.Tunnel) { t.Helper() @@ -211,10 +206,15 @@ func TestIngressReachesTheApp(t *testing.T) { s.send(t, open(1, port)) s.send(t, data(1, "hello")) var echoed []byte - for len(echoed) < 5 { + granted := 0 + for len(echoed) < 5 || granted < 5 { msg := s.expect(t) - require.Nil(t, msg.GetWindow(), "pacing is the client's own") - echoed = append(echoed, msg.GetData().GetPayload()...) + if d := msg.GetData(); d != nil { + echoed = append(echoed, d.GetPayload()...) + } + if w := msg.GetWindow(); w != nil { + granted += int(w.GetBytes()) + } } assert.Equal(t, "hello", string(echoed)) @@ -241,17 +241,16 @@ func TestIngressToAnotherPortIsReset(t *testing.T) { assert.Equal(t, "bridge serves only port 3000", reset.GetReason()) } -func TestRelayDataIsTakenAsFastAsItComes(t *testing.T) { +func TestIngressSendsNoMoreThanTheWindow(t *testing.T) { r := startRelay(t) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) t.Cleanup(func() { _ = lis.Close() }) - got := make(chan int, 1) + body := bytes.Repeat([]byte("x"), connWindow+frameBytes) go func() { nc, err := lis.Accept() if err == nil { - n, _ := io.Copy(io.Discard, nc) - got <- int(n) + _, _ = nc.Write(body) } }() port := lis.Addr().(*net.TCPAddr).Port @@ -260,63 +259,17 @@ func TestRelayDataIsTakenAsFastAsItComes(t *testing.T) { s.hold(t) s.send(t, open(1, port)) - // Far past what any window would allow, with nothing granted. - const total = 4 << 20 - chunk := string(bytes.Repeat([]byte("x"), frameBytes)) - for sent := 0; sent < total; sent += frameBytes { - s.send(t, data(1, chunk)) - } - s.send(t, closeConn(1)) + assert.Len(t, s.expectData(t, 1, connWindow), connWindow) select { - case n := <-got: - assert.Equal(t, total, n) - case <-time.After(testTimeout): - t.Fatal("the app didn't get it all") - } -} - -func TestADevServerOutrunningTheStreamIsHeldBack(t *testing.T) { - gate := make(chan struct{}) - r := startRelay(t, func(r *fakeRelay) { r.gate = gate }) - lis, err := net.Listen("tcp", "127.0.0.1:0") - require.NoError(t, err) - t.Cleanup(func() { _ = lis.Close() }) - const total = 8 << 20 - written := make(chan struct{}) - go func() { - nc, err := lis.Accept() - if err == nil { - _, _ = nc.Write(bytes.Repeat([]byte("x"), total)) - close(written) + case msg := <-s.recv: + if msg.GetData() != nil { + t.Fatal("sent past the window") } - }() - port := lis.Addr().(*net.TCPAddr).Port - b, _ := newTunnel(t, r, "tok", port, func(tm *timings) { tm.silent = time.Minute }) - s := r.next(t) - s.hold(t) - s.send(t, open(1, port)) - - // The relay isn't reading, so the conn's sends wait once it has a - // megabyte queued, and the dev server's writes back up. - unsent := func() (n int64) { - b.conns.Range(func(_ string, f *relayFrames) bool { - n = f.unsent.Load() - return true - }) - return n - } - waitFor(t, func() bool { return unsent() >= maxUnsentBytes }) - time.Sleep(200 * time.Millisecond) - assert.LessOrEqual(t, unsent(), int64(maxUnsentBytes+maxPayload)) - select { - case <-written: - t.Fatal("the dev server wasn't held back") - default: + case <-time.After(200 * time.Millisecond): } - close(gate) - assert.Len(t, s.expectData(t, 1, total), total) - <-written + s.send(t, window(1, frameBytes)) + assert.Len(t, s.expectData(t, 1, frameBytes), frameBytes) } func TestEgressLeavesThroughTheDeployment(t *testing.T) { @@ -390,14 +343,13 @@ func TestReconnectKeepsTheOldStreamsConns(t *testing.T) { } func TestRefusedTokenWaitsForANewOne(t *testing.T) { - r := startRelay(t, func(r *fakeRelay) { - r.refuse = func(md metadata.MD) error { - if md.Get("authorization")[0] == "Bearer old" { - return status.Error(codes.Unauthenticated, "missing or invalid dev tunnel token") - } - return nil + r := startRelay(t) + r.refuse = func(md metadata.MD) error { + if md.Get("authorization")[0] == "Bearer old" { + return status.Error(codes.Unauthenticated, "missing or invalid dev tunnel token") } - }) + return nil + } c, _ := newTunnel(t, r, "old", 3000) waitFor(t, func() bool { return c.Status().GetState() == bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN }) @@ -421,11 +373,10 @@ func TestNoTokenWaitsForOne(t *testing.T) { } func TestFatalRefusalStops(t *testing.T) { - r := startRelay(t, func(r *fakeRelay) { - r.refuse = func(metadata.MD) error { - return status.Error(codes.PermissionDenied, "not allowed") - } - }) + r := startRelay(t) + r.refuse = func(metadata.MD) error { + return status.Error(codes.PermissionDenied, "not allowed") + } c, tun := newTunnel(t, r, "tok", 3000) select { diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go index 9eb28f8..edffed3 100644 --- a/pkg/devtunnel/conn.go +++ b/pkg/devtunnel/conn.go @@ -1,22 +1,13 @@ package devtunnel import ( - "net" - "sync/atomic" - "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/tunnel" ) -const ( - // Per Data frame. - maxPayload = 64 << 10 - // What a conn may have queued for the relay, not yet on the stream, before - // its sends wait. A dev server writing faster than the stream drains is - // held back by TCP, while the other conns keep flowing. - maxUnsentBytes = 1 << 20 -) +// Per Data frame. +const maxPayload = 64 << 10 type direction int @@ -27,18 +18,14 @@ const ( egress ) -// relayFrames writes one conn's frames on a Connect stream: Data, Close and -// Reset. The relay's Window frames are ignored: what the relay sends is taken -// as fast as it comes. +// relayFrames writes one conn's frames on a Connect stream: Data, Close, +// Reset, and a Window for each payload received. type relayFrames struct { s *stream id string wire uint64 dir direction stream *plumbing.MuxStream - - unsent atomic.Int64 - drained chan struct{} } // newConn opens id on the stream as the relay's conn wire, or returns nil @@ -47,9 +34,10 @@ func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Ad if s.closed.Load() { return nil } - f := &relayFrames{s: s, id: id, wire: wire, dir: dir, drained: make(chan struct{}, 1)} + f := &relayFrames{s: s, id: id, wire: wire, dir: dir} f.stream = plumbing.NewMuxStream(f, plumbing.MuxConfig{ MaxFrame: maxPayload, + Window: connWindow, Local: addr("bridge"), Remote: addr(remote.String()), }) @@ -63,25 +51,9 @@ func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Ad return f } -// Data waits while the conn has maxUnsentBytes queued for the relay. func (f *relayFrames) Data(payload []byte) error { - for f.unsent.Load() >= maxUnsentBytes { - select { - case <-f.drained: - case <-f.stream.Done(): - return net.ErrClosed - } - } - n := int64(len(payload)) - f.unsent.Add(n) - f.s.b.bytesSent.Add(n) - f.s.sendThen(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: f.wire, Payload: payload}}}, func() { - f.unsent.Add(-n) - select { - case f.drained <- struct{}{}: - default: - } - }) + f.s.b.bytesSent.Add(int64(len(payload))) + f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: f.wire, Payload: payload}}}) return nil } @@ -95,6 +67,11 @@ func (f *relayFrames) Reset(reason string) error { return nil } +func (f *relayFrames) Window(n int) error { + f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: f.wire, Bytes: uint32(n)}}}) + return nil +} + // Release forgets the conn. func (f *relayFrames) Release() { f.s.b.conns.Compute(f.id, func(cur *relayFrames, loaded bool) (*relayFrames, bool) { diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index f753c3c..6e6c3a2 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -67,17 +67,14 @@ func (s *stream) run(token string, moving func()) error { s.send(heartbeat) go func() { for { - entry, ok := s.out.pop(s.ctx) + frame, ok := s.out.pop(s.ctx) if !ok { return } // A failed send ends the call, which Recv reports. - if err := call.Send(entry.frame); err != nil { + if err := call.Send(frame); err != nil { return } - if entry.sent != nil { - entry.sent() - } } }() go s.heartbeats() @@ -142,6 +139,10 @@ func (s *stream) handle(msg *devtunnel.ConnectResponse, moving func()) { s.logger.Debug("Relay reset the connection", "conn_id", c.id, "reason", f.Reset_.GetReason()) c.stream.PeerReset() } + case *devtunnel.ConnectResponse_Window: + if c := s.conn(f.Window.GetConnId()); c != nil { + c.stream.Conn().Grant(int(f.Window.GetBytes())) + } } } @@ -191,30 +192,20 @@ func (s *stream) endConns() { } func (s *stream) send(frame *devtunnel.ConnectRequest) { - s.out.push(outboxEntry{frame: frame}) -} - -// sendThen queues frame, calling sent once it's on the stream. -func (s *stream) sendThen(frame *devtunnel.ConnectRequest, sent func()) { - s.out.push(outboxEntry{frame: frame, sent: sent}) + s.out.push(frame) } -// outbox queues frames for the relay in order. Each conn has at most -// maxUnsentBytes of Data in it; the rest is small. +// outbox queues frames for the relay in order. Data is bounded by each conn's +// window; the rest is small. type outbox struct { - mu sync.Mutex - entries []outboxEntry - wake chan struct{} -} - -type outboxEntry struct { - frame *devtunnel.ConnectRequest - sent func() + mu sync.Mutex + frames []*devtunnel.ConnectRequest + wake chan struct{} } -func (o *outbox) push(e outboxEntry) { +func (o *outbox) push(frame *devtunnel.ConnectRequest) { o.mu.Lock() - o.entries = append(o.entries, e) + o.frames = append(o.frames, frame) o.mu.Unlock() select { case o.wake <- struct{}{}: @@ -222,21 +213,21 @@ func (o *outbox) push(e outboxEntry) { } } -func (o *outbox) pop(ctx context.Context) (outboxEntry, bool) { +func (o *outbox) pop(ctx context.Context) (*devtunnel.ConnectRequest, bool) { for { o.mu.Lock() - if len(o.entries) > 0 { - e := o.entries[0] - o.entries[0] = outboxEntry{} - o.entries = o.entries[1:] + if len(o.frames) > 0 { + frame := o.frames[0] + o.frames[0] = nil + o.frames = o.frames[1:] o.mu.Unlock() - return e, true + return frame, true } o.mu.Unlock() select { case <-o.wake: case <-ctx.Done(): - return outboxEntry{}, false + return nil, false } } } diff --git a/pkg/plumbing/metered.go b/pkg/plumbing/metered.go index 56d5570..6d87a92 100644 --- a/pkg/plumbing/metered.go +++ b/pkg/plumbing/metered.go @@ -3,34 +3,66 @@ package plumbing import ( "errors" "net" + "sync" "sync/atomic" "time" ) -// MeteredConn is a net.Conn that counts what passes through it. +// MeteredConn is a net.Conn that counts what passes through it. Given a +// window, Write sends no more than the peer has granted with Grant: the flow +// control a conn multiplexed with others needs, since the transport's own can +// only slow all of them. type MeteredConn interface { net.Conn + // Grant lets Write send n more bytes. + Grant(n int) // CloseWrite half-closes the wrapped conn, if it can. CloseWrite() error // Stats reports what has passed through so far. Stats() *MeteredStats } +type meteredConn struct { + net.Conn + window int + + credit atomic.Int64 + wake chan struct{} + closed chan struct{} + closeOnce sync.Once + // The wrapped conn's, if it has a Done method, so a write waiting for + // credit returns once that conn is over. + innerDone <-chan struct{} + + bytesRead, bytesWritten atomic.Int64 + writeWait, writeWaits atomic.Int64 +} + // MeteredStats is what has passed through a MeteredConn. type MeteredStats struct { BytesRead int64 BytesWritten int64 - // How long writes took, which grows when the transport holds them back. - WriteTime time.Duration + // What Write may still send. Zero without a window. + Credit int64 + // How long writes waited for credit, over how many waits. + WriteWait time.Duration + WriteWaits int64 } -type meteredConn struct { - net.Conn - bytesRead, bytesWritten, writeTime atomic.Int64 -} - -func NewMeteredConn(conn net.Conn) MeteredConn { - return &meteredConn{Conn: conn} +// NewMeteredConn wraps conn. Write may send window bytes before Grant gives +// more; 0 for no limit. +func NewMeteredConn(conn net.Conn, window int) MeteredConn { + m := &meteredConn{ + Conn: conn, + window: window, + wake: make(chan struct{}, 1), + closed: make(chan struct{}), + } + m.credit.Store(int64(window)) + if d, ok := conn.(interface{ Done() <-chan struct{} }); ok { + m.innerDone = d.Done() + } + return m } func (m *meteredConn) Read(p []byte) (int, error) { @@ -40,11 +72,35 @@ func (m *meteredConn) Read(p []byte) (int, error) { } func (m *meteredConn) Write(p []byte) (int, error) { - start := time.Now() - n, err := m.Conn.Write(p) - m.writeTime.Add(int64(time.Since(start))) - m.bytesWritten.Add(int64(n)) - return n, err + if m.window <= 0 { + n, err := m.Conn.Write(p) + m.bytesWritten.Add(int64(n)) + return n, err + } + written := 0 + for written < len(p) { + credit, err := m.awaitCredit() + if err != nil { + return written, err + } + size := min(len(p)-written, int(credit)) + n, err := m.Conn.Write(p[written : written+size]) + m.credit.Add(-int64(n)) + m.bytesWritten.Add(int64(n)) + written += n + if err != nil { + return written, err + } + } + return written, nil +} + +func (m *meteredConn) Grant(n int) { + m.credit.Add(int64(n)) + select { + case m.wake <- struct{}{}: + default: + } } func (m *meteredConn) CloseWrite() error { @@ -54,10 +110,45 @@ func (m *meteredConn) CloseWrite() error { return errors.ErrUnsupported } +func (m *meteredConn) Close() error { + m.closeOnce.Do(func() { close(m.closed) }) + return m.Conn.Close() +} + func (m *meteredConn) Stats() *MeteredStats { - return &MeteredStats{ + st := &MeteredStats{ BytesRead: m.bytesRead.Load(), BytesWritten: m.bytesWritten.Load(), - WriteTime: time.Duration(m.writeTime.Load()), + WriteWait: time.Duration(m.writeWait.Load()), + WriteWaits: m.writeWaits.Load(), + } + if m.window > 0 { + st.Credit = m.credit.Load() + } + return st +} + +func (m *meteredConn) awaitCredit() (int64, error) { + var start time.Time + defer func() { + if !start.IsZero() { + m.writeWait.Add(int64(time.Since(start))) + } + }() + for { + if credit := m.credit.Load(); credit > 0 { + return credit, nil + } + if start.IsZero() { + start = time.Now() + m.writeWaits.Add(1) + } + select { + case <-m.wake: + case <-m.closed: + return 0, net.ErrClosed + case <-m.innerDone: + return 0, net.ErrClosed + } } } diff --git a/pkg/plumbing/metered_test.go b/pkg/plumbing/metered_test.go index 7b7fe0e..2b0831b 100644 --- a/pkg/plumbing/metered_test.go +++ b/pkg/plumbing/metered_test.go @@ -16,9 +16,10 @@ import ( // recordingConn keeps what's written to it and reads from r. type recordingConn struct { net.Conn - r io.Reader - mu sync.Mutex - w bytes.Buffer + r io.Reader + mu sync.Mutex + w bytes.Buffer + done chan struct{} } func (c *recordingConn) Read(p []byte) (int, error) { return c.r.Read(p) } @@ -37,13 +38,25 @@ func (c *recordingConn) written() string { func (c *recordingConn) Close() error { return nil } +func (c *recordingConn) Done() <-chan struct{} { return c.done } + func newRecordingConn(r string) *recordingConn { - return &recordingConn{r: bytes.NewReader([]byte(r))} + return &recordingConn{r: bytes.NewReader([]byte(r)), done: make(chan struct{})} +} + +// writeAsync starts a Write and returns its result. +func writeAsync(m MeteredConn, p string) <-chan error { + errc := make(chan error, 1) + go func() { + _, err := m.Write([]byte(p)) + errc <- err + }() + return errc } -func TestMeteredConnCounts(t *testing.T) { +func TestMeteredConnCountsWithoutAWindow(t *testing.T) { inner := newRecordingConn("hello") - m := NewMeteredConn(inner) + m := NewMeteredConn(inner, 0) _, err := m.Write([]byte("abc")) require.NoError(t, err) @@ -51,14 +64,47 @@ func TestMeteredConnCounts(t *testing.T) { require.NoError(t, err) assert.Equal(t, "abc", inner.written()) + assert.Equal(t, &MeteredStats{BytesRead: 5, BytesWritten: 3}, m.Stats()) +} + +func TestMeteredConnHoldsWritesToTheWindow(t *testing.T) { + inner := newRecordingConn("") + m := NewMeteredConn(inner, 4) + + errc := writeAsync(m, "abcdefgh") + require.Eventually(t, func() bool { return inner.written() == "abcd" }, time.Second, time.Millisecond) + select { + case <-errc: + t.Fatal("wrote past the window") + case <-time.After(50 * time.Millisecond): + } + + m.Grant(4) + require.NoError(t, <-errc) + assert.Equal(t, "abcdefgh", inner.written()) st := m.Stats() - assert.Equal(t, int64(5), st.BytesRead) - assert.Equal(t, int64(3), st.BytesWritten) - assert.Greater(t, st.WriteTime, time.Duration(0)) + assert.Equal(t, int64(0), st.Credit) + assert.Equal(t, int64(1), st.WriteWaits) + assert.Greater(t, st.WriteWait, time.Duration(0)) +} + +func TestMeteredConnWriteWaitingEndsWithTheConn(t *testing.T) { + m := NewMeteredConn(newRecordingConn(""), 1) + errc := writeAsync(m, "ab") + time.Sleep(10 * time.Millisecond) + require.NoError(t, m.Close()) + assert.ErrorIs(t, <-errc, net.ErrClosed) + + inner := newRecordingConn("") + m = NewMeteredConn(inner, 1) + errc = writeAsync(m, "ab") + time.Sleep(10 * time.Millisecond) + close(inner.done) + assert.ErrorIs(t, <-errc, net.ErrClosed) } func TestMeteredConnCloseWrite(t *testing.T) { - assert.True(t, errors.Is(NewMeteredConn(newRecordingConn("")).CloseWrite(), errors.ErrUnsupported)) + assert.True(t, errors.Is(NewMeteredConn(newRecordingConn(""), 0).CloseWrite(), errors.ErrUnsupported)) lis, err := net.Listen("tcp", "127.0.0.1:0") require.NoError(t, err) @@ -73,7 +119,7 @@ func TestMeteredConnCloseWrite(t *testing.T) { nc, err := net.Dial("tcp", lis.Addr().String()) require.NoError(t, err) defer nc.Close() - m := NewMeteredConn(nc) + m := NewMeteredConn(nc, 0) require.NoError(t, m.CloseWrite()) // The peer saw EOF and closed its side. _, err = nc.Read(make([]byte, 1)) diff --git a/pkg/plumbing/mux.go b/pkg/plumbing/mux.go index 32fe3a7..daee405 100644 --- a/pkg/plumbing/mux.go +++ b/pkg/plumbing/mux.go @@ -20,6 +20,9 @@ type Frames interface { CloseWrite() error // Reset abandons the conn on the peer, for reason. Reset(reason string) error + // Window grants the peer n more bytes, what Recv returned. A no-op + // without flow control. + Window(n int) error // Release forgets the conn once it's over. Release() } @@ -29,14 +32,17 @@ type MuxConfig struct { // MaxFrame caps each Data's payload; 0 for no cap. MaxFrame int // Queue is how many payloads from the peer may wait unread before Deliver - // waits; 0 for no limit. - Queue int + // waits; 0 for no limit, for a transport whose window bounds them. + Queue int + // Window is what the conn may send before the peer grants more; 0 for no + // limit. + Window int Local, Remote net.Addr } // MuxStream is one conn's Stream on a multiplexed transport: Recv returns what // the transport Delivers for it, and Send, CloseSend and Close write its -// frames. Conn is a metered net.Conn over it. +// frames. Conn is a net.Conn over it, metered to its window. type MuxStream struct { frames Frames cfg MuxConfig @@ -64,11 +70,11 @@ func NewMuxStream(frames Frames, cfg MuxConfig) *MuxStream { room: make(chan struct{}, 1), done: make(chan struct{}), } - s.conn = NewMeteredConn(NewStreamConn(s, cfg.Local, cfg.Remote)) + s.conn = NewMeteredConn(NewStreamConn(s, cfg.Local, cfg.Remote), cfg.Window) return s } -// Conn is the net.Conn over the stream. +// Conn is the net.Conn over the stream. Grant it the peer's window. func (s *MuxStream) Conn() MeteredConn { return s.conn } // Deliver queues payload from the peer, waiting while a bounded queue is @@ -125,6 +131,9 @@ func (s *MuxStream) Recv() ([]byte, error) { s.in = s.in[1:] s.mu.Unlock() signal(s.room) + if err := s.frames.Window(len(payload)); err != nil { + return nil, err + } return payload, nil } eof := s.eof diff --git a/pkg/plumbing/mux_test.go b/pkg/plumbing/mux_test.go index 42d534f..4d9a72d 100644 --- a/pkg/plumbing/mux_test.go +++ b/pkg/plumbing/mux_test.go @@ -17,6 +17,7 @@ import ( type recordingFrames struct { mu sync.Mutex data []string + windows []int closes int resets []string releases int @@ -47,13 +48,20 @@ func (f *recordingFrames) Reset(reason string) error { return nil } +func (f *recordingFrames) Window(n int) error { + f.mu.Lock() + defer f.mu.Unlock() + f.windows = append(f.windows, n) + return nil +} + func (f *recordingFrames) Release() { f.mu.Lock() defer f.mu.Unlock() f.releases++ } -func TestMuxStreamReceivesInOrder(t *testing.T) { +func TestMuxStreamReceivesInOrderAndGrantsWhatItReturns(t *testing.T) { f := &recordingFrames{halfClose: true} s := NewMuxStream(f, MuxConfig{}) ctx := context.Background() @@ -64,6 +72,7 @@ func TestMuxStreamReceivesInOrder(t *testing.T) { got, err := io.ReadAll(s.Conn()) require.NoError(t, err) assert.Equal(t, "abcde", string(got)) + assert.Equal(t, []int{2, 3}, f.windows) } func TestMuxStreamBoundedQueueWaitsForRecv(t *testing.T) { diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go index c1b6285..33eaf5d 100644 --- a/pkg/tunnel/stream.go +++ b/pkg/tunnel/stream.go @@ -185,6 +185,9 @@ func (f *messageFrames) Reset(reason string) error { return f.b.send(&bridgev1.TunnelNetworkMessage{ConnectionId: f.id, Error: reason}) } +// Window: the stream has no flow control; the queue holds the peer back. +func (f *messageFrames) Window(int) error { return nil } + // Release forgets the conn, unless another has since taken its id. func (f *messageFrames) Release() { f.b.conns.Compute(f.id, func(cur *messageFrames, loaded bool) (*messageFrames, bool) { diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index 7063444..df1ece5 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -187,7 +187,8 @@ func (t *tunnelImpl) pump(local, remote net.Conn) { "remote", remote.RemoteAddr().String(), "bytes_sent", st.BytesWritten, "bytes_received", st.BytesRead, - "write_time", st.WriteTime, + "window_wait", st.WriteWait, + "window_waits", st.WriteWaits, ) } } diff --git a/proto/vercel/api/dev_tunnel/v1/tunnel.proto b/proto/vercel/api/dev_tunnel/v1/tunnel.proto index 8009fd3..67252cd 100644 --- a/proto/vercel/api/dev_tunnel/v1/tunnel.proto +++ b/proto/vercel/api/dev_tunnel/v1/tunnel.proto @@ -83,6 +83,7 @@ message Open { string host = 4; } +// No more than the conn's window allows (see Window). message Data { uint64 conn_id = 1; bytes payload = 2; @@ -100,11 +101,10 @@ message Reset { string reason = 2; } -// The deployment has read `bytes` more of the Data the client sent on a conn. -// A client may pace its sends on these, as `vc dev` has: each conn starting -// able to send 256 KiB, and waiting for the service to grant more. Pacing is -// the client's; the service holds it to no window, sends Data as fast as the -// stream takes it, and ignores a client's Window. +// Lets the peer send `bytes` more of a conn's Data, as the sender of this +// consumes what it received. Each side starts each conn able to send 256 KiB. +// A client that sends past its window, or grants back more than it was sent, +// has the conn reset. message Window { uint64 conn_id = 1; uint32 bytes = 2; From bace08298f5c7fa8329d10a6ff87121155a13efa Mon Sep 17 00:00:00 2001 From: edwardowens Date: Mon, 26 Oct 2026 18:52:37 -0700 Subject: [PATCH 11/11] refactor(tunnel): speak tunnel Messages, with a Stream adapter per protocol pkg/tunnel has its own protocol, Message (Open, Data, CloseWrite, Reset, Window), and multiplexes conns over any Stream of them. Each transport translates to it: - bridgestream: a bridge TunnelNetwork stream, whose first message to an unknown conn opens it, with no half-close or flow control. - devtunnel.Stream: a dev tunnel's Connect stream, with its reconnects, heartbeats, token and status. tunnel.Backend goes, and pkg/tunnel imports no protos: the hijacker now takes a hostname and a destination. Co-Authored-By: Claude Opus 5.5 (1M context) --- pkg/commands/intercept.go | 10 +- pkg/commands/intercept_server.go | 4 +- pkg/devtunnel/backend.go | 500 ------------ pkg/devtunnel/call.go | 263 +++++++ pkg/devtunnel/conn.go | 89 --- pkg/devtunnel/stream.go | 716 +++++++++++++----- .../{backend_test.go => stream_test.go} | 23 +- pkg/mitm/hijacker.go | 31 +- pkg/proxy/grpcserver.go | 3 +- pkg/tunnel/backend.go | 40 - pkg/tunnel/bridgestream/bridgestream.go | 125 +++ .../bridgestream_test.go} | 14 +- pkg/tunnel/message.go | 82 ++ pkg/tunnel/pump_test.go | 127 ++-- pkg/tunnel/stream.go | 199 ----- pkg/tunnel/tunnel.go | 288 +++++-- 16 files changed, 1336 insertions(+), 1178 deletions(-) delete mode 100644 pkg/devtunnel/backend.go create mode 100644 pkg/devtunnel/call.go delete mode 100644 pkg/devtunnel/conn.go rename pkg/devtunnel/{backend_test.go => stream_test.go} (96%) delete mode 100644 pkg/tunnel/backend.go create mode 100644 pkg/tunnel/bridgestream/bridgestream.go rename pkg/tunnel/{tunnel_test.go => bridgestream/bridgestream_test.go} (96%) create mode 100644 pkg/tunnel/message.go delete mode 100644 pkg/tunnel/stream.go diff --git a/pkg/commands/intercept.go b/pkg/commands/intercept.go index 55a6c94..adb3740 100644 --- a/pkg/commands/intercept.go +++ b/pkg/commands/intercept.go @@ -27,6 +27,7 @@ import ( "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/probe" "github.com/vercel/bridge/pkg/tunnel" + "github.com/vercel/bridge/pkg/tunnel/bridgestream" "google.golang.org/grpc" ) @@ -188,7 +189,7 @@ func doIntercept(ctx context.Context, c *cli.Command) error { dnsResolver hostResolver metadata *bridgev1.GetMetadataResponse mountServers []*fuseServer - devTunnel *devtunnel.Backend + devTunnel *devtunnel.Stream ) if tunnelProtocol == tunnelProtocolDevTunnel { if len(mountPaths) > 0 { @@ -206,7 +207,10 @@ func doIntercept(ctx context.Context, c *cli.Command) error { if err != nil { return err } - tun = tunnel.NewWithBackend(plumbing.NewStaticPortDialer(appPort, nil), devTunnel) + devTunnelCtx, stopDevTunnel := context.WithCancel(ctx) + defer stopDevTunnel() + devTunnel.Start(devTunnelCtx) + tun = tunnel.New(plumbing.NewStaticPortDialer(appPort, nil), devTunnel) tun.Start(ctx) defer tun.Close() dnsResolver = deploymentResolver{} @@ -410,7 +414,7 @@ func connectBridge(ctx context.Context, serverAddr string, appPort int, mountPat } dialer := plumbing.NewStaticPortDialer(appPort, nil) - tun := tunnel.New(dialer, stream) + tun := tunnel.New(dialer, bridgestream.New(stream)) tun.Start(ctx) slog.Info("Tunnel connected", "app_port", appPort) diff --git a/pkg/commands/intercept_server.go b/pkg/commands/intercept_server.go index fca9a66..1157399 100644 --- a/pkg/commands/intercept_server.go +++ b/pkg/commands/intercept_server.go @@ -35,7 +35,7 @@ type interceptServer struct { ready atomic.Bool monitors atomic.Pointer[monitorSet] - tunnel atomic.Pointer[devtunnel.Backend] + tunnel atomic.Pointer[devtunnel.Stream] } // monitorSet is the immutable bundle stored in interceptServer.monitors. @@ -80,7 +80,7 @@ func (s *interceptServer) SetMonitors(liveness, readiness, startup probe.Monitor // SetTunnel publishes the dev tunnel GetStatus reports on and SetTunnelToken // hands tokens to. -func (s *interceptServer) SetTunnel(t *devtunnel.Backend) { +func (s *interceptServer) SetTunnel(t *devtunnel.Stream) { s.tunnel.Store(t) } diff --git a/pkg/devtunnel/backend.go b/pkg/devtunnel/backend.go deleted file mode 100644 index 38e6bbf..0000000 --- a/pkg/devtunnel/backend.go +++ /dev/null @@ -1,500 +0,0 @@ -// Package devtunnel is a tunnel.Backend over a dev tunnel: the Connect stream -// to api-dev-tunnel that `vc dev --tunnel` holds otherwise. The deployment's -// conns reach the app, and the app's outbound conns leave through the -// deployment. -package devtunnel - -import ( - "context" - "crypto/tls" - "errors" - "fmt" - "log/slog" - "net" - "net/url" - "regexp" - "strconv" - "sync" - "sync/atomic" - "time" - - "github.com/puzpuzpuz/xsync/v3" - "github.com/segmentio/ksuid" - bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" - "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" - "github.com/vercel/bridge/pkg/plumbing" - "github.com/vercel/bridge/pkg/tunnel" - "google.golang.org/grpc" - "google.golang.org/grpc/codes" - "google.golang.org/grpc/credentials" - "google.golang.org/grpc/credentials/insecure" - "google.golang.org/grpc/keepalive" - "google.golang.org/grpc/status" -) - -const ( - // What each side of a conn may send before the other grants more. - connWindow = 256 << 10 - // A client numbers its conns from here; the relay numbers its own from 1. - clientConnIDFloor = uint64(1) << 63 - - instanceHeader = "x-vercel-dev-tunnel-instance" - - retryMin = 250 * time.Millisecond - retryMax = 5 * time.Second -) - -var instancePattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) - -// Codes reconnecting can't fix. -var fatal = map[codes.Code]bool{ - codes.FailedPrecondition: true, - codes.InvalidArgument: true, - codes.PermissionDenied: true, - codes.Unimplemented: true, -} - -// Config configures a Backend. -type Config struct { - // RelayURL is api-dev-tunnel's URL, from create-session. - RelayURL string - // Token is the client token, from create-session. Without one the - // backend waits for SetToken. - Token string - // Instance tells the relay this process's reconnects from another - // client's, so a restart can take the tunnel back while its old stream is - // still open. Random if empty. - Instance string - // Port is the only port of the app the deployment's conns may reach. - Port int - // RelayDialer connects to the relay. Defaults to a net.Dialer. - RelayDialer plumbing.ContextDialer - Logger *slog.Logger -} - -// timings are the relay's protocol intervals, shortened by tests. -type timings struct { - // The relay needs a heartbeat at least every 30s. - heartbeat time.Duration - // The relay heartbeats every 30s; three missed and the stream is dead. - silent time.Duration - // The relay holds the deployment's conns this long for a client, and an - // outbound conn waits as long for a stream. - liveWait time.Duration -} - -var defaultTimings = timings{ - heartbeat: 15 * time.Second, - silent: 90 * time.Second, - liveWait: 5 * time.Second, -} - -// Backend holds a dev tunnel's stream to its relay, replacing it whenever it -// drops. -type Backend struct { - cfg Config - timings timings - logger *slog.Logger - instance string - target string - creds credentials.TransportCredentials - - // Set by Run, before any stream. - rpc devtunnel.TunnelServiceClient - accept func(net.Conn, *tunnel.ConnInfo) - ctx context.Context - - // Each open conn, by an id unique across streams. - conns *xsync.MapOf[string, *relayFrames] - - mu sync.Mutex - token string - tokenSet chan struct{} - // The newest stream holding the tunnel, which new outbound conns take. - live *stream - liveCh chan struct{} - liveClosed bool - // Streams holding the tunnel, including one finishing its conns after the - // relay said to reconnect. - held int - state bridgev1.TunnelState - lastErr string - liveSince time.Time - reconnects int64 - - ingressConns, egressConns, ingressTotal, egressTotal atomic.Int64 - bytesSent, bytesReceived atomic.Int64 - nextEgress, nextStream atomic.Uint64 -} - -var _ tunnel.Backend = (*Backend)(nil) - -// New returns a Backend for cfg. Nothing connects until Run. -func New(cfg Config) (*Backend, error) { - target, creds, err := relayTarget(cfg.RelayURL) - if err != nil { - return nil, err - } - instance := cfg.Instance - if instance == "" { - instance = "bridge-" + ksuid.New().String() - } - if !instancePattern.MatchString(instance) { - return nil, fmt.Errorf("tunnel instance must be 1 to 64 letters, digits or dashes: %q", instance) - } - if cfg.RelayDialer == nil { - cfg.RelayDialer = &net.Dialer{Timeout: 10 * time.Second} - } - logger := cfg.Logger - if logger == nil { - logger = slog.Default() - } - return &Backend{ - cfg: cfg, - timings: defaultTimings, - logger: logger.With("component", "dev-tunnel", "relay", cfg.RelayURL, "instance", instance), - instance: instance, - target: target, - creds: creds, - conns: xsync.NewMapOf[string, *relayFrames](), - token: cfg.Token, - tokenSet: make(chan struct{}), - liveCh: make(chan struct{}), - state: bridgev1.TunnelState_TUNNEL_STATE_CONNECTING, - }, nil -} - -// relayTarget turns the relay's URL into a gRPC target. Run dials it with -// RelayDialer rather than resolving it through gRPC, so an intercept whose DNS -// forwards every domain still reaches the relay directly. -func relayTarget(relayURL string) (string, credentials.TransportCredentials, error) { - u, err := url.Parse(relayURL) - if err != nil { - return "", nil, fmt.Errorf("invalid relay URL %q: %w", relayURL, err) - } - if u.Path != "" && u.Path != "/" { - return "", nil, fmt.Errorf("relay URL %q must have no path", relayURL) - } - var creds credentials.TransportCredentials - port := u.Port() - switch u.Scheme { - case "https": - creds = credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) - if port == "" { - port = "443" - } - case "http": - creds = insecure.NewCredentials() - if port == "" { - port = "80" - } - default: - return "", nil, fmt.Errorf("relay URL %q must be http or https", relayURL) - } - if u.Hostname() == "" { - return "", nil, fmt.Errorf("relay URL %q has no host", relayURL) - } - return "passthrough:///" + net.JoinHostPort(u.Hostname(), port), creds, nil -} - -// Run holds the tunnel until ctx ends, replacing its stream whenever it -// drops. It returns an error only if the relay refuses the tunnel for good. -func (b *Backend) Run(ctx context.Context, accept func(net.Conn, *tunnel.ConnInfo)) error { - relayDialer := b.cfg.RelayDialer - cc, err := grpc.NewClient(b.target, - grpc.WithTransportCredentials(b.creds), - grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { - return relayDialer.DialContext(ctx, "tcp", addr) - }), - // Notices a dead connection, e.g. after sleep, sooner than the - // relay's heartbeats would. - grpc.WithKeepaliveParams(keepalive.ClientParameters{ - Time: 10 * time.Second, - Timeout: 20 * time.Second, - PermitWithoutStream: true, - }), - ) - if err != nil { - return fmt.Errorf("creating relay client: %w", err) - } - defer cc.Close() - b.rpc = devtunnel.NewTunnelServiceClient(cc) - b.accept = accept - b.ctx = ctx - // A stream the relay moved off may still be finishing. - defer b.conns.Range(func(_ string, f *relayFrames) bool { - f.stream.PeerReset() - return true - }) - - err = b.run() - if err != nil { - b.mu.Lock() - b.state = bridgev1.TunnelState_TUNNEL_STATE_FAILED - b.lastErr = err.Error() - b.mu.Unlock() - b.logger.Error("Tunnel stopped", "error", err) - } - return err -} - -// Open sends an outbound conn out through the deployment, to its hostname if -// the intercept's DNS gave it one, or else to the address it was headed for. -// The deployment resolves the hostname as its own code would. It waits a -// while for a stream to hold the tunnel. -func (b *Backend) Open(ctx context.Context, info *tunnel.ConnInfo) (net.Conn, error) { - s := b.awaitLive(ctx) - if s == nil { - return nil, errors.New("no stream holds the tunnel") - } - n := b.nextEgress.Add(1) - f := s.newConn("e"+strconv.FormatUint(n, 10), clientConnIDFloor|n, egress, info.Dest) - if f == nil { - return nil, errors.New("the tunnel's stream ended") - } - host := info.Hostname - if host == "" { - host = info.Dest.Host - } - s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Open{Open: &devtunnel.Open{ - ConnId: f.wire, - Protocol: devtunnel.Protocol_PROTOCOL_TCP, - Host: host, - Port: uint32(info.Dest.Port), - }}}) - return f.stream.Conn(), nil -} - -// counters tracks conns opening (delta 1) and ending (delta -1). -func (b *Backend) counters(dir direction, delta int64) { - open, total := &b.ingressConns, &b.ingressTotal - if dir == egress { - open, total = &b.egressConns, &b.egressTotal - } - open.Add(delta) - if delta > 0 { - total.Add(delta) - } -} - -// SetToken replaces the client token. A backend waiting for one connects with -// it; a live one uses it on its next stream. -func (b *Backend) SetToken(token string) { - b.mu.Lock() - defer b.mu.Unlock() - b.token = token - close(b.tokenSet) - b.tokenSet = make(chan struct{}) -} - -// Status reports the tunnel's state and traffic. -func (b *Backend) Status() *bridgev1.TunnelStatus { - b.mu.Lock() - defer b.mu.Unlock() - st := &bridgev1.TunnelStatus{ - State: b.state, - LastError: b.lastErr, - Reconnects: b.reconnects, - IngressConns: b.ingressConns.Load(), - EgressConns: b.egressConns.Load(), - IngressConnsTotal: b.ingressTotal.Load(), - EgressConnsTotal: b.egressTotal.Load(), - BytesSent: b.bytesSent.Load(), - BytesReceived: b.bytesReceived.Load(), - } - if !b.liveSince.IsZero() { - st.LiveSinceUnixSeconds = b.liveSince.Unix() - } - return st -} - -// awaitLive returns the stream new outbound conns take, waiting a while for -// one, or nil. -func (b *Backend) awaitLive(ctx context.Context) *stream { - timer := time.NewTimer(b.timings.liveWait) - defer timer.Stop() - for { - b.mu.Lock() - s, ch := b.live, b.liveCh - b.mu.Unlock() - if s != nil { - return s - } - select { - case <-ch: - case <-timer.C: - return nil - case <-ctx.Done(): - return nil - } - } -} - -// run replaces the tunnel's stream whenever it ends, for as long as it takes: -// the relay's pods roll often. A stream the relay says to move off keeps its -// open conns while the next one opens. -func (b *Backend) run() error { - delay := retryMin - first := true - warnedTaken := false - for { - token, ok := b.awaitToken() - if !ok { - return nil - } - if !first { - b.mu.Lock() - b.reconnects++ - b.mu.Unlock() - } - first = false - - s := b.newStream() - ended := make(chan error, 1) - moved := make(chan struct{}) - go func() { ended <- s.run(token, sync.OnceFunc(func() { close(moved) })) }() - - var err error - select { - case <-moved: - b.logger.Info("Relay pod is shutting down, opening a new stream") - continue - case err = <-ended: - } - if b.ctx.Err() != nil { - return nil - } - if s.held.Load() { - delay = retryMin - warnedTaken = false - } - code := status.Code(err) - b.mu.Lock() - if err != nil { - b.lastErr = err.Error() - } else { - b.lastErr = "the relay ended the stream" - } - b.mu.Unlock() - if fatal[code] { - return fmt.Errorf("the relay refused the tunnel: %w", err) - } - - logger := b.logger.With("code", code.String(), "error", err) - switch code { - case codes.Unauthenticated: - // Each create-session replaces the client token. - b.refuseToken(token) - logger.Warn("The relay refused the client token, waiting for a new one") - continue - case codes.AlreadyExists: - b.setState(bridgev1.TunnelState_TUNNEL_STATE_TAKEN) - if !warnedTaken { - warnedTaken = true - logger.Warn("Another client holds the tunnel, waiting for it to let go") - } - default: - b.mu.Lock() - if b.held == 0 { - b.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING - } - b.mu.Unlock() - logger.Info("Tunnel stream ended, reconnecting", "delay", delay) - } - - select { - case <-time.After(delay): - case <-b.ctx.Done(): - return nil - } - delay = min(delay*2, retryMax) - } -} - -// awaitToken returns the client token, waiting for SetToken while there is -// none. -func (b *Backend) awaitToken() (string, bool) { - for { - b.mu.Lock() - token, set := b.token, b.tokenSet - if token == "" { - b.state = bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN - } - b.mu.Unlock() - if token != "" { - return token, true - } - select { - case <-set: - case <-b.ctx.Done(): - return "", false - } - } -} - -// refuseToken forgets token, unless SetToken already replaced it. -func (b *Backend) refuseToken(token string) { - b.mu.Lock() - defer b.mu.Unlock() - if b.token == token { - b.token = "" - } -} - -func (b *Backend) setState(state bridgev1.TunnelState) { - b.mu.Lock() - defer b.mu.Unlock() - b.state = state -} - -// setLive makes s the stream new outbound conns take; nil for none. -func (b *Backend) setLive(s *stream) { - b.live = s - if s != nil && !b.liveClosed { - close(b.liveCh) - b.liveClosed = true - } else if s == nil && b.liveClosed { - b.liveCh = make(chan struct{}) - b.liveClosed = false - } -} - -// holding: the relay routes the tunnel's conns to s. -func (b *Backend) holding(s *stream) { - b.mu.Lock() - defer b.mu.Unlock() - b.held++ - b.setLive(s) - if b.state != bridgev1.TunnelState_TUNNEL_STATE_LIVE { - b.state = bridgev1.TunnelState_TUNNEL_STATE_LIVE - b.liveSince = time.Now() - b.logger.Info("Tunnel live", "reconnects", b.reconnects) - } -} - -// moving: s gets no new conns, but finishes the ones open on it. -func (b *Backend) moving(s *stream) { - b.mu.Lock() - defer b.mu.Unlock() - if b.live == s { - b.setLive(nil) - } -} - -// lost: s ended. -func (b *Backend) lost(s *stream) { - b.mu.Lock() - defer b.mu.Unlock() - if s.held.Load() { - b.held-- - } - if b.live == s { - b.setLive(nil) - } - if b.held == 0 && b.state == bridgev1.TunnelState_TUNNEL_STATE_LIVE { - b.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING - b.liveSince = time.Time{} - } -} - -var errSilent = errors.New("no heartbeat from the relay") diff --git a/pkg/devtunnel/call.go b/pkg/devtunnel/call.go new file mode 100644 index 0000000..0c57cc9 --- /dev/null +++ b/pkg/devtunnel/call.go @@ -0,0 +1,263 @@ +package devtunnel + +import ( + "context" + "errors" + "fmt" + "io" + "log/slog" + "strconv" + "sync" + "sync/atomic" + "time" + + "github.com/puzpuzpuz/xsync/v3" + "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/tunnel" + "google.golang.org/grpc/metadata" +) + +var heartbeat = &devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Heartbeat{Heartbeat: &devtunnel.Heartbeat{}}} + +// call is one Connect call. The relay ends every conn on it with it. +type call struct { + d *Stream + logger *slog.Logger + // The relay numbers each call's conns from 1, so the tunnel's id for one + // starts with this. + prefix string + ctx context.Context + cancel context.CancelCauseFunc + out outbox + // The tunnel's id for each conn on the call, by its id here. + ids *xsync.MapOf[uint64, string] + + // Set once the relay routes the tunnel's conns to this call. + held atomic.Bool + lastHeard atomic.Int64 + closed atomic.Bool +} + +func (d *Stream) newCall() *call { + ctx, cancel := context.WithCancelCause(d.ctx) + return &call{ + d: d, + logger: d.logger, + prefix: strconv.FormatUint(d.nextCall.Add(1), 10) + ".", + ctx: ctx, + cancel: cancel, + out: outbox{wake: make(chan struct{}, 1)}, + ids: xsync.NewMapOf[uint64, string](), + } +} + +// run holds the call until the relay ends it. moving is called if the relay +// says to reconnect elsewhere; the call keeps its conns until then. +func (c *call) run(token string, moving func()) error { + defer c.cancel(nil) + ctx := metadata.AppendToOutgoingContext(c.ctx, + "authorization", "Bearer "+token, + instanceHeader, c.d.instance, + ) + stream, err := c.d.rpc.Connect(ctx) + if err != nil { + return err + } + defer c.d.lost(c) + defer c.endConns() + + c.lastHeard.Store(time.Now().UnixNano()) + // The call starts with its first message. + c.send(heartbeat) + go func() { + for { + frame, ok := c.out.pop(c.ctx) + if !ok { + return + } + // A failed send ends the call, which Recv reports. + if err := stream.Send(frame); err != nil { + return + } + } + }() + go c.heartbeats() + + for { + msg, err := stream.Recv() + if err != nil { + if cause := context.Cause(c.ctx); errors.Is(cause, errSilent) { + return cause + } + if errors.Is(err, io.EOF) { + return nil + } + return err + } + c.lastHeard.Store(time.Now().UnixNano()) + c.handle(msg, moving) + } +} + +func (c *call) heartbeats() { + ticker := time.NewTicker(c.d.timings.heartbeat) + defer ticker.Stop() + for { + select { + case <-c.ctx.Done(): + return + case <-ticker.C: + if time.Since(time.Unix(0, c.lastHeard.Load())) > c.d.timings.silent { + c.logger.Info("Relay went silent, reconnecting") + c.cancel(errSilent) + return + } + c.send(heartbeat) + } + } +} + +// handle hands one of the relay's frames to the tunnel as a Message. +func (c *call) handle(msg *devtunnel.ConnectResponse, moving func()) { + d := c.d + switch f := msg.GetFrame().(type) { + case *devtunnel.ConnectResponse_Heartbeat: + if c.held.CompareAndSwap(false, true) { + d.holding(c) + } + case *devtunnel.ConnectResponse_Reconnect: + d.moving(c) + moving() + case *devtunnel.ConnectResponse_Open: + c.open(f.Open) + case *devtunnel.ConnectResponse_Data: + if id, _, ok := c.route(f.Data.GetConnId()); ok { + d.bytesReceived.Add(int64(len(f.Data.GetPayload()))) + d.deliver(&tunnel.Message{Kind: tunnel.MessageData, ConnID: id, Payload: f.Data.GetPayload()}) + } + case *devtunnel.ConnectResponse_Close: + if id, r, ok := c.route(f.Close.GetConnId()); ok { + r.peerClosed.Store(true) + d.deliver(&tunnel.Message{Kind: tunnel.MessageCloseWrite, ConnID: id}) + if r.closeSent.Load() { + d.forget(id, r) + } + } + case *devtunnel.ConnectResponse_Reset_: + if id, r, ok := c.route(f.Reset_.GetConnId()); ok && d.forget(id, r) { + c.logger.Debug("Relay reset the connection", "conn_id", id, "reason", f.Reset_.GetReason()) + d.deliver(&tunnel.Message{Kind: tunnel.MessageReset, ConnID: id, Reason: f.Reset_.GetReason()}) + } + case *devtunnel.ConnectResponse_Window: + if id, _, ok := c.route(f.Window.GetConnId()); ok { + d.deliver(&tunnel.Message{Kind: tunnel.MessageWindow, ConnID: id, Bytes: int(f.Window.GetBytes())}) + } + } +} + +// open hands one of the deployment's conns to the tunnel, which connects it +// to the app. +func (c *call) open(open *devtunnel.Open) { + wire := open.GetConnId() + port := c.d.cfg.Port + if open.GetProtocol() != devtunnel.Protocol_PROTOCOL_TCP || int(open.GetPort()) != port { + c.send(resetFrame(wire, fmt.Sprintf("bridge serves only port %d", port))) + return + } + id := c.prefix + strconv.FormatUint(wire, 10) + if !c.track(id, &route{c: c, wire: wire, dir: ingress}) { + return + } + c.d.deliver(&tunnel.Message{ + Kind: tunnel.MessageOpen, + ConnID: id, + Conn: &tunnel.ConnInfo{Dest: tunnel.Addr{Host: "127.0.0.1", Port: port}}, + }) +} + +// route returns the tunnel's id and route for the conn the relay calls wire, +// if it's open on this call. +func (c *call) route(wire uint64) (string, *route, bool) { + id, ok := c.ids.Load(wire) + if !ok { + return "", nil, false + } + r, ok := c.d.routes.Load(id) + if !ok || r.c != c { + return "", nil, false + } + return id, r, true +} + +// track routes id to this call, or reports false once the call has ended. +func (c *call) track(id string, r *route) bool { + if c.closed.Load() { + return false + } + // Counted before it's stored, so whichever forget removes it uncounts it. + c.d.counters(r.dir, 1) + c.d.routes.Store(id, r) + c.ids.Store(r.wire, id) + // endConns may have swept the routes before the Store. + if c.closed.Load() { + c.d.forget(id, r) + return false + } + return true +} + +// endConns resets every conn on the call, which the relay ended with it. +func (c *call) endConns() { + c.closed.Store(true) + c.d.routes.Range(func(id string, r *route) bool { + if r.c == c && c.d.forget(id, r) { + c.d.deliver(&tunnel.Message{Kind: tunnel.MessageReset, ConnID: id, Reason: "the tunnel's stream ended"}) + } + return true + }) +} + +func (c *call) send(frame *devtunnel.ConnectRequest) { + c.out.push(frame) +} + +// outbox queues frames for the relay in order. Data is bounded by each conn's +// window; the rest is small. +type outbox struct { + mu sync.Mutex + frames []*devtunnel.ConnectRequest + wake chan struct{} +} + +func (o *outbox) push(frame *devtunnel.ConnectRequest) { + o.mu.Lock() + o.frames = append(o.frames, frame) + o.mu.Unlock() + select { + case o.wake <- struct{}{}: + default: + } +} + +func (o *outbox) pop(ctx context.Context) (*devtunnel.ConnectRequest, bool) { + for { + o.mu.Lock() + if len(o.frames) > 0 { + frame := o.frames[0] + o.frames[0] = nil + o.frames = o.frames[1:] + o.mu.Unlock() + return frame, true + } + o.mu.Unlock() + select { + case <-o.wake: + case <-ctx.Done(): + return nil, false + } + } +} + +func resetFrame(id uint64, reason string) *devtunnel.ConnectRequest { + return &devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Reset_{Reset_: &devtunnel.Reset{ConnId: id, Reason: reason}}} +} diff --git a/pkg/devtunnel/conn.go b/pkg/devtunnel/conn.go deleted file mode 100644 index edffed3..0000000 --- a/pkg/devtunnel/conn.go +++ /dev/null @@ -1,89 +0,0 @@ -package devtunnel - -import ( - "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" - "github.com/vercel/bridge/pkg/plumbing" - "github.com/vercel/bridge/pkg/tunnel" -) - -// Per Data frame. -const maxPayload = 64 << 10 - -type direction int - -const ( - // From the deployment to the app. - ingress direction = iota - // From the app out through the deployment. - egress -) - -// relayFrames writes one conn's frames on a Connect stream: Data, Close, -// Reset, and a Window for each payload received. -type relayFrames struct { - s *stream - id string - wire uint64 - dir direction - stream *plumbing.MuxStream -} - -// newConn opens id on the stream as the relay's conn wire, or returns nil -// once the stream has ended. -func (s *stream) newConn(id string, wire uint64, dir direction, remote tunnel.Addr) *relayFrames { - if s.closed.Load() { - return nil - } - f := &relayFrames{s: s, id: id, wire: wire, dir: dir} - f.stream = plumbing.NewMuxStream(f, plumbing.MuxConfig{ - MaxFrame: maxPayload, - Window: connWindow, - Local: addr("bridge"), - Remote: addr(remote.String()), - }) - s.b.counters(dir, 1) - s.b.conns.Store(id, f) - // endConns may have swept the conns before the Store. - if s.closed.Load() { - f.stream.PeerReset() - return nil - } - return f -} - -func (f *relayFrames) Data(payload []byte) error { - f.s.b.bytesSent.Add(int64(len(payload))) - f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: f.wire, Payload: payload}}}) - return nil -} - -func (f *relayFrames) CloseWrite() error { - f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: f.wire}}}) - return nil -} - -func (f *relayFrames) Reset(reason string) error { - f.s.send(resetFrame(f.wire, reason)) - return nil -} - -func (f *relayFrames) Window(n int) error { - f.s.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: f.wire, Bytes: uint32(n)}}}) - return nil -} - -// Release forgets the conn. -func (f *relayFrames) Release() { - f.s.b.conns.Compute(f.id, func(cur *relayFrames, loaded bool) (*relayFrames, bool) { - if !loaded { - return nil, true - } - return cur, cur == f - }) - f.s.b.counters(f.dir, -1) -} - -type addr string - -func (a addr) Network() string { return "dev-tunnel" } -func (a addr) String() string { return string(a) } diff --git a/pkg/devtunnel/stream.go b/pkg/devtunnel/stream.go index 6e6c3a2..c500a2c 100644 --- a/pkg/devtunnel/stream.go +++ b/pkg/devtunnel/stream.go @@ -1,237 +1,599 @@ +// Package devtunnel carries a tunnel over a dev tunnel: the Connect stream to +// api-dev-tunnel that `vc dev --tunnel` holds otherwise. Stream translates the +// relay's frames to and from tunnel Messages, so the deployment's conns reach +// the app and the app's outbound conns leave through the deployment. package devtunnel import ( "context" + "crypto/tls" "errors" "fmt" "io" "log/slog" - "strconv" + "net" + "net/url" + "regexp" "sync" "sync/atomic" "time" + "github.com/puzpuzpuz/xsync/v3" + "github.com/segmentio/ksuid" + bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" "github.com/vercel/bridge/api/go/vercel/api/dev_tunnel/v1" + "github.com/vercel/bridge/pkg/plumbing" "github.com/vercel/bridge/pkg/tunnel" - "google.golang.org/grpc/metadata" + "google.golang.org/grpc" + "google.golang.org/grpc/codes" + "google.golang.org/grpc/credentials" + "google.golang.org/grpc/credentials/insecure" + "google.golang.org/grpc/keepalive" + "google.golang.org/grpc/status" ) -var heartbeat = &devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Heartbeat{Heartbeat: &devtunnel.Heartbeat{}}} - -// stream is one Connect call. The relay ends every conn on it with it. -type stream struct { - b *Backend - logger *slog.Logger - // The relay numbers each stream's conns from 1, so the tunnel's id for one - // starts with this. - prefix string - ctx context.Context - cancel context.CancelCauseFunc - out outbox - - // Set once the relay routes the tunnel's conns to this stream. - held atomic.Bool - lastHeard atomic.Int64 - closed atomic.Bool -} - -func (b *Backend) newStream() *stream { - ctx, cancel := context.WithCancelCause(b.ctx) - return &stream{ - b: b, - logger: b.logger, - prefix: strconv.FormatUint(b.nextStream.Add(1), 10) + ".", - ctx: ctx, - cancel: cancel, - out: outbox{wake: make(chan struct{}, 1)}, - } -} - -// run holds the stream until the relay ends it. moving is called if the -// relay says to reconnect elsewhere; the stream keeps its conns until then. -func (s *stream) run(token string, moving func()) error { - defer s.cancel(nil) - ctx := metadata.AppendToOutgoingContext(s.ctx, - "authorization", "Bearer "+token, - instanceHeader, s.b.instance, - ) - call, err := s.b.rpc.Connect(ctx) +const ( + // What each side of a conn may send before the other grants more. + connWindow = 256 << 10 + // A client numbers its conns from here; the relay numbers its own from 1. + clientConnIDFloor = uint64(1) << 63 + // The relay's frames waiting for the tunnel to take them. + queuedMessages = 256 + + instanceHeader = "x-vercel-dev-tunnel-instance" + + retryMin = 250 * time.Millisecond + retryMax = 5 * time.Second +) + +var instancePattern = regexp.MustCompile(`^[A-Za-z0-9-]{1,64}$`) + +// Codes reconnecting can't fix. +var fatal = map[codes.Code]bool{ + codes.FailedPrecondition: true, + codes.InvalidArgument: true, + codes.PermissionDenied: true, + codes.Unimplemented: true, +} + +// Config configures a Stream. +type Config struct { + // RelayURL is api-dev-tunnel's URL, from create-session. + RelayURL string + // Token is the client token, from create-session. Without one the stream + // waits for SetToken. + Token string + // Instance tells the relay this process's reconnects from another + // client's, so a restart can take the tunnel back while its old stream is + // still open. Random if empty. + Instance string + // Port is the only port of the app the deployment's conns may reach. + Port int + // RelayDialer connects to the relay. Defaults to a net.Dialer. + RelayDialer plumbing.ContextDialer + Logger *slog.Logger +} + +// timings are the relay's protocol intervals, shortened by tests. +type timings struct { + // The relay needs a heartbeat at least every 30s. + heartbeat time.Duration + // The relay heartbeats every 30s; three missed and the stream is dead. + silent time.Duration + // The relay holds the deployment's conns this long for a client, and an + // outbound conn waits as long for a stream. + liveWait time.Duration +} + +var defaultTimings = timings{ + heartbeat: 15 * time.Second, + silent: 90 * time.Second, + liveWait: 5 * time.Second, +} + +// Stream holds a dev tunnel's Connect stream to its relay, replacing it +// whenever it drops, as one tunnel.Stream. +type Stream struct { + cfg Config + timings timings + logger *slog.Logger + instance string + target string + creds credentials.TransportCredentials + + // Set by Start, before any call. + rpc devtunnel.TunnelServiceClient + ctx context.Context + + msgs chan *tunnel.Message + done chan struct{} + // Why it stopped; read after done. + err error + + // Each open conn's route, by the tunnel's id for it. + routes *xsync.MapOf[string, *route] + + mu sync.Mutex + token string + tokenSet chan struct{} + // The newest call holding the tunnel, which new outbound conns take. + live *call + liveCh chan struct{} + liveClosed bool + // Calls holding the tunnel, including one finishing its conns after the + // relay said to reconnect. + held int + state bridgev1.TunnelState + lastErr string + liveSince time.Time + reconnects int64 + + ingressConns, egressConns, ingressTotal, egressTotal atomic.Int64 + bytesSent, bytesReceived atomic.Int64 + nextEgress, nextCall atomic.Uint64 +} + +// route is where one of the tunnel's conns lives on the relay's protocol. +type route struct { + c *call + // The conn's id on c. + wire uint64 + dir direction + // Both sides have half-closed once both are set. + closeSent, peerClosed atomic.Bool +} + +type direction int + +const ( + // From the deployment to the app. + ingress direction = iota + // From the app out through the deployment. + egress +) + +var _ tunnel.Stream = (*Stream)(nil) + +// New returns a Stream for cfg. Nothing connects until Start. +func New(cfg Config) (*Stream, error) { + target, creds, err := relayTarget(cfg.RelayURL) if err != nil { - return err + return nil, err + } + instance := cfg.Instance + if instance == "" { + instance = "bridge-" + ksuid.New().String() } - defer s.b.lost(s) - defer s.endConns() + if !instancePattern.MatchString(instance) { + return nil, fmt.Errorf("tunnel instance must be 1 to 64 letters, digits or dashes: %q", instance) + } + if cfg.RelayDialer == nil { + cfg.RelayDialer = &net.Dialer{Timeout: 10 * time.Second} + } + logger := cfg.Logger + if logger == nil { + logger = slog.Default() + } + return &Stream{ + cfg: cfg, + timings: defaultTimings, + logger: logger.With("component", "dev-tunnel", "relay", cfg.RelayURL, "instance", instance), + instance: instance, + target: target, + creds: creds, + msgs: make(chan *tunnel.Message, queuedMessages), + done: make(chan struct{}), + routes: xsync.NewMapOf[string, *route](), + token: cfg.Token, + tokenSet: make(chan struct{}), + liveCh: make(chan struct{}), + state: bridgev1.TunnelState_TUNNEL_STATE_CONNECTING, + }, nil +} - s.lastHeard.Store(time.Now().UnixNano()) - // The call starts with its first message. - s.send(heartbeat) - go func() { - for { - frame, ok := s.out.pop(s.ctx) - if !ok { - return - } - // A failed send ends the call, which Recv reports. - if err := call.Send(frame); err != nil { - return - } +// relayTarget turns the relay's URL into a gRPC target. Start dials it with +// RelayDialer rather than resolving it through gRPC, so an intercept whose DNS +// forwards every domain still reaches the relay directly. +func relayTarget(relayURL string) (string, credentials.TransportCredentials, error) { + u, err := url.Parse(relayURL) + if err != nil { + return "", nil, fmt.Errorf("invalid relay URL %q: %w", relayURL, err) + } + if u.Path != "" && u.Path != "/" { + return "", nil, fmt.Errorf("relay URL %q must have no path", relayURL) + } + var creds credentials.TransportCredentials + port := u.Port() + switch u.Scheme { + case "https": + creds = credentials.NewTLS(&tls.Config{MinVersion: tls.VersionTLS12}) + if port == "" { + port = "443" } - }() - go s.heartbeats() - - for { - msg, err := call.Recv() - if err != nil { - if cause := context.Cause(s.ctx); errors.Is(cause, errSilent) { - return cause - } - if errors.Is(err, io.EOF) { - return nil - } - return err + case "http": + creds = insecure.NewCredentials() + if port == "" { + port = "80" } - s.lastHeard.Store(time.Now().UnixNano()) - s.handle(msg, moving) + default: + return "", nil, fmt.Errorf("relay URL %q must be http or https", relayURL) + } + if u.Hostname() == "" { + return "", nil, fmt.Errorf("relay URL %q has no host", relayURL) } + return "passthrough:///" + net.JoinHostPort(u.Hostname(), port), creds, nil } -func (s *stream) heartbeats() { - ticker := time.NewTicker(s.b.timings.heartbeat) - defer ticker.Stop() - for { - select { - case <-s.ctx.Done(): - return - case <-ticker.C: - if time.Since(time.Unix(0, s.lastHeard.Load())) > s.b.timings.silent { - s.logger.Info("Relay went silent, reconnecting") - s.cancel(errSilent) - return - } - s.send(heartbeat) +// Start holds the tunnel until ctx ends, replacing its Connect stream +// whenever it drops. Recv fails once it stops: when ctx ends, or the relay +// refuses the tunnel for good. +func (d *Stream) Start(ctx context.Context) { + d.ctx = ctx + go func() { + err := d.hold() + if err != nil { + d.mu.Lock() + d.state = bridgev1.TunnelState_TUNNEL_STATE_FAILED + d.lastErr = err.Error() + d.mu.Unlock() + d.logger.Error("Tunnel stopped", "error", err) } + d.err = err + close(d.done) + }() +} + +func (d *Stream) hold() error { + relayDialer := d.cfg.RelayDialer + cc, err := grpc.NewClient(d.target, + grpc.WithTransportCredentials(d.creds), + grpc.WithContextDialer(func(ctx context.Context, addr string) (net.Conn, error) { + return relayDialer.DialContext(ctx, "tcp", addr) + }), + // Notices a dead connection, e.g. after sleep, sooner than the + // relay's heartbeats would. + grpc.WithKeepaliveParams(keepalive.ClientParameters{ + Time: 10 * time.Second, + Timeout: 20 * time.Second, + PermitWithoutStream: true, + }), + ) + if err != nil { + return fmt.Errorf("creating relay client: %w", err) } + defer cc.Close() + d.rpc = devtunnel.NewTunnelServiceClient(cc) + return d.run() } -// handle passes one of the relay's frames to the tunnel. -func (s *stream) handle(msg *devtunnel.ConnectResponse, moving func()) { - switch f := msg.GetFrame().(type) { - case *devtunnel.ConnectResponse_Heartbeat: - if s.held.CompareAndSwap(false, true) { - s.b.holding(s) - } - case *devtunnel.ConnectResponse_Reconnect: - s.b.moving(s) - moving() - case *devtunnel.ConnectResponse_Open: - s.open(f.Open) - case *devtunnel.ConnectResponse_Data: - if c := s.conn(f.Data.GetConnId()); c != nil { - s.b.bytesReceived.Add(int64(len(f.Data.GetPayload()))) - c.stream.Deliver(s.ctx, f.Data.GetPayload()) - } - case *devtunnel.ConnectResponse_Close: - if c := s.conn(f.Close.GetConnId()); c != nil { - c.stream.PeerClosed() - } - case *devtunnel.ConnectResponse_Reset_: - if c := s.conn(f.Reset_.GetConnId()); c != nil { - s.logger.Debug("Relay reset the connection", "conn_id", c.id, "reason", f.Reset_.GetReason()) - c.stream.PeerReset() - } - case *devtunnel.ConnectResponse_Window: - if c := s.conn(f.Window.GetConnId()); c != nil { - c.stream.Conn().Grant(int(f.Window.GetBytes())) +func (d *Stream) Features() tunnel.Features { + return tunnel.Features{Window: connWindow, HalfClose: true} +} + +func (d *Stream) Recv() (*tunnel.Message, error) { + select { + case m := <-d.msgs: + return m, nil + case <-d.done: + if d.err != nil { + return nil, d.err } + return nil, io.EOF } } -// open hands one of the deployment's conns to the tunnel, which connects it -// to the app. -func (s *stream) open(open *devtunnel.Open) { - wire := open.GetConnId() - port := s.b.cfg.Port - if open.GetProtocol() != devtunnel.Protocol_PROTOCOL_TCP || int(open.GetPort()) != port { - s.send(resetFrame(wire, fmt.Sprintf("bridge serves only port %d", port))) - return +func (d *Stream) Send(m *tunnel.Message) error { + if m.Kind == tunnel.MessageOpen { + return d.open(m) } - dest := tunnel.Addr{Host: "127.0.0.1", Port: port} - if f := s.newConn(s.idFor(wire), wire, ingress, dest); f != nil { - s.b.accept(f.stream.Conn(), &tunnel.ConnInfo{Dest: dest}) + r, ok := d.routes.Load(m.ConnID) + if !ok { + // The conn is over. + return nil + } + switch m.Kind { + case tunnel.MessageData: + d.bytesSent.Add(int64(len(m.Payload))) + r.c.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Data{Data: &devtunnel.Data{ConnId: r.wire, Payload: m.Payload}}}) + case tunnel.MessageCloseWrite: + r.closeSent.Store(true) + r.c.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Close{Close: &devtunnel.Close{ConnId: r.wire}}}) + if r.peerClosed.Load() { + d.forget(m.ConnID, r) + } + case tunnel.MessageReset: + r.c.send(resetFrame(r.wire, m.Reason)) + d.forget(m.ConnID, r) + case tunnel.MessageWindow: + r.c.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Window{Window: &devtunnel.Window{ConnId: r.wire, Bytes: uint32(m.Bytes)}}}) } + return nil } -// idFor is the backend's id for the conn the relay calls wire on this stream. -// The relay numbers each stream's conns from 1; outbound conns are numbered -// across streams. -func (s *stream) idFor(wire uint64) string { - if wire >= clientConnIDFloor { - return "e" + strconv.FormatUint(wire&^clientConnIDFloor, 10) +// open sends an outbound conn out through the deployment, to its hostname if +// the intercept's DNS gave it one, or else to the address it was headed for. +// The deployment resolves the hostname as its own code would. It waits a +// while for a call to hold the tunnel. +func (d *Stream) open(m *tunnel.Message) error { + c := d.awaitLive() + if c == nil { + return errors.New("no stream holds the tunnel") + } + r := &route{c: c, wire: clientConnIDFloor | d.nextEgress.Add(1), dir: egress} + if !c.track(m.ConnID, r) { + return errors.New("the tunnel's stream ended") } - return s.prefix + strconv.FormatUint(wire, 10) + host := m.Conn.Hostname + if host == "" { + host = m.Conn.Dest.Host + } + c.send(&devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Open{Open: &devtunnel.Open{ + ConnId: r.wire, + Protocol: devtunnel.Protocol_PROTOCOL_TCP, + Host: host, + Port: uint32(m.Conn.Dest.Port), + }}}) + return nil } -// conn returns the conn the relay calls wire, if it's open on this stream. -func (s *stream) conn(wire uint64) *relayFrames { - f, ok := s.b.conns.Load(s.idFor(wire)) - if !ok || f.s != s { - return nil +// deliver hands the tunnel one of the relay's messages. +func (d *Stream) deliver(m *tunnel.Message) { + select { + case d.msgs <- m: + case <-d.done: } - return f } -// endConns ends every conn on the stream, which the relay ended with it. -func (s *stream) endConns() { - s.closed.Store(true) - s.b.conns.Range(func(_ string, f *relayFrames) bool { - if f.s == s { - f.stream.PeerReset() +// forget drops id's route if it's still r. Reports whether it did. +func (d *Stream) forget(id string, r *route) bool { + removed := false + d.routes.Compute(id, func(cur *route, loaded bool) (*route, bool) { + if !loaded { + return nil, true + } + if cur != r { + return cur, false } - return true + removed = true + return cur, true }) + if removed { + r.c.ids.Delete(r.wire) + d.counters(r.dir, -1) + } + return removed } -func (s *stream) send(frame *devtunnel.ConnectRequest) { - s.out.push(frame) +// counters tracks conns opening (delta 1) and ending (delta -1). +func (d *Stream) counters(dir direction, delta int64) { + open, total := &d.ingressConns, &d.ingressTotal + if dir == egress { + open, total = &d.egressConns, &d.egressTotal + } + open.Add(delta) + if delta > 0 { + total.Add(delta) + } } -// outbox queues frames for the relay in order. Data is bounded by each conn's -// window; the rest is small. -type outbox struct { - mu sync.Mutex - frames []*devtunnel.ConnectRequest - wake chan struct{} +// SetToken replaces the client token. A stream waiting for one connects with +// it; a live one uses it on its next call. +func (d *Stream) SetToken(token string) { + d.mu.Lock() + defer d.mu.Unlock() + d.token = token + close(d.tokenSet) + d.tokenSet = make(chan struct{}) } -func (o *outbox) push(frame *devtunnel.ConnectRequest) { - o.mu.Lock() - o.frames = append(o.frames, frame) - o.mu.Unlock() - select { - case o.wake <- struct{}{}: - default: +// Status reports the tunnel's state and traffic. +func (d *Stream) Status() *bridgev1.TunnelStatus { + d.mu.Lock() + defer d.mu.Unlock() + st := &bridgev1.TunnelStatus{ + State: d.state, + LastError: d.lastErr, + Reconnects: d.reconnects, + IngressConns: d.ingressConns.Load(), + EgressConns: d.egressConns.Load(), + IngressConnsTotal: d.ingressTotal.Load(), + EgressConnsTotal: d.egressTotal.Load(), + BytesSent: d.bytesSent.Load(), + BytesReceived: d.bytesReceived.Load(), + } + if !d.liveSince.IsZero() { + st.LiveSinceUnixSeconds = d.liveSince.Unix() } + return st } -func (o *outbox) pop(ctx context.Context) (*devtunnel.ConnectRequest, bool) { +// awaitLive returns the call new outbound conns take, waiting a while for +// one, or nil. +func (d *Stream) awaitLive() *call { + timer := time.NewTimer(d.timings.liveWait) + defer timer.Stop() for { - o.mu.Lock() - if len(o.frames) > 0 { - frame := o.frames[0] - o.frames[0] = nil - o.frames = o.frames[1:] - o.mu.Unlock() - return frame, true - } - o.mu.Unlock() + d.mu.Lock() + c, ch := d.live, d.liveCh + d.mu.Unlock() + if c != nil { + return c + } select { - case <-o.wake: - case <-ctx.Done(): - return nil, false + case <-ch: + case <-timer.C: + return nil + case <-d.done: + return nil } } } -func resetFrame(id uint64, reason string) *devtunnel.ConnectRequest { - return &devtunnel.ConnectRequest{Frame: &devtunnel.ConnectRequest_Reset_{Reset_: &devtunnel.Reset{ConnId: id, Reason: reason}}} +// run replaces the tunnel's call whenever it ends, for as long as it takes: +// the relay's pods roll often. A call the relay says to move off keeps its +// open conns while the next one opens. +func (d *Stream) run() error { + delay := retryMin + first := true + warnedTaken := false + for { + token, ok := d.awaitToken() + if !ok { + return nil + } + if !first { + d.mu.Lock() + d.reconnects++ + d.mu.Unlock() + } + first = false + + c := d.newCall() + ended := make(chan error, 1) + moved := make(chan struct{}) + go func() { ended <- c.run(token, sync.OnceFunc(func() { close(moved) })) }() + + var err error + select { + case <-moved: + d.logger.Info("Relay pod is shutting down, opening a new stream") + continue + case err = <-ended: + } + if d.ctx.Err() != nil { + return nil + } + if c.held.Load() { + delay = retryMin + warnedTaken = false + } + code := status.Code(err) + d.mu.Lock() + if err != nil { + d.lastErr = err.Error() + } else { + d.lastErr = "the relay ended the stream" + } + d.mu.Unlock() + if fatal[code] { + return fmt.Errorf("the relay refused the tunnel: %w", err) + } + + logger := d.logger.With("code", code.String(), "error", err) + switch code { + case codes.Unauthenticated: + // Each create-session replaces the client token. + d.refuseToken(token) + logger.Warn("The relay refused the client token, waiting for a new one") + continue + case codes.AlreadyExists: + d.setState(bridgev1.TunnelState_TUNNEL_STATE_TAKEN) + if !warnedTaken { + warnedTaken = true + logger.Warn("Another client holds the tunnel, waiting for it to let go") + } + default: + d.mu.Lock() + if d.held == 0 { + d.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING + } + d.mu.Unlock() + logger.Info("Tunnel stream ended, reconnecting", "delay", delay) + } + + select { + case <-time.After(delay): + case <-d.ctx.Done(): + return nil + } + delay = min(delay*2, retryMax) + } +} + +// awaitToken returns the client token, waiting for SetToken while there is +// none. +func (d *Stream) awaitToken() (string, bool) { + for { + d.mu.Lock() + token, set := d.token, d.tokenSet + if token == "" { + d.state = bridgev1.TunnelState_TUNNEL_STATE_NEEDS_TOKEN + } + d.mu.Unlock() + if token != "" { + return token, true + } + select { + case <-set: + case <-d.ctx.Done(): + return "", false + } + } +} + +// refuseToken forgets token, unless SetToken already replaced it. +func (d *Stream) refuseToken(token string) { + d.mu.Lock() + defer d.mu.Unlock() + if d.token == token { + d.token = "" + } +} + +func (d *Stream) setState(state bridgev1.TunnelState) { + d.mu.Lock() + defer d.mu.Unlock() + d.state = state } + +// setLive makes c the call new outbound conns take; nil for none. +func (d *Stream) setLive(c *call) { + d.live = c + if c != nil && !d.liveClosed { + close(d.liveCh) + d.liveClosed = true + } else if c == nil && d.liveClosed { + d.liveCh = make(chan struct{}) + d.liveClosed = false + } +} + +// holding: the relay routes the tunnel's conns to c. +func (d *Stream) holding(c *call) { + d.mu.Lock() + defer d.mu.Unlock() + d.held++ + d.setLive(c) + if d.state != bridgev1.TunnelState_TUNNEL_STATE_LIVE { + d.state = bridgev1.TunnelState_TUNNEL_STATE_LIVE + d.liveSince = time.Now() + d.logger.Info("Tunnel live", "reconnects", d.reconnects) + } +} + +// moving: c gets no new conns, but finishes the ones open on it. +func (d *Stream) moving(c *call) { + d.mu.Lock() + defer d.mu.Unlock() + if d.live == c { + d.setLive(nil) + } +} + +// lost: c ended. +func (d *Stream) lost(c *call) { + d.mu.Lock() + defer d.mu.Unlock() + if c.held.Load() { + d.held-- + } + if d.live == c { + d.setLive(nil) + } + if d.held == 0 && d.state == bridgev1.TunnelState_TUNNEL_STATE_LIVE { + d.state = bridgev1.TunnelState_TUNNEL_STATE_CONNECTING + d.liveSince = time.Time{} + } +} + +var errSilent = errors.New("no heartbeat from the relay") diff --git a/pkg/devtunnel/backend_test.go b/pkg/devtunnel/stream_test.go similarity index 96% rename from pkg/devtunnel/backend_test.go rename to pkg/devtunnel/stream_test.go index c8fba0a..1bbcf34 100644 --- a/pkg/devtunnel/backend_test.go +++ b/pkg/devtunnel/stream_test.go @@ -151,19 +151,24 @@ func window(id uint64, n int) *devtunnel.ConnectResponse { return &devtunnel.ConnectResponse{Frame: &devtunnel.ConnectResponse_Window{Window: &devtunnel.Window{ConnId: id, Bytes: uint32(n)}}} } -// newTunnel runs a tunnel over a Backend for r. -func newTunnel(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) (*Backend, tunnel.Tunnel) { +// newTunnel runs a tunnel over a Stream for r. +func newTunnel(t *testing.T, r *fakeRelay, token string, port int, tune ...func(*timings)) (*Stream, tunnel.Tunnel) { t.Helper() - b, err := New(Config{RelayURL: r.url, Token: token, Instance: "test-instance", Port: port}) + d, err := New(Config{RelayURL: r.url, Token: token, Instance: "test-instance", Port: port}) require.NoError(t, err) - b.timings = timings{heartbeat: 50 * time.Millisecond, silent: time.Second, liveWait: 2 * time.Second} + d.timings = timings{heartbeat: 50 * time.Millisecond, silent: time.Second, liveWait: 2 * time.Second} for _, f := range tune { - f(&b.timings) + f(&d.timings) } - tun := tunnel.NewWithBackend(&net.Dialer{Timeout: time.Second}, b) - tun.Start(context.Background()) - t.Cleanup(func() { _ = tun.Close() }) - return b, tun + ctx, cancel := context.WithCancel(context.Background()) + d.Start(ctx) + tun := tunnel.New(&net.Dialer{Timeout: time.Second}, d) + tun.Start(ctx) + t.Cleanup(func() { + cancel() + _ = tun.Close() + }) + return d, tun } // echoApp echoes each connection, then half-closes it once the peer has. diff --git a/pkg/mitm/hijacker.go b/pkg/mitm/hijacker.go index 1daacee..de9fe82 100644 --- a/pkg/mitm/hijacker.go +++ b/pkg/mitm/hijacker.go @@ -14,13 +14,14 @@ import ( // ShouldHijack for each new connection; if it returns true the tunnel calls // Hijack instead of dialing the real destination. type Hijacker interface { - // ShouldHijack returns true if the hijacker wants to handle this connection. - ShouldHijack(msg *bridgev1.TunnelNetworkMessage) bool + // ShouldHijack returns true if the hijacker wants to handle a connection + // to hostname (may be empty). + ShouldHijack(hostname string) bool // Hijack returns a net.Conn that the tunnel reads from and writes to, - // just like a dialed connection. The hijacker is responsible for - // servicing the other end (e.g. via a net.Pipe goroutine). - Hijack(ctx context.Context, msg *bridgev1.TunnelNetworkMessage) (net.Conn, error) + // just like a connection dialed to dest ("host:port"). The hijacker is + // responsible for servicing the other end (e.g. via a net.Pipe goroutine). + Hijack(ctx context.Context, hostname, dest string) (net.Conn, error) } // FacadeHijacker implements Hijacker by matching incoming connections against @@ -54,10 +55,9 @@ func NewFacadeHijacker(facades []*bridgev1.ServerFacade, certPEM, keyPEM []byte) return &FacadeHijacker{ca: ca, facades: compiled}, nil } -// ShouldHijack returns true if the message's hostname matches any facade's -// host glob pattern. -func (h *FacadeHijacker) ShouldHijack(msg *bridgev1.TunnelNetworkMessage) bool { - hostname := msg.GetHostname() +// ShouldHijack returns true if hostname matches any facade's host glob +// pattern. +func (h *FacadeHijacker) ShouldHijack(hostname string) bool { if hostname == "" { return false } @@ -71,9 +71,7 @@ func (h *FacadeHijacker) ShouldHijack(msg *bridgev1.TunnelNetworkMessage) bool { // Hijack returns a net.Conn for the tunnel to read/write. It finds all facades // matching the hostname and creates a Handler to serve the connection. -func (h *FacadeHijacker) Hijack(ctx context.Context, msg *bridgev1.TunnelNetworkMessage) (net.Conn, error) { - hostname := msg.GetHostname() - +func (h *FacadeHijacker) Hijack(ctx context.Context, hostname, dest string) (net.Conn, error) { matched := h.matchingFacades(hostname) if len(matched) == 0 { return nil, fmt.Errorf("no matching facades for hostname %q", hostname) @@ -81,13 +79,10 @@ func (h *FacadeHijacker) Hijack(ctx context.Context, msg *bridgev1.TunnelNetwork tunnelConn, handlerConn := net.Pipe() - dest := msg.GetDest() - destAddr := fmt.Sprintf("%s:%d", dest.GetIp(), dest.GetPort()) - handler := &Handler{ - CA: h.ca, - Facades: matched, - DestAddr: destAddr, + CA: h.ca, + Facades: matched, + DestAddr: dest, } go func() { diff --git a/pkg/proxy/grpcserver.go b/pkg/proxy/grpcserver.go index 75e37fa..96411af 100644 --- a/pkg/proxy/grpcserver.go +++ b/pkg/proxy/grpcserver.go @@ -15,6 +15,7 @@ import ( "github.com/vercel/bridge/pkg/grpcutil" "github.com/vercel/bridge/pkg/mitm" "github.com/vercel/bridge/pkg/tunnel" + "github.com/vercel/bridge/pkg/tunnel/bridgestream" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" "google.golang.org/grpc" @@ -264,7 +265,7 @@ func (s *GRPCServer) TunnelNetwork(stream grpc.BidiStreamingServer[bridgev1.Tunn if s.hijacker != nil { opts = append(opts, tunnel.WithHijacker(s.hijacker)) } - tun := tunnel.New(&net.Dialer{Timeout: egressDialTimeout}, stream, opts...) + tun := tunnel.New(&net.Dialer{Timeout: egressDialTimeout}, bridgestream.New(stream), opts...) s.tunnelMu.Lock() s.tunnel = tun diff --git a/pkg/tunnel/backend.go b/pkg/tunnel/backend.go deleted file mode 100644 index dcb75c7..0000000 --- a/pkg/tunnel/backend.go +++ /dev/null @@ -1,40 +0,0 @@ -package tunnel - -import ( - "context" - "net" - "strconv" -) - -// Backend carries a tunnel's conns over one multiplexed transport: a bridge -// server's TunnelNetwork stream (NewStreamBackend), or a dev tunnel's Connect -// stream (devtunnel.Backend). Each conn is a net.Conn over one of its streams; -// its CloseWrite half-closes it and Close resets it. -type Backend interface { - // Run carries the transport until ctx ends or the transport does, handing - // accept each conn the peer opens, with where it goes. It ends every conn - // before returning. - Run(ctx context.Context, accept func(conn net.Conn, info *ConnInfo)) error - // Open opens a conn through the transport to info.Dest. - Open(ctx context.Context, info *ConnInfo) (net.Conn, error) -} - -// Addr is a host (an IP address or a hostname) and a port. -type Addr struct { - Host string - Port int -} - -func (a Addr) String() string { return net.JoinHostPort(a.Host, strconv.Itoa(a.Port)) } - -// ConnInfo addresses a conn. -type ConnInfo struct { - Source, Dest Addr - // The DNS name Dest was looked up as, if any. - Hostname string -} - -type addr string - -func (a addr) Network() string { return "tunnel" } -func (a addr) String() string { return string(a) } diff --git a/pkg/tunnel/bridgestream/bridgestream.go b/pkg/tunnel/bridgestream/bridgestream.go new file mode 100644 index 0000000..8f93721 --- /dev/null +++ b/pkg/tunnel/bridgestream/bridgestream.go @@ -0,0 +1,125 @@ +// Package bridgestream carries a tunnel over a bridge TunnelNetwork stream, +// translating its TunnelNetworkMessages to and from tunnel Messages. +// +// Every TunnelNetworkMessage addresses its conn, the first one to an unknown +// conn opens it, and one with an error ends it. The stream has no half-close +// and no flow control. +package bridgestream + +import ( + "log/slog" + "sync" + + "github.com/puzpuzpuz/xsync/v3" + bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + "github.com/vercel/bridge/pkg/tunnel" +) + +// GRPCStream is a TunnelNetwork stream. Both the server-side +// BidiStreamingServer and the client-side TunnelNetworkClient satisfy it. +type GRPCStream interface { + Send(*bridgev1.TunnelNetworkMessage) error + Recv() (*bridgev1.TunnelNetworkMessage, error) +} + +// New adapts stream to a tunnel.Stream. +func New(stream GRPCStream) tunnel.Stream { + return &adapter{stream: stream, conns: xsync.NewMapOf[string, *tunnel.ConnInfo]()} +} + +type adapter struct { + stream GRPCStream + sendMu sync.Mutex + // Each open conn's addresses, as this side sends them. + conns *xsync.MapOf[string, *tunnel.ConnInfo] + // The Data that came with an Open, for the next Recv. Only Recv touches it. + pending *tunnel.Message +} + +func (a *adapter) Features() tunnel.Features { return tunnel.Features{} } + +func (a *adapter) Recv() (*tunnel.Message, error) { + if m := a.pending; m != nil { + a.pending = nil + return m, nil + } + for { + msg, err := a.stream.Recv() + if err != nil { + return nil, err + } + id := msg.GetConnectionId() + _, open := a.conns.Load(id) + if msg.GetError() != "" { + if open { + a.conns.Delete(id) + return &tunnel.Message{Kind: tunnel.MessageReset, ConnID: id, Reason: msg.GetError()}, nil + } + continue + } + data := &tunnel.Message{Kind: tunnel.MessageData, ConnID: id, Payload: msg.GetData()} + if open { + return data, nil + } + dest := msg.GetDest() + if dest == nil { + slog.Info("Tunnel: ignoring message with no dest", "conn_id", id) + continue + } + info := &tunnel.ConnInfo{ + Source: address(msg.GetSource()), + Dest: address(dest), + Hostname: msg.GetHostname(), + } + a.conns.Store(id, info.Reply()) + if len(data.Payload) > 0 { + a.pending = data + } + return &tunnel.Message{Kind: tunnel.MessageOpen, ConnID: id, Conn: info}, nil + } +} + +func (a *adapter) Send(m *tunnel.Message) error { + switch m.Kind { + case tunnel.MessageOpen: + // The conn's first Data opens it. + a.conns.Store(m.ConnID, m.Conn) + return nil + case tunnel.MessageData: + info, ok := a.conns.Load(m.ConnID) + if !ok { + return nil + } + return a.send(&bridgev1.TunnelNetworkMessage{ + ConnectionId: m.ConnID, + Source: tunnelAddress(info.Source), + Dest: tunnelAddress(info.Dest), + Hostname: info.Hostname, + Data: m.Payload, + }) + case tunnel.MessageCloseWrite: + // No half-close: the conn ends here, and the peer isn't told. + a.conns.Delete(m.ConnID) + return nil + case tunnel.MessageReset: + a.conns.Delete(m.ConnID) + return a.send(&bridgev1.TunnelNetworkMessage{ConnectionId: m.ConnID, Error: m.Reason}) + default: + return nil + } +} + +// send serializes Sends, which the stream doesn't take concurrently. +func (a *adapter) send(msg *bridgev1.TunnelNetworkMessage) error { + a.sendMu.Lock() + defer a.sendMu.Unlock() + return a.stream.Send(msg) +} + +func address(a *bridgev1.TunnelAddress) tunnel.Addr { + return tunnel.Addr{Host: a.GetIp(), Port: int(a.GetPort())} +} + +func tunnelAddress(a tunnel.Addr) *bridgev1.TunnelAddress { + return &bridgev1.TunnelAddress{Ip: a.Host, Port: int32(a.Port)} +} diff --git a/pkg/tunnel/tunnel_test.go b/pkg/tunnel/bridgestream/bridgestream_test.go similarity index 96% rename from pkg/tunnel/tunnel_test.go rename to pkg/tunnel/bridgestream/bridgestream_test.go index f612139..541986f 100644 --- a/pkg/tunnel/tunnel_test.go +++ b/pkg/tunnel/bridgestream/bridgestream_test.go @@ -1,4 +1,4 @@ -package tunnel +package bridgestream import ( "context" @@ -11,6 +11,7 @@ import ( "time" bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + "github.com/vercel/bridge/pkg/tunnel" ) const testTimeout = 5 * time.Second @@ -148,17 +149,18 @@ func waitFor(t *testing.T, what string, cond func() bool) { t.Fatalf("timed out waiting for %s", what) } -func startTunnel(t *testing.T, dialer *blockingDialer) (*fakeStream, Tunnel) { +func startTunnel(t *testing.T, dialer *blockingDialer) (*fakeStream, *adapter) { t.Helper() stream := newFakeStream() - tun := New(dialer, stream) + a := New(stream).(*adapter) + tun := tunnel.New(dialer, a) ctx, cancel := context.WithCancel(context.Background()) tun.Start(ctx) t.Cleanup(func() { cancel() tun.Close() }) - return stream, tun + return stream, a } func TestChunksArrivingDuringDialShareOneConnection(t *testing.T) { @@ -214,14 +216,14 @@ func TestDialFailureReportsErrorAndForgetsConnection(t *testing.T) { func TestPeerErrorDuringDialClosesTheDialedConnection(t *testing.T) { dialer := newBlockingDialer() - stream, tun := startTunnel(t, dialer) + stream, a := startTunnel(t, dialer) const connID = "172.17.0.3:55988->172.20.143.45:80" stream.in <- dataMsg(connID, "hello") waitFor(t, "first dial", func() bool { return dialer.dials.Load() == 1 }) stream.in <- &bridgev1.TunnelNetworkMessage{ConnectionId: connID, Error: "peer closed"} waitFor(t, "the peer error to drop the buffered connection", func() bool { - _, loaded := tun.(*tunnelImpl).backend.(*streamBackend).conns.Load(connID) + _, loaded := a.conns.Load(connID) return !loaded }) diff --git a/pkg/tunnel/message.go b/pkg/tunnel/message.go new file mode 100644 index 0000000..90d864c --- /dev/null +++ b/pkg/tunnel/message.go @@ -0,0 +1,82 @@ +package tunnel + +import ( + "net" + "strconv" +) + +// Stream carries a tunnel's Messages over one transport: a bridge +// TunnelNetwork stream (bridgestream), or a dev tunnel's Connect stream +// (devtunnel). It translates them to and from the transport's own. +type Stream interface { + // Send sends m to the peer. It's called from many goroutines at once. + Send(m *Message) error + // Recv returns the peer's next Message. An error ends the tunnel. + Recv() (*Message, error) + // Features says what the transport carries beyond Open, Data and Reset. + Features() Features +} + +// Features are what a transport carries beyond Open, Data and Reset. +type Features struct { + // Window is what each side of a conn may send before the other grants + // more with a Window message; 0 for no flow control. + Window int + // HalfClose: a CloseWrite reaches the peer, which can still send. Without + // it, a conn ends at either side's EOF. + HalfClose bool +} + +type MessageKind uint8 + +const ( + // A new conn to Conn.Dest. + MessageOpen MessageKind = iota + 1 + // Payload for a conn. + MessageData + // The sender has nothing more to send on the conn. + MessageCloseWrite + // The conn is abandoned in both directions, for Reason. + MessageReset + // The receiver may send Bytes more of the conn's Data. + MessageWindow +) + +// Message is one message of the tunnel's own protocol, about one conn. +type Message struct { + Kind MessageKind + ConnID string + // Open only. + Conn *ConnInfo + Payload []byte + Reason string + Bytes int +} + +// Addr is a host (an IP address or a hostname) and a port. +type Addr struct { + Host string + Port int +} + +func (a Addr) String() string { return net.JoinHostPort(a.Host, strconv.Itoa(a.Port)) } + +// ConnInfo addresses a conn, as the side opening it sees it. +type ConnInfo struct { + Source, Dest Addr + // The DNS name Dest was looked up as, if any. + Hostname string +} + +// Reply addresses the other way: from where the conn was headed, back to +// where it came from. +func (c *ConnInfo) Reply() *ConnInfo { + return &ConnInfo{Source: c.Dest, Dest: c.Source, Hostname: c.Hostname} +} + +// SplitAddr parses "host:port". +func SplitAddr(s string) Addr { + host, portStr, _ := net.SplitHostPort(s) + port, _ := strconv.Atoi(portStr) + return Addr{Host: host, Port: port} +} diff --git a/pkg/tunnel/pump_test.go b/pkg/tunnel/pump_test.go index 6fca973..675abdf 100644 --- a/pkg/tunnel/pump_test.go +++ b/pkg/tunnel/pump_test.go @@ -8,50 +8,52 @@ import ( "time" ) -// acceptingBackend hands the tunnel the conns a test gives it. -type acceptingBackend struct { - conns chan net.Conn - dest Addr +const testTimeout = 5 * time.Second + +// fakeStream hands the tunnel the messages a test gives it, and keeps what the +// tunnel sends. +type fakeStream struct { + features Features + in chan *Message + out chan *Message +} + +func newFakeStream(f Features) *fakeStream { + return &fakeStream{features: f, in: make(chan *Message, 64), out: make(chan *Message, 1024)} } -func (b *acceptingBackend) Run(ctx context.Context, accept func(net.Conn, *ConnInfo)) error { +func (s *fakeStream) Send(m *Message) error { s.out <- m; return nil } +func (s *fakeStream) Features() Features { return s.features } + +func (s *fakeStream) Recv() (*Message, error) { + m, ok := <-s.in + if !ok { + return nil, io.EOF + } + return m, nil +} + +// next returns the next message the tunnel sent other than a window grant. +func (s *fakeStream) next(t *testing.T) *Message { + t.Helper() for { select { - case c := <-b.conns: - accept(c, &ConnInfo{Dest: b.dest}) - case <-ctx.Done(): - return nil + case m := <-s.out: + if m.Kind != MessageWindow { + return m + } + case <-time.After(testTimeout): + t.Fatal("the tunnel sent nothing") } } } -func (b *acceptingBackend) Open(context.Context, *ConnInfo) (net.Conn, error) { - return nil, net.ErrClosed -} - -// tcpPair returns both ends of a TCP connection. -func tcpPair(t *testing.T) (net.Conn, net.Conn) { +func startTunnel(t *testing.T, s Stream) Tunnel { t.Helper() - lis, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - t.Fatal(err) - } - defer lis.Close() - accepted := make(chan net.Conn, 1) - go func() { - c, _ := lis.Accept() - accepted <- c - }() - a, err := net.Dial("tcp", lis.Addr().String()) - if err != nil { - t.Fatal(err) - } - b := <-accepted - t.Cleanup(func() { - a.Close() - b.Close() - }) - return a, b + tun := New(&net.Dialer{Timeout: time.Second}, s) + tun.Start(context.Background()) + t.Cleanup(func() { tun.Close() }) + return tun } func TestHalfCloseCrossesThePump(t *testing.T) { @@ -72,16 +74,12 @@ func TestHalfCloseCrossesThePump(t *testing.T) { _, _ = c.Write([]byte("bye")) _ = c.Close() }() + s := newFakeStream(Features{Window: 1 << 20, HalfClose: true}) + startTunnel(t, s) - b := &acceptingBackend{conns: make(chan net.Conn, 1), dest: splitAddr(app.Addr().String())} - tun := NewWithBackend(&net.Dialer{Timeout: time.Second}, b) - tun.Start(context.Background()) - t.Cleanup(func() { tun.Close() }) - - peer, remote := tcpPair(t) - b.conns <- remote - _, _ = peer.Write([]byte("hi")) - _ = peer.(*net.TCPConn).CloseWrite() + s.in <- &Message{Kind: MessageOpen, ConnID: "1", Conn: &ConnInfo{Dest: SplitAddr(app.Addr().String())}} + s.in <- &Message{Kind: MessageData, ConnID: "1", Payload: []byte("hi")} + s.in <- &Message{Kind: MessageCloseWrite, ConnID: "1"} select { case req := <-got: @@ -91,12 +89,41 @@ func TestHalfCloseCrossesThePump(t *testing.T) { case <-time.After(testTimeout): t.Fatal("the app never saw the peer's half-close") } - _ = peer.SetReadDeadline(time.Now().Add(testTimeout)) - resp, err := io.ReadAll(peer) - if err != nil { - t.Fatal(err) + if m := s.next(t); m.Kind != MessageData || string(m.Payload) != "bye" { + t.Fatalf("expected the app's answer, got %+v", m) + } + if m := s.next(t); m.Kind != MessageCloseWrite { + t.Fatalf("expected the app's half-close, got %+v", m) + } +} + +func TestWritesWaitForWindow(t *testing.T) { + s := newFakeStream(Features{Window: 10, HalfClose: true}) + tun := startTunnel(t, s) + app, intercepted := net.Pipe() + defer app.Close() + tun.AddConn(intercepted, "10.0.0.1:443", "example.com") + open := s.next(t) + if open.Kind != MessageOpen || open.Conn.Hostname != "example.com" || open.Conn.Dest.Port != 443 { + t.Fatalf("expected the open, got %+v", open) + } + + go func() { _, _ = app.Write([]byte("0123456789abcdefghijklmno")) }() + sent := "" + for len(sent) < 10 { + sent += string(s.next(t).Payload) + } + select { + case m := <-s.out: + t.Fatalf("sent past the window: %+v", m) + case <-time.After(100 * time.Millisecond): + } + + s.in <- &Message{Kind: MessageWindow, ConnID: open.ConnID, Bytes: 15} + for len(sent) < 25 { + sent += string(s.next(t).Payload) } - if string(resp) != "bye" { - t.Fatalf("the peer read %q", resp) + if sent != "0123456789abcdefghijklmno" { + t.Fatalf("sent %q", sent) } } diff --git a/pkg/tunnel/stream.go b/pkg/tunnel/stream.go deleted file mode 100644 index 33eaf5d..0000000 --- a/pkg/tunnel/stream.go +++ /dev/null @@ -1,199 +0,0 @@ -package tunnel - -import ( - "context" - "errors" - "io" - "log/slog" - "net" - - "github.com/puzpuzpuz/xsync/v3" - bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" - "github.com/vercel/bridge/pkg/plumbing" -) - -// What a conn may have queued from the peer before the stream waits for it -// to be read. The stream has no flow control of its own. -const queuedMessages = 64 - -// Stream abstracts a bidirectional gRPC stream. Both the server-side -// BidiStreamingServer and client-side TunnelNetworkClient satisfy this -// interface directly since both directions use TunnelNetworkMessage. -type Stream interface { - Send(*bridgev1.TunnelNetworkMessage) error - Recv() (*bridgev1.TunnelNetworkMessage, error) -} - -// NewStreamBackend carries a tunnel over a bridge TunnelNetwork stream. Every -// message addresses its conn, the first one to an unknown conn opens it, and -// an error ends it. The stream has no half-close and no flow control. -func NewStreamBackend(stream Stream) Backend { - return &streamBackend{ - stream: stream, - sendCh: make(chan *bridgev1.TunnelNetworkMessage, 64), - stop: make(chan struct{}), - conns: xsync.NewMapOf[string, *messageFrames](), - } -} - -type streamBackend struct { - stream Stream - sendCh chan *bridgev1.TunnelNetworkMessage - // Closed once Run returns, so a Send waiting for room gives up. - stop chan struct{} - conns *xsync.MapOf[string, *messageFrames] -} - -func (b *streamBackend) Run(ctx context.Context, accept func(net.Conn, *ConnInfo)) error { - defer close(b.stop) - defer b.conns.Range(func(_ string, f *messageFrames) bool { - f.stream.PeerReset() - return true - }) - - // Single goroutine drains the send channel onto the stream. - go func() { - for { - select { - case msg := <-b.sendCh: - if err := b.stream.Send(msg); err != nil { - slog.Info("Tunnel: stream send error", "conn_id", msg.GetConnectionId(), "error", err) - return - } - slog.Debug("Tunnel: sent", "conn_id", msg.GetConnectionId(), "bytes", len(msg.GetData())) - case <-ctx.Done(): - return - } - } - }() - - recvCh := make(chan *bridgev1.TunnelNetworkMessage) - recvErr := make(chan error, 1) - go func() { - for { - msg, err := b.stream.Recv() - if err != nil { - recvErr <- err - return - } - select { - case recvCh <- msg: - case <-ctx.Done(): - return - } - } - }() - - for { - select { - case msg := <-recvCh: - b.handle(ctx, msg, accept) - case err := <-recvErr: - if err == io.EOF { - slog.Info("Tunnel: stream closed (EOF)") - return nil - } - slog.Info("Tunnel: stream recv error", "error", err) - return err - case <-ctx.Done(): - return nil - } - } -} - -// handle routes one of the peer's messages to its conn. The first message of -// an unknown conn opens it, before the dial, so the chunks that arrive during -// the dial queue on it instead of each opening another. -func (b *streamBackend) handle(ctx context.Context, msg *bridgev1.TunnelNetworkMessage, accept func(net.Conn, *ConnInfo)) { - id := msg.GetConnectionId() - f, ok := b.conns.Load(id) - if msg.GetError() != "" { - if ok { - f.stream.PeerReset() - } - return - } - if !ok { - dest := msg.GetDest() - if dest == nil { - slog.Info("Tunnel: ignoring message with no dest", "conn_id", id) - return - } - info := &ConnInfo{ - Source: Addr{Host: msg.GetSource().GetIp(), Port: int(msg.GetSource().GetPort())}, - Dest: Addr{Host: dest.GetIp(), Port: int(dest.GetPort())}, - Hostname: msg.GetHostname(), - } - // This side sends from where the peer's conn was headed, back to it. - f = b.newConn(id, &ConnInfo{Source: info.Dest, Dest: info.Source, Hostname: info.Hostname}) - accept(f.stream.Conn(), info) - } - f.stream.Deliver(ctx, msg.GetData()) -} - -func (b *streamBackend) Open(_ context.Context, info *ConnInfo) (net.Conn, error) { - id := plumbing.ConnectionID(info.Source.Host, info.Source.Port, info.Dest.Host, info.Dest.Port) - // Its first Data opens it. - return b.newConn(id, info).stream.Conn(), nil -} - -// newConn registers a conn that sends as info says. -func (b *streamBackend) newConn(id string, info *ConnInfo) *messageFrames { - f := &messageFrames{b: b, id: id, info: info} - f.stream = plumbing.NewMuxStream(f, plumbing.MuxConfig{ - Queue: queuedMessages, - Local: addr(info.Source.String()), - Remote: addr(info.Dest.String()), - }) - b.conns.Store(id, f) - return f -} - -func (b *streamBackend) send(msg *bridgev1.TunnelNetworkMessage) error { - select { - case b.sendCh <- msg: - return nil - case <-b.stop: - return net.ErrClosed - } -} - -// messageFrames writes one conn's messages: every one addresses the conn, and -// an error ends it. -type messageFrames struct { - b *streamBackend - id string - // As this side sends it. - info *ConnInfo - stream *plumbing.MuxStream -} - -func (f *messageFrames) Data(payload []byte) error { - return f.b.send(&bridgev1.TunnelNetworkMessage{ - ConnectionId: f.id, - Source: tunnelAddress(f.info.Source), - Dest: tunnelAddress(f.info.Dest), - Hostname: f.info.Hostname, - Data: payload, - }) -} - -// CloseWrite: the stream can't carry a half-close. -func (f *messageFrames) CloseWrite() error { return errors.ErrUnsupported } - -func (f *messageFrames) Reset(reason string) error { - return f.b.send(&bridgev1.TunnelNetworkMessage{ConnectionId: f.id, Error: reason}) -} - -// Window: the stream has no flow control; the queue holds the peer back. -func (f *messageFrames) Window(int) error { return nil } - -// Release forgets the conn, unless another has since taken its id. -func (f *messageFrames) Release() { - f.b.conns.Compute(f.id, func(cur *messageFrames, loaded bool) (*messageFrames, bool) { - if !loaded { - return nil, true - } - return cur, cur == f - }) -} diff --git a/pkg/tunnel/tunnel.go b/pkg/tunnel/tunnel.go index df1ece5..3d0ea9c 100644 --- a/pkg/tunnel/tunnel.go +++ b/pkg/tunnel/tunnel.go @@ -2,36 +2,42 @@ package tunnel import ( "context" + "errors" "io" "log/slog" "net" - "strconv" "sync" - bridgev1 "github.com/vercel/bridge/api/go/bridge/v1" + "github.com/puzpuzpuz/xsync/v3" "github.com/vercel/bridge/pkg/mitm" "github.com/vercel/bridge/pkg/plumbing" ) -// Each direction of a conn copies through one buffer of this size. -const copyBufferSize = 32 << 10 +const ( + // Each direction of a conn copies through one buffer of this size. + copyBufferSize = 32 << 10 + // Per Data message. + maxPayload = 64 << 10 + // What a conn may have queued from a peer without flow control before the + // tunnel waits for it to be read. + queuedPayloads = 64 +) -// Tunnel multiplexes conns over one transport, its Backend. It is used on both -// ends: the intercept and the bridge server, or the intercept and a dev -// tunnel's relay. +// Tunnel multiplexes conns over one Stream. It is used on both ends: the +// intercept and the bridge server, or the intercept and a dev tunnel's relay. type Tunnel interface { // AddConn sends conn through the tunnel, to destOverride if given or else // where conn was headed. hostname is the DNS name the destination was // looked up as (may be empty). AddConn(conn net.Conn, destOverride string, hostname string) - // Start runs the backend, dialing a local conn for each the peer opens. + // Start reads the stream, dialing a local conn for each the peer opens. Start(ctx context.Context) - // Close stops the backend, which ends every conn. + // Close cancels the tunnel context, closing all connections and stopping all goroutines. Close() error - // Done returns a channel that is closed when the backend stops. + // Done returns a channel that is closed when the tunnel stops. Done() <-chan struct{} } @@ -44,21 +50,17 @@ func WithHijacker(h mitm.Hijacker) Option { return func(t *tunnelImpl) { t.hijacker = h } } -// New creates a Tunnel over a bridge TunnelNetwork stream. A conn the peer -// opens is dialed with dialer: the client side uses the StaticPortDialer while -// the server side uses a standard ContextDialer. +// New creates a Tunnel over stream. A conn the peer opens is dialed with +// dialer: the client side uses the StaticPortDialer while the server side uses +// a standard ContextDialer. func New(dialer plumbing.ContextDialer, stream Stream, opts ...Option) Tunnel { - return NewWithBackend(dialer, NewStreamBackend(stream), opts...) -} - -// NewWithBackend creates a Tunnel over backend, dialing the conns the peer -// opens with dialer. -func NewWithBackend(dialer plumbing.ContextDialer, backend Backend, opts ...Option) Tunnel { t := &tunnelImpl{ - dialer: dialer, - backend: backend, - done: make(chan struct{}), - logger: slog.Default(), + dialer: dialer, + stream: stream, + features: stream.Features(), + conns: xsync.NewMapOf[string, *conn](), + done: make(chan struct{}), + logger: slog.Default(), } t.bufs.New = func() any { buf := make([]byte, copyBufferSize) @@ -72,91 +74,155 @@ func NewWithBackend(dialer plumbing.ContextDialer, backend Backend, opts ...Opti } type tunnelImpl struct { - dialer plumbing.ContextDialer - hijacker mitm.Hijacker - backend Backend - bufs sync.Pool - logger *slog.Logger - done chan struct{} - ctx context.Context - cancel context.CancelFunc + dialer plumbing.ContextDialer + hijacker mitm.Hijacker + stream Stream + features Features + conns *xsync.MapOf[string, *conn] + bufs sync.Pool + logger *slog.Logger + done chan struct{} + ctx context.Context + cancel context.CancelFunc + closeOnce sync.Once } func (t *tunnelImpl) Done() <-chan struct{} { return t.done } func (t *tunnelImpl) Start(ctx context.Context) { stop := context.AfterFunc(ctx, t.cancel) + recv := make(chan *Message) + recvErr := make(chan error, 1) + go func() { + for { + m, err := t.stream.Recv() + if err != nil { + recvErr <- err + return + } + select { + case recv <- m: + case <-t.ctx.Done(): + return + } + } + }() go func() { defer stop() - defer close(t.done) - defer t.cancel() - if err := t.backend.Run(t.ctx, t.accept); err != nil { - t.logger.Info("Tunnel: backend stopped", "error", err) + defer t.Close() + for { + select { + case m := <-recv: + t.handle(m) + case err := <-recvErr: + if errors.Is(err, io.EOF) { + t.logger.Info("Tunnel: stream closed (EOF)") + } else { + t.logger.Info("Tunnel: stream recv error", "error", err) + } + return + case <-t.ctx.Done(): + return + } } }() } func (t *tunnelImpl) Close() error { - t.cancel() + t.closeOnce.Do(func() { + t.cancel() + t.conns.Range(func(_ string, c *conn) bool { + c.stream.PeerReset() + return true + }) + close(t.done) + }) return nil } -func (t *tunnelImpl) AddConn(conn net.Conn, destOverride string, hostname string) { - srcAddr := conn.RemoteAddr().String() - dstAddr := conn.LocalAddr().String() +func (t *tunnelImpl) AddConn(local net.Conn, destOverride string, hostname string) { + srcAddr := local.RemoteAddr().String() + dstAddr := local.LocalAddr().String() if destOverride != "" { dstAddr = destOverride } - info := &ConnInfo{Source: splitAddr(srcAddr), Dest: splitAddr(dstAddr), Hostname: hostname} - logger := t.logger.With("src", srcAddr, "dst", dstAddr, "hostname", hostname) + info := &ConnInfo{Source: SplitAddr(srcAddr), Dest: SplitAddr(dstAddr), Hostname: hostname} + id := plumbing.ConnectionID(info.Source.Host, info.Source.Port, info.Dest.Host, info.Dest.Port) + logger := t.logger.With("conn_id", id, "src", srcAddr, "dst", dstAddr, "hostname", hostname) logger.Debug("Tunnel: adding connection") + c := t.newConn(id, info) go func() { - remote, err := t.backend.Open(t.ctx, info) - if err != nil { + if err := t.stream.Send(&Message{Kind: MessageOpen, ConnID: id, Conn: info}); err != nil { logger.Info("Tunnel: open failed", "error", err) - _ = conn.Close() + c.stream.PeerReset() + _ = local.Close() return } - t.pump(conn, remote) + t.pump(local, c) }() } -// accept dials a conn the peer opened, giving the hijacker first shot at it. -func (t *tunnelImpl) accept(remote net.Conn, info *ConnInfo) { +// handle routes one of the peer's messages to its conn. +func (t *tunnelImpl) handle(m *Message) { + if m.Kind == MessageOpen { + t.open(m) + return + } + c, ok := t.conns.Load(m.ConnID) + if !ok { + return + } + switch m.Kind { + case MessageData: + c.stream.Deliver(t.ctx, m.Payload) + case MessageCloseWrite: + c.stream.PeerClosed() + case MessageReset: + c.stream.PeerReset() + case MessageWindow: + c.stream.Conn().Grant(m.Bytes) + } +} + +// open registers a conn the peer opened before dialing it, so its Data queues +// on it during the dial. +func (t *tunnelImpl) open(m *Message) { + if m.Conn == nil { + t.logger.Info("Tunnel: ignoring open with no dest", "conn_id", m.ConnID) + return + } + c := t.newConn(m.ConnID, m.Conn.Reply()) go func() { - local, err := t.dial(info) + local, err := t.dial(m.Conn) if err != nil { - t.logger.Info("Tunnel: connect failed", "dest", info.Dest.String(), "hostname", info.Hostname, "error", err) - _ = remote.Close() + t.logger.Info("Tunnel: connect failed", "conn_id", m.ConnID, "hostname", m.Conn.Hostname, "error", err) + _ = c.stream.Close() return } - t.pump(local, remote) + t.pump(local, c) }() } +// dial gives the hijacker first shot at a conn the peer opened, then falls +// through to the dialer. func (t *tunnelImpl) dial(info *ConnInfo) (net.Conn, error) { - if t.hijacker != nil { - msg := &bridgev1.TunnelNetworkMessage{ - Source: tunnelAddress(info.Source), - Dest: tunnelAddress(info.Dest), - Hostname: info.Hostname, - } - if t.hijacker.ShouldHijack(msg) { - conn, err := t.hijacker.Hijack(t.ctx, msg) - if err == nil { - return conn, nil - } - t.logger.Info("Tunnel: hijack failed, falling back to dialer", "hostname", info.Hostname, "error", err) + dest := info.Dest.String() + if t.hijacker != nil && t.hijacker.ShouldHijack(info.Hostname) { + local, err := t.hijacker.Hijack(t.ctx, info.Hostname, dest) + if err == nil { + return local, nil } + t.logger.Info("Tunnel: hijack failed, falling back to dialer", "hostname", info.Hostname, "error", err) } - return t.dialer.DialContext(t.ctx, "tcp", info.Dest.String()) + return t.dialer.DialContext(t.ctx, "tcp", dest) } -// pump copies between local and remote in both directions, each through a -// buffer from the pool, then closes both. Either side's EOF half-closes the -// other; a failure either way closes both, which resets remote. -func (t *tunnelImpl) pump(local, remote net.Conn) { +// pump copies between local and c in both directions, each through a buffer +// from the pool, then closes both. Either side's EOF half-closes the other; a +// failure either way closes both, which resets c. +func (t *tunnelImpl) pump(local net.Conn, c *conn) { + remote := c.stream.Conn() toRemote := t.bufs.Get().(*[]byte) fromRemote := t.bufs.Get().(*[]byte) defer func() { @@ -164,10 +230,12 @@ func (t *tunnelImpl) pump(local, remote net.Conn) { t.bufs.Put(fromRemote) }() + // Once the conn is over, reading remote fails and aborts too. abort := sync.OnceFunc(func() { _ = remote.Close() _ = local.Close() }) + var wg sync.WaitGroup wg.Add(2) go func() { @@ -181,16 +249,14 @@ func (t *tunnelImpl) pump(local, remote net.Conn) { wg.Wait() abort() - if m, ok := remote.(plumbing.MeteredConn); ok { - st := m.Stats() - t.logger.Debug("Tunnel: connection closed", - "remote", remote.RemoteAddr().String(), - "bytes_sent", st.BytesWritten, - "bytes_received", st.BytesRead, - "window_wait", st.WriteWait, - "window_waits", st.WriteWaits, - ) - } + st := remote.Stats() + t.logger.Debug("Tunnel: connection closed", + "conn_id", c.id, + "bytes_sent", st.BytesWritten, + "bytes_received", st.BytesRead, + "window_wait", st.WriteWait, + "window_waits", st.WriteWaits, + ) } // copyHalf copies src to dst until src ends, then half-closes dst. @@ -206,12 +272,66 @@ func copyHalf(dst, src net.Conn, buf []byte, abort func()) { } } -func splitAddr(s string) Addr { - host, portStr, _ := net.SplitHostPort(s) - port, _ := strconv.Atoi(portStr) - return Addr{Host: host, Port: port} +// conn is one of the tunnel's conns, as the plumbing.Frames its MuxStream +// writes: each frame is a Message on the tunnel's Stream. +type conn struct { + t *tunnelImpl + id string + stream *plumbing.MuxStream } -func tunnelAddress(a Addr) *bridgev1.TunnelAddress { - return &bridgev1.TunnelAddress{Ip: a.Host, Port: int32(a.Port)} +// newConn registers a conn that sends as info addresses it. +func (t *tunnelImpl) newConn(id string, info *ConnInfo) *conn { + c := &conn{t: t, id: id} + queue := 0 + if t.features.Window == 0 { + queue = queuedPayloads + } + c.stream = plumbing.NewMuxStream(c, plumbing.MuxConfig{ + MaxFrame: maxPayload, + Queue: queue, + Window: t.features.Window, + Local: addr(info.Source.String()), + Remote: addr(info.Dest.String()), + }) + t.conns.Store(id, c) + return c } + +func (c *conn) Data(payload []byte) error { + return c.t.stream.Send(&Message{Kind: MessageData, ConnID: c.id, Payload: payload}) +} + +func (c *conn) CloseWrite() error { + err := c.t.stream.Send(&Message{Kind: MessageCloseWrite, ConnID: c.id}) + if !c.t.features.HalfClose { + return errors.ErrUnsupported + } + return err +} + +func (c *conn) Reset(reason string) error { + return c.t.stream.Send(&Message{Kind: MessageReset, ConnID: c.id, Reason: reason}) +} + +func (c *conn) Window(n int) error { + if c.t.features.Window == 0 { + return nil + } + return c.t.stream.Send(&Message{Kind: MessageWindow, ConnID: c.id, Bytes: n}) +} + +// Release forgets the conn, unless another has since taken its id. +func (c *conn) Release() { + c.t.conns.Compute(c.id, func(cur *conn, loaded bool) (*conn, bool) { + if !loaded { + return nil, true + } + return cur, cur == c + }) +} + +type addr string + +func (a addr) Network() string { return "tunnel" } +func (a addr) String() string { return string(a) }