From ab95e363e3f54e8f5ef79c627fb70698026e9c7f Mon Sep 17 00:00:00 2001 From: Alex R Date: Thu, 28 Sep 2023 15:00:04 +0200 Subject: [PATCH 1/2] Allow disabling internalIP routing via arg --- cmd/kg/main.go | 5 ++++- docs/annotations.md | 4 ++-- pkg/k8s/backend.go | 2 +- pkg/mesh/mesh.go | 5 ++++- pkg/mesh/routes.go | 4 ++-- 5 files changed, 13 insertions(+), 7 deletions(-) diff --git a/cmd/kg/main.go b/cmd/kg/main.go index 66e39a09..a9719fd7 100644 --- a/cmd/kg/main.go +++ b/cmd/kg/main.go @@ -117,6 +117,7 @@ var ( resyncPeriod time.Duration iptablesForwardRule bool prioritisePrivateAddr bool + routeInternalIP bool printVersion bool logLevel string @@ -150,6 +151,8 @@ func init() { cmd.Flags().BoolVar(&iptablesForwardRule, "iptables-forward-rules", false, "Add default accept rules to the FORWARD chain in iptables. Warning: this may break firewalls with a deny all policy and is potentially insecure!") cmd.Flags().BoolVar(&prioritisePrivateAddr, "prioritise-private-addresses", false, "Prefer to assign a private IP address to the node's endpoint.") + cmd.Flags().BoolVar(&routeInternalIP, "route-internal-ip", true, "Route internal traffic through WireGuard") + cmd.PersistentFlags().BoolVar(&printVersion, "version", false, "Print version and exit") cmd.PersistentFlags().StringVar(&logLevel, "log-level", logLevelInfo, fmt.Sprintf("Log level to use. Possible values: %s", availableLogLevels)) } @@ -259,7 +262,7 @@ func runRoot(_ *cobra.Command, _ []string) error { serviceCIDRs = append(serviceCIDRs, s) } - m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, serviceCIDRs, log.With(logger, "component", "kilo"), registry) + m, err := mesh.New(b, enc, gr, hostname, port, s, local, cni, cniPath, iface, cleanUp, cleanUpIface, createIface, mtu, resyncPeriod, prioritisePrivateAddr, iptablesForwardRule, routeInternalIP, serviceCIDRs, log.With(logger, "component", "kilo"), registry) if err != nil { return fmt.Errorf("failed to create Kilo mesh: %v", err) } diff --git a/docs/annotations.md b/docs/annotations.md index 29037bf3..7850cf08 100644 --- a/docs/annotations.md +++ b/docs/annotations.md @@ -18,7 +18,7 @@ The Kilo agent running on each node will use heuristics to automatically detect * _no automatic public IP on ethernet device_: on some cloud providers it is common for nodes to be allocated a public IP address but for the Ethernet devices to only be automatically configured with the private network address; in this case the allocated public IP address should be specified; * _multiple public IP addresses_: if a node has multiple public IPs but one is preferred, then the preferred IP address should be specified; * _IPv6_: if a node has both public IPv4 and IPv6 addresses and the Kilo network should operate over IPv6, then the IPv6 address should be specified; - * _dynamic IP address_: if a node has a dynamically allocated public IP address, for example an IP leased from a network provider, then a dynamic DNS name can be given can be given and Kilo will periodically lookup the IP to keep the endpoint up-to-date; + * _dynamic IP address_: if a node has a dynamically allocated public IP address, for example an IP leased from a network provider, then a dynamic DNS name can be given and Kilo will periodically lookup the IP to keep the endpoint up-to-date; * _override port_: if a node should listen on a specific port that is different from the mesh's default WireGuard port, then this annotation can be used to override the port; this can be useful, for example, to ensure that two nodes operating behind the same port-forwarded NAT gateway can each be allocated a different port. ### force-internal-ip @@ -51,7 +51,7 @@ In certain deployments, cluster nodes may be located behind NAT or a firewall, e In these scenarios, the nodes behind NAT can send packets to the nodes outside of the NATed network, however the outside nodes can only send packets into the NATed network as long as the NAT mapping remains valid. In order for a node behind NAT to receive packets from nodes outside of the NATed network, it must maintain the NAT mapping by regularly sending packets to those nodes, ie by sending _keepalives_. The frequency of emission of these keepalive packets can be controlled by setting the persistent-keepalive annotation on the node behind NAT. -The annotated node will use the specified value will as the persistent-keepalive interval for all of its peers. +The annotated node will use the specified value as the persistent-keepalive interval for all of its peers. For more background, [see the WireGuard documentation on NAT and firewall traversal](https://www.wireguard.com/quickstart/#nat-and-firewall-traversal-persistence). ### allowed-location-ips diff --git a/pkg/k8s/backend.go b/pkg/k8s/backend.go index e94a6646..c98ef677 100644 --- a/pkg/k8s/backend.go +++ b/pkg/k8s/backend.go @@ -291,7 +291,7 @@ func translateNode(node *v1.Node, topologyLabel string) *mesh.Node { if internalIP == nil { internalIP = normalizeIP(node.ObjectMeta.Annotations[internalIPAnnotationKey]) } - // Set the ForceInternalIP flag, if force-internal-ip annotation was set to "". + // Set the noInternalIP flag, if force-internal-ip annotation was set to "". noInternalIP := false if s, ok := node.ObjectMeta.Annotations[forceInternalIPAnnotationKey]; ok && (s == "" || s == "-") { noInternalIP = true diff --git a/pkg/mesh/mesh.go b/pkg/mesh/mesh.go index 3057d2a2..6d7e8251 100644 --- a/pkg/mesh/mesh.go +++ b/pkg/mesh/mesh.go @@ -89,7 +89,7 @@ type Mesh struct { } // New returns a new Mesh instance. -func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule bool, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { +func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularity, hostname string, port int, subnet *net.IPNet, local, cni bool, cniPath, iface string, cleanup bool, cleanUpIface bool, createIface bool, mtu uint, resyncPeriod time.Duration, prioritisePrivateAddr, iptablesForwardRule, routeInternalIP bool, serviceCIDRs []*net.IPNet, logger log.Logger, registerer prometheus.Registerer) (*Mesh, error) { if err := os.MkdirAll(kiloPath, 0700); err != nil { return nil, fmt.Errorf("failed to create directory to store configuration: %v", err) } @@ -138,6 +138,9 @@ func New(backend Backend, enc encapsulation.Encapsulator, granularity Granularit if err != nil { return nil, fmt.Errorf("failed to find public IP: %v", err) } + if !routeInternalIP { + privateIP = nil + } var privIface int if privateIP != nil { ifaces, err := interfacesForIP(privateIP) diff --git a/pkg/mesh/routes.go b/pkg/mesh/routes.go index bff920ea..4b8dc914 100644 --- a/pkg/mesh/routes.go +++ b/pkg/mesh/routes.go @@ -35,7 +35,7 @@ func (t *Topology) Routes(kiloIfaceName string, kiloIface, privIface, tunlIface var rules []*netlink.Rule if !t.leader { // Find the GW for this segment. - // This will be the an IP of the leader. + // This will be the IP of the leader. // In an IPIP encapsulated mesh it is the leader's private IP. var gw net.IP for _, segment := range t.segments { @@ -347,7 +347,7 @@ func (t *Topology) Rules(cni, iptablesForwardRule bool) iptables.RuleSet { } } for _, s := range t.segments { - rules.AddToPrepend(iptables.NewRule(iptables.GetProtocol(s.wireGuardIP), "nat", "KILO-NAT", "-d", oneAddressCIDR(s.wireGuardIP).String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuared IPs", "-j", "RETURN")) + rules.AddToPrepend(iptables.NewRule(iptables.GetProtocol(s.wireGuardIP), "nat", "KILO-NAT", "-d", oneAddressCIDR(s.wireGuardIP).String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for WireGuard IPs", "-j", "RETURN")) for _, aip := range s.allowedIPs { rules.AddToPrepend(iptables.NewRule(iptables.GetProtocol(aip.IP), "nat", "KILO-NAT", "-d", aip.String(), "-m", "comment", "--comment", "Kilo: do not NAT packets destined for known IPs", "-j", "RETURN")) } From 0ddeafc904f0405e8b475e8f66695f0059021dcc Mon Sep 17 00:00:00 2001 From: Alex R Date: Mon, 30 Oct 2023 21:34:28 +0100 Subject: [PATCH 2/2] fix help --- docs/kg.md | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/kg.md b/docs/kg.md index 3f395c52..66d33da5 100644 --- a/docs/kg.md +++ b/docs/kg.md @@ -54,6 +54,7 @@ Flags: --port int The port over which WireGuard peers should communicate. (default 51820) --prioritise-private-addresses Prefer to assign a private IP address to the node's endpoint. --resync-period duration How often should the Kilo controllers reconcile? (default 30s) + --route-internal-ip Route internal traffic through WireGuard (default true) --service-cidr strings The service CIDR for the Kubernetes cluster. Can be provided optionally to avoid masquerading packets sent to service IPs. Can be specified multiple times. --subnet string CIDR from which to allocate addresses for WireGuard interfaces. (default "10.4.0.0/16") --topology-label string Kubernetes node label used to group nodes into logical locations. (default "topology.kubernetes.io/region")