Refactor anycast setup

- Link interface presence with lifetime of service
- Remove autoservice and NAT scripting; import device routes instead

Author: jlu5

anycast-dns.yml

@@ -22,7 +22,6 @@
     - role: config-loopback
       dummy_iface: "{{ dummy_interfaces.anycast_recursors }}"
     - role: config-iptables
-    - config-bird2-autoservice
 
     # Install & configure PowerDNS authoritative server
     - role: config-powerdns

dn42-registry

@@ -14,17 +14,17 @@ dns_records:
     # DNS & web anycast
     "ns":
       type: host_record
-      ip4: "{{ dummy_interfaces.anycast_auth_dns.ip4_natmap[0] }}"
+      ip4: "{{ dummy_interfaces.anycast_auth_dns.ip4[0] }}"
       ip6: "{{ dummy_interfaces.anycast_auth_dns.ip6[0] }}"
     "dns":
       type: host_record
-      ip4: "{{ dummy_interfaces.anycast_recursors.ip4_natmap[0] }}"
+      ip4: "{{ dummy_interfaces.anycast_recursors.ip4[0] }}"
       ip6: "{{ dummy_interfaces.anycast_recursors.ip6[0] }}"
     "@":
       type: multi
       records:
         - type: A
-          target: "{{ dummy_interfaces.anycast_recursors.ip4_natmap[0] }}"
+          target: "{{ dummy_interfaces.anycast_recursors.ip4[0] }}"
         - type: AAAA
           target: "{{ dummy_interfaces.anycast_recursors.ip6[0] }}"
     "www":

global-config/dns-entries.yml

@@ -17,26 +17,17 @@ dummy_interfaces:
     ifname: "igp-dummy0"
     ip4: ["{{ ownip }}"]
     ip6: ["{{ ownip6 }}"]
-    igp_announce: yes
+    track_service: no
   anycast_auth_dns:
     ifname: "igp-dummy1"
-    ip4: ["169.254.208.21"]
-    ip4_natmap: ["172.22.108.21"]
+    ip4: ["172.22.108.21"]
     ip6: ["fd86:bad:11b7:53::1"]
-    igp_announce: no
-    track_service: pdns
+    track_service: yes
   anycast_recursors:
     ifname: "igp-dummy2"
-    # HACK: 169.254.208.53 et al. are internal IPs with 1-to-1 NAT to anycast service IPs. This works around
-    # Linux unconditionally dropping IPv4 "Martian packets": i.e. those with a source IP matching
-    # a locally bound address, but not sent from that interface
-    # These NAT rules are coded in roles/config-iptables/templates and triggered by a
-    # pdns-recursor service override
-    ip4: ["169.254.208.22", "169.254.208.53", "169.254.208.81"]
-    ip4_natmap: ["172.22.108.22", "172.23.0.53", "172.20.0.81"]
+    ip4: ["172.22.108.22", "172.23.0.53", "172.20.0.81"]
     ip6: ["fd86:bad:11b7:53::2", "fd42:d42:d42:53::1/64", "fd42:d42:d42:81::1/64"]
-    igp_announce: no
-    track_service: pdns-recursor
+    track_service: yes
 
 # Interfaces to clean up (e.g. for decommissioned nodes)
 cleanup_remove_ifaces: []

global-config/general.yml

@@ -6,4 +6,3 @@
   become: yes
   roles:
     - config-bird2
-    - config-bird2-autoservice

reconfigure-bird.yml

@@ -16,4 +16,3 @@
     - config-gre-plain
     - config-igpping
     - config-bird2
-    - config-bird2-autoservice

reconfigure.yml

@@ -0,0 +1,9 @@
+{% if 'anycast_pool' in group_names %}
+protocol direct anycast_import {
+    ipv4;
+    ipv6;
+    interface {{ dummy_interfaces.values() | list | selectattr("track_service") | map(attribute='ifname') | map("to_json") | sort | join(", ") }};
+}
+{% else %}
+# Anycast not enabled on this node
+{% endif %}

roles/config-bird2-autoservice/handlers

@@ -143,6 +143,4 @@ template bgp dnpeers {
 include "/etc/bird/ospf.conf";
 include "/etc/bird/ibgp.conf";
 include "/etc/bird/local*.conf";
-{% if 'anycast_pool' in group_names %}
-include "/etc/bird/autoservice-enabled/*.conf";
-{% endif %}
+include "/etc/bird/anycast_services.conf";

roles/config-bird2-autoservice/tasks/main.yml

@@ -71,8 +71,8 @@ function dn42_import_filter(int link_latency; int link_bandwidth; int link_crypt
 }
 
 function dn42_export_filter(int link_latency; int link_bandwidth; int link_crypto) {
-    if ((is_valid_network() || is_valid_network_v6()) && source ~ [RTS_STATIC, RTS_BGP]) then {
-        if source = RTS_STATIC then {
+    if ((is_valid_network() || is_valid_network_v6()) && source ~ [RTS_STATIC, RTS_BGP, RTS_DEVICE]) then {
+        if source ~ [RTS_STATIC, RTS_DEVICE] then {
             bgp_community.add((64511, DN42_REGION));
         }
         update_flags(link_latency, link_bandwidth, link_crypto);
@@ -121,7 +121,7 @@ function dn42_export_noexport(int link_latency; int link_bandwidth; int link_cry
 
 # Only export static routes (aggregates, mostly)
 function dn42_export_peer_only(int link_latency; int link_bandwidth; int link_crypto) {
-    if (source = RTS_STATIC) then {
+    if (source ~ [RTS_STATIC, RTS_DEVICE]) then {
         dn42_export_filter(link_latency, link_bandwidth, link_crypto);
     } else {
         reject;

roles/config-bird2-autoservice/templates/autoservice-route.conf.j2

@@ -188,7 +188,7 @@ function ebgp_export_filter() {
     if (bgp_path.len = 0) then {
         bgp_large_community.add((OWNAS, LC_ORIGIN_NODEID, NODEID));
     }
-    if source = RTS_STATIC then {
+    if source ~ [RTS_STATIC, RTS_DEVICE] then {
 {% for region_tag in dn42_regions[1:] %}
         bgp_community.add((64511, {{ region_tag }}));
 {% endfor %}
@@ -222,10 +222,5 @@ function ibgp_export_filter() {
     if (source != RTS_BGP && !is_ibgp_network()) then {
         reject;
     }
-    # Don't export device routes for dn42 anycast networks. These will be filled in via a static route config
-    # triggered by service start/stop
-    if (is_ibgp_network() && source = RTS_DEVICE) then {
-        reject;
-    }
     accept;
 }

roles/config-bird2/config/anycast_services.conf.j2

@@ -13,8 +13,8 @@ area 0 {
         type ptp;
     };
 {% endfor %}
-{# In order: global stub ifname globs, local stub ifnames (appended), dummy interfaces with igp_announce=yes #}
-{% for entry in stub_ifnames + stub_ifnames_append|default([]) + (dummy_interfaces.values() | list | selectattr("igp_announce") | map(attribute='ifname') | list | sort) %}
+{# In order: global stub ifname globs, local stub ifnames (appended), dummy interfaces with track_service=no #}
+{% for entry in stub_ifnames + stub_ifnames_append|default([]) + (dummy_interfaces.values() | list | rejectattr("track_service") | map(attribute='ifname') | list | sort) %}
     interface "{{ entry }}" {
         stub on;
     };

roles/config-bird2/config/bird.conf.j2

@@ -8,12 +8,12 @@ protocol bgp burble_grc_2602 {
     ipv4 {
         add paths tx;
         import none;
-        export where is_valid_network() && source ~ [RTS_STATIC, RTS_BGP];
+        export where is_valid_network() && source ~ [RTS_STATIC, RTS_BGP, RTS_DEVICE];
     };
 
     ipv6 {
         add paths tx;
         import none;
-        export where is_valid_network_v6() && source ~ [RTS_STATIC, RTS_BGP];
+        export where is_valid_network_v6() && source ~ [RTS_STATIC, RTS_BGP, RTS_DEVICE];
     };
 }

roles/config-bird2/config/community_filters.conf

@@ -46,15 +46,6 @@
   notify:
   - "Reload iptables rules"
 
-- name: "Template iptables NAT support script"
-  template:
-    src: apply-anycast.j2
-    dest: "{{ iptables_dir }}/apply-anycast"
-    mode: '755'
-  notify:
-  - "Reload iptables rules"
-  when: "'anycast_pool' in group_names"
-
 - name: "Write iptables rules for IPv4"
   blockinfile:
     dest: "{{ iptables_rules_path }}"

roles/config-bird2/config/custom_filters.conf.j2

@@ -7,15 +7,3 @@ fi
 
 iptables-restore  < {{ iptables_rules_path  | quote }} && echo 'applied IPv4 rules'
 ip6tables-restore < {{ ip6tables_rules_path | quote }} && echo 'applied IPv6 rules'
-
-{% if 'anycast_pool' in group_names %}
-{% for service_name, anycast_iface in dummy_interfaces.items() %}
-{% if 'track_service' in anycast_iface %}
-if systemctl is-active --quiet "{{ anycast_iface.track_service }}"; then
-	/etc/iptables/apply-anycast start "{{ service_name }}"
-else
-	/etc/iptables/apply-anycast stop "{{ service_name }}"
-fi
-{% endif %}
-{% endfor %}
-{% endif %}

roles/config-bird2/config/ospf_interfaces.conf.j2

@@ -1,11 +1,21 @@
+{% if not dummy_iface.track_service %}
 auto {{ dummy_iface.ifname | mandatory }}
+{% endif %}
 {#
 Use a namespace variable to track state across scopes: https://stackoverflow.com/a/61342337
 In particular we only want to attach the interface creation pre-up command to the first IP associated with
 an interface, or that command will run multiple times and cause errors.
 Note: this requires Jinja2 2.10+
 #}
 {% set ns = namespace(has_init_line=false) %}
+{% macro init_lines(dummy_iface) %}
+{% if not ns.has_init_line %}
+{# Cap MTU at IPv6 minimum as PMTUD setup is tricky... #}
+    mtu 1280
+    pre-up ip link add {{ dummy_iface.ifname }} type dummy
+{% set ns.has_init_line = true %}
+{% endif %}
+{% endmacro %}
 
 {% for ip in dummy_iface.ip4 %}
 iface {{ dummy_iface.ifname | mandatory }} inet static
@@ -14,11 +24,7 @@ iface {{ dummy_iface.ifname | mandatory }} inet static
 {% set ip = ip | ipaddr('host') %}
 {% endif %}
     address {{ ip }}
-{% if not ns.has_init_line %}
-    mtu 1280
-    pre-up ip link add {{ dummy_iface.ifname }} type dummy
-{% set ns.has_init_line = true %}
-{% endif %}
+{{ init_lines(dummy_iface) }}
 {% endfor %}
 
 {% for ip in dummy_iface.ip6 %}
@@ -27,10 +33,6 @@ iface {{ dummy_iface.ifname }} inet6 static
 {% if ip | ipaddr('address') %}
 {% set ip = ip | ipaddr('host') %}
 {% endif %}
-    mtu 1280
     address {{ ip }}
-{% if not ns.has_init_line %}
-    pre-up ip link add {{ dummy_iface.ifname }} type dummy
-{% set ns.has_init_line = true %}
-{% endif %}
+{{ init_lines(dummy_iface) }}
 {% endfor %}

roles/config-bird2/config/peers/grc_export.conf

@@ -47,6 +47,5 @@
       User: "{{ pdns_rec_user }}"
       Group: "{{ pdns_rec_group }}"
       # Start/stop
-      # XXX: 2nd arg (autoservice name) is hardcoded here
-      ExecStartPost: "+-{{ iptables_dir }}/apply-anycast start anycast_recursors"
-      ExecStopPost: "+-{{ iptables_dir }}/apply-anycast stop anycast_recursors"
+      ExecStartPre: "+ifup {{ dummy_interfaces.anycast_recursors.ifname }} --force"
+      ExecStopPost: "+ip link del {{ dummy_interfaces.anycast_recursors.ifname }}"

roles/config-iptables/tasks/main.yml

@@ -67,9 +67,8 @@
       User: "{{ pdns_user }}"
       Group: "{{ pdns_group }}"
       # Start/stop
-      # XXX: 2nd arg (autoservice name) is hardcoded here
-      ExecStartPost: "+-{{ iptables_dir }}/apply-anycast start anycast_auth_dns"
-      ExecStopPost: "+-{{ iptables_dir }}/apply-anycast stop anycast_auth_dns"
+      ExecStartPre: "+ifup {{ dummy_interfaces.anycast_auth_dns.ifname }} --force"
+      ExecStopPost: "+ip link del {{ dummy_interfaces.anycast_auth_dns.ifname }}"
 
 - name: "Synchronize DNS zones"
   synchronize:

roles/config-iptables/templates/apply-anycast.j2

@@ -25,7 +25,7 @@ ptr_zones["31.172.in-addr.arpa"]="$REGISTRY_ROOT/inetnum/172.31.0.0_16"
 ptr_zones["10.in-addr.arpa"]="$REGISTRY_ROOT/inetnum/10.0.0.0_8"
 ptr_zones["d.f.ip6.arpa"]="$REGISTRY_ROOT/inet6num/fd00::_8"
 
-local_authserver_ip4="$(yq -r .dummy_interfaces.anycast_auth_dns.ip4_natmap[0] "$DNS_CONFIG")"
+local_authserver_ip4="$(yq -r .dummy_interfaces.anycast_auth_dns.ip4[0] "$DNS_CONFIG")"
 local_authserver_ip6="$(yq -r .dummy_interfaces.anycast_auth_dns.ip6[0] "$DNS_CONFIG")"
 
 addrtype() {