Merge pull request #13 from rackerlabs/iterative-updates

adjust networking to use expected MAC addresses for traffic

Author: cardoe

README.md

@@ -62,3 +62,11 @@ source .venv/bin/activate
 pip install esxi-img
 esxi-img --output esxi.img path/to/esxi.iso
 ```
+
+## ESXi Network Interfaces
+
+ESXi has physical network interfaces and logical interfaces. The `vmnicX`
+interfaces are the physical interfaces and the `vmkX` interfaces are the
+logical interfaces, which are also known as VMkernel interfaces.
+
+![Diagram of ESXi interfaces](/docs/assets/esxi-interfaces.png "Explanation of interfaces")

docs/assets/esxi-interfaces.png

@@ -1,23 +1,20 @@
+import logging
 from functools import cached_property
 
 from .esxhost import ESXHost
 from .network_data import NetworkData
 from .nic import NIC
 from .nic_list import NICList
 
+logger = logging.getLogger(__name__)
+
 
 class ESXConfig:
     def __init__(self, network_data: NetworkData, dry_run=False) -> None:
         self.network_data = network_data
         self.dry_run = dry_run
         self.host = ESXHost(dry_run)
 
-    def add_default_mgmt_interface(
-        self, portgroup_name, switch_name, interface_name="vmk0"
-    ):
-        self.host.portgroup_add(portgroup_name=portgroup_name, switch_name=switch_name)
-        self.host.add_ip_interface(name=interface_name, portgroup_name=portgroup_name)
-
     def clean_default_network_setup(self, portgroup_name, switch_name):
         """Removes default networking setup left by the installer."""
         self.host.delete_vmknic(portgroup_name=portgroup_name)
@@ -26,6 +23,11 @@ def clean_default_network_setup(self, portgroup_name, switch_name):
         )
         self.host.destroy_vswitch(name=switch_name)
 
+    def configure_portgroups(self, switch_name: str, portgroups):
+        """Adds each requested portgroup to the specified switch."""
+        for portgroup_name in portgroups:
+            self.host.portgroup_add(portgroup_name, switch_name)
+
     def configure_default_route(self):
         """Configures default route.
 
@@ -34,7 +36,7 @@ def configure_default_route(self):
         route = self.network_data.default_route()
         self.host.configure_default_route(route.gateway)
 
-    def configure_portgroups(self, switch_name="vSwitch0"):
+    def configure_vlans(self, switch_name="vSwitch0"):
         portgroups = []
         for link in self.network_data.links:
             if link.type == "vlan":
@@ -45,13 +47,23 @@ def configure_portgroups(self, switch_name="vSwitch0"):
                 portgroups.append(pg_name)
         return portgroups
 
-    def configure_management_interface(self):
-        mgmt_network = next(
-            net for net in self.network_data.networks if net.default_routes()
-        )
-        return self.host.change_ip(
-            "vmk0", mgmt_network.ip_address, mgmt_network.netmask
-        )
+    def configure_management_interface(self, mgmt_portgroup: str):
+        for net in self.network_data.networks:
+            logger.info(
+                "Creating %s with MAC %s for network %s",
+                net.id,
+                net.link.ethernet_mac_address,
+                net.network_id,
+            )
+            self.host.add_ip_interface(
+                net.id, mgmt_portgroup, net.link.ethernet_mac_address, net.link.mtu
+            )
+            if net.type == "ipv4":
+                self.host.set_static_ipv4(net.id, net.ip_address, net.netmask)
+            elif net.type == "ipv4_dhcp":
+                self.host.set_dhcp_ipv4(net.id)
+            else:
+                raise NotImplementedError(f"net type {net.type}")
 
     def configure_vswitch(self, uplink: NIC, switch_name: str, mtu: int):
         """Sets up vSwitch."""

packages/esxi-netinit/esxi_netinit/esxconfig.py

@@ -1,5 +1,8 @@
+import logging
 import subprocess
 
+logger = logging.getLogger(__name__)
+
 
 class ESXHost:
     """Low level commands for configuring various aspects of ESXi hypervisor."""
@@ -9,13 +12,17 @@ def __init__(self, dry_run=False) -> None:
 
     def __execute(self, cmd: list):
         if self.dry_run:
-            print(f"Would execute: {' '.join(cmd)}")
+            logger.info("Would execute: %s", " ".join(cmd))
             return cmd
         else:
+            logger.debug("Executing %s", cmd)
             subprocess.run(cmd, check=True)  # noqa: S603
 
-    def add_ip_interface(self, name, portgroup_name):
+    def add_ip_interface(self, inf: str, portgroup_name: str, mac: str, mtu: int):
         """Adds IP interface."""
+        logger.info(
+            "Adding IP interface %s (%s) for portgroup %s", inf, mac, portgroup_name
+        )
         return self.__execute(
             [
                 "/bin/esxcli",
@@ -24,7 +31,11 @@ def add_ip_interface(self, name, portgroup_name):
                 "interface",
                 "add",
                 "--interface-name",
-                name,
+                inf,
+                "--mac-address",
+                mac,
+                "--mtu",
+                str(mtu),
                 "--portgroup-name",
                 portgroup_name,
             ]
@@ -65,6 +76,43 @@ def change_ip(self, interface, ip, netmask):
         ]
         return self.__execute(cmd)
 
+    def set_dhcp_ipv4(self, inf: str):
+        """Configures DHCP (IPv4) on an interface."""
+        logger.info("Configuring IPv4 interface %s with DHCP", inf)
+        cmd = [
+            "/bin/esxcli",
+            "network",
+            "ip",
+            "interface",
+            "ipv4",
+            "set",
+            "--interface-name",
+            inf,
+            "--peer-dns=true",
+            "--type=dhcp",
+        ]
+        return self.__execute(cmd)
+
+    def set_static_ipv4(self, inf: str, ip_addr: str, netmask: str):
+        """Configures a static IPv4 address on an interface."""
+        logger.info("Configuring IPv4 interface %s with static IP %s", inf, ip_addr)
+        cmd = [
+            "/bin/esxcli",
+            "network",
+            "ip",
+            "interface",
+            "ipv4",
+            "set",
+            "--interface-name",
+            inf,
+            "--type=static",
+            "--ipv4",
+            ip_addr,
+            "--netmask",
+            netmask,
+        ]
+        return self.__execute(cmd)
+
     def configure_dns(self, servers=None, search=None):
         """Sets up arbitrary DNS servers."""
         if not servers:

packages/esxi-netinit/esxi_netinit/esxhost.py

@@ -1,4 +1,7 @@
 import argparse
+import logging
+import logging.handlers
+import os
 import sys
 
 from esxi_netinit.esxconfig import ESXConfig
@@ -10,6 +13,31 @@
 NEW_VSWITCH = "vSwitch22"
 
 
+logger = logging.getLogger(__name__)
+
+
+def setup_logger(log_level=logging.INFO):
+    """Set up the root logger.
+
+    Output to stdout and to syslog at the requested log_level.
+    """
+    logging.basicConfig(
+        format="%(asctime)s - %(name)s - %(levelname)s - %(message)s",
+        level=log_level,
+    )
+
+    app_name = os.path.basename(sys.argv[0])
+
+    syslog_fmt = logging.Formatter(f"{app_name}: %(levelname)s - %(message)s")
+    try:
+        syslog_handler = logging.handlers.SysLogHandler()
+        syslog_handler.setLevel(log_level)
+        syslog_handler.setFormatter(syslog_fmt)
+        logging.getLogger().addHandler(syslog_handler)
+    except Exception:
+        logger.error("Failed to setup syslog for logging")
+
+
 def main(json_file, dry_run):
     network_data = NetworkData.from_json_file(json_file)
     esx = ESXConfig(network_data, dry_run=dry_run)
@@ -18,9 +46,10 @@ def main(json_file, dry_run):
         uplink=esx.identify_uplink(), switch_name=NEW_VSWITCH, mtu=9000
     )
 
-    esx.configure_portgroups()
-    esx.add_default_mgmt_interface(NEW_MGMT_PG, NEW_VSWITCH)
-    esx.configure_management_interface()
+    # this configures the Management Network to the default vSwitch
+    esx.configure_portgroups(NEW_VSWITCH, [NEW_MGMT_PG])
+    esx.configure_vlans()
+    esx.configure_management_interface(NEW_MGMT_PG)
     esx.configure_default_route()
     esx.configure_requested_dns()
 
@@ -35,8 +64,10 @@ def main(json_file, dry_run):
     )
     args = parser.parse_args()
 
+    setup_logger()
+
     try:
         main(args.json_file, args.dry_run)
-    except Exception as e:
-        print(f"Error configuring network: {str(e)}")
+    except Exception:
+        logger.exception("Error configuring network")
         sys.exit(1)

packages/esxi-netinit/esxi_netinit/main.py

@@ -13,12 +13,23 @@ def __init__(self, data: dict) -> None:
         self.data = data
         self.links = self._init_links(data.get("links", []))
         self.networks = []
+        counter = 0
 
         for net_data in data.get("networks", []):
             net_data = net_data.copy()
             routes_data = net_data.pop("routes", [])
             routes = [Route(**route) for route in routes_data]
-            link_id = net_data.pop("link", [])
+            # VMware wants vmkX to be the pattern for logical interfaces
+            # networks map to logical interfaces so ensure we use the right
+            # value
+            net_data["id"] = f"vmk{counter}"
+            # increment for the next one
+            counter = counter + 1
+            link_id = net_data.pop("link")
+            if not link_id:
+                raise ValueError(
+                    f"Network {net_data.get('network_id')} is invalid, no link supplied"
+                ) from None
             try:
                 relevant_link = next(link for link in self.links if link.id == link_id)
             except StopIteration:

packages/esxi-netinit/esxi_netinit/network_data.py

@@ -31,4 +31,7 @@ def _esxi_nics(self) -> str:
         ).stdout.decode()
 
     def find_by_mac(self, mac) -> NIC:
-        return next(nic for nic in self if nic.mac == mac)
+        try:
+            return next(nic for nic in self if nic.mac == mac)
+        except StopIteration:
+            raise ValueError(f"No NIC with MAC {mac}") from None