NAS-135917 / 25.10 / SPDK backend for NVMe-oF

src/middlewared/debian/control

@@ -63,6 +63,7 @@ Build-Depends: alembic,
                python3-truenas-api-client,
                python3-truenas-connect-utils,
                python3-truenas-crypto-utils,
+               python3-truenas-spdk,
                python3-truenas-verify,
                python3-websocket,
                python3-zeroconf,
@@ -156,6 +157,7 @@ Depends: alembic,
          python3-truenas-api-client,
          python3-truenas-connect-utils,
          python3-truenas-crypto-utils,
+         python3-truenas-spdk,
          python3-truenas-verify,
          python3-websocket,
          python3-zeroconf,

src/middlewared/middlewared/alembic/versions/25.10/2025-07-16_16-23_nvmet_dedicated_nic.py

@@ -0,0 +1,28 @@
+"""Add dedicated_nic to nvmet port config
+
+This will be used with SPDK.
+
+Revision ID: 72b63cd393d3
+Revises: 3d738dbd75ef
+Create Date: 2025-07-16 16:23:12.160536+00:00
+
+"""
+from alembic import op
+import sqlalchemy as sa
+
+
+# revision identifiers, used by Alembic.
+revision = '72b63cd393d3'
+down_revision = '3d738dbd75ef'
+branch_labels = None
+depends_on = None
+
+
+def upgrade():
+    with op.batch_alter_table('services_nvmet_port', schema=None) as batch_op:
+        batch_op.add_column(sa.Column('nvmet_port_dedicated_nic', sa.Boolean(), server_default='0', nullable=False))
+
+
+def downgrade():
+    with op.batch_alter_table('services_nvmet_port', schema=None) as batch_op:
+        batch_op.drop_column('nvmet_port_dedicated_nic')

src/middlewared/middlewared/api/v25_10_0/nvmet_port.py

@@ -48,6 +48,12 @@ class NVMetPortEntry(BaseModel):
     # """ Transport Requirements codes for Discovery Log Page entry TREQ field. """
     enabled: bool = True
     """ Port enabled.  When NVMe target is running, cannot make changes to an enabled port. """
+    dedicated_nic: bool = False
+    """
+    NIC dedicated.  If supported by the underlying NVMe-oF implementation, then
+    the NIC will be dedicated to NVMe-oF, and not available for use by other
+    protocols.
+    """
 
 
 class NVMetPortCreateTemplate(NVMetPortEntry, ABC):

src/middlewared/middlewared/etc_files/nvmet_kernel.py

@@ -1,4 +1,4 @@
-from middlewared.plugins.nvmet.kernel import write_config
+from middlewared.utils.nvmet.kernel import write_config
 
 
 def render(service, middleware, render_ctx):

src/middlewared/middlewared/etc_files/nvmet_spdk.py

@@ -0,0 +1,35 @@
+from middlewared.utils.nvmet.spdk import write_config
+from middlewared.plugins.nvmet.constants import NAMESPACE_DEVICE_TYPE
+
+
+def render(service, middleware, render_ctx):
+    if middleware.call_sync('nvmet.spdk.nvmf_ready', True):
+
+        # If we have any namespaces that are configured which are FILE
+        # type, then we need to work out the blocksize for each one.
+        # This will be the recordsize of the underlying dataset.
+        fns = {ns['device_path'] for ns in filter(lambda ns: ns.get('device_type') == NAMESPACE_DEVICE_TYPE.FILE.api,
+                                                  render_ctx['nvmet.namespace.query'])}
+        if fns:
+            record_sizes = {f'{item["mountpoint"]}/': int(item['recordsize']['rawvalue']) for item in
+                            middleware.call_sync('pool.dataset.query',
+                                                 [["mountpoint", "!=", None]],
+                                                 {"select": ["name",
+                                                             "children",
+                                                             "mountpoint",
+                                                             "recordsize.rawvalue"]})}
+            path_to_recordsize = {}
+            for path in fns:
+                longest_match = 0
+                matched_value = None
+                for key, value in record_sizes.items():
+                    if path.startswith(key):
+                        if (length := len(key)) > longest_match:
+                            longest_match = length
+                            matched_value = value
+                if matched_value:
+                    path_to_recordsize[path] = matched_value
+            # Inject into context
+            render_ctx['path_to_recordsize'] = path_to_recordsize
+
+        write_config(render_ctx)

src/middlewared/middlewared/plugins/etc.py

@@ -201,6 +201,7 @@ class EtcService(Service):
             ],
             'entries': [
                 {'type': 'py', 'path': 'nvmet_kernel'},
+                {'type': 'py', 'path': 'nvmet_spdk'},
             ]
         },
         'pam': {

src/middlewared/middlewared/plugins/nvmet/constants.py

@@ -1,9 +1,7 @@
 import enum
 
 NVMET_KERNEL_CONFIG_DIR = '/sys/kernel/config/nvmet'
-NVMET_NODE_A_ANA_GRPID = 2
-NVMET_NODE_B_ANA_GRPID = 3
-
+NVMET_DISCOVERY_NQN = 'nqn.2014-08.org.nvmexpress.discovery'
 NVMET_NQN_UUID = 'nqn.2011-06.com.truenas:uuid'
 NVMET_SERVICE_NAME = 'nvmet'
 
@@ -24,6 +22,10 @@ def api(self):
     def sysfs(self):
         return self.value[2]
 
+    @property
+    def spdk(self):
+        return self.value[3]
+
     @classmethod
     def by_db(cls, needle, raise_exception=True):
         for x in cls.__members__.values():
@@ -68,10 +70,10 @@ class PORT_TRTYPE(ApiMapper):
 
 
 class PORT_ADDR_FAMILY(ApiMapper):
-    IPV4 = (1, 'IPV4', 'ipv4')
-    IPV6 = (2, 'IPV6', 'ipv6')
-    IB = (3, 'IB', 'ib')
-    FC = (4, 'FC', 'fc')
+    IPV4 = (1, 'IPV4', 'ipv4', 'IPv4')
+    IPV6 = (2, 'IPV6', 'ipv6', 'IPv6')
+    IB = (3, 'IB', 'ib', 'IB')
+    FC = (4, 'FC', 'fc', 'FC')
 
 
 def port_transport_family_generator():

src/middlewared/middlewared/plugins/nvmet/global.py

@@ -14,11 +14,14 @@
 from middlewared.service import SystemServiceService, ValidationErrors, filterable_api_method, private
 from middlewared.utils import filter_list
 from .constants import NVMET_SERVICE_NAME
-from .kernel import clear_config, load_modules, nvmet_kernel_module_loaded, unload_module
+from middlewared.utils.nvmet.kernel import clear_config, load_modules, nvmet_kernel_module_loaded, unload_module
+from middlewared.utils.nvmet.spdk import make_client, nvmf_subsystem_get_qpairs
 from .mixin import NVMetStandbyMixin
 from .utils import uuid_nqn
 
 NVMET_DEBUG_DIR = '/sys/kernel/debug/nvmet'
+NVMF_SERVICE = 'nvmf'
+AVX2_FLAG = 'avx2'
 
 
 class NVMetGlobalModel(sa.Model):
@@ -69,9 +72,11 @@ async def do_update(self, data):
     async def __ana_forbidden(self):
         return not await self.middleware.call('failover.licensed')
 
+    async def __spdk_forbidden(self):
+        # For now we'll disallow SPDK if any CPU is missing avx2
+        return any(AVX2_FLAG not in flags for flags in (await self.middleware.call('system.cpu_flags')).values())
+
     async def __validate(self, verrors, data, schema_name, old=None):
-        if not data.get('kernel', False):
-            verrors.add(f'{schema_name}.kernel', 'Cannot disable kernel mode.')
         if data['rdma'] and old['rdma'] != data['rdma']:
             available_rdma_protocols = await self.middleware.call('rdma.capable_protocols')
             if RDMAprotocols.NVMET.value not in available_rdma_protocols:
@@ -85,6 +90,17 @@ async def __validate(self, verrors, data, schema_name, old=None):
                     f'{schema_name}.ana',
                     'This platform does not support Asymmetric Namespace Access(ANA).'
                 )
+        if old['kernel'] != data['kernel']:
+            if not data['kernel'] and await self.__spdk_forbidden():
+                verrors.add(
+                    f'{schema_name}.kernel',
+                    'Cannot switch nvmet backend because CPU lacks required capabilities.'
+                )
+            elif await self.running():
+                verrors.add(
+                    f'{schema_name}.kernel',
+                    'Cannot switch nvmet backend while the service is running.'
+                )
 
     @api_method(
         NVMetGlobalAnaEnabledArgs,
@@ -133,9 +149,9 @@ async def rdma_enabled(self):
     async def sessions(self, filters, options):
         sessions = []
         subsys_id = None
-        for filter in filters:
-            if len(filter) == 3 and filter[0] == 'subsys_id' and filter[1] == '=':
-                subsys_id = filter[2]
+        for _filter in filters:
+            if len(_filter) == 3 and _filter[0] == 'subsys_id' and _filter[1] == '=':
+                subsys_id = _filter[2]
                 break
         sessions = await self.middleware.call('nvmet.global.local_sessions', subsys_id)
         if await self.ana_enabled():
@@ -171,33 +187,74 @@ def local_sessions(self, subsys_id=None):
         sessions = []
         global_info = self.middleware.call_sync('nvmet.global.config')
         subsystems = self.middleware.call_sync('nvmet.subsys.query')
+        ports = self.middleware.call_sync('nvmet.port.query')
+        ha = self.middleware.call_sync('failover.licensed')
 
         if global_info['kernel']:
             nvmet_debug_path = pathlib.Path(NVMET_DEBUG_DIR)
             if not nvmet_debug_path.exists():
                 return sessions
 
-        port_index_to_id = {port['index']: port['id'] for port in self.middleware.call_sync('nvmet.port.query')}
+            port_index_to_id = {port['index']: port['id'] for port in ports}
 
-        if subsys_id is None:
-            basenqn = global_info['basenqn']
-            subsys_name_to_subsys_id = {f'{basenqn}:{subsys["name"]}': subsys['id'] for subsys in subsystems}
-            for subsys in nvmet_debug_path.iterdir():
-                if subsys_id := subsys_name_to_subsys_id.get(subsys.name):
-                    for ctrl in subsys.iterdir():
-                        if session := self.__parse_session_dir(ctrl, port_index_to_id):
-                            session['subsys_id'] = subsys_id
-                            sessions.append(session)
-        else:
-            for subsys in subsystems:
-                if subsys['id'] == subsys_id:
-                    subnqn = f'{global_info["basenqn"]}:{subsys["name"]}'
-                    path = nvmet_debug_path / subnqn
-                    if path.is_dir():
-                        for ctrl in path.iterdir():
+            if subsys_id is None:
+                subsys_name_to_subsys_id = {subsys['subnqn']: subsys['id'] for subsys in subsystems}
+                for subsys in nvmet_debug_path.iterdir():
+                    if subsys_id := subsys_name_to_subsys_id.get(subsys.name):
+                        for ctrl in subsys.iterdir():
                             if session := self.__parse_session_dir(ctrl, port_index_to_id):
                                 session['subsys_id'] = subsys_id
                                 sessions.append(session)
+            else:
+                for subsys in subsystems:
+                    if subsys['id'] == subsys_id:
+                        subnqn = subsys['subnqn']
+                        path = nvmet_debug_path / subnqn
+                        if path.is_dir():
+                            for ctrl in path.iterdir():
+                                if session := self.__parse_session_dir(ctrl, port_index_to_id):
+                                    session['subsys_id'] = subsys_id
+                                    sessions.append(session)
+        else:
+            if not self.middleware.call_sync('nvmet.spdk.nvmf_ready'):
+                return sessions
+            client = make_client()
+            choices = {}
+            port_to_portid = {}
+            for port in ports:
+                addr = port['addr_traddr']
+                trtype = port['addr_trtype']
+                addrs = [addr]
+                if ha:
+                    try:
+                        mychoices = choices[trtype]
+                    except KeyError:
+                        mychoices = self.middleware.call_sync('nvmet.port.transport_address_choices', trtype, True)
+                        mychoices[trtype] = mychoices
+                    if pair := mychoices.get(addr, '').split('/'):
+                        addrs.extend(pair)
+
+                for addr in addrs:
+                    port_to_portid[f"{trtype}:{addr}:{port['addr_trsvcid']}"] = port['id']
+            for subsys in subsystems:
+                if subsys_id is not None and subsys_id != subsys['id']:
+                    continue
+                for entry in nvmf_subsystem_get_qpairs(client, subsys['subnqn']):
+                    laddr = entry['listen_address']
+                    key = f"{laddr['trtype']}:{laddr['traddr']}:{laddr['trsvcid']}"
+                    if port_id := port_to_portid.get(key):
+                        session = {
+                            'host_traddr': entry.get('peer_address', {}).get('traddr'),
+                            'hostnqn': entry['hostnqn'],
+                            'subsys_id': subsys['id'],
+                            'port_id': port_id,
+                            'ctrl': entry['cntlid']
+                        }
+                        # Do we really care that we might have multiple duplicate connections
+                        # (only visible delta being a different qid and peer_address.trsvcid)
+                        # If so, replace this with a simple append.
+                        if session not in sessions:
+                            sessions.append(session)
 
         return sessions
 
@@ -228,7 +285,7 @@ async def running(self):
         if (await self.config())['kernel']:
             return await self.middleware.run_in_thread(nvmet_kernel_module_loaded)
         else:
-            return False
+            return await self.middleware.call('service.started', NVMF_SERVICE)
 
     @private
     async def reload(self):
@@ -237,14 +294,22 @@ async def reload(self):
 
     @private
     async def start(self):
-        await self.middleware.call('nvmet.global.load_kernel_modules')
+        if (await self.config())['kernel']:
+            await self.middleware.call('nvmet.global.load_kernel_modules')
+        else:
+            await self.middleware.call('nvmet.spdk.slots')
+            if await self.middleware.call('service.start', NVMF_SERVICE):
+                await self.middleware.call('nvmet.spdk.wait_nvmf_ready')
         await self.middleware.call('etc.generate', 'nvmet')
 
     @private
     async def stop(self):
         if await self.running():
-            await self.middleware.run_in_thread(clear_config)
-            await self.middleware.call('nvmet.global.unload_kernel_modules')
+            if (await self.config())['kernel']:
+                await self.middleware.run_in_thread(clear_config)
+                await self.middleware.call('nvmet.global.unload_kernel_modules')
+            else:
+                return await self.middleware.call('service.stop', NVMF_SERVICE)
 
     @private
     async def system_ready(self):

src/middlewared/middlewared/plugins/nvmet/namespace.py

@@ -17,9 +17,9 @@
 from middlewared.service import SharingService, ValidationErrors, private
 from middlewared.service_exception import CallError, MatchNotFound
 from .constants import NAMESPACE_DEVICE_TYPE
-from .kernel import lock_namespace as kernel_lock_namespace
-from .kernel import unlock_namespace as kernel_unlock_namespace
-from .kernel import resize_namespace as kernel_resize_namespace
+from middlewared.utils.nvmet.kernel import lock_namespace as kernel_lock_namespace
+from middlewared.utils.nvmet.kernel import unlock_namespace as kernel_unlock_namespace
+from middlewared.utils.nvmet.kernel import resize_namespace as kernel_resize_namespace
 
 UUID_GENERATE_RETRIES = 10
 NSID_SEARCH_RANGE = 0xFFFF  # This is much less than NSID, but good enough for practical purposes.

src/middlewared/middlewared/plugins/nvmet/port.py

@@ -36,6 +36,7 @@ class NVMetPortModel(sa.Model):
     nvmet_port_max_queue_size = sa.Column(sa.Integer(), nullable=True, default=None)
     nvmet_port_pi_enable = sa.Column(sa.Boolean(), nullable=True, default=None)
     nvmet_port_enabled = sa.Column(sa.Boolean())
+    nvmet_port_dedicated_nic = sa.Column(sa.Boolean(), default=False)
 
 
 class NVMetPortService(CRUDService):
@@ -276,6 +277,25 @@ async def __validate(self, verrors, data, schema_name, old=None):
         except MatchNotFound:
             existing = None
 
+        # Dedicate NIC checks only apply if SPDK is running:
+        if running := await self.middleware.call('nvmet.global.running'):
+            if not (await self.middleware.call('nvmet.global.config'))['kernel']:
+                # SPDK is running
+                if data['dedicated_nic'] and not old:
+                    # Cannot add a *new* dedicated NIC
+                    # Check to see if any ports already have this NIC as dedicated
+                    filters = [
+                        ['addr_trtype', '=', data['addr_trtype']],
+                        ['addr_traddr', '=', data['addr_traddr']],
+                        ['dedicated_nic', '=', True],
+                    ]
+                    if not await self.middleware.call('nvmet.port.query', filters):
+                        verrors.add(f'{schema_name}.dedicated_nic',
+                                    'Cannot add dedicated_nic when service is running.')
+                elif old and old['dedicated_nic'] != data['dedicated_nic']:
+                    verrors.add(f'{schema_name}.dedicated_nic',
+                                'Cannot modify dedicated_nic when service is running')
+
         if old is None:
             # Create
             # Ensure that we're not duplicating an existing entry
@@ -295,11 +315,11 @@ async def __validate(self, verrors, data, schema_name, old=None):
                                           [['port.id', '=', old['id']]],
                                           {'count': True}):
                 # Have some subsystems attached to the port
-                if old['enabled'] and await self.middleware.call('nvmet.global.running'):
-                    # port is enabled and running
-                    # Ensure we're only changing enabled
+                if old['enabled'] and running:
+                    # port is enabled and running. Ensure we're only changing enabled
+                    # (or dedicated_nic which is checked separately above)
                     for key, oldvalue in old.items():
-                        if key == 'enabled':
+                        if key in ['enabled', 'dedicated_nic']:
                             continue
                         if data[key] == oldvalue:
                             continue