UCT/GGA: enable testing for md, iface, ep_connect, CR2

src/uct/ib/base/ib_iface.c

@@ -701,29 +701,39 @@ uct_ib_iface_roce_is_reachable(const uct_ib_device_gid_info_t *local_gid_info,
     return ret;
 }
 
-int uct_ib_iface_is_same_device(const uct_ib_address_t *ib_addr, uint16_t dlid,
-                                const union ibv_gid *dgid)
+static int
+uct_ib_iface_is_same_device_unpacked(
+        const uct_ib_address_pack_params_t *unpacked_addr, uint16_t dlid,
+        const union ibv_gid *dgid)
 {
-    uct_ib_address_pack_params_t params;
-
-    uct_ib_address_unpack(ib_addr, &params);
-
-    if (!(params.flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) &&
-        (dlid != params.lid)) {
+    if (!(unpacked_addr->flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) &&
+        (dlid != unpacked_addr->lid)) {
         return 0;
     }
 
     if (dgid == NULL) {
-        return !(params.flags & (UCT_IB_ADDRESS_PACK_FLAG_ETH |
-                                 UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID));
+        return !(unpacked_addr->flags &
+                 (UCT_IB_ADDRESS_PACK_FLAG_ETH |
+                  UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID));
     }
 
-    if (params.flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) {
-        return !memcmp(dgid->raw, params.gid.raw, sizeof(params.gid.raw));
+    if (unpacked_addr->flags & UCT_IB_ADDRESS_PACK_FLAG_ETH) {
+        return !memcmp(dgid->raw, unpacked_addr->gid.raw,
+                       sizeof(unpacked_addr->gid.raw));
     }
 
-    return !(params.flags & UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID) ||
-           (params.gid.global.interface_id == dgid->global.interface_id);
+    return !(unpacked_addr->flags & UCT_IB_ADDRESS_PACK_FLAG_INTERFACE_ID) ||
+           (unpacked_addr->gid.global.interface_id ==
+            dgid->global.interface_id);
+}
+
+int uct_ib_iface_is_same_device(const uct_ib_address_t *ib_addr, uint16_t dlid,
+                                const union ibv_gid *dgid)
+{
+    uct_ib_address_pack_params_t params;
+
+    uct_ib_address_unpack(ib_addr, &params);
+    return uct_ib_iface_is_same_device_unpacked(&params, dlid, dgid);
 }
 
 static int uct_ib_iface_gid_extract_flid(const union ibv_gid *gid)
@@ -742,14 +752,22 @@ static int uct_ib_iface_is_flid_enabled(uct_ib_iface_t *iface)
            (uct_ib_iface_gid_extract_flid(&iface->gid_info.gid) != 0);
 }
 
-static int uct_ib_iface_dev_addr_is_reachable(uct_ib_iface_t *iface,
-                                              const uct_ib_address_t *ib_addr)
+int uct_ib_iface_dev_addr_is_reachable(uct_ib_iface_t *iface,
+                                       const uct_ib_address_t *ib_addr,
+                                       uct_iface_reachability_scope_t scope)
 {
-    int is_local_eth                = uct_ib_iface_is_roce(iface);
+    int is_local_eth = uct_ib_iface_is_roce(iface);
     uct_ib_address_pack_params_t params;
 
     uct_ib_address_unpack(ib_addr, &params);
 
+    if ((scope == UCT_IFACE_REACHABILITY_SCOPE_DEVICE) &&
+        !uct_ib_iface_is_same_device_unpacked(&params,
+                                              uct_ib_iface_port_attr(iface)->lid,
+                                              &iface->gid_info.gid)) {
+        return 0;
+    }
+
     if (/* at least one PKEY has to be with full membership */
         !((params.pkey | iface->pkey) & UCT_IB_PKEY_MEMBERSHIP_MASK) ||
         /* PKEY values have to be equal */
@@ -797,16 +815,9 @@ int uct_ib_iface_is_reachable_v2(const uct_iface_h tl_iface,
         return 0;
     }
 
-    if (!uct_ib_iface_dev_addr_is_reachable(iface, device_addr)) {
-        return 0;
-    }
-
     scope = UCS_PARAM_VALUE(UCT_IFACE_IS_REACHABLE_FIELD, params, scope, SCOPE,
                             UCT_IFACE_REACHABILITY_SCOPE_NETWORK);
-    return (scope == UCT_IFACE_REACHABILITY_SCOPE_NETWORK) ||
-           uct_ib_iface_is_same_device(device_addr,
-                                       uct_ib_iface_port_attr(iface)->lid,
-                                       &iface->gid_info.gid);
+    return uct_ib_iface_dev_addr_is_reachable(iface, device_addr, scope);
 }
 
 ucs_status_t uct_ib_iface_create_ah(uct_ib_iface_t *iface,

src/uct/ib/base/ib_iface.h

@@ -718,6 +718,10 @@ uct_ib_iface_roce_dscp(uct_ib_iface_t *iface)
     return iface->config.traffic_class >> 2;
 }
 
+int uct_ib_iface_dev_addr_is_reachable(uct_ib_iface_t *iface,
+                                       const uct_ib_address_t *ib_addr,
+                                       uct_iface_reachability_scope_t scope);
+
 #if HAVE_DECL_IBV_CREATE_CQ_EX
 static UCS_F_ALWAYS_INLINE void
 uct_ib_fill_cq_attr(struct ibv_cq_init_attr_ex *cq_attr,

src/uct/ib/dc/dc_mlx5.c

@@ -646,6 +646,7 @@ void uct_dc_mlx5_destroy_dct(uct_dc_mlx5_iface_t *iface)
         uct_ib_mlx5_devx_obj_destroy(iface->rx.dct.devx.obj, "DCT");
 #endif
         break;
+    case UCT_IB_MLX5_OBJ_TYPE_NULL:
     case UCT_IB_MLX5_OBJ_TYPE_LAST:
         break;
     }

src/uct/ib/mlx5/dv/ib_mlx5_dv.c

@@ -202,7 +202,9 @@ ucs_status_t uct_ib_mlx5_devx_create_qp(uct_ib_iface_t *iface,
     UCT_IB_MLX5DV_SET(qpc, qpc, pd, uct_ib_mlx5_devx_md_get_pdn(md));
     UCT_IB_MLX5DV_SET(qpc, qpc, uar_page, uar->uar->page_id);
     ucs_assert((attr->super.srq == NULL) || (attr->super.srq_num != 0));
-    UCT_IB_MLX5DV_SET(qpc, qpc, rq_type, !!attr->super.srq_num);
+    UCT_IB_MLX5DV_SET(qpc, qpc, rq_type,
+                      attr->super.srq_num ? 1 /* SRQ */ :
+                      max_rx ? 0 /* RQ */ : 3 /* no RQ */);
     UCT_IB_MLX5DV_SET(qpc, qpc, srqn_rmpn_xrqn, attr->super.srq_num);
     UCT_IB_MLX5DV_SET(qpc, qpc, cqn_snd, send_cq->cq_num);
     UCT_IB_MLX5DV_SET(qpc, qpc, cqn_rcv, recv_cq->cq_num);
@@ -319,6 +321,8 @@ ucs_status_t uct_ib_mlx5_devx_modify_qp(uct_ib_mlx5_qp_t *qp,
     case UCT_IB_MLX5_OBJ_TYPE_DEVX:
         return uct_ib_mlx5_devx_obj_modify(qp->devx.obj, in, inlen, out, outlen,
                                            opcode_str);
+    case UCT_IB_MLX5_OBJ_TYPE_NULL:
+        return UCS_ERR_INVALID_PARAM;
     case UCT_IB_MLX5_OBJ_TYPE_LAST:
         return UCS_ERR_UNSUPPORTED;
     }
@@ -352,6 +356,8 @@ uct_ib_mlx5_devx_query_qp(uct_ib_mlx5_qp_t *qp, void *in, size_t inlen,
             return UCS_ERR_IO_ERROR;
         }
         break;
+    case UCT_IB_MLX5_OBJ_TYPE_NULL:
+        return UCS_ERR_INVALID_PARAM;
     case UCT_IB_MLX5_OBJ_TYPE_LAST:
         return UCS_ERR_UNSUPPORTED;
     }

src/uct/ib/mlx5/ib_mlx5.c

@@ -786,6 +786,8 @@ void uct_ib_mlx5_qp_mmio_cleanup(uct_ib_mlx5_qp_t *qp,
         uct_ib_mlx5_iface_put_res_domain(qp);
         uct_worker_tl_data_put(reg, uct_ib_mlx5_mmio_cleanup);
         break;
+    case UCT_IB_MLX5_OBJ_TYPE_NULL:
+        ucs_fatal("qp %p: TYPE_NULL", qp);
     case UCT_IB_MLX5_OBJ_TYPE_LAST:
         if (reg != NULL) {
             uct_worker_tl_data_put(reg, uct_ib_mlx5_mmio_cleanup);
@@ -941,6 +943,8 @@ void uct_ib_mlx5_destroy_qp(uct_ib_mlx5_md_t *md, uct_ib_mlx5_qp_t *qp)
     case UCT_IB_MLX5_OBJ_TYPE_DEVX:
         uct_ib_mlx5_devx_destroy_qp(md, qp);
         break;
+    case UCT_IB_MLX5_OBJ_TYPE_NULL:
+        ucs_fatal("md %p: qp %p: TYPE_NULL", md, qp);
     case UCT_IB_MLX5_OBJ_TYPE_LAST:
         break;
     }

src/uct/ib/mlx5/ib_mlx5.h

@@ -437,6 +437,7 @@ typedef struct uct_ib_mlx5_dbrec {
 typedef enum {
     UCT_IB_MLX5_OBJ_TYPE_VERBS,
     UCT_IB_MLX5_OBJ_TYPE_DEVX,
+    UCT_IB_MLX5_OBJ_TYPE_NULL,
     UCT_IB_MLX5_OBJ_TYPE_LAST
 } uct_ib_mlx5_obj_type_t;