ukaea · stephen-dixon · Jan 23, 2025 · Sep 11, 2024 · Oct 30, 2024 · Nov 20, 2024
diff --git a/.github/workflows/build_wheels.yml b/.github/workflows/build_wheels.yml
@@ -28,20 +28,20 @@ jobs:
           # which are only executed on push to tag. 
           # Test wheels are also generated for the other platforms (and uploaded to testpypi)
           # on pushes to release branches and to main
-          - [ubuntu-latest, x86_64, manylinux2014_x86_64]
-          - [ubuntu-latest, x86_64, manylinux_2_28_x86_64]
-          - [ubuntu-latest, aarch64, manylinux_2_28_aarch64]
-          - [macos-13, x86_64, macosx_x86_64]
-          - [macos-14, arm64, macosx_arm64]
-          - [windows-latest, AMD64, win_amd64]
+          - [ubuntu-latest, x86_64, manylinux2014_x86_64, 0.0]
+          - [ubuntu-latest, x86_64, manylinux_2_28_x86_64, 0.0]
+          - [ubuntu-latest, aarch64, manylinux_2_28_aarch64, 0.0]
+          - [macos-13, x86_64, macosx_x86_64, 13.0]
+          - [macos-14, arm64, macosx_arm64, 14.0]
+          - [windows-latest, AMD64, win_amd64, 0.0]
 
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         # need git tags available for setuptools_scm to grab tags
         with:
           fetch-depth: 0
 
-      - uses: actions/github-script@v6
+      - uses: actions/github-script@v7
         with:
           script: |
             core.exportVariable('ACTIONS_CACHE_URL', process.env.ACTIONS_CACHE_URL || '');
@@ -107,15 +107,15 @@ jobs:
 
       - name: build windows wheels
         if: runner.OS == 'Windows'
-        uses: pypa/cibuildwheel@v2.17.0
+        uses: pypa/cibuildwheel@v2.22.0
         with:
           package-dir: ${{github.workspace}}/install/python_installer
           config-file: ${{github.workspace}}/install/python_installer/pyproject.toml
         env:
           CIBW_ARCHS: ${{matrix.build-platform[1]}}
           CIBW_PLATFORM: windows
           CIBW_BUILD: cp3*-${{matrix.build-platform[2]}}
-          CIBW_SKIP: cp36* cp37*   
+          CIBW_SKIP: cp36* cp37* cp313*  
           CIBW_REPAIR_WHEEL_COMMAND_WINDOWS: "delvewheel repair -w {dest_dir} {wheel} --add-path ${{github.workspace}}/install/lib"
           CIBW_BEFORE_BUILD_WINDOWS: "pip install delvewheel"
 
@@ -125,7 +125,7 @@ jobs:
           startswith(matrix.build-platform[2], 'manylinux2014') && 
           github.event_name == 'push' && 
           startsWith(github.ref, 'refs/tags/')
-        uses: pypa/cibuildwheel@v2.17.0
+        uses: pypa/cibuildwheel@v2.22.0
         with:
           package-dir: ./source/wrappers/python
           config-file: ./source/wrappers/python/pyproject.toml
@@ -176,7 +176,7 @@ jobs:
           startswith(matrix.build-platform[2], 'manylinux_2_28') && 
           ! ( startswith(matrix.build-platform[1], 'aarch64') && 
           !startsWith(github.ref, 'refs/tags/') )
-        uses: pypa/cibuildwheel@v2.17.0
+        uses: pypa/cibuildwheel@v2.22.0
         with:
           package-dir: ./source/wrappers/python
           config-file: ./source/wrappers/python/pyproject.toml
@@ -189,7 +189,7 @@ jobs:
           CIBW_BEFORE_ALL: >
             dnf update -y &&
             dnf install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-8.noarch.rpm &&
-            dnf install -y boost-devel openssl-devel libxml2-devel libtirpc-devel 
+            dnf install -y boost1.78-devel openssl-devel libxml2-devel libtirpc-devel 
             fmt fmt-devel spdlog spdlog-devel capnproto capnproto-devel &&
             cd /project &&
             cmake -B build 
@@ -203,6 +203,10 @@ jobs:
 
       - name: build uda on macos
         if: runner.os == 'macOS'
+        # note: it's recommended in cibuildwheel to build all deps from source in macos 
+        # instead of using homebrew as this way the wheel will be back-compatible with 
+        # other macos versions. MACOSX_DEPLOYMENT_TARGET will no longer be required if
+        # this change is made.
         run: >
           brew update-reset && brew install
           git
@@ -222,18 +226,20 @@ jobs:
           -DCMAKE_INSTALL_PREFIX=$PWD/install
           -DCLIENT_ONLY=ON &&
           cmake --build build -j --config Release --target install &&
-          cp -r $PWD/install/python_installer/* ${{github.workspace}}/source/wrappers/python/ 
+          cp -r $PWD/install/python_installer/* ${{github.workspace}}/source/wrappers/python/
 
       - name: Build macos wheels
         if: runner.os == 'macOS'
-        uses: pypa/cibuildwheel@v2.17.0
+        uses: pypa/cibuildwheel@v2.22.0
         with:
           package-dir: ./source/wrappers/python
           config-file: ./source/wrappers/python/pyproject.toml
         env:
           CIBW_ARCHS: ${{matrix.build-platform[1]}}
           CIBW_PLATFORM: macos
+          CIBW_SKIP: cp313*
           CIBW_BUILD: cp*-${{matrix.build-platform[2]}}
+          MACOSX_DEPLOYMENT_TARGET: ${{matrix.build-platform[3]}}
 
       - uses: actions/upload-artifact@v4
         with:

diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -1,4 +1,4 @@
-cmake_minimum_required( VERSION 3.0 )
+cmake_minimum_required( VERSION 3.12 )
 
 ########################################################################################################################
 # Macro used to convert Windows path to Unix

diff --git a/docs/client_installation.md b/docs/client_installation.md
@@ -76,6 +76,7 @@ Install dependencies from system package manager.
 ```sh
 sudo apt update && sudo apt install -y \
 git \
+libtirpc-dev \
 libboost-dev \
 libboost-program-options-dev \
 libssl-dev \

diff --git a/source/cache/CMakeLists.txt b/source/cache/CMakeLists.txt
@@ -24,6 +24,7 @@ set( SRC_FILES
 
 include_directories(
   ${OPENSSL_INCLUDE_DIR}
+  ${Boost_INCLUDE_DIRS}
   ${Boost_INCLUDE_DIR}
   ${CMAKE_SOURCE_DIR}/source
   ${LIBXML2_INCLUDE_DIR}

diff --git a/source/client/CMakeLists.txt b/source/client/CMakeLists.txt
@@ -4,6 +4,7 @@
 find_package( OpenSSL REQUIRED )
 find_package( fmt REQUIRED )
 find_package( LibXml2 REQUIRED )
+find_package( Boost REQUIRED )
 
 if( WIN32 OR MINGW )
   find_package( XDR REQUIRED )
@@ -75,6 +76,7 @@ include_directories(
   ${OPENSSL_INCLUDE_DIR}
   ${LIBXML2_INCLUDE_DIR}
   ${fmt_SOURCE_DIR}/include
+  ${Boost_INCLUDE_DIRS}
 )
 
 add_library( client-objects OBJECT ${SRC_FILES} ${HEADER_FILES} )

diff --git a/source/client/udaClient.cpp b/source/client/udaClient.cpp
@@ -41,7 +41,7 @@
 #if !defined(FATCLIENT) || !defined(NOLIBMEMCACHED)
 //static int protocol_version = 9;
 #endif
-int client_version = 9;          // previous version
+int client_version = 10;          // previous version
 
 //----------------------------------------------------------------------------------------------------------------------
 // FATCLIENT objects shared with server code
@@ -1042,6 +1042,8 @@ int idamClient(REQUEST_BLOCK* request_block, int* indices)
             }
 
             printDataBlock(*data_block);
+            data_received = true;
+            data_block_indices[i] = data_block_idx;
         }
 
 #endif      // <========================== End of FatClient Code Only

diff --git a/source/client2/CMakeLists.txt b/source/client2/CMakeLists.txt
@@ -4,6 +4,7 @@
 find_package( OpenSSL REQUIRED )
 find_package( LibXml2 REQUIRED )
 find_package( fmt REQUIRED )
+find_package( Boost REQUIRED )
 
 if( WIN32 OR MINGW )
   find_package( XDR REQUIRED )
@@ -73,6 +74,7 @@ include_directories(
   ${OPENSSL_INCLUDE_DIR}
   ${LIBXML2_INCLUDE_DIR}
   ${fmt_SOURCE_DIR}/include
+  ${Boost_INCLUDE_DIRS}
 )
 
 add_library( client2-objects OBJECT ${SRC_FILES} ${HEADER_FILES} )

diff --git a/source/clientserver/CMakeLists.txt b/source/clientserver/CMakeLists.txt
@@ -6,6 +6,7 @@ find_package( fmt REQUIRED )
 if( NOT CLIENT_ONLY )
   find_package( LibXml2 REQUIRED )
 endif()
+find_package( Boost REQUIRED )
 
 if( WIN32 OR MINGW )
   find_package( XDR REQUIRED )
@@ -105,6 +106,7 @@ include_directories(
   ${OPENSSL_INCLUDE_DIR}
   ${LIBXML2_INCLUDE_DIR}
   ${fmt_SOURCE_DIR}/include
+  ${Boost_INCLUDE_DIRS}
 )
 
 if( MINGW OR WIN32 )

diff --git a/source/clientserver/protocol.cpp b/source/clientserver/protocol.cpp
@@ -221,7 +221,7 @@ int protocol(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS
 
                     if ((err = allocData(data_block)) != 0) break;        // Allocate Heap Memory
 
-                    if (!xdr_data_block2(xdrs, data_block)) {
+                    if (!xdr_data_block2(xdrs, data_block, protocolVersion)) {
                         err = UDA_PROTOCOL_ERROR_62;
                         break;
                     }
@@ -309,7 +309,7 @@ int protocol(XDR* xdrs, int protocol_id, int direction, int* token, LOGMALLOCLIS
                         break;
                     }
 
-                    if (!xdr_data_block2(xdrs, data_block)) {
+                    if (!xdr_data_block2(xdrs, data_block, protocolVersion)) {
                         err = UDA_PROTOCOL_ERROR_62;
                         break;
                     }

diff --git a/source/clientserver/protocol2.cpp b/source/clientserver/protocol2.cpp
@@ -742,7 +742,7 @@ static int handle_data_block(XDR* xdrs, int direction, const void* str, int prot
 
             if ((err = allocData(data_block)) != 0) break;        // Allocate Heap Memory
 
-            if (!xdr_data_block2(xdrs, data_block)) {
+            if (!xdr_data_block2(xdrs, data_block, protocolVersion)) {
                 err = UDA_PROTOCOL_ERROR_62;
                 break;
             }
@@ -838,7 +838,7 @@ static int handle_data_block(XDR* xdrs, int direction, const void* str, int prot
                 break;
             }
 
-            if (!xdr_data_block2(xdrs, data_block)) {
+            if (!xdr_data_block2(xdrs, data_block, protocolVersion)) {
                 err = UDA_PROTOCOL_ERROR_62;
                 break;
             }

diff --git a/source/clientserver/udaTypes.cpp b/source/clientserver/udaTypes.cpp
@@ -37,6 +37,8 @@ size_t getSizeOf(UDA_TYPE data_type)
             return sizeof(DCOMPLEX);
         case UDA_TYPE_CAPNP:
             return sizeof(char);
+        case UDA_TYPE_OPAQUE:
+            return sizeof(char);
         default:
             return 0;
     }
@@ -78,4 +80,4 @@ size_t getPtrSizeOf(UDA_TYPE data_type)
         default:
             return 0;
     }
-}
+}
diff --git a/source/clientserver/xdrlib.cpp b/source/clientserver/xdrlib.cpp
@@ -616,7 +616,7 @@ bool_t xdr_data_block1(XDR* xdrs, DATA_BLOCK* str, int protocolVersion)
     return rc;
 }
 
-bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str)
+bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str, int protocolVersion)
 {
     switch (str->data_type) {
         case UDA_TYPE_FLOAT:
@@ -662,7 +662,13 @@ bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str)
             return 1;    // Nothing to send so retain good return code
 
         case UDA_TYPE_CAPNP:
-            return xdr_vector(xdrs, str->data, (u_int)str->data_n, sizeof(char), (xdrproc_t)xdr_char);
+            if (protocolVersion >= 10) {
+                return xdr_opaque(xdrs, str->data, (u_int)str->data_n * sizeof(char));
+            } else {
+                return xdr_vector(xdrs, str->data, (u_int)str->data_n, sizeof(char), (xdrproc_t)xdr_char);
+            }
+        case UDA_TYPE_OPAQUE:
+            return xdr_opaque(xdrs, str->data, (u_int)str->data_n * sizeof(char));
 
         default:
             return 0;

diff --git a/source/clientserver/xdrlib.h b/source/clientserver/xdrlib.h
@@ -43,7 +43,7 @@ bool_t xdr_putdata_block1(XDR* xdrs, PUTDATA_BLOCK* str);
 bool_t xdr_putdata_block2(XDR* xdrs, PUTDATA_BLOCK* str);
 bool_t xdr_data_block_list(XDR* xdrs, DATA_BLOCK_LIST* str, int protocolVersion);
 bool_t xdr_data_block1(XDR* xdrs, DATA_BLOCK* str, int protocolVersion);
-bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str);
+bool_t xdr_data_block2(XDR* xdrs, DATA_BLOCK* str, int protocolVersion);
 bool_t xdr_data_block3(XDR* xdrs, DATA_BLOCK* str);
 bool_t xdr_data_block4(XDR* xdrs, DATA_BLOCK* str);
 bool_t xdr_data_dim1(XDR* xdrs, DATA_BLOCK* str);

diff --git a/source/logging/logging.h b/source/logging/logging.h
@@ -22,7 +22,7 @@
         strftime(uix_buf, 30, "%Y:%m:%dT%H:%M:%S", uix_tm);
 #  define uix_printtime() { \
             UIX_DEFINETIME \
-            printf("%s.%dZ, ", uix_buf,(int32_t)uix_tmnow.tv_usec); \
+            printf("%s.%06dZ, ", uix_buf,(int32_t)uix_tmnow.tv_usec); \
         }
 #  define UDA_LOG(LEVEL, FMT, ...) do{ if (LEVEL >= udaGetLogLevel()) {UIX_DEFINETIME udaLog(LEVEL, "%s.%dZ, %s:%d >> " FMT, uix_buf, (int32_t)uix_tmnow.tv_usec, basename((char *)__FILE__), __LINE__, ##__VA_ARGS__); }} while(0)
 #endif

diff --git a/source/plugins/bytes/CMakeLists.txt b/source/plugins/bytes/CMakeLists.txt
@@ -24,6 +24,7 @@ uda_plugin(
   EXAMPLE "BYTES::read()"
   LIBNAME bytes_plugin
   SOURCES bytesPlugin.cpp readBytesNonOptimally.cpp md5Sum.cpp
+  CONFIG_FILE bytesPlugin.cfg
   EXTRA_INCLUDE_DIRS
     ${LIBXML2_INCLUDE_DIR}
   EXTRA_LINK_LIBS

diff --git a/source/plugins/bytes/bytesPlugin.cfg.in b/source/plugins/bytes/bytesPlugin.cfg.in
@@ -0,0 +1 @@
+export UDA_BYTES_PLUGIN_ALLOWED_PATHS=