[tmva][sofie] Disentangle PyMVA and SOFIE

cmake/modules/RootBuildOptions.cmake

@@ -175,7 +175,7 @@ ROOT_BUILD_OPTION(tmva-gpu OFF "Build TMVA with GPU support for deep learning (r
 ROOT_BUILD_OPTION(tmva-pymva OFF "Enable usage of Python ML libraries in TMVA (requires NumPy, works only with TensorFlow<=2.15)")
 ROOT_BUILD_OPTION(tmva-rmva OFF "Enable support for R in TMVA")
 ROOT_BUILD_OPTION(tmva-sofie OFF "Build TMVA with support for sofie - fast inference code generation (requires protobuf 3)")
-ROOT_BUILD_OPTION(tpython ON "Build the TPython class that allows you to run Python code from C++")
+ROOT_BUILD_OPTION(tpython ON "Build the TPython class that allows you to run Python code from C++, as well as other ROOT libraries that link against libpython")
 ROOT_BUILD_OPTION(unfold OFF "Enable the unfold package [GPL]")
 ROOT_BUILD_OPTION(unuran OFF "Enable support for UNURAN (package for generating non-uniform random numbers) [GPL]")
 ROOT_BUILD_OPTION(uring OFF "Enable support for io_uring (requires liburing and Linux kernel >= 5.1)")

cmake/modules/RootMacros.cmake

@@ -2008,14 +2008,16 @@ endmacro()
 # ROOT_FIND_PYTHON_MODULE(module [REQUIRED] [QUIET])
 # Try importing the python dependency and cache the result in
 # ROOT_TEST_<MODULE> (all upper case).
-# Also set ROOT_<MODULE>_FOUND (all upper case) as well as ROOT_<module>_FOUND
-# (the original spelling of the argument) in the parent scope of this function
-# for convenient testing in subsequent if().
+# Also set ROOT_<MODULE>_FOUND and ROOT_<module>_FOUND (the original spelling)
+# in the parent scope for convenient testing in subsequent if() statements.
+# Additionally, sets ROOT_<MODULE>_VERSION (and ROOT_<module>_VERSION)
+# if the version could be determined.
 #----------------------------------------------------------------------------
 function(ROOT_FIND_PYTHON_MODULE module)
   CMAKE_PARSE_ARGUMENTS(ARG "REQUIRED;QUIET" "" "" ${ARGN})
   string(TOUPPER ${module} module_upper)
   set(CACHE_VAR ROOT_TEST_${module_upper})
+  set(CACHE_VAR_VERSION "${CACHE_VAR}_VERSION")
 
   if(NOT DEFINED ${CACHE_VAR})
     execute_process(COMMAND "${Python3_EXECUTABLE}" "-c"
@@ -2025,10 +2027,25 @@ function(ROOT_FIND_PYTHON_MODULE module)
       OUTPUT_STRIP_TRAILING_WHITESPACE
       ERROR_QUIET)
 
-    if(${status} EQUAL 0)
+    if(status EQUAL 0)
       set(${CACHE_VAR} ON CACHE BOOL "Enable tests depending on '${module}'")
+      # Only cache a non-empty, non-'unknown' version string.
+      if(module_version AND NOT module_version STREQUAL "unknown")
+        set(${CACHE_VAR_VERSION} "${module_version}" CACHE STRING "Detected version of python module ${module}")
+      else()
+        # ensure no stale version remains in cache
+        if(DEFINED ${CACHE_VAR_VERSION})
+          unset(${CACHE_VAR_VERSION} CACHE)
+        endif()
+        unset(module_version)
+      endif()
     else()
       set(${CACHE_VAR} OFF CACHE BOOL "Enable tests depending on '${module}'")
+      # ensure version cache entry is removed on failure
+      if(DEFINED ${CACHE_VAR_VERSION})
+        unset(${CACHE_VAR_VERSION} CACHE)
+      endif()
+      unset(module_version)
     endif()
 
     if(NOT ARG_QUIET)
@@ -2038,12 +2055,25 @@ function(ROOT_FIND_PYTHON_MODULE module)
         message(STATUS "Could NOT find Python module ${module}. Corresponding tests will be disabled.")
       endif()
     endif()
+  else()
+    # Cache exists; if a cached version string exists, read it into module_version.
+    if(DEFINED ${CACHE_VAR_VERSION})
+      set(module_version ${${CACHE_VAR_VERSION}})
+    endif()
   endif()
 
   # Set the ROOT_xxx_FOUND to the (cached) result of the search:
   set(ROOT_${module_upper}_FOUND ${${CACHE_VAR}} PARENT_SCOPE)
   set(ROOT_${module}_FOUND ${${CACHE_VAR}} PARENT_SCOPE)
 
+  # Expose version only if module was found and a version string is available.
+  if(${CACHE_VAR})
+    if(DEFINED module_version AND NOT module_version STREQUAL "" AND NOT module_version STREQUAL "unknown")
+      set(ROOT_${module_upper}_VERSION "${module_version}" PARENT_SCOPE)
+      set(ROOT_${module}_VERSION "${module_version}" PARENT_SCOPE)
+    endif()
+  endif()
+
   if(ARG_REQUIRED AND NOT ${CACHE_VAR})
     message(FATAL_ERROR "Python module ${module} is required.")
   endif()

cmake/modules/SearchInstalledSoftware.cmake

@@ -705,7 +705,7 @@ if(pyroot AND NOT (tpython OR tmva-pymva))
 elseif(tpython OR tmva-pymva)
   list(APPEND python_components Development)
 endif()
-if(tmva-pymva)
+if(tmva-pymva OR (tpython AND tmva))
   list(APPEND python_components NumPy)
 endif()
 find_package(Python3 3.9 COMPONENTS ${python_components})
@@ -1796,7 +1796,7 @@ if(tmva)
       endif()
     endif()
   endif()
-  if(tmva-pymva)
+  if(tmva-pymva OR (tpython AND tmva))
     if(fail-on-missing AND (NOT Python3_NumPy_FOUND OR NOT Python3_Development_FOUND))
       message(SEND_ERROR "TMVA: numpy python package or Python development package not found and tmva-pymva component required"
                           " (python executable: ${Python3_EXECUTABLE})")

tmva/pymva/CMakeLists.txt

@@ -17,17 +17,13 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     TMVA/MethodPyKeras.h
     TMVA/MethodPyRandomForest.h
     TMVA/MethodPyTorch.h
-    TMVA/RModelParser_Keras.h
-    TMVA/RModelParser_PyTorch.h
     TMVA/PyMethodBase.h
   SOURCES
     src/MethodPyAdaBoost.cxx
     src/MethodPyGTB.cxx
     src/MethodPyKeras.cxx
     src/MethodPyRandomForest.cxx
     src/MethodPyTorch.cxx
-    src/RModelParser_Keras.cxx
-    src/RModelParser_PyTorch.cxx
     src/PyMethodBase.cxx
   LIBRARIES
     Python3::NumPy
@@ -38,7 +34,6 @@ ROOT_STANDARD_LIBRARY_PACKAGE(PyMVA
     Thread
     RIO
     TMVA
-    ROOTTMVASofie
 )
 
 ROOT_ADD_TEST_SUBDIRECTORY(test)

tmva/pymva/inc/LinkDef.h

@@ -15,8 +15,4 @@
 #pragma link C++ class TMVA::MethodPyGTB+;
 #pragma link C++ class TMVA::MethodPyKeras+;
 #pragma link C++ class TMVA::MethodPyTorch+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyKeras;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyKeras::Parse+;
-#pragma link C++ namespace TMVA::Experimental::SOFIE::PyTorch;
-#pragma link C++ function TMVA::Experimental::SOFIE::PyTorch::Parse+;
 #endif

tmva/pymva/test/CMakeLists.txt

@@ -11,7 +11,7 @@
 
 project(pymva-tests)
 
-set(Libraries Core MathCore TMVA PyMVA ROOTTMVASofie)
+set(Libraries Core MathCore TMVA PyMVA)
 
 # Look for needed python modules
 ROOT_FIND_PYTHON_MODULE(torch)
@@ -55,10 +55,6 @@ endif(ROOT_SKLEARN_FOUND)
 
 # Enable tests based on available python modules
 if(ROOT_TORCH_FOUND)
-   configure_file(generatePyTorchModelClassification.py generatePyTorchModelClassification.py COPYONLY)
-   configure_file(generatePyTorchModelMulticlass.py generatePyTorchModelMulticlass.py COPYONLY)
-   configure_file(generatePyTorchModelRegression.py generatePyTorchModelRegression.py COPYONLY)
-   configure_file(generatePyTorchModels.py generatePyTorchModels.py COPYONLY)
    # Test PyTorch: Binary classification
 
    if (ROOT_SKLEARN_FOUND)
@@ -80,27 +76,9 @@ if(ROOT_TORCH_FOUND)
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Torch-Multiclass COMMAND testPyTorchMulticlass DEPENDS ${PyMVA-Torch-Multiclass-depends})
 
-   # Test RModelParser_PyTorch
-
-   if(BLAS_FOUND)
-     ROOT_ADD_GTEST(TestRModelParserPyTorch TestRModelParserPyTorch.C
-           LIBRARIES
-           ROOTTMVASofie
-           TMVA
-           Python3::NumPy
-           Python3::Python
-           BLAS::BLAS
-           INCLUDE_DIRS
-           SYSTEM
-           ${CMAKE_CURRENT_BINARY_DIR}
-        )
-   endif()
-
 endif(ROOT_TORCH_FOUND)
 
 if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSORFLOW_FOUND))
-   configure_file(generateKerasModels.py generateKerasModels.py COPYONLY)
-   configure_file(scale_by_2_op.hxx scale_by_2_op.hxx COPYONLY)
 
    if (ROOT_TORCH_FOUND)
       set(PyMVA-Keras-Classification-depends PyMVA-Torch-Classification)
@@ -127,19 +105,4 @@ if((ROOT_KERAS_FOUND AND ROOT_THEANO_FOUND) OR (ROOT_KERAS_FOUND AND ROOT_TENSOR
    ROOT_EXECUTABLE(testPyKerasMulticlass testPyKerasMulticlass.C
       LIBRARIES ${Libraries})
    ROOT_ADD_TEST(PyMVA-Keras-Multiclass COMMAND testPyKerasMulticlass DEPENDS ${PyMVA-Keras-Multiclass-depends})
-
-   if(BLAS_FOUND)
-      ROOT_ADD_GTEST(TestRModelParserKeras TestRModelParserKeras.C
-         LIBRARIES
-         ROOTTMVASofie
-         PyMVA
-         Python3::NumPy
-         Python3::Python
-         BLAS::BLAS
-         INCLUDE_DIRS
-         SYSTEM
-         ${CMAKE_CURRENT_BINARY_DIR}
-     )
-  endif()
-
 endif()

tmva/sofie/CMakeLists.txt

@@ -74,6 +74,9 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
    TMVA/RFunction_MLP.hxx
    TMVA/RFunction_Sum.hxx
    TMVA/RFunction_Mean.hxx
+
+   TMVA/RModelParser_Keras.h
+   TMVA/RModelParser_PyTorch.h
   SOURCES
     src/RModel_Base.cxx
     src/RModel.cxx
@@ -90,6 +93,17 @@ ROOT_STANDARD_LIBRARY_PACKAGE(ROOTTMVASofie
     ${EXTRA_SOFIE_DEPENDENCIES}
 )
 
+if(tpython)
+  ROOT_LINKER_LIBRARY(ROOTTMVASofiePyParsers
+      src/RModelParser_Keras.cxx
+      src/RModelParser_PyTorch.cxx
+    LIBRARIES
+      Python3::NumPy
+      Python3::Python
+      ROOTTMVASofie
+  )
+endif()
+
 target_include_directories(ROOTTMVASofie PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_BINARY_DIR}>)
 set_target_properties(ROOTTMVASofie PROPERTIES

tmva/pymva/inc/TMVA/RModelParser_Keras.h → tmva/sofie/inc/TMVA/RModelParser_Keras.h

@@ -30,17 +30,11 @@
 #include "TMVA/Types.h"
 #include "TMVA/OperatorList.hxx"
 
-#include "TMVA/PyMethodBase.h"
-
 #include "Rtypes.h"
 #include "TString.h"
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyKeras{
-
+namespace TMVA::Experimental::SOFIE::PyKeras {
 
 /// Parser function for translatng Keras .h5 model into a RModel object.
 /// Accepts the file location of a Keras model and returns the
@@ -49,8 +43,6 @@ namespace PyKeras{
 /// has not a defined input batch size : e.g. for input = (input_dim,)
 RModel Parse(std::string filename, int batch_size = -1);
 
-}//PyKeras
-}//SOFIE
-}//Experimental
-}//TMVA
+} // namespace TMVA::Experimental::SOFIE::PyKeras
+
 #endif //TMVA_PYMVA_RMODELPARSER_KERAS

tmva/pymva/inc/TMVA/RModelParser_PyTorch.h → tmva/sofie/inc/TMVA/RModelParser_PyTorch.h

@@ -30,31 +30,23 @@
 #include "TMVA/Types.h"
 #include "TMVA/OperatorList.hxx"
 
-#include "TMVA/PyMethodBase.h"
-
 #include "Rtypes.h"
 #include "TString.h"
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyTorch{
+namespace TMVA::Experimental::SOFIE::PyTorch {
 
 /// Parser function for translating PyTorch .pt model into a RModel object.
 /// Accepts the file location of a PyTorch model, shapes and data-types of input tensors
 /// and returns the equivalent RModel object.
-RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
+RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes, std::vector<ETensorType> dtype);
 
 /// Overloaded Parser function for translating PyTorch .pt model into a RModel object.
 /// Accepts the file location of a PyTorch model and the shapes of input tensors.
 /// Builds the vector of data-types for input tensors and calls the `Parse()` function to
 /// return the equivalent RModel object.
-RModel Parse(std::string filepath,std::vector<std::vector<size_t>> inputShapes);
+RModel Parse(std::string filepath, std::vector<std::vector<size_t>> inputShapes);
 
-}//PyTorch
-}//SOFIE
-}//Experimental
-}//TMVA
+} // namespace TMVA::Experimental::SOFIE::PyTorch
 
 #endif //TMVA_PYMVA_RMODELPARSER_PYTORCH

tmva/pymva/src/RModelParser_Keras.cxx → tmva/sofie/src/RModelParser_Keras.cxx

@@ -25,16 +25,48 @@
 #include <numpy/arrayobject.h>
 
 
-namespace TMVA{
-namespace Experimental{
-namespace SOFIE{
-namespace PyKeras{
+namespace TMVA::Experimental::SOFIE::PyKeras {
 
-// Referencing Python utility functions present in PyMethodBase
-static void(& PyRunString)(TString, PyObject*, PyObject*) = PyMethodBase::PyRunString;
-static const char*(& PyStringAsString)(PyObject*) = PyMethodBase::PyStringAsString;
-static std::vector<size_t>(& GetDataFromTuple)(PyObject*) = PyMethodBase::GetDataFromTuple;
-static PyObject*(& GetValueFromDict)(PyObject*, const char*) = PyMethodBase::GetValueFromDict;
+namespace {
+
+// Utility functions (taken from PyMethodBase in PyMVA)
+
+void PyRunString(TString code, PyObject *globalNS, PyObject *localNS)
+{
+   PyObject *fPyReturn = PyRun_String(code, Py_single_input, globalNS, localNS);
+   if (!fPyReturn) {
+      std::cout << "\nPython error message:\n";
+      PyErr_Print();
+      throw std::runtime_error("\nFailed to run python code: " + code);
+   }
+}
+
+const char *PyStringAsString(PyObject *string)
+{
+   PyObject *encodedString = PyUnicode_AsUTF8String(string);
+   const char *cstring = PyBytes_AsString(encodedString);
+   return cstring;
+}
+
+std::vector<size_t> GetDataFromTuple(PyObject *tupleObject)
+{
+   std::vector<size_t> tupleVec;
+   for (Py_ssize_t tupleIter = 0; tupleIter < PyTuple_Size(tupleObject); ++tupleIter) {
+      auto itemObj = PyTuple_GetItem(tupleObject, tupleIter);
+      if (itemObj == Py_None)
+         tupleVec.push_back(0); // case shape is for example (None,2,3)
+      else
+         tupleVec.push_back((size_t)PyLong_AsLong(itemObj));
+   }
+   return tupleVec;
+}
+
+PyObject *GetValueFromDict(PyObject *dict, const char *key)
+{
+   return PyDict_GetItemWithError(dict, PyUnicode_FromString(key));
+}
+
+} // namespace
 
 namespace INTERNAL{
 
@@ -1036,7 +1068,5 @@ RModel Parse(std::string filename, int batch_size){
 
    return rmodel;
 }
-}//PyKeras
-}//SOFIE
-}//Experimental
-}//TMVA
+
+} // namespace TMVA::Experimental::SOFIE::PyKeras