Merge pull request #230 from PrincetonUniversity/issue-170

Issue 170

CMakeLists.txt

@@ -1,6 +1,6 @@
 cmake_minimum_required(VERSION 3.17.5)
 
-project(specfem2d_kokkos VERSION 0.1.0)
+project(SPECFEMPP VERSION 0.1.0)
 
 set(CMAKE_CXX_STANDARD 17)
 option(HDF5_CXX_BUILD "Build HDF5 C++" ON)
@@ -10,19 +10,33 @@ option(BUILD_TESTS "Tests included" OFF)
 option(BUILD_EXAMPLES "Examples included" ON)
 option(ENABLE_SIMD "Enable SIMD" OFF)
 option(ENABLE_PROFILING "Enable profiling" OFF)
+option(SPECFEMPP_ENABLE_PYTHON "Enable Python binding" OFF)
 # set(CMAKE_BUILD_TYPE Release)
 set(CHUNK_SIZE 32)
 set(NUM_CHUNKS 1)
 set(NUM_THREADS 160)
 set(NUM_VECTOR_LANES 1)
 
-if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
-        set(CMAKE_CXX_FLAGS "-fp-model=precise")
-elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
-        message("-- Detected Intel classic compiler which will be deprecated soon.")
-        message("-- It is recommended you use IntelLLVM compiler.")
-        set(CMAKE_CXX_FLAGS "-diag-disable=10441 -fp-model=precise")
-endif()
+if (SPECFEMPP_ENABLE_PYTHON)
+    message("-- Adding -fPIC flag for Python binding.")
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+            set(CMAKE_CXX_FLAGS "-fp-model=precise -fPIC")
+    elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+            message("-- Detected Intel classic compiler which will be deprecated soon.")
+            message("-- It is recommended you use IntelLLVM compiler.")
+            set(CMAKE_CXX_FLAGS "-diag-disable=10441 -fp-model=precise -fPIC")
+    else()
+            set(CMAKE_CXX_FLAGS "-fPIC")
+    endif()
+else (SPECFEMPP_ENABLE_PYTHON)
+    if(CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM")
+            set(CMAKE_CXX_FLAGS "-fp-model=precise")
+    elseif(CMAKE_CXX_COMPILER_ID STREQUAL "Intel")
+            message("-- Detected Intel classic compiler which will be deprecated soon.")
+            message("-- It is recommended you use IntelLLVM compiler.")
+            set(CMAKE_CXX_FLAGS "-diag-disable=10441 -fp-model=precise")
+    endif()
+endif (SPECFEMPP_ENABLE_PYTHON)
 
 if (DEFINED Kokkos_ENABLE_CUDA)
         if (Kokkos_ENABLE_CUDA)
@@ -611,3 +625,36 @@ if (DOXYGEN_FOUND)
 else (DOXYGEN_FOUND)
   message("Doxygen need to be installed to generate the doxygen documentation")
 endif (DOXYGEN_FOUND)
+
+if (SPECFEMPP_USE_SKBUILD AND EXISTS ${SKBUILD_SCRIPTS_DIR})
+    install(TARGETS specfem2d DESTINATION ${SKBUILD_SCRIPTS_DIR})
+    install(FILES ${CMAKE_BINARY_DIR}/xmeshfem2D DESTINATION ${SKBUILD_SCRIPTS_DIR} PERMISSIONS OWNER_READ OWNER_WRITE OWNER_EXECUTE)
+endif ()
+
+if (SPECFEMPP_ENABLE_PYTHON)
+    find_package(Python REQUIRED COMPONENTS Interpreter Development.Module)
+    find_package(pybind11 CONFIG REQUIRED)
+    python_add_library(_core MODULE src/specfem2d.cpp WITH_SOABI)
+    target_link_libraries(
+        _core PRIVATE
+        specfem_mpi
+        Kokkos::kokkos
+        mesh
+        quadrature
+        compute
+        source_class
+        parameter_reader
+        receiver_class
+        writer
+        reader
+        domain
+        coupled_interface
+        kernels
+        solver
+        Boost::program_options
+        pybind11::headers
+    )
+    target_compile_definitions(_core PRIVATE SPECFEMPP_ENABLE_PYTHON)
+    target_compile_definitions(_core PRIVATE VERSION_INFO=${PROJECT_VERSION})
+    install(TARGETS _core DESTINATION specfempp_core)
+endif (SPECFEMPP_ENABLE_PYTHON)

include/specfem_mpi/specfem_mpi.hpp

@@ -18,7 +18,10 @@ namespace MPI {
  * Incase specfem is compiled without MPI then I need placeholders for reducer
  * types
  */
-enum reduce_type { sum = MPI_SUM, min = MPI_MIN, max = MPI_MAX };
+using reduce_type = MPI_Op;
+const static reduce_type sum = MPI_SUM;
+const static reduce_type min = MPI_MIN;
+const static reduce_type max = MPI_MAX;
 #else
 /**
  * @brief MPI reducer type
@@ -224,8 +227,9 @@ class MPI {
   void bcast(double &val, int root) const;
 
 private:
-  int world_size; ///< total number of MPI processes
-  int my_rank;    ///< rank of my process
+  int world_size;  ///< total number of MPI processes
+  int my_rank;     ///< rank of my process
+  int extern_init; ///< flag to check if MPI was initialized outside SPECFEM
 #ifdef MPI_PARALLEL
   MPI_Comm comm; ///< MPI communicator
 #endif

pyproject.toml

@@ -1,8 +1,27 @@
-[tool.poetry]
-name = "SPECFEMPP"
+[build-system]
+requires = ["scikit-build-core>=0.10", "pybind11"]
+build-backend = "scikit_build_core.build"
+
+[project]
+name = "specfempp_core"
 version = "0.1.0"
 description = "Kokkos implementation of SpecFEM2D code"
-authors = ["Your Name <[email protected]>"]
+readme = "README.md"
+authors = [
+  { name = "Rohit R Kakodkar", email = "[email protected]" },
+  { name = "Lucas Sawade", email = "[email protected]" },
+  { name = "Congyue Cui", email = "[email protected]" }
+]
+requires-python = ">=3.9"
+
+[tool.scikit-build]
+cmake.define.CMAKE_BUILD_TYPE = "Release"
+cmake.define.BUILD_TESTS = "ON"
+cmake.define.SPECFEMPP_USE_SKBUILD = "ON"
+cmake.define.SPECFEMPP_ENABLE_PYTHON = "ON"
+build-dir = "./build"
+
+[tool.poetry]
 package-mode = false
 
 [tool.poetry.dependencies]

python/specfempp_core/__init__.py

@@ -0,0 +1,5 @@
+from __future__ import annotations
+
+from ._core import __doc__, __version__, _run
+
+__all__ = ["__doc__", "__version__", "_run"]

python/specfempp_core/__init__.pyi

@@ -0,0 +1,20 @@
+"""
+SPECFEM++ Python
+-----------------------
+
+.. currentmodule:: specfempp
+
+.. autosummary::
+    :toctree: _generate
+
+    _run
+    subtract
+"""
+
+def _run(argv: list[str]) -> int:
+    """
+    Run main specfem workflow
+
+    Args:
+        argv: list of command line arguments for initializing MPI and Kokkos
+    """

src/specfem2d.cpp

@@ -20,6 +20,12 @@
 #include <stdexcept>
 #include <string>
 #include <vector>
+#ifdef SPECFEMPP_ENABLE_PYTHON
+#include <pybind11/pybind11.h>
+#include <pybind11/stl.h>
+#define STRINGIFY(x) #x
+#define MACRO_STRINGIFY(x) STRINGIFY(x)
+#endif
 // Specfem2d driver
 
 std::string
@@ -253,8 +259,7 @@ void execute(const std::string &parameter_file, const std::string &default_file,
   return;
 }
 
-int main(int argc, char **argv) {
-
+int run(int argc, char **argv) {
   // Initialize MPI
   specfem::MPI::MPI *mpi = new specfem::MPI::MPI(&argc, &argv);
   // Initialize Kokkos
@@ -274,3 +279,61 @@ int main(int argc, char **argv) {
   delete mpi;
   return 0;
 }
+
+#ifdef SPECFEMPP_ENABLE_PYTHON
+
+namespace py = pybind11;
+
+int _run(py::list argv_py) {
+  // parse argc and argv from Python
+  int argc = argv_py.size();
+  char **argv = new char *[argc + 1];
+
+  for (size_t i = 0; i < argc; i++) {
+    std::string str =
+        argv_py[i].cast<std::string>(); // Convert Python string to std::string
+    argv[i] =
+        new char[str.length() + 1]; // Allocate memory for each C-style string
+    std::strcpy(argv[i], str.c_str()); // Copy the string content
+  }
+
+  argv[argc] = nullptr; // Null-terminate argv following the specification
+
+  int return_code = run(argc, argv);
+
+  for (int i = 0; i < argc; i++) {
+    delete[] argv[i]; // Free each individual string
+  }
+
+  delete[] argv;
+
+  return return_code;
+}
+
+PYBIND11_MODULE(_core, m) {
+    m.doc() = R"pbdoc(
+        SPECfem++ core module
+        -----------------------
+
+        .. currentmodule:: specfempp
+
+        .. autosummary::
+           :toctree: _generate
+
+           _run
+    )pbdoc";
+
+    m.def("_run", &_run, R"pbdoc(
+        Execute the main SPECFEM++ workflow.
+    )pbdoc");
+
+#ifdef VERSION_INFO
+    m.attr("__version__") = MACRO_STRINGIFY(VERSION_INFO);
+#else
+    m.attr("__version__") = "dev";
+#endif
+}
+
+#endif
+
+int main(int argc, char **argv) { return run(argc, argv); }

src/specfem_mpi/specfem_mpi.cpp

@@ -7,13 +7,17 @@
 
 specfem::MPI::MPI::MPI(int *argc, char ***argv) {
 #ifdef MPI_PARALLEL
-  MPI_Init(argc, argv);
+  MPI_Initialized(&this->extern_init);
+  if (!this->extern_init) {
+    MPI_Init(argc, argv);
+  }
   this->comm = MPI_COMM_WORLD;
   MPI_Comm_size(this->comm, &this->world_size);
   MPI_Comm_rank(this->comm, &this->my_rank);
 #else
   this->world_size = 1;
   this->my_rank = 0;
+  this->extern_init = 0;
 #endif
 }
 
@@ -25,7 +29,9 @@ void specfem::MPI::MPI::sync_all() const {
 
 specfem::MPI::MPI::~MPI() {
 #ifdef MPI_PARALLEL
-  MPI_Finalize();
+  if (!this->extern_init) {
+    MPI_Finalize();
+  }
 #endif
 }