209-in-situ-vis

.gitignore

@@ -7,6 +7,22 @@ manual/*.idx
 manual/*.lof
 manual/*.lot
 manual/ippl_user_guide.pdf
-*~
-*.~
 
+# ignure vtk dump stuff
+data/
+datasets*/
+*.vtk
+
+# CMake and Build folder
+CMakeFiles/
+build*
+tags
+compile_commands.json
+
+# temporary folder
+.cache/
+.idea/
+*.dat
+__pycache__
+
+*.~

CMakeLists.txt

@@ -1,5 +1,6 @@
-cmake_minimum_required (VERSION 3.20.3)
-project (IPPL CXX)
+cmake_minimum_required (VERSION 3.24)
+project (IPPL LANGUAGES C CXX)
+
 set (IPPL_VERSION_MAJOR 3)
 set (IPPL_VERSION_MINOR 0.2)
 include(FetchContent)
@@ -142,8 +143,10 @@ endif()
 set (CMAKE_MODULE_PATH "${CMAKE_SOURCE_DIR}/CMakeModules")
 
 find_package (MPI REQUIRED)
+message (STATUS "The C compiler identification is: ${CMAKE_C_COMPILER_ID}")
 message (STATUS "The C++ compiler identification is: ${CMAKE_CXX_COMPILER_ID}")
 message (STATUS "The C++ compiler version is: ${CMAKE_CXX_COMPILER_VERSION}")
+message (STATUS "The MPI C compiler is: ${MPI_C_COMPILER}")
 message (STATUS "The MPI C++ compiler is: ${MPI_CXX_COMPILER}")
 message (STATUS "The underlying C++ compiler is: ${CMAKE_CXX_COMPILER}")
 
@@ -154,6 +157,7 @@ endif()
 
 
 option (ENABLE_FFT "Enable FFT transform" OFF)
+
 if (ENABLE_FFT)
     add_definitions (-DENABLE_FFT)
     if(NOT Heffte_VERSION)
@@ -200,6 +204,7 @@ if (ENABLE_FFT)
 endif ()
 
 option (ENABLE_SOLVERS "Enable IPPL solvers" OFF)
+option (ENABLE_CATALYST "Build example with Catalyst enabled" OFF)
 
 add_subdirectory (src)
 
@@ -220,6 +225,7 @@ if (ENABLE_ALPINE)
     add_subdirectory (alpine)
 endif ()
 
+
 option (ENABLE_UNIT_TESTS "Enable unit tests" OFF)
 if (ENABLE_UNIT_TESTS)
     if (NOT ENABLE_FFT)
@@ -252,7 +258,7 @@ install (
     RENAME ${PROJECT_NAME}Config.cmake
     )
 
-# vi: set et ts=4 sw=4 sts=4:
+# vim: set et ts=4 sw=4 sts=4:
 
 # Local Variables:
 # mode: cmake

IN-SITU.md

@@ -0,0 +1,90 @@
+# In Situ Visualization with Catalyst
+If Catalyst is not available as a module on the machine, this is the first to build.
+For a more indepth documentation on how to build catalyst check the documentation, here are only minimal steps described.
+## Build and Install Catalyst
+### Requirements for Catalyst
+* MPI
+
+1. Configuration
+```sh
+git clone https://gitlab.kitware.com/paraview/catalyst.git
+cd catalyst
+git checkout v2.0.0-rc4
+cmake -S . \
+	-B build \
+	-DCMAKE_BUILD_TYPE=RelWithDebInfo \
+  -DCATALYST_USE_MPI=ON \
+	-DCMAKE_INSTALL_PREFIX=../catalyst_install
+```
+2. Build and Test
+`-j` flag is for parallel build and parallel test
+```sh
+cmake --build build -j 4
+cmake --install build
+ctest --test-dir build/tests -j 4
+```
+The tests should all pass, however for some of them you need to export the `catalyst_DIR` path.
+So inspect the output of the failed ones why they failed!
+
+## Build IPPL with Catalyst
+I would recommend to use the install script from IPPL for building IPPL and then after that setting `ENABLE_CATALYST` to `ON`, this procedure would then be:
+```sh
+./ippl-build-scripts/999-build-everything -t serial -i
+cmake ippl/build_serial
+```
+Now in order you need to pass the `Catalyst_Dir` path to the install script.
+You can do that in the cli over:
+```sh
+cmake build_serial\
+  -DENABLE_CATALYST=ON \
+  -DCATALYST_DIR=<absolute_path_to_catalyst_install>
+```
+or using the cmake gui over:
+```sh
+ccmake build_serial
+```
+
+Now build again and you will have catalyst enabled.
+
+
+## Starting a simulation with Catalyst
+An general set for examples can be seen on [https://gitlab.kitware.com/paraview/paraview/-/tree/master/Examples/Catalyst2](https://gitlab.kitware.com/paraview/paraview/-/tree/master/Examples/Catalyst2).
+Generally the setup is always similar and starting a simulation is usually as:
+```sh
+./some-program <program_args> ./<path_to_python_script>
+```
+Some sample scripts for ippl do lay under the `./test/stream/` directory.
+For all scripts the `CATALYST_IMPLEMENTATION_NAME` should be set to `paraview` and `CATALYST_IMPLEMENTATION_PATHS` should be set to the proper library in the ParaView folder.
+```sh
+export CATALYST_IMPLEMENTATION_PATHS="<path_to_downlaod_folder_of_paraview>/ParaView-5.11.1-MPI-Linux-Python3.9-x86_64/lib/catalyst"
+export CATALYST_IMPLEMENTATION_NAME="ParaView"
+```
+
+I assume that you are familiar with the examples and manage to start them with catalyst.
+
+# Tips and Tricks
+## ParaView Live
+In order to use the ParaView live feature with an existing Catalyst script the following options need to be set
+```python
+# Catalyst options
+from paraview import catalyst
+options = catalyst.Options()
+options.GlobalTrigger = 'TimeStep'
+options.EnableCatalystLive = 1
+options.CatalystLiveTrigger = 'TimeStep'
+```
+After these settings you can start `ParaView -> Catalyst -> Connect`, accept the port and then ParaView is waiting for catalyst to connect.
+Now also set the simulation to pause over `ParaView -> Catalyst -> Pause Simulation`, then start the simulation.
+Now when it worked out you can see on Pipeline Browser of ParaView a small symbol left from catalyst stuff, this needs to be activated to then access the catalyst data.
+
+## Creating a new script
+Using ParaView Live you can quite easy get the data into the position and angle that you like for the simulation and depending on that you can then create different filter to exporting data.
+Some choices do include dumping a csv, creating a vtp or also dumping images.
+
+General workflow would be:
+1. Add `CatalystAdaptor::Initialize()` directly after the MPI/IPPL initialize
+2. Add `CatalystAdaptor::Execute.....()` where you want your data to be extracted
+3. Add `CatalystAdaptor::Finalize()` directly before the MPI/IPPL finalize
+4. Start simulation with matching channel names in live mode
+5. Generate extraction / dump script
+6. Pass the script as argument to the program executable

adios2.xml

@@ -0,0 +1,37 @@
+<?xml version="1.0"?>
+
+<!-- For more details about how configure it : -->
+<!-- https://adios2.readthedocs.io/en/latest/engines/engines.html#sst-sustainable-staging-transport -->
+<adios-config>
+    <!-- For Simulation Nodes -->
+    <io name="Writer">
+        <engine type="SST">
+            <!-- Define the number of readers needed to start the simulation -->
+            <!-- This should correspond to as many readers launched an not the number of processes per reader -->
+            <parameter key="RendezvousReaderCount" value="1" />
+
+            <!-- Define max number of step cached for the Reader -->
+            <parameter key="QueueLimit" value="5" />
+            <!-- Define the number of step stored for late-arriving readers -->
+            <parameter key="ReserveQueueLimit" value="3" />
+
+            <!-- Define writer's behavior when the queue is full -->
+            <parameter key="QueueFullPolicy" value="Block" />
+
+            <!-- First and last step will never be discarded -->
+            <parameter key="FirstTimestepPrecious" value="true" />
+            <parameter key="AlwaysProvideLatestTimestep" value="true" />
+
+            <!-- Specify the timeout -->
+            <parameter key="OpenTimeoutSecs" value="100.0" />
+        </engine>
+    </io>
+
+    <!--  For Analysis Nodes -->
+    <io name="Reader">
+        <engine type="SST">
+            <!-- Specify the timeout -->
+            <parameter key="OpenTimeoutSecs" value="100.0" />
+        </engine>
+    </io>
+</adios-config>

alpine/CMakeLists.txt

@@ -16,7 +16,7 @@ set (IPPL_LIBS ippl ${MPI_CXX_LIBRARIES})
 set (COMPILE_FLAGS ${OPAL_CXX_FLAGS})
 
 add_executable (PenningTrap PenningTrap.cpp)
-target_link_libraries (PenningTrap ${IPPL_LIBS})
+target_link_libraries (PenningTrap PUBLIC ${IPPL_LIBS})
 
 add_executable (LandauDamping LandauDamping.cpp)
 target_link_libraries (LandauDamping ${IPPL_LIBS})

alpine/ParticleContainer.hpp

@@ -6,8 +6,8 @@
 
 // Define the ParticlesContainer class
 template <typename T, unsigned Dim = 3>
-class ParticleContainer : public ippl::ParticleBase<ippl::ParticleSpatialLayout<T, Dim>>{
-    using Base = ippl::ParticleBase<ippl::ParticleSpatialLayout<T, Dim>>;
+class ParticleContainer : public ippl::ParticleBase<ippl::ParticleSpatialLayout<T, Dim>,Kokkos::DefaultExecutionSpace>{
+    using Base = ippl::ParticleBase<ippl::ParticleSpatialLayout<T, Dim>, Kokkos::DefaultExecutionSpace>;
 
     public:
         ippl::ParticleAttrib<double> q;                 // charge

alpine/PenningTrap.cpp

@@ -30,10 +30,7 @@ const char* TestName   = "PenningTrap";
 #include <Kokkos_Random.hpp>
 #include <chrono>
 #include <iostream>
-#include <random>
-#include <set>
 #include <string>
-#include <vector>
 
 #include "datatypes.h"
 
@@ -42,9 +39,18 @@ const char* TestName   = "PenningTrap";
 #include "Manager/PicManager.h"
 #include "PenningTrapManager.h"
 
+#ifdef ENABLE_CATALYST
+#include "Stream/InSitu/CatalystAdaptor.h"
+#endif
+
 int main(int argc, char* argv[]) {
     ippl::initialize(argc, argv);
     {
+
+#ifdef ENABLE_CATALYST
+        CatalystAdaptor::Initialize(argc, argv);
+#endif
+
         Inform msg(TestName);
         Inform msg2all(TestName, INFORM_ALL_NODES);
 
@@ -72,12 +78,19 @@ int main(int argc, char* argv[]) {
 
         manager.setTime(0.0);
 
+
         msg << "Starting iterations ..." << endl;
 
+
         manager.run(manager.getNt());
 
         msg << "End." << endl;
 
+#ifdef ENABLE_CATALYST
+        CatalystAdaptor::Finalize();
+#endif
+
+
         IpplTimings::stopTimer(mainTimer);
         IpplTimings::print();
         IpplTimings::print(std::string("timing.dat"));

alpine/PenningTrapManager.h

@@ -14,6 +14,11 @@
 #include "Random/NormalDistribution.h"
 #include "Random/Randn.h"
 
+#ifdef ENABLE_CATALYST
+#include <optional>
+#include "Stream/InSitu/CatalystAdaptor.h"
+#endif
+
 using view_type = typename ippl::detail::ViewType<ippl::Vector<double, Dim>, 1>::view_type;
 
 template <typename T, unsigned Dim>
@@ -279,6 +284,10 @@ class PenningTrapManager : public AlpineManager<T, Dim> {
 
         // scatter the charge onto the underlying grid
         this->par2grid();
+#ifdef ENABLE_CATALYST
+        std::optional<conduit_cpp::Node> node = std::nullopt;
+        CatalystAdaptor::Execute_Particle(it, this->time_m, ippl::Comm->rank(),  pc, node);
+#endif
 
         // Field solve
         IpplTimings::startTimer(SolveTimer);

src/CMakeLists.txt

@@ -64,10 +64,9 @@ add_ippl_sources (Ippl.cpp)
 
 add_subdirectory (Communicate)
 if (ENABLE_FFT)
+
     set (HEFFTE_LIBRARY Heffte)
     add_subdirectory (FFT)
-else()
-    set (HEFFTE_LIBRARY "")
 endif()
 add_subdirectory (Field)
 add_subdirectory (FieldLayout)
@@ -82,6 +81,8 @@ add_subdirectory (Utility)
 add_subdirectory (Expression)
 add_subdirectory (Types)
 add_subdirectory (Partition)
+add_subdirectory (Stream)
+add_subdirectory (Stream/InSitu)
 add_subdirectory (Random)
 
 if (ENABLE_SOLVERS)
@@ -91,6 +92,15 @@ if (ENABLE_SOLVERS)
     add_subdirectory (LinearSolvers)
 endif ()
 
+if (ENABLE_CATALYST)
+    find_package(catalyst 2.0 REQUIRED)
+    message (STATUS "Enable Catalyst")
+    message (STATUS "Found catalyst_DIR: ${catalyst_DIR}")
+endif()
+
+if (ENABLE_AMR)
+    add_subdirectory(AmrParticle)
+endif ()
 include_directories (
     BEFORE ${CMAKE_CURRENT_SOURCE_DIR}
     )
@@ -103,9 +113,20 @@ string(REPLACE "SERIAL;" "" KokkosAccelerators "${KokkosAccelerators}")
 if ("${KokkosAccelerators}" MATCHES ".+;.+")
     set(TimerFences false)
 endif()
+
 message("Timer fences for this build: ${TimerFences}")
-target_compile_definitions(ippl PUBLIC IPPL_ENABLE_TIMER_FENCES=${TimerFences})
-target_link_libraries(ippl PUBLIC Kokkos::kokkos ${HEFFTE_LIBRARY})
+
+target_compile_definitions(ippl PUBLIC
+  $<$<BOOL:${ENABLE_FFT}>:-DENABLE_FFT>
+  $<$<BOOL:${ENABLE_CATALYST}>:-DENABLE_CATALYST>
+  IPPL_ENABLE_TIMER_FENCES=${TimerFences}
+)
+
+target_link_libraries(ippl PUBLIC
+  Kokkos::kokkos
+  $<$<BOOL:${ENABLE_FFT}>:Heffte::Heffte>
+  $<$<BOOL:${ENABLE_CATALYST}>:catalyst::catalyst>
+  )
 
 install (TARGETS ippl DESTINATION lib)
 install (FILES ${IPPL_BASEDIR_HDRS} DESTINATION include)

src/Stream/CMakeLists.txt

@@ -0,0 +1,24 @@
+set (_SRCS
+)
+
+set (_HDRS
+)
+
+include_DIRECTORIES (
+        ${CMAKE_CURRENT_SOURCE_DIR}
+)
+
+add_ippl_sources (${_SRCS})
+add_ippl_headers (${_HDRS})
+
+install (FILES ${_HDRS} DESTINATION include/Stream)
+
+
+# vi: set et ts=4 sw=4 sts=4:
+
+# Local Variables:
+# mode: cmake
+# cmake-tab-width: 4
+# indent-tabs-mode: nil
+# require-final-newline: nil
+# End: