Feature celeritas integration

CMakeLists.txt

@@ -70,6 +70,7 @@ math(EXPR DD4HEP_HIGH_MEM_POOL_DEPTH "${HOST_RAM_MB} / 2000" OUTPUT_FORMAT DECIM
 
 option(DD4HEP_USE_XERCESC       "Enable 'Detector Builders' based on XercesC"   OFF)
 option(DD4HEP_USE_GEANT4        "Enable the simulation part based on Geant4"    OFF)
+option(DD4HEP_USE_CELERITAS     "Enable offloading tracks to Celeritas" OFF)
 option(DD4HEP_IGNORE_GEANT4_TLS "Ignore the tls flag Geant4 was compiled with"  OFF)
 option(DD4HEP_USE_GEAR          "Build gear wrapper for backward compatibility" OFF)
 option(DD4HEP_USE_LCIO          "Build lcio extensions"                         OFF)
@@ -154,6 +155,10 @@ if(DD4HEP_USE_GEANT4)
   SET_DIRECTORY_PROPERTIES(PROPERTIES INCLUDE_DIRECTORIES "")
 endif()
 
+if(DD4HEP_USE_CELERITAS)
+  find_package(Celeritas REQUIRED)
+endif()
+
 if(DD4HEP_USE_LCIO)
   find_package(LCIO REQUIRED CONFIG)
   DD4HEP_SETUP_LCIO_TARGETS()

DDG4/CMakeLists.txt

@@ -25,6 +25,8 @@ target_link_libraries(DDG4
   PUBLIC
   DD4hep::DDCore
   Geant4::Interface
+  Celeritas::corecel
+  Celeritas::accel
   )
 #
 #Ensure our own includes come before those of the system

DDG4/include/DDG4/Celeritas.h

@@ -0,0 +1,33 @@
+#ifndef Celeritas_h
+#define Celeritas_h 1
+
+#include <accel/SimpleOffload.hh>
+#include <G4EmStandardPhysics.hh>
+
+/// Namespace for the AIDA detector description toolkit
+namespace dd4hep  {
+
+  /// Namespace for the Geant4 based simulation part of the AIDA detector description toolkit
+  namespace sim  {
+
+    class EMPhysicsConstructor final : public G4EmStandardPhysics
+    {
+    public:
+        using G4EmStandardPhysics::G4EmStandardPhysics;
+
+        void ConstructProcess() override;
+    };
+
+  } /* End namespace sim */  
+} /* End namespace dd4hep*/
+
+// Global shared setup options
+celeritas::SetupOptions& CelerSetupOptions();
+// Shared data and GPU setup
+celeritas::SharedParams& CelerSharedParams();
+// Thread-local transporter
+celeritas::LocalTransporter& CelerLocalTransporter();
+// Thread-local offload
+celeritas::SimpleOffload& CelerSimpleOffload();
+
+#endif

DDG4/include/DDG4/Geant4PhysicsList.h

@@ -292,6 +292,8 @@ namespace dd4hep {
       virtual void enable(G4VUserPhysicsList* physics);
       /// Extend physics list from factory:
       G4VUserPhysicsList* extensionList();
+      /// Activate Celeritas tracking offload within EM physics constructor
+      G4VUserPhysicsList* activateCeleritas();
     };
 
   }    // End namespace sim

DDG4/src/Celeritas.cpp

@@ -0,0 +1,86 @@
+#include <DDG4/Celeritas.h>
+
+#include <G4Threading.hh>
+#include <accel/AlongStepFactory.hh>
+#include <celeritas/field/UniformFieldData.hh>
+#include <celeritas/io/ImportData.hh>
+#include <accel/LocalTransporter.hh>
+#include <accel/SetupOptions.hh>
+#include <accel/SharedParams.hh>
+#include <accel/TrackingManagerOffload.hh>
+#include <G4Electron.hh>
+#include <G4Positron.hh>
+#include <G4Gamma.hh>
+
+#include <memory>
+
+
+using namespace dd4hep::sim;
+
+// Global shared setup options
+celeritas::SetupOptions& CelerSetupOptions()
+{
+  static celeritas::SetupOptions options = [] {
+    // Construct setup options the first time CelerSetupOptions is invoked
+    celeritas::SetupOptions so;
+
+    // Set along-step factory
+    so.make_along_step = celeritas::UniformAlongStepFactory();
+    // NOTE: these numbers are appropriate for CPU execution
+    so.max_num_tracks = 1024;
+    so.initializer_capacity = 1024 * 128;
+    // Celeritas does not support EmStandard MSC physics above 100 MeV
+    so.ignore_processes = {"CoulombScat"};
+
+    // Use Celeritas "hit processor" to call back to Geant4 SDs.
+    so.sd.enabled = false;
+
+    // Only call back for nonzero energy depositions: this is currently a
+    // global option for all detectors, so if any SDs extract data from tracks
+    // with no local energy deposition over the step, it must be set to false.
+    so.sd.ignore_zero_deposition = false;
+
+    // Using the pre-step point, reconstruct the G4 touchable handle.
+    so.sd.locate_touchable = true;
+
+    // Pre-step time is used
+    so.sd.pre.global_time = true;
+    return so;
+  }();
+  return options;
+}
+
+// Shared data and GPU setup
+celeritas::SharedParams& CelerSharedParams()
+{
+  static celeritas::SharedParams sp;
+  return sp;
+}
+
+// Thread-local transporter
+celeritas::LocalTransporter& CelerLocalTransporter()
+{
+  static G4ThreadLocal celeritas::LocalTransporter lt;
+  return lt;
+}
+
+// Thread-local offload interface
+celeritas::SimpleOffload& CelerSimpleOffload()
+{
+  static G4ThreadLocal celeritas::SimpleOffload so;
+  return so;
+}
+
+void EMPhysicsConstructor::ConstructProcess()
+{
+    CELER_LOG_LOCAL(status) << "Setting up tracking manager offload";
+    G4EmStandardPhysics::ConstructProcess();
+
+    // Add Celeritas tracking manager to electron, positron, gamma.
+    auto* celer_tracking = new celeritas::TrackingManagerOffload(
+        &CelerSharedParams(), &CelerLocalTransporter());
+
+    G4Electron::Definition()->SetTrackingManager(celer_tracking);
+    G4Positron::Definition()->SetTrackingManager(celer_tracking);
+    G4Gamma::Definition()->SetTrackingManager(celer_tracking);
+}

DDG4/src/Geant4PhysicsList.cpp

@@ -19,6 +19,7 @@
 #include <DD4hep/InstanceCount.h>
 #include <DD4hep/Printout.h>
 #include <DD4hep/Plugins.h>
+#include <DDG4/Celeritas.h>
 
 // Geant4 include files
 #include <G4VPhysicsConstructor.hh>
@@ -374,6 +375,17 @@ G4VUserPhysicsList* Geant4PhysicsListActionSequence::extensionList()    {
   return physics;
 }
 
+G4VUserPhysicsList* Geant4PhysicsListActionSequence::activateCeleritas()    {
+  G4VModularPhysicsList* physics = ( m_extends.empty() )
+    ? new EmptyPhysics()
+    : G4PhysListFactory().GetReferencePhysList(m_extends);
+
+physics->ReplacePhysics(new EMPhysicsConstructor);
+
+return physics;
+}
+
+
 /// Install command control messenger if wanted
 void Geant4PhysicsListActionSequence::installCommandMessenger()   {
   control()->addCall("dump", "Dump content of " + name(), Callback(this).make(&Geant4PhysicsListActionSequence::dump));