From 2639e878013c4713d750011d2ae732342eab130f Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 26 Dec 2024 17:43:10 +0100 Subject: [PATCH 1/2] Add central synchronize configuration parameter to Alpaka modules --- HeterogeneousCore/AlpakaCore/README.md | 28 +++++++++++-- .../alpaka/EDMetadataAcquireSentry.h | 5 ++- .../interface/alpaka/EDMetadataSentry.h | 6 ++- .../AlpakaCore/interface/alpaka/ESProducer.h | 4 +- .../interface/alpaka/ProducerBase.h | 16 +++++++- .../interface/alpaka/global/EDProducer.h | 7 +++- .../interface/alpaka/stream/EDProducer.h | 7 +++- .../alpaka/stream/SynchronizingEDProducer.h | 9 ++++- .../AlpakaCore/interface/modulePrevalidate.h | 10 +++++ .../interface/module_backend_config.h | 10 ----- .../python/ProcessAcceleratorAlpaka.py | 15 ++++++- .../src/alpaka/EDMetadataAcquireSentry.cc | 18 ++++++--- .../AlpakaCore/src/alpaka/EDMetadataSentry.cc | 15 +++++-- .../AlpakaCore/src/modulePrevalidate.cc | 40 +++++++++++++++++++ .../AlpakaCore/src/module_backend_config.cc | 35 ---------------- .../alpaka/TestAlpakaGlobalProducer.cc | 3 +- .../alpaka/TestAlpakaGlobalProducerE.cc | 3 +- .../TestAlpakaGlobalProducerNoOutput.cc | 2 +- .../alpaka/TestAlpakaGlobalProducerNullES.cc | 3 +- .../alpaka/TestAlpakaGlobalProducerOffset.cc | 3 +- .../alpaka/TestAlpakaGlobalProducerWithPtr.cc | 2 +- .../plugins/alpaka/TestAlpakaProducer.cc | 3 +- .../alpaka/TestAlpakaStreamProducer.cc | 3 +- .../TestAlpakaStreamSynchronizingProducer.cc | 3 +- ...pakaStreamSynchronizingProducerToDevice.cc | 3 +- .../AlpakaTest/test/testAlpakaModules.sh | 4 ++ .../AlpakaTest/test/testAlpakaModules_cfg.py | 35 +++++++++++----- 27 files changed, 200 insertions(+), 92 deletions(-) create mode 100644 HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h delete mode 100644 HeterogeneousCore/AlpakaCore/interface/module_backend_config.h create mode 100644 HeterogeneousCore/AlpakaCore/src/modulePrevalidate.cc delete mode 100644 HeterogeneousCore/AlpakaCore/src/module_backend_config.cc diff --git a/HeterogeneousCore/AlpakaCore/README.md b/HeterogeneousCore/AlpakaCore/README.md index 844b14a8be92b..4e0dad334c3e9 100644 --- a/HeterogeneousCore/AlpakaCore/README.md +++ b/HeterogeneousCore/AlpakaCore/README.md @@ -146,7 +146,9 @@ The `...` can in principle be any of the module abilities listed in the linked T New base classes (or other functionality) can be added based on new use cases that come up. -The Alpaka-based ESProducers should use the `ESProducer` base class (`#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h"`). Note that the Alpaka-based ESProducer constructor must pass the argument `edm::ParameterSet` object to the constructor of the `ESProducer` base class. +The Alpaka-based ESProducers should use the `ESProducer` base class (`#include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h"`). + +Note that both the Alpaka-based EDProducer and ESProducer constructors must pass the argument `edm::ParameterSet` object to the constructor of their base class. Note that currently Alpaka-based ESSources are not supported. If you need to produce EventSetup data products into a Record for which there is no ESSource yet, use [`EmptyESSource`](https://twiki.cern.ch/twiki/bin/view/CMSPublic/SWGuideEDMParametersForModules#EmptyESSource). @@ -237,8 +239,10 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class ExampleAlpakaProducer : public global::EDProducer<> { public: ExampleAlpakaProducer(edm::ParameterSet const& iConfig) - // produces() must not specify the product type, it is deduced from deviceToken_ - : deviceToken_{produces()}, size_{iConfig.getParameter("size")} {} + : EDProducer<>(iConfig), + // produces() must not specify the product type, it is deduced from deviceToken_ + deviceToken_{produces()}, + size_{iConfig.getParameter("size")} {} // device::Event and device::EventSetup are defined in ALPAKA_ACCELERATOR_NAMESPACE as well void produce(edm::StreamID sid, device::Event& iEvent, device::EventSetup const& iSetup) const override { @@ -479,6 +483,24 @@ process.ProcessAcceleratorAlpaka.setBackend("serial_sync") # or "cuda_async" or process.options.accelerators = ["cpu"] # or "gpu-nvidia" or "gpu-amd" ``` +### Blocking synchronization (for testing) + +While the general approach is to favor asynchronous operations with non-blocking synchronization, for testing purposes it can be useful to synchronize the EDModule's `acquire()` / `produce()` or ESProducer's production functions in a blocking way. Such a blocking synchronization can be specified for individual modules via the `alpaka` `PSet` along +```python +process.producer = cms.EDProducer("ExampleAlpakaProducer@alpaka", + ... + alpaka = cms.untracked.PSet( + synchronize = cms.untracked.bool(True) + ) +) +``` + +The blocking synchronization can be specified for all Alpaka modules via the `ProcessAcceleratorAlpaka` along +```python +process.ProcessAcceleratorAlpaka.setSynchronize(True) +``` +Note that the possible per-module parameter overrides this global setting. + ## Unit tests diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataAcquireSentry.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataAcquireSentry.h index 6495d2127ff5e..a8f285372bf77 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataAcquireSentry.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataAcquireSentry.h @@ -16,10 +16,10 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: // TODO: WaitingTaskWithArenaHolder not really needed for host synchronous case // Constructor overload to be called from acquire() - EDMetadataAcquireSentry(edm::StreamID stream, edm::WaitingTaskWithArenaHolder holder); + EDMetadataAcquireSentry(edm::StreamID stream, edm::WaitingTaskWithArenaHolder holder, bool synchronize); // Constructor overload to be called from registerTransformAsync() - EDMetadataAcquireSentry(Device const& device, edm::WaitingTaskWithArenaHolder holder); + EDMetadataAcquireSentry(Device const& device, edm::WaitingTaskWithArenaHolder holder, bool synchronize = false); EDMetadataAcquireSentry(EDMetadataAcquireSentry const&) = delete; EDMetadataAcquireSentry& operator=(EDMetadataAcquireSentry const&) = delete; @@ -40,6 +40,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { std::shared_ptr metadata_; edm::WaitingTaskWithArenaHolder waitingTaskHolder_; + bool const synchronize_; }; } // namespace detail } // namespace ALPAKA_ACCELERATOR_NAMESPACE diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataSentry.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataSentry.h index a56670af92210..11ec11c9ba3d0 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataSentry.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataSentry.h @@ -14,10 +14,11 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class EDMetadataSentry { public: // For normal module - EDMetadataSentry(edm::StreamID stream); + EDMetadataSentry(edm::StreamID stream, bool synchronize); // For ExternalWork-module's produce() - EDMetadataSentry(std::shared_ptr metadata) : metadata_(std::move(metadata)) {} + EDMetadataSentry(std::shared_ptr metadata, bool synchronize) + : metadata_(std::move(metadata)), synchronize_(synchronize) {} EDMetadataSentry(EDMetadataSentry const&) = delete; EDMetadataSentry& operator=(EDMetadataSentry const&) = delete; @@ -31,6 +32,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { private: std::shared_ptr metadata_; + bool const synchronize_; }; } // namespace detail } // namespace ALPAKA_ACCELERATOR_NAMESPACE diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h index d4e9f2bb3ba28..83cecf0f77b36 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/ESProducer.h @@ -4,7 +4,7 @@ #include "FWCore/Framework/interface/ESProducer.h" #include "FWCore/Framework/interface/MakeDataException.h" #include "FWCore/Framework/interface/produce_helpers.h" -#include "HeterogeneousCore/AlpakaCore/interface/module_backend_config.h" +#include "HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESDeviceProduct.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/ESDeviceProductType.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/Record.h" @@ -30,7 +30,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { public: static void prevalidate(edm::ConfigurationDescriptions& descriptions) { Base::prevalidate(descriptions); - cms::alpakatools::module_backend_config(descriptions); + cms::alpakatools::modulePrevalidate(descriptions); } protected: diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/ProducerBase.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/ProducerBase.h index a770397765b6f..b0be0e2ab476b 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/ProducerBase.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/ProducerBase.h @@ -5,13 +5,14 @@ #include "FWCore/Framework/interface/FrameworkfwdMostUsed.h" #include "FWCore/Framework/interface/moduleAbilities.h" #include "FWCore/Framework/interface/Event.h" +#include "FWCore/ParameterSet/interface/ParameterSet.h" #include "FWCore/Utilities/interface/EDPutToken.h" #include "FWCore/Utilities/interface/Transition.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/DeviceProductType.h" #include "HeterogeneousCore/AlpakaCore/interface/alpaka/EDMetadataAcquireSentry.h" #include "HeterogeneousCore/AlpakaCore/interface/EventCache.h" #include "HeterogeneousCore/AlpakaCore/interface/QueueCache.h" -#include "HeterogeneousCore/AlpakaCore/interface/module_backend_config.h" +#include "HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h" #include "HeterogeneousCore/AlpakaInterface/interface/Backend.h" #include "HeterogeneousCore/AlpakaInterface/interface/CopyToHost.h" @@ -46,7 +47,15 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { using Base = BaseT; public: + // TODO: default constructor to be removed after all derived classes have been migrated ProducerBase() : backendToken_(Base::produces("backend")) {} + ProducerBase(edm::ParameterSet const& iConfig) + : backendToken_(Base::produces("backend")), + // The 'synchronize' parameter can be unset in Alpaka + // modules specified with the namespace prefix instead if + // '@alpaka' suffix + synchronize_(iConfig.getUntrackedParameter("alpaka").getUntrackedParameter( + "synchronize", false)) {} template [[nodiscard]] auto produces() noexcept { @@ -60,7 +69,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { static void prevalidate(edm::ConfigurationDescriptions& descriptions) { Base::prevalidate(descriptions); - cms::alpakatools::module_backend_config(descriptions); + cms::alpakatools::modulePrevalidate(descriptions); } protected: @@ -68,8 +77,11 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { iEvent.emplace(this->backendToken_, static_cast(kBackend)); } + bool synchronize() const { return synchronize_; } + private: edm::EDPutTokenT const backendToken_; + bool const synchronize_ = false; template friend class ProducerBaseAdaptor; diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h index 374ee0001f376..9a34a90699bbf 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/global/EDProducer.h @@ -15,10 +15,15 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { static_assert(not edm::CheckAbility::kHasIt, "ALPAKA_ACCELERATOR_NAMESPACE::global::EDProducer may not be used with ExternalWork ability. " "Please use ALPAKA_ACCELERATOR_NAMESPACE::stream::SynchronizingEDProducer instead."); + using Base = ProducerBase; + + protected: + EDProducer() = default; // to be removed in the near future + EDProducer(edm::ParameterSet const iConfig) : Base(iConfig) {} public: void produce(edm::StreamID sid, edm::Event& iEvent, edm::EventSetup const& iSetup) const final { - detail::EDMetadataSentry sentry(sid); + detail::EDMetadataSentry sentry(sid, this->synchronize()); device::Event ev(iEvent, sentry.metadata()); device::EventSetup const es(iSetup, ev.device()); produce(sid, ev, es); diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/EDProducer.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/EDProducer.h index 08687b0b804b3..b8297a145aaf6 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/EDProducer.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/EDProducer.h @@ -15,10 +15,15 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { static_assert(not edm::CheckAbility::kHasIt, "ALPAKA_ACCELERATOR_NAMESPACE::stream::EDProducer may not be used with ExternalWork ability. " "Please use ALPAKA_ACCELERATOR_NAMESPACE::stream::SynchronizingEDProducer instead."); + using Base = ProducerBase; + + protected: + EDProducer() = default; // to be removed in the near future + EDProducer(edm::ParameterSet const iConfig) : Base(iConfig) {} public: void produce(edm::Event& iEvent, edm::EventSetup const& iSetup) final { - detail::EDMetadataSentry sentry(iEvent.streamID()); + detail::EDMetadataSentry sentry(iEvent.streamID(), this->synchronize()); device::Event ev(iEvent, sentry.metadata()); device::EventSetup const es(iSetup, ev.device()); produce(ev, es); diff --git a/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/SynchronizingEDProducer.h b/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/SynchronizingEDProducer.h index 28abd77be20db..e0aa364ef4f51 100644 --- a/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/SynchronizingEDProducer.h +++ b/HeterogeneousCore/AlpakaCore/interface/alpaka/stream/SynchronizingEDProducer.h @@ -18,12 +18,17 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { not edm::CheckAbility::kHasIt, "ExternalWork ability is redundant with ALPAKA_ACCELERATOR_NAMESPACE::stream::SynchronizingEDProducer." "Please remove it."); + using Base = ProducerBase; + + protected: + SynchronizingEDProducer() = default; // to be removed in the near future + SynchronizingEDProducer(edm::ParameterSet const iConfig) : Base(iConfig) {} public: void acquire(edm::Event const& iEvent, edm::EventSetup const& iSetup, edm::WaitingTaskWithArenaHolder holder) final { - detail::EDMetadataAcquireSentry sentry(iEvent.streamID(), std::move(holder)); + detail::EDMetadataAcquireSentry sentry(iEvent.streamID(), std::move(holder), this->synchronize()); device::Event const ev(iEvent, sentry.metadata()); device::EventSetup const es(iSetup, ev.device()); acquire(ev, es); @@ -31,7 +36,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } void produce(edm::Event& iEvent, edm::EventSetup const& iSetup) final { - detail::EDMetadataSentry sentry(std::move(metadata_)); + detail::EDMetadataSentry sentry(std::move(metadata_), this->synchronize()); device::Event ev(iEvent, sentry.metadata()); device::EventSetup const es(iSetup, ev.device()); produce(ev, es); diff --git a/HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h b/HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h new file mode 100644 index 0000000000000..22f2d889421fe --- /dev/null +++ b/HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h @@ -0,0 +1,10 @@ +#ifndef HeterogeneousCore_AlpakaCore_interface_modulePrevalidate_h +#define HeterogeneousCore_AlpakaCore_interface_modulePrevalidate_h + +#include "FWCore/Framework/interface/FrameworkfwdMostUsed.h" + +namespace cms::alpakatools { + void modulePrevalidate(edm::ConfigurationDescriptions& iDesc); +} // namespace cms::alpakatools + +#endif diff --git a/HeterogeneousCore/AlpakaCore/interface/module_backend_config.h b/HeterogeneousCore/AlpakaCore/interface/module_backend_config.h deleted file mode 100644 index 5fc9afb8b60b1..0000000000000 --- a/HeterogeneousCore/AlpakaCore/interface/module_backend_config.h +++ /dev/null @@ -1,10 +0,0 @@ -#ifndef HeterogeneousCore_AlpakaCore_interface_module_backend_config_h -#define HeterogeneousCore_AlpakaCore_interface_module_backend_config_h - -#include "FWCore/Framework/interface/FrameworkfwdMostUsed.h" - -namespace cms::alpakatools { - void module_backend_config(edm::ConfigurationDescriptions& iDesc); -} - -#endif diff --git a/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py b/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py index 673a479d8eb2b..5c1f724749da7 100644 --- a/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py +++ b/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py @@ -5,7 +5,7 @@ from HeterogeneousCore.Common.PlatformStatus import PlatformStatus class ModuleTypeResolverAlpaka: - def __init__(self, accelerators, backend): + def __init__(self, accelerators, backend, synchronize): # first element is used as the default if nothing is set self._valid_backends = [] if "gpu-nvidia" in accelerators: @@ -23,6 +23,7 @@ def __init__(self, accelerators, backend): if backend != self._valid_backends[0]: self._valid_backends.remove(backend) self._valid_backends.insert(0, backend) + self._synchronize = synchronize def plugin(self): return "ModuleTypeResolverAlpaka" @@ -42,6 +43,12 @@ def setModuleVariant(self, module): module.alpaka = cms.untracked.PSet( backend = cms.untracked.string(defaultBackend) ) + isDefaultValue = lambda v: \ + isinstance(v, type(cms.optional.untracked.bool)) \ + and not v.isTracked() \ + and v.isCompatibleCMSType(cms.bool) + if not hasattr(module.alpaka, "synchronize") or isDefaultValue(module.alpaka.synchronize): + module.alpaka.synchronize = cms.untracked.bool(self._synchronize) class ProcessAcceleratorAlpaka(cms.ProcessAccelerator): """ProcessAcceleratorAlpaka itself does not define or inspect @@ -53,14 +60,18 @@ class ProcessAcceleratorAlpaka(cms.ProcessAccelerator): def __init__(self): super(ProcessAcceleratorAlpaka, self).__init__() self._backend = None + self._synchronize = False # User-facing interface def setBackend(self, backend): self._backend = backend + def setSynchronize(self, synchronize): + self._synchronize = synchronize + # Framework-facing interface def moduleTypeResolver(self, accelerators): - return ModuleTypeResolverAlpaka(accelerators, self._backend) + return ModuleTypeResolverAlpaka(accelerators, self._backend, self._synchronize) def apply(self, process, accelerators): # Propagate the AlpakaService messages through the MessageLogger diff --git a/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataAcquireSentry.cc b/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataAcquireSentry.cc index 923c0d52d5cf6..ef1a6472e2a5f 100644 --- a/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataAcquireSentry.cc +++ b/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataAcquireSentry.cc @@ -5,11 +5,15 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { namespace detail { - EDMetadataAcquireSentry::EDMetadataAcquireSentry(edm::StreamID streamID, edm::WaitingTaskWithArenaHolder holder) - : EDMetadataAcquireSentry(detail::chooseDevice(streamID), std::move(holder)) {} + EDMetadataAcquireSentry::EDMetadataAcquireSentry(edm::StreamID streamID, + edm::WaitingTaskWithArenaHolder holder, + bool synchronize) + : EDMetadataAcquireSentry(detail::chooseDevice(streamID), std::move(holder), synchronize) {} - EDMetadataAcquireSentry::EDMetadataAcquireSentry(Device const& device, edm::WaitingTaskWithArenaHolder holder) - : waitingTaskHolder_(std::move(holder)) { + EDMetadataAcquireSentry::EDMetadataAcquireSentry(Device const& device, + edm::WaitingTaskWithArenaHolder holder, + bool synchronize) + : waitingTaskHolder_(std::move(holder)), synchronize_(synchronize) { #ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED // all synchronous backends metadata_ = std::make_shared(cms::alpakatools::getQueueCache().get(device)); @@ -23,7 +27,11 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { #ifndef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED // all asynchronous backends std::shared_ptr EDMetadataAcquireSentry::finish() { - metadata_->enqueueCallback(std::move(waitingTaskHolder_)); + if (synchronize_) { + alpaka::wait(metadata_->queue()); + } else { + metadata_->enqueueCallback(std::move(waitingTaskHolder_)); + } return std::move(metadata_); } #endif diff --git a/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataSentry.cc b/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataSentry.cc index da5e9065e9c12..74dda0811684c 100644 --- a/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataSentry.cc +++ b/HeterogeneousCore/AlpakaCore/src/alpaka/EDMetadataSentry.cc @@ -5,7 +5,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { namespace detail { - EDMetadataSentry::EDMetadataSentry(edm::StreamID streamID) { + EDMetadataSentry::EDMetadataSentry(edm::StreamID streamID, bool synchronize) : synchronize_(synchronize) { auto const& device = detail::chooseDevice(streamID); #ifdef ALPAKA_ACC_CPU_B_SEQ_T_SEQ_ENABLED metadata_ = std::make_shared(cms::alpakatools::getQueueCache().get(device)); @@ -16,12 +16,19 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { } void EDMetadataSentry::finish(bool launchedAsyncWork) { - if (launchedAsyncWork) { + if constexpr (not std::is_same_v>) { + if (launchedAsyncWork and synchronize_) { + alpaka::wait(metadata_->queue()); + } + } + + if (launchedAsyncWork and not synchronize_) { metadata_->recordEvent(); } else { // If we are certain no asynchronous work was launched (i.e. - // the Queue was not used in any way), there is no need to - // synchronize, and the Event can be discarded. + // the Queue was not used in any way), or a blocking + // synchronization was explicitly requested, there is no need + // to synchronize later, and the Event can be discarded. metadata_->discardEvent(); } } diff --git a/HeterogeneousCore/AlpakaCore/src/modulePrevalidate.cc b/HeterogeneousCore/AlpakaCore/src/modulePrevalidate.cc new file mode 100644 index 0000000000000..984dd0cc799e4 --- /dev/null +++ b/HeterogeneousCore/AlpakaCore/src/modulePrevalidate.cc @@ -0,0 +1,40 @@ +#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" +#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" +#include "HeterogeneousCore/AlpakaCore/interface/modulePrevalidate.h" + +namespace { + const std::string kPSetName("alpaka"); + const char* const kComment = "PSet allows to override some Alpaka module settings per module instance."; +} // namespace + +namespace cms::alpakatools { + void modulePrevalidate(edm::ConfigurationDescriptions& iDesc) { + edm::ParameterSetDescription descAlpaka; + descAlpaka.addUntracked("backend", "") + ->setComment( + "Set the Alpaka backend for this module instance. This parameter has an effect only when the module class " + "name has '@alpaka' suffix, i.e. it has no effect when the Alpaka backend namespace is used explicitly. " + "Can be empty string (for the global default), 'serial_sync', or - depending on the architecture and " + "available hardware - 'cuda_async', 'rocm_async'"); + // Optional in order to have unset state so that python-side + // ModuleTypeResolverAlpaka can set the global default only if the + // per-module parameter is not set + descAlpaka.addOptionalUntracked("synchronize") + ->setComment( + "On backends with an asynchronous Queue, synchronize the host and device in a blocking way at the end of " + "acquire() and produce() functions. Has no effect on backends with synchronous Queue."); + + if (iDesc.defaultDescription()) { + if (iDesc.defaultDescription()->isLabelUnused(kPSetName)) { + iDesc.defaultDescription() + ->addUntracked(kPSetName, descAlpaka) + ->setComment(kComment); + } + } + for (auto& v : iDesc) { + if (v.second.isLabelUnused(kPSetName)) { + v.second.addUntracked(kPSetName, descAlpaka)->setComment(kComment); + } + } + } +} // namespace cms::alpakatools diff --git a/HeterogeneousCore/AlpakaCore/src/module_backend_config.cc b/HeterogeneousCore/AlpakaCore/src/module_backend_config.cc deleted file mode 100644 index 55b3ee5e11b30..0000000000000 --- a/HeterogeneousCore/AlpakaCore/src/module_backend_config.cc +++ /dev/null @@ -1,35 +0,0 @@ -#include "FWCore/ParameterSet/interface/ConfigurationDescriptions.h" -#include "FWCore/ParameterSet/interface/ParameterSetDescription.h" -#include "HeterogeneousCore/AlpakaCore/interface/module_backend_config.h" - -namespace { - const std::string kPSetName("alpaka"); - const char* const kComment = - "PSet allows to override the Alpaka backend per module instance. Has an effect only when the module class name " - "has '@alpaka' suffix, i.e. has no effect when the Alpaka backend namespace is used explicitly."; -} // namespace - -namespace cms::alpakatools { - void module_backend_config(edm::ConfigurationDescriptions& iDesc) { - // the code below leads to 'alpaka = untracked.PSet(backend = untracked.string)' to be added to the generated cfi files - // TODO: I don't know if this is a desired behavior for HLT - edm::ParameterSetDescription descAlpaka; - descAlpaka.addUntracked("backend", "") - ->setComment( - "Alpaka backend for this module. Can be empty string (for the global default), 'serial_sync', or " - " - depending on the architecture and available hardware - 'cuda_async', 'rocm_async'"); - - if (iDesc.defaultDescription()) { - if (iDesc.defaultDescription()->isLabelUnused(kPSetName)) { - iDesc.defaultDescription() - ->addUntracked(kPSetName, descAlpaka) - ->setComment(kComment); - } - } - for (auto& v : iDesc) { - if (v.second.isLabelUnused(kPSetName)) { - v.second.addUntracked(kPSetName, descAlpaka)->setComment(kComment); - } - } - } -} // namespace cms::alpakatools diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducer.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducer.cc index 19340231732b8..58dac53421a4e 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducer.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducer.cc @@ -21,7 +21,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaGlobalProducer : public global::EDProducer<> { public: TestAlpakaGlobalProducer(edm::ParameterSet const& config) - : esToken_(esConsumes(config.getParameter("eventSetupSource"))), + : EDProducer<>(config), + esToken_(esConsumes(config.getParameter("eventSetupSource"))), esMultiToken_(esConsumes(config.getParameter("eventSetupSourceMulti"))), deviceToken_{produces()}, deviceTokenMulti2_{produces()}, diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerE.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerE.cc index 253b8dcad8988..8012dac2b20a2 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerE.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerE.cc @@ -22,7 +22,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaGlobalProducerE : public global::EDProducer<> { public: TestAlpakaGlobalProducerE(edm::ParameterSet const& config) - : esToken_(esConsumes(config.getParameter("eventSetupSource"))), + : EDProducer<>(config), + esToken_(esConsumes(config.getParameter("eventSetupSource"))), getToken_(consumes(config.getParameter("source"))), getTokenMulti2_(consumes(config.getParameter("source"))), getTokenMulti3_(consumes(config.getParameter("source"))), diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNoOutput.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNoOutput.cc index d7d361b6ab8d1..37f219413a081 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNoOutput.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNoOutput.cc @@ -16,7 +16,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaGlobalProducerNoOutput : public global::EDProducer<> { public: TestAlpakaGlobalProducerNoOutput(edm::ParameterSet const& config) - : getToken_(consumes(config.getParameter("source"))) {} + : EDProducer<>(config), getToken_(consumes(config.getParameter("source"))) {} void produce(edm::StreamID, device::Event& iEvent, device::EventSetup const& iSetup) const override { [[maybe_unused]] auto const& input = iEvent.get(getToken_); diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNullES.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNullES.cc index d8ae5ac5f366e..42a08d0e8e81b 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNullES.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerNullES.cc @@ -15,7 +15,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaGlobalProducerNullES : public global::EDProducer<> { public: TestAlpakaGlobalProducerNullES(edm::ParameterSet const& config) - : esTokenA_(esConsumes(config.getParameter("eventSetupSource"))), + : EDProducer<>(config), + esTokenA_(esConsumes(config.getParameter("eventSetupSource"))), esTokenC_(esConsumes(config.getParameter("eventSetupSource"))), esTokenCNotExist_(esConsumes(edm::ESInputTag("", "doesNotExist"))) {} diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerOffset.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerOffset.cc index 2e2637d050fe0..689da8e77467d 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerOffset.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerOffset.cc @@ -22,7 +22,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaGlobalProducerOffset : public global::EDProducer<> { public: TestAlpakaGlobalProducerOffset(edm::ParameterSet const& config) - : esToken_(esConsumes()), + : EDProducer<>(config), + esToken_(esConsumes()), deviceToken_{produces()}, x_(config.getParameter("xvalue").getParameter( EDM_STRINGIZE(ALPAKA_ACCELERATOR_NAMESPACE))) {} diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerWithPtr.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerWithPtr.cc index e09bccb50a712..ad6fc518ea465 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerWithPtr.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaGlobalProducerWithPtr.cc @@ -19,7 +19,7 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaGlobalProducerWithPtr : public global::EDProducer<> { public: TestAlpakaGlobalProducerWithPtr(edm::ParameterSet const& config) - : token_{produces()}, size_{config.getParameter("size")} {} + : EDProducer<>(config), token_{produces()}, size_{config.getParameter("size")} {} static void fillDescriptions(edm::ConfigurationDescriptions& descriptions) { edm::ParameterSetDescription desc; diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaProducer.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaProducer.cc index 27013ffb72c7a..3f29ff8b88b9f 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaProducer.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaProducer.cc @@ -19,7 +19,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaProducer : public global::EDProducer<> { public: TestAlpakaProducer(edm::ParameterSet const& config) - : objectToken_{produces()}, + : EDProducer<>(config), + objectToken_{produces()}, collectionToken_{produces()}, deviceTokenMulti2_{produces()}, deviceTokenMulti3_{produces()}, diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamProducer.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamProducer.cc index 74cd08e39f56a..f5998ed980ba4 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamProducer.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamProducer.cc @@ -24,7 +24,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaStreamProducer : public stream::EDProducer<> { public: TestAlpakaStreamProducer(edm::ParameterSet const& config) - : size_{config.getParameter("size").getParameter( + : EDProducer<>(config), + size_{config.getParameter("size").getParameter( EDM_STRINGIZE(ALPAKA_ACCELERATOR_NAMESPACE))}, size2_{config.getParameter("size").getParameter( EDM_STRINGIZE(ALPAKA_ACCELERATOR_NAMESPACE))}, diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducer.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducer.cc index 613c31498746a..3badb1bb75b45 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducer.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducer.cc @@ -25,7 +25,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaStreamSynchronizingProducer : public stream::SynchronizingEDProducer<> { public: TestAlpakaStreamSynchronizingProducer(edm::ParameterSet const& iConfig) - : esTokenDevice_(esConsumes()), + : SynchronizingEDProducer<>(iConfig), + esTokenDevice_(esConsumes()), putToken_{produces()}, putTokenMulti2_{produces()}, putTokenMulti3_{produces()}, diff --git a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducerToDevice.cc b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducerToDevice.cc index 913636f686805..67a972a7b9163 100644 --- a/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducerToDevice.cc +++ b/HeterogeneousCore/AlpakaTest/plugins/alpaka/TestAlpakaStreamSynchronizingProducerToDevice.cc @@ -19,7 +19,8 @@ namespace ALPAKA_ACCELERATOR_NAMESPACE { class TestAlpakaStreamSynchronizingProducerToDevice : public stream::SynchronizingEDProducer<> { public: TestAlpakaStreamSynchronizingProducerToDevice(edm::ParameterSet const& iConfig) - : putToken_{produces()}, + : SynchronizingEDProducer<>(iConfig), + putToken_{produces()}, size_{iConfig.getParameter("size").getParameter( EDM_STRINGIZE(ALPAKA_ACCELERATOR_NAMESPACE))} {} diff --git a/HeterogeneousCore/AlpakaTest/test/testAlpakaModules.sh b/HeterogeneousCore/AlpakaTest/test/testAlpakaModules.sh index e0e6c8f1db96d..b3f8e62488930 100755 --- a/HeterogeneousCore/AlpakaTest/test/testAlpakaModules.sh +++ b/HeterogeneousCore/AlpakaTest/test/testAlpakaModules.sh @@ -59,6 +59,8 @@ function runForGPU { runSuccess "--processAcceleratorBackend=$BACKEND --moduleBackend=serial_sync --expectBackend=serial_sync" runSuccess "--processAcceleratorBackend=serial_sync --moduleBackend=$BACKEND --expectBackend=$BACKEND" + runSuccess "--moduleSynchronize --expectBackend=$BACKEND" + runSuccess "--processAcceleratorSynchronize --expectBackend=$BACKEND" runFailure "--accelerators=$ACCELERATOR --processAcceleratorBackend=serial_sync --expectBackend=serial_sync" runFailure "--accelerators=$ACCELERATOR --moduleBackend=serial_sync --expectBackend=serial_sync" @@ -78,6 +80,8 @@ runSuccess "--moduleBackend=serial_sync --expectBackend=serial_sync" if [ "${TARGET}" == "cpu" ]; then runSuccess "--expectBackend=serial_sync" + runSuccess "--moduleSynchronize --expectBackend=serial_sync" + runSuccess "--processAcceleratorSynchronize --expectBackend=serial_sync" runFailure "--accelerators=gpu-nvidia --expectBackend=cuda_async" runFailure "--processAcceleratorBackend=cuda_async --expectBackend=cuda_async" diff --git a/HeterogeneousCore/AlpakaTest/test/testAlpakaModules_cfg.py b/HeterogeneousCore/AlpakaTest/test/testAlpakaModules_cfg.py index ac39117119cce..42028d9f9ddc3 100644 --- a/HeterogeneousCore/AlpakaTest/test/testAlpakaModules_cfg.py +++ b/HeterogeneousCore/AlpakaTest/test/testAlpakaModules_cfg.py @@ -8,6 +8,8 @@ parser.add_argument("--moduleBackend", type=str, help="Set Alpaka backend via module instances", default="") parser.add_argument("--processAcceleratorBackend", type=str, help="Set Alpaka backend via ProcessAcceleratorAlpaka", default="") parser.add_argument("--expectBackend", type=str, help="Expect this backend to run") +parser.add_argument("--moduleSynchronize", action="store_true", help="Set synchronize parameter via module instances", default="") +parser.add_argument("--processAcceleratorSynchronize", action="store_true", help="Set synchronize parameter via ProcessAcceleratorAlpaka", default="") parser.add_argument("--run", type=int, help="Run number (default: 1)", default=1) args = parser.parse_args() @@ -168,20 +170,21 @@ eventSetupSource = cms.ESInputTag("", "null") ) +_postfixes = ["ESProducerA", "ESProducerB", "ESProducerC", "ESProducerD", "ESProducerE", "ESProducerAMulti", + "ESProducerNull", + "GlobalProducer", "GlobalProducerE", + "GlobalProducerCopyToDeviceCache", "GlobalProducerMoveToDeviceCache", + "StreamProducer", "StreamInstanceProducer", + "StreamSynchronizingProducer", "StreamSynchronizingProducerToDevice", + "GlobalDeviceConsumer", "StreamDeviceConsumer", + "StreamSynchronizingProducerToDeviceDeviceConsumer1", "StreamSynchronizingProducerToDeviceDeviceConsumer2", + "NullESConsumer"] +alpakaModules = ["alpaka"+x for x in _postfixes] if args.processAcceleratorBackend != "": process.ProcessAcceleratorAlpaka.setBackend(args.processAcceleratorBackend) if args.moduleBackend != "": - for name in ["ESProducerA", "ESProducerB", "ESProducerC", "ESProducerD", "ESProducerE", "ESProducerAMulti", - "ESProducerNull", - "GlobalProducer", "GlobalProducerE", - "GlobalProducerCopyToDeviceCache", "GlobalProducerMoveToDeviceCache", - "StreamProducer", "StreamInstanceProducer", - "StreamSynchronizingProducer", "StreamSynchronizingProducerToDevice", - "GlobalDeviceConsumer", "StreamDeviceConsumer", - "StreamSynchronizingProducerToDeviceDeviceConsumer1", "StreamSynchronizingProducerToDeviceDeviceConsumer2", - "NullESConsumer"]: - mod = getattr(process, "alpaka"+name) - mod.alpaka = cms.untracked.PSet(backend = cms.untracked.string(args.moduleBackend)) + for name in alpakaModules: + getattr(process, name).alpaka = cms.untracked.PSet(backend = cms.untracked.string(args.moduleBackend)) if args.expectBackend == "cuda_async": def setExpect(m, size): m.expectSize = size @@ -211,6 +214,16 @@ def setExpect(m, size): setExpect(process.alpakaStreamInstanceConsumer, size = 216) setExpect(process.alpakaStreamSynchronizingConsumer, size = 30) +if args.processAcceleratorSynchronize: + process.ProcessAcceleratorAlpaka.setSynchronize(True) +if args.moduleSynchronize: + for name in alpakaModules: + mod = getattr(process, name) + if hasattr(mod, "alpaka"): + mod.alpaka = dict(synchronize = cms.untracked.bool(True)) + else: + mod.alpaka = cms.untracked.PSet(synchronize = cms.untracked.bool(True)) + process.output = cms.OutputModule('PoolOutputModule', fileName = cms.untracked.string('testAlpaka.root'), outputCommands = cms.untracked.vstring( From 5469807b8457e539c648017430cbd5c2dcf88471 Mon Sep 17 00:00:00 2001 From: Matti Kortelainen Date: Thu, 26 Dec 2024 18:32:05 +0100 Subject: [PATCH 2/2] Raise an exception with a clear error message if alpaka PSet is tracked --- .../AlpakaCore/python/ProcessAcceleratorAlpaka.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py b/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py index 5c1f724749da7..910f7ecd58ce8 100644 --- a/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py +++ b/HeterogeneousCore/AlpakaCore/python/ProcessAcceleratorAlpaka.py @@ -32,6 +32,11 @@ def setModuleVariant(self, module): if module.type_().endswith("@alpaka"): defaultBackend = self._valid_backends[0] if hasattr(module, "alpaka"): + # Ensure the untrackedness already here, because the + # C++ ModuleTypeResolverAlpaka relies on the + # untrackedness (before the configuration validation) + if module.alpaka.isTracked(): + raise cms.EDMException(cms.edm.errors.Configuration, "The 'alpaka' PSet in module '{}' is tracked, but it should be untracked".format(module.label())) if hasattr(module.alpaka, "backend"): if module.alpaka.backend == "": module.alpaka.backend = defaultBackend