From bf1a608e70e9702e621a32b7bbb94bc4441acc7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Wed, 28 Feb 2024 12:41:06 +0100 Subject: [PATCH 1/3] Allow combining tape values locally. Refactor combineData() -> combineDataMPI(). --- include/codi/tapes/misc/tapeValues.hpp | 30 +++++++++++++++++++++++--- 1 file changed, 27 insertions(+), 3 deletions(-) diff --git a/include/codi/tapes/misc/tapeValues.hpp b/include/codi/tapes/misc/tapeValues.hpp index bf370b26..bf24c473 100644 --- a/include/codi/tapes/misc/tapeValues.hpp +++ b/include/codi/tapes/misc/tapeValues.hpp @@ -66,8 +66,9 @@ namespace codi { * - formatRow(): Output the data in this object in one row. One column per entry. * * - Misc: - * - combineData(): Perform a MPI_Allreduce on MPI_COMM_WORLD. - * - getAllocatedMemorySize: Get the allocated memory size. + * - combineData(): Perform element-wise addition with other tape values. + * - combineDataMPI(): Perform an MPI_Allreduce on MPI_COMM_WORLD. + * - getAllocatedMemorySize(): Get the allocated memory size. * - getUsedMemorySize(): Get the used memory size. */ struct TapeValues { @@ -223,8 +224,31 @@ namespace codi { /// @name Misc. /// @{ + /// Perform entry-wise additions. + void combineData(TapeValues const& other) { + + // Basic checks to ensure that we add tape values of the same tape type. + codiAssert(this->sections.size() == other.sections.size()); + codiAssert(this->sections.size() == 0 || this->sections[0].name == other.sections[0].name); + + // Size checks for the subsequent loops. + codiAssert(this->doubleData.size() == other.doubleData.size()); + codiAssert(this->longData.size() == other.longData.size()); + codiAssert(this->unsignedLongData.size() == other.unsignedLongData.size()); + + for (size_t i = 0; i < this->doubleData.size(); ++i) { + this->doubleData[i] += other.doubleData[i]; + } + for (size_t i = 0; i < this->longData.size(); ++i) { + this->longData[i] += other.longData[i]; + } + for (size_t i = 0; i < this->unsignedLongData.size(); ++i) { + this->unsignedLongData[i] += other.unsignedLongData[i]; + } + } + /// Perform an MPI_Allreduce with MPI_COMM_WORLD. - void combineData() { + void combineDataMPI() { #ifdef MPI_VERSION MPI_Allreduce(MPI_IN_PLACE, doubleData.data(), doubleData.size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); MPI_Allreduce(MPI_IN_PLACE, longData.data(), longData.size(), MPI_LONG, MPI_SUM, MPI_COMM_WORLD); From c25565429fe54a02c6020a080470a7dc05269288 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Wed, 28 Feb 2024 14:58:38 +0100 Subject: [PATCH 2/3] Keep prior combineData for backwards compatibility. --- include/codi/tapes/misc/tapeValues.hpp | 23 ++++++++++++++++------- 1 file changed, 16 insertions(+), 7 deletions(-) diff --git a/include/codi/tapes/misc/tapeValues.hpp b/include/codi/tapes/misc/tapeValues.hpp index bf24c473..42b30e2b 100644 --- a/include/codi/tapes/misc/tapeValues.hpp +++ b/include/codi/tapes/misc/tapeValues.hpp @@ -66,8 +66,9 @@ namespace codi { * - formatRow(): Output the data in this object in one row. One column per entry. * * - Misc: - * - combineData(): Perform element-wise addition with other tape values. - * - combineDataMPI(): Perform an MPI_Allreduce on MPI_COMM_WORLD. + * - combineData(TapeVales const& other): Perform element-wise addition with other tape values. + * - combineData(): Deprecated. Kept for backwards compatibility. Perform an MPI_Allreduce on MPI_COMM_WORLD. + * - combineDataMPI(): Perform an MPI_Allreduce on a given communicator. * - getAllocatedMemorySize(): Get the allocated memory size. * - getUsedMemorySize(): Get the used memory size. */ @@ -248,15 +249,23 @@ namespace codi { } /// Perform an MPI_Allreduce with MPI_COMM_WORLD. - void combineDataMPI() { + /// This method is deprecated and only kept for backwards compatibility. combineDataMPI should be used instead. + void combineData() { #ifdef MPI_VERSION - MPI_Allreduce(MPI_IN_PLACE, doubleData.data(), doubleData.size(), MPI_DOUBLE, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, longData.data(), longData.size(), MPI_LONG, MPI_SUM, MPI_COMM_WORLD); - MPI_Allreduce(MPI_IN_PLACE, unsignedLongData.data(), unsignedLongData.size(), MPI_UNSIGNED_LONG, MPI_SUM, - MPI_COMM_WORLD); + combineDataMPI(MPI_COMM_WORLD); #endif } +#ifdef MPI_VERSION + /// Perform an MPI_Allreduce with the given communicator. + void combineDataMPI(MPI_Comm communicator) { + MPI_Allreduce(MPI_IN_PLACE, doubleData.data(), doubleData.size(), MPI_DOUBLE, MPI_SUM, communicator); + MPI_Allreduce(MPI_IN_PLACE, longData.data(), longData.size(), MPI_LONG, MPI_SUM, communicator); + MPI_Allreduce(MPI_IN_PLACE, unsignedLongData.data(), unsignedLongData.size(), MPI_UNSIGNED_LONG, MPI_SUM, + communicator); + } +#endif + /// Get the allocated memory in bytes. double getAllocatedMemorySize() { return doubleData[allocatedMemoryIndex]; From 123d7e0e30bbdf0a9f55bfa718d07f2ec65e4a7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Johannes=20Bl=C3=BChdorn?= Date: Wed, 28 Feb 2024 18:41:36 +0100 Subject: [PATCH 3/3] Dummy implementation without MPI. --- include/codi/tapes/misc/tapeValues.hpp | 13 ++++++++++--- 1 file changed, 10 insertions(+), 3 deletions(-) diff --git a/include/codi/tapes/misc/tapeValues.hpp b/include/codi/tapes/misc/tapeValues.hpp index 42b30e2b..9b6862a4 100644 --- a/include/codi/tapes/misc/tapeValues.hpp +++ b/include/codi/tapes/misc/tapeValues.hpp @@ -248,22 +248,29 @@ namespace codi { } } - /// Perform an MPI_Allreduce with MPI_COMM_WORLD. - /// This method is deprecated and only kept for backwards compatibility. combineDataMPI should be used instead. + /** @brief Perform an MPI_Allreduce with MPI_COMM_WORLD. + * + * This method is deprecated and only kept for backwards compatibility. combineDataMPI should be used instead. + */ void combineData() { #ifdef MPI_VERSION combineDataMPI(MPI_COMM_WORLD); #endif } -#ifdef MPI_VERSION /// Perform an MPI_Allreduce with the given communicator. +#ifdef MPI_VERSION void combineDataMPI(MPI_Comm communicator) { MPI_Allreduce(MPI_IN_PLACE, doubleData.data(), doubleData.size(), MPI_DOUBLE, MPI_SUM, communicator); MPI_Allreduce(MPI_IN_PLACE, longData.data(), longData.size(), MPI_LONG, MPI_SUM, communicator); MPI_Allreduce(MPI_IN_PLACE, unsignedLongData.data(), unsignedLongData.size(), MPI_UNSIGNED_LONG, MPI_SUM, communicator); } +#else + template + void combineDataMPI(Comm communicator) { + CODI_UNUSED(communicator); + } #endif /// Get the allocated memory in bytes.