From 05c886eb7a8abcc41cfe21a2bc6eea96dd988ac7 Mon Sep 17 00:00:00 2001 From: Arne Morten Kvarving Date: Wed, 29 Jan 2025 13:41:07 +0100 Subject: [PATCH] added: callback for changing number of threads application may have to change this and this would create problems in basis function cache since it is sized according to number of max threads. this adds a callback so application can signal and have the buffers resize accordingly --- src/ASM/ASMbase.h | 2 ++ src/ASM/ASMs2D.C | 7 +++++++ src/ASM/ASMs2D.h | 3 +++ src/ASM/ASMs3D.C | 7 +++++++ src/ASM/ASMs3D.h | 3 +++ src/ASM/BasisFunctionCache.C | 26 ++++++++++++++++++-------- src/ASM/BasisFunctionCache.h | 3 +++ src/ASM/LR/ASMu2D.C | 7 +++++++ src/ASM/LR/ASMu2D.h | 3 +++ src/ASM/LR/ASMu3D.C | 7 +++++++ src/ASM/LR/ASMu3D.h | 3 +++ src/SIM/SIMbase.C | 8 ++++++++ src/SIM/SIMbase.h | 3 +++ 13 files changed, 74 insertions(+), 8 deletions(-) diff --git a/src/ASM/ASMbase.h b/src/ASM/ASMbase.h index f9f59de52..cff388846 100644 --- a/src/ASM/ASMbase.h +++ b/src/ASM/ASMbase.h @@ -485,6 +485,8 @@ class ASMbase //! \brief Generate element-groups for multi-threading based on a partition. virtual void generateThreadGroupsFromElms(const IntVec&) {} + //! \brief Hook for changing number of threads. + virtual void changeNumThreads() {} // Methods for integration of finite element quantities. // These are the main computational methods of the ASM class hierarchy. diff --git a/src/ASM/ASMs2D.C b/src/ASM/ASMs2D.C index 1553743dd..de697db3e 100644 --- a/src/ASM/ASMs2D.C +++ b/src/ASM/ASMs2D.C @@ -3098,6 +3098,13 @@ void ASMs2D::generateThreadGroups (size_t strip1, size_t strip2, } +void ASMs2D::changeNumThreads () +{ + for (std::unique_ptr& c : myCache) + c->resizeThreadBuffers(); +} + + bool ASMs2D::getNoStructElms (int& n1, int& n2, int& n3) const { n1 = surf->numCoefs_u() - surf->order_u() + 1; diff --git a/src/ASM/ASMs2D.h b/src/ASM/ASMs2D.h index 124e57308..67a759db9 100644 --- a/src/ASM/ASMs2D.h +++ b/src/ASM/ASMs2D.h @@ -688,6 +688,9 @@ class ASMs2D : public ASMstruct, public ASM2D virtual void generateThreadGroups(const Integrand& integrand, bool silence, bool ignoreGlobalLM); + //! \brief Hook for changing number of threads. + virtual void changeNumThreads(); + //! \brief Generates element groups for multi-threading of interior integrals. //! \param[in] strip1 Strip width in first direction //! \param[in] strip2 Strip width in second direction diff --git a/src/ASM/ASMs3D.C b/src/ASM/ASMs3D.C index 9a4c2030c..bc76288c7 100644 --- a/src/ASM/ASMs3D.C +++ b/src/ASM/ASMs3D.C @@ -3531,6 +3531,13 @@ void ASMs3D::generateThreadGroups (size_t strip1, size_t strip2, size_t strip3, } +void ASMs3D::changeNumThreads () +{ + for (std::unique_ptr& c : myCache) + c->resizeThreadBuffers(); +} + + void ASMs3D::generateThreadGroups (char lIndex, bool silence, bool) { std::map::iterator tit = threadGroupsFace.find(lIndex); diff --git a/src/ASM/ASMs3D.h b/src/ASM/ASMs3D.h index 47cc2e924..fe7d346aa 100644 --- a/src/ASM/ASMs3D.h +++ b/src/ASM/ASMs3D.h @@ -768,6 +768,9 @@ class ASMs3D : public ASMstruct, public ASM3D void generateThreadGroups(size_t strip1, size_t strip2, size_t strip3, bool silence, bool ignoreGlobalLM); + //! \brief Hook for changing number of threads. + virtual void changeNumThreads(); + //! \brief Returns 0-based index of first node on integration basis. virtual int getFirstItgElmNode() const { return 0; } //! \brief Returns 0-based index of last node on integration basis. diff --git a/src/ASM/BasisFunctionCache.C b/src/ASM/BasisFunctionCache.C index d189cf014..5bd6f2b63 100644 --- a/src/ASM/BasisFunctionCache.C +++ b/src/ASM/BasisFunctionCache.C @@ -46,16 +46,10 @@ bool BasisFunctionCache::init (int nd) return false; if (ASM::cachePolicy == ASM::NO_CACHE) { -#ifdef USE_OPENMP - size_t size = omp_get_max_threads(); -#else - size_t size = 1; -#endif - values.resize(size); - if (this->hasReduced()) - valuesRed.resize(size); + this->resizeThreadBuffers(); return true; } + values.resize(nTotal); if (this->hasReduced()) valuesRed.resize(nTotalRed); @@ -135,5 +129,21 @@ BasisFunctionCache::gpIndex (size_t gp, bool reduced) const } +template +void BasisFunctionCache::resizeThreadBuffers () +{ + if (ASM::cachePolicy == ASM::NO_CACHE) { +#ifdef USE_OPENMP + size_t size = omp_get_max_threads(); +#else + size_t size = 1; +#endif + values.resize(size); + if (this->hasReduced()) + valuesRed.resize(size); + } +} + + template class BasisFunctionCache<2>; template class BasisFunctionCache<3>; diff --git a/src/ASM/BasisFunctionCache.h b/src/ASM/BasisFunctionCache.h index ecb52020e..d2f0d294d 100644 --- a/src/ASM/BasisFunctionCache.h +++ b/src/ASM/BasisFunctionCache.h @@ -84,6 +84,9 @@ template class BasisFunctionCache int basis = 1; //!< Basis to use + //! \brief Called if application changes number of threads. + void resizeThreadBuffers(); + protected: //! \brief Template struct holding information about a quadrature. struct Quadrature { diff --git a/src/ASM/LR/ASMu2D.C b/src/ASM/LR/ASMu2D.C index 00ad620dd..667929939 100644 --- a/src/ASM/LR/ASMu2D.C +++ b/src/ASM/LR/ASMu2D.C @@ -2643,6 +2643,13 @@ void ASMu2D::generateThreadGroups (const Integrand& integrand, bool silence, } +void ASMu2D::changeNumThreads () +{ + for (std::unique_ptr& c : myCache) + c->resizeThreadBuffers(); +} + + void ASMu2D::remapErrors (RealArray& errors, const RealArray& origErr, bool elemErrors) const { diff --git a/src/ASM/LR/ASMu2D.h b/src/ASM/LR/ASMu2D.h index 27084d978..0126b8014 100644 --- a/src/ASM/LR/ASMu2D.h +++ b/src/ASM/LR/ASMu2D.h @@ -674,6 +674,9 @@ class ASMu2D : public ASMLRSpline, public ASM2D //! \brief Generate element groups from a partition. virtual void generateThreadGroupsFromElms(const std::vector& elms); + //! \brief Hook for changing number of threads. + virtual void changeNumThreads(); + //! \brief Remap element wise errors to basis functions. //! \param errors The remapped errors //! \param[in] origErr The element wise errors on the geometry mesh diff --git a/src/ASM/LR/ASMu3D.C b/src/ASM/LR/ASMu3D.C index c3e48be02..c6a26fdd7 100644 --- a/src/ASM/LR/ASMu3D.C +++ b/src/ASM/LR/ASMu3D.C @@ -1916,6 +1916,13 @@ void ASMu3D::generateThreadGroups (const Integrand& integrand, bool silence, } +void ASMu3D::changeNumThreads () +{ + for (std::unique_ptr& c : myCache) + c->resizeThreadBuffers(); +} + + bool ASMu3D::updateDirichlet (const std::map& func, const std::map& vfunc, double time, const std::map* g2l) diff --git a/src/ASM/LR/ASMu3D.h b/src/ASM/LR/ASMu3D.h index 4734163dc..9d83e0963 100644 --- a/src/ASM/LR/ASMu3D.h +++ b/src/ASM/LR/ASMu3D.h @@ -680,6 +680,9 @@ class ASMu3D : public ASMLRSpline, public ASM3D //! \brief Generate element groups from a partition. virtual void generateThreadGroupsFromElms(const std::vector& elms); + //! \brief Hook for changing number of threads. + virtual void changeNumThreads(); + //! \brief Remap element wise errors to basis functions. //! \param errors The remapped errors //! \param[in] origErr The element wise errors on the geometry mesh diff --git a/src/SIM/SIMbase.C b/src/SIM/SIMbase.C index ac2f6898e..092ded35b 100644 --- a/src/SIM/SIMbase.C +++ b/src/SIM/SIMbase.C @@ -491,6 +491,14 @@ void SIMbase::generateThreadGroups (const Property& p, bool silence) } +void SIMbase::changeNumThreads () +{ + for (ASMbase* pch : myModel) + if (!pch->empty()) + pch->changeNumThreads(); +} + + VecFunc* SIMbase::getVecFunc (size_t patch, Property::Type ptype) const { for (PropertyVec::const_iterator p = myProps.begin(); p != myProps.end(); ++p) diff --git a/src/SIM/SIMbase.h b/src/SIM/SIMbase.h index 3f3b017a2..af07c8ae6 100644 --- a/src/SIM/SIMbase.h +++ b/src/SIM/SIMbase.h @@ -758,6 +758,9 @@ class SIMbase : public SIMadmin, public SIMdependency //! \param[in] silence If \e true, suppress threading group outprint void generateThreadGroups(const Property& p, bool silence = false); + //! \brief Called if number of threads changes. + void changeNumThreads(); + //! \brief Adds a MADOF with an extraordinary number of DOFs on a given basis. //! \param[in] basis The basis to specify number of DOFs for //! \param[in] nndof Number of nodal DOFs on the given basis