Make Zoltan's Hyperedge Size Threshold Runtime Controllable

opm/simulators/flow/CpGridVanguard.hpp

@@ -316,6 +316,12 @@ class CpGridVanguard : public FlowBaseVanguard<TypeTag>
     {
         return this->zoltanParams_;
     }
+
+    double zoltanPhgEdgeSizeThreshold() const override
+    {
+        return this->zoltanPhgEdgeSizeThreshold_;
+    }
+
     const std::string& metisParams() const override
     {
         return this->metisParams_;

opm/simulators/flow/FlowGenericVanguard.cpp

@@ -124,7 +124,7 @@ FlowGenericVanguard::FlowGenericVanguard(SimulationModelParams&& params)
     partitionMethod_ = Dune::PartitionMethod(Parameters::Get<Parameters::PartitionMethod>());
     serialPartitioning_ = Parameters::Get<Parameters::SerialPartitioning>();
     zoltanParams_ = Parameters::Get<Parameters::ZoltanParams>();
-
+    zoltanPhgEdgeSizeThreshold_ = Parameters::Get<Parameters::ZoltanPhgEdgeSizeThreshold>();
     metisParams_ = Parameters::Get<Parameters::MetisParams>();
 
     externalPartitionFile_ = Parameters::Get<Parameters::ExternalPartition>();
@@ -468,21 +468,26 @@ void FlowGenericVanguard::registerParameters_()
     Parameters::Register<Parameters::SerialPartitioning>
         ("Perform partitioning for parallel runs on a single process.");
     Parameters::Register<Parameters::ZoltanImbalanceTol<Scalar>>
-        ("Tolerable imbalance of the loadbalancing provided by Zoltan. DEPRECATED: Use --imbalance-tol instead");
+        ("Tolerable imbalance of the loadbalancing provided by Zoltan. "
+         "DEPRECATED: Use --imbalance-tol instead.");
     Parameters::Register<Parameters::ZoltanParams>
         ("Configuration of Zoltan partitioner. "
          "Valid options are: graph, hypergraph or scotch. "
          "Alternatively, you can request a configuration to be read "
-         "from a JSON file by giving the filename here, ending with '.json.' "
+         "from a JSON file by giving the filename here, with extension '.json'. "
          "See https://sandialabs.github.io/Zoltan/ug_html/ug.html "
          "for available Zoltan options.");
     Parameters::Register<Parameters::ImbalanceTol<Scalar>>
-        ("Tolerable imbalance of the loadbalancing (default: 1.1).");
+        ("Tolerable imbalance of the loadbalancing.");
+    Parameters::Register<Parameters::ZoltanPhgEdgeSizeThreshold>
+        ("Low-level threshold fraction in the range [0,1] controlling "
+         "which hypergraph edge to omit. Used if --zoltan-params=\"graph\" "
+         "or if --zoltan-params=\"hypergraph\".");
     Parameters::Register<Parameters::MetisParams>
         ("Configuration of Metis partitioner. "
          "You can request a configuration to be read "
-         "from a JSON file by giving the filename here, ending with '.json.' "
-         "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf"
+         "from a JSON file by giving the filename here, with extension '.json'. "
+         "See http://glaros.dtc.umn.edu/gkhome/fetch/sw/metis/manual.pdf "
          "for available METIS options.");
     Parameters::Register<Parameters::ExternalPartition>
         ("Name of file from which to load an externally generated "
@@ -493,6 +498,7 @@ void FlowGenericVanguard::registerParameters_()
 
     Parameters::Hide<Parameters::ZoltanImbalanceTol<Scalar>>();
     Parameters::Hide<Parameters::ZoltanParams>();
+    Parameters::Hide<Parameters::ZoltanPhgEdgeSizeThreshold>();
 #endif // HAVE_MPI
 
     Parameters::Register<Parameters::AllowDistributedWells>

opm/simulators/flow/FlowGenericVanguard.hpp

@@ -84,6 +84,8 @@ struct SerialPartitioning{ static constexpr bool value = false; };
 template<class Scalar>
 struct ZoltanImbalanceTol { static constexpr Scalar value = 1.1; };
 
+struct ZoltanPhgEdgeSizeThreshold { static constexpr auto value = 0.35; };
+
 struct ZoltanParams { static constexpr auto value = "graph"; };
 
 } // namespace Opm::Parameters
@@ -380,6 +382,7 @@ class FlowGenericVanguard {
 
     bool zoltanImbalanceTolSet_;
     double zoltanImbalanceTol_;
+    double zoltanPhgEdgeSizeThreshold_;
     std::string zoltanParams_;
 
     std::string metisParams_;

opm/simulators/flow/GenericCpGridVanguard.cpp

@@ -158,11 +158,21 @@ doLoadBalance_(const Dune::EdgeWeightMethod             edgeWeightsMethod,
                const int                                numJacobiBlocks,
                const bool                               enableEclOutput)
 {
-    if ((partitionMethod == Dune::PartitionMethod::zoltan
-         || partitionMethod == Dune::PartitionMethod::zoltanGoG) && !this->zoltanParams().empty())
-        this->grid_->setPartitioningParams(setupZoltanParams(this->zoltanParams()));
-    if (partitionMethod == Dune::PartitionMethod::metis && !this->metisParams().empty())
-        this->grid_->setPartitioningParams(setupMetisParams(this->metisParams()));
+    if (((partitionMethod == Dune::PartitionMethod::zoltan) ||
+         (partitionMethod == Dune::PartitionMethod::zoltanGoG)) &&
+        !this->zoltanParams().empty())
+    {
+        this->grid_->setPartitioningParams
+            (setupZoltanParams(this->zoltanParams(),
+                               this->zoltanPhgEdgeSizeThreshold()));
+    }
+
+    if ((partitionMethod == Dune::PartitionMethod::metis) &&
+        !this->metisParams().empty())
+    {
+        this->grid_->setPartitioningParams
+            (setupMetisParams(this->metisParams()));
+    }
 
     const auto mpiSize = this->grid_->comm().size();
 

opm/simulators/flow/GenericCpGridVanguard.hpp

@@ -209,6 +209,7 @@ class GenericCpGridVanguard {
 
 protected:
     virtual const std::string& zoltanParams() const = 0;
+    virtual double zoltanPhgEdgeSizeThreshold() const = 0;
     virtual const std::string& metisParams() const = 0;
 
 #endif  // HAVE_MPI

opm/simulators/utils/SetupPartitioningParams.cpp

@@ -32,6 +32,7 @@
 
 #include <filesystem>
 #include <map>
+#include <optional>
 #include <stdexcept>
 #include <string>
 #include <string_view>
@@ -99,6 +100,18 @@ namespace {
 
 namespace {
 
+    std::map<std::string, std::string>
+    applyEdgeSizeThreshold(std::map<std::string, std::string>&& params,
+                           const std::optional<double>&         edgeSizeThreshold)
+    {
+        if (edgeSizeThreshold.has_value()) {
+            params.emplace("PHG_EDGE_SIZE_THRESHOLD",
+                           fmt::format("{}", *edgeSizeThreshold));
+        }
+
+        return params;
+    }
+
     std::map<std::string, std::string>
     zoltanGraphParameters(std::string_view graphPackage)
     {
@@ -108,9 +121,10 @@ namespace {
         };
     }
 
-    auto zoltanGraphParameters()
+    auto zoltanGraphParameters(const std::optional<double>& edgeSizeThreshold)
     {
-        return zoltanGraphParameters("PHG");
+        return applyEdgeSizeThreshold(zoltanGraphParameters("PHG"),
+                                      edgeSizeThreshold);
     }
 
     auto zoltanScotchParameters()
@@ -119,26 +133,28 @@ namespace {
     }
 
     std::map<std::string, std::string>
-    zoltanHyperGraphParameters()
+    zoltanHyperGraphParameters(const std::optional<double>& edgeSizeThreshold)
     {
-        return {
+        return applyEdgeSizeThreshold({
             { "LB_METHOD", "HYPERGRAPH" },
-        };
+            { "HYPERGRAPH_PACKAGE", "PHG" },
+        }, edgeSizeThreshold);
     }
 
 } // Anonymous namespace
 
 // ===========================================================================
 
 std::map<std::string, std::string>
-Opm::setupZoltanParams(const std::string& conf)
+Opm::setupZoltanParams(const std::string&           conf,
+                       const std::optional<double>& edgeSizeThreshold)
 {
     if (conf == "graph") {
-        return zoltanGraphParameters();
+        return zoltanGraphParameters(edgeSizeThreshold);
     }
 
     else if (conf == "hypergraph") {
-        return zoltanHyperGraphParameters();
+        return zoltanHyperGraphParameters(edgeSizeThreshold);
     }
 
     else if (conf == "scotch") {

opm/simulators/utils/SetupPartitioningParams.hpp

@@ -21,12 +21,47 @@
 #define OPM_SETUP_PARTITIONING_PARAMS_HPP
 
 #include <map>
+#include <optional>
 #include <string>
 
 namespace Opm {
 
-std::map<std::string,std::string> setupZoltanParams(const std::string& conf);
-std::map<std::string,std::string> setupMetisParams(const std::string& conf);
+/// Form collection of Zoltan partitioning parameters from named configuration
+///
+/// \param[in] conf Named Zoltan configuration. Must either be the name of a
+/// JSON configuration file with the filename extension ".json", or one of
+/// the known configuration names
+///
+///   -* graph Generates configuration parameters for the "GRAPH"
+///            load-balancing method, using the "PHG" graph package.
+///
+///   -* hypergraph Generates configuration parameters for the "HYPERGRAPH"
+///            load-balancing method.
+///
+///   -* scotch Generates configuration parameters for the "GRAPH"
+///            load-balancing method, using the "Scotch" graph package.
+///
+/// \param[in] edgeSizeThreshold Low-level Zoltan partitioning control
+/// parameter for when to omit a hyperedge in a hypergraph.  Fraction in the
+/// range [0,1] representing a threshold above which to omit discard
+/// hyperedges.  Used for conf="graph" and conf="hypergraph".  Nullopt to
+/// use the built-in default value.
+///
+/// \return Collection of Zoltan partitioning parameters.
+std::map<std::string, std::string>
+setupZoltanParams(const std::string&           conf,
+                  const std::optional<double>& edgeSizeThreshold = {});
+
+/// Form collection of METIS partitioning parameters from named configuration
+///
+/// \param[in] conf Named METIS configuration. Must either be the name of a
+/// JSON configuration file with the filename extension ".json", or the
+/// known configuration name "default" which uses the built-in default
+/// partitioning parameters.
+///
+/// \return Collection of METIS partitioning parameters.
+std::map<std::string, std::string>
+setupMetisParams(const std::string& conf);
 
 } // namespace Opm