Make Zoltan's Hyperedge Size Threshold Runtime Controllable

This commit introduces a new hidden runtime parameter,

    ZoltanPhgEdgeSizeThreshold (--zoltan-phg-edge-size-threshold,
    double, default value = 0.35)

which maps to Zoltan's low-level control parameter

    PHG_EDGE_SIZE_THRESHOLD

that controls which hypergraph edges to omit/discard.  We add a new
parameter to 'setupZoltanParameters()' and thread the command line
parameter through as an argument to this function.

This is to enable runtime experimentation with this aspect of the
partitioning algorithm, but end-users should typically not alter the
default value without good reason.

Author: bska

opm/simulators/flow/CpGridVanguard.hpp

@@ -316,6 +316,12 @@ class CpGridVanguard : public FlowBaseVanguard<TypeTag>
     {
         return this->zoltanParams_;
     }
+
+    double zoltanPhgEdgeSizeThreshold() const override
+    {
+        return this->zoltanPhgEdgeSizeThreshold_;
+    }
+
     const std::string& metisParams() const override
     {
         return this->metisParams_;

opm/simulators/flow/FlowGenericVanguard.cpp

@@ -124,7 +124,7 @@ FlowGenericVanguard::FlowGenericVanguard(SimulationModelParams&& params)
     partitionMethod_ = Dune::PartitionMethod(Parameters::Get<Parameters::PartitionMethod>());
     serialPartitioning_ = Parameters::Get<Parameters::SerialPartitioning>();
     zoltanParams_ = Parameters::Get<Parameters::ZoltanParams>();
-
+    zoltanPhgEdgeSizeThreshold_ = Parameters::Get<Parameters::ZoltanPhgEdgeSizeThreshold>();
     metisParams_ = Parameters::Get<Parameters::MetisParams>();
 
     externalPartitionFile_ = Parameters::Get<Parameters::ExternalPartition>();
@@ -479,6 +479,10 @@ void FlowGenericVanguard::registerParameters_()
          "for available Zoltan options.");
     Parameters::Register<Parameters::ImbalanceTol<Scalar>>
         ("Tolerable imbalance of the loadbalancing.");
+    Parameters::Register<Parameters::ZoltanPhgEdgeSizeThreshold>
+        ("Low-level threshold fraction in the range [0,1] controlling "
+         "which hypergraph edge to omit. Used if --zoltan-params=\"graph\" "
+         "or if --zoltan-params=\"hypergraph\".");
     Parameters::Register<Parameters::MetisParams>
         ("Configuration of Metis partitioner. "
          "You can request a configuration to be read "
@@ -494,6 +498,7 @@ void FlowGenericVanguard::registerParameters_()
 
     Parameters::Hide<Parameters::ZoltanImbalanceTol<Scalar>>();
     Parameters::Hide<Parameters::ZoltanParams>();
+    Parameters::Hide<Parameters::ZoltanPhgEdgeSizeThreshold>();
 #endif // HAVE_MPI
 
     Parameters::Register<Parameters::AllowDistributedWells>

opm/simulators/flow/FlowGenericVanguard.hpp

@@ -84,6 +84,8 @@ struct SerialPartitioning{ static constexpr bool value = false; };
 template<class Scalar>
 struct ZoltanImbalanceTol { static constexpr Scalar value = 1.1; };
 
+struct ZoltanPhgEdgeSizeThreshold { static constexpr auto value = 0.35; };
+
 struct ZoltanParams { static constexpr auto value = "graph"; };
 
 } // namespace Opm::Parameters
@@ -380,6 +382,7 @@ class FlowGenericVanguard {
 
     bool zoltanImbalanceTolSet_;
     double zoltanImbalanceTol_;
+    double zoltanPhgEdgeSizeThreshold_;
     std::string zoltanParams_;
 
     std::string metisParams_;

opm/simulators/flow/GenericCpGridVanguard.cpp

@@ -158,11 +158,21 @@ doLoadBalance_(const Dune::EdgeWeightMethod             edgeWeightsMethod,
                const int                                numJacobiBlocks,
                const bool                               enableEclOutput)
 {
-    if ((partitionMethod == Dune::PartitionMethod::zoltan
-         || partitionMethod == Dune::PartitionMethod::zoltanGoG) && !this->zoltanParams().empty())
-        this->grid_->setPartitioningParams(setupZoltanParams(this->zoltanParams()));
-    if (partitionMethod == Dune::PartitionMethod::metis && !this->metisParams().empty())
-        this->grid_->setPartitioningParams(setupMetisParams(this->metisParams()));
+    if (((partitionMethod == Dune::PartitionMethod::zoltan) ||
+         (partitionMethod == Dune::PartitionMethod::zoltanGoG)) &&
+        !this->zoltanParams().empty())
+    {
+        this->grid_->setPartitioningParams
+            (setupZoltanParams(this->zoltanParams(),
+                               this->zoltanPhgEdgeSizeThreshold()));
+    }
+
+    if ((partitionMethod == Dune::PartitionMethod::metis) &&
+        !this->metisParams().empty())
+    {
+        this->grid_->setPartitioningParams
+            (setupMetisParams(this->metisParams()));
+    }
 
     const auto mpiSize = this->grid_->comm().size();
 

opm/simulators/flow/GenericCpGridVanguard.hpp

@@ -209,6 +209,7 @@ class GenericCpGridVanguard {
 
 protected:
     virtual const std::string& zoltanParams() const = 0;
+    virtual double zoltanPhgEdgeSizeThreshold() const = 0;
     virtual const std::string& metisParams() const = 0;
 
 #endif  // HAVE_MPI

opm/simulators/utils/SetupPartitioningParams.cpp

@@ -32,6 +32,7 @@
 
 #include <filesystem>
 #include <map>
+#include <optional>
 #include <stdexcept>
 #include <string>
 #include <string_view>
@@ -99,6 +100,18 @@ namespace {
 
 namespace {
 
+    std::map<std::string, std::string>
+    applyEdgeSizeThreshold(std::map<std::string, std::string>&& params,
+                           const std::optional<double>&         edgeSizeThreshold)
+    {
+        if (edgeSizeThreshold.has_value()) {
+            params.emplace("PHG_EDGE_SIZE_THRESHOLD",
+                           fmt::format("{}", *edgeSizeThreshold));
+        }
+
+        return params;
+    }
+
     std::map<std::string, std::string>
     zoltanGraphParameters(std::string_view graphPackage)
     {
@@ -108,9 +121,10 @@ namespace {
         };
     }
 
-    auto zoltanGraphParameters()
+    auto zoltanGraphParameters(const std::optional<double>& edgeSizeThreshold)
     {
-        return zoltanGraphParameters("PHG");
+        return applyEdgeSizeThreshold(zoltanGraphParameters("PHG"),
+                                      edgeSizeThreshold);
     }
 
     auto zoltanScotchParameters()
@@ -119,26 +133,28 @@ namespace {
     }
 
     std::map<std::string, std::string>
-    zoltanHyperGraphParameters()
+    zoltanHyperGraphParameters(const std::optional<double>& edgeSizeThreshold)
     {
-        return {
+        return applyEdgeSizeThreshold({
             { "LB_METHOD", "HYPERGRAPH" },
-        };
+            { "HYPERGRAPH_PACKAGE", "PHG" },
+        }, edgeSizeThreshold);
     }
 
 } // Anonymous namespace
 
 // ===========================================================================
 
 std::map<std::string, std::string>
-Opm::setupZoltanParams(const std::string& conf)
+Opm::setupZoltanParams(const std::string&           conf,
+                       const std::optional<double>& edgeSizeThreshold)
 {
     if (conf == "graph") {
-        return zoltanGraphParameters();
+        return zoltanGraphParameters(edgeSizeThreshold);
     }
 
     else if (conf == "hypergraph") {
-        return zoltanHyperGraphParameters();
+        return zoltanHyperGraphParameters(edgeSizeThreshold);
     }
 
     else if (conf == "scotch") {

opm/simulators/utils/SetupPartitioningParams.hpp

@@ -21,12 +21,47 @@
 #define OPM_SETUP_PARTITIONING_PARAMS_HPP
 
 #include <map>
+#include <optional>
 #include <string>
 
 namespace Opm {
 
-std::map<std::string,std::string> setupZoltanParams(const std::string& conf);
-std::map<std::string,std::string> setupMetisParams(const std::string& conf);
+/// Form collection of Zoltan partitioning parameters from named configuration
+///
+/// \param[in] conf Named Zoltan configuration. Must either be the name of a
+/// JSON configuration file with the filename extension ".json", or one of
+/// the known configuration names
+///
+///   -* graph Generates configuration parameters for the "GRAPH"
+///            load-balancing method, using the "PHG" graph package.
+///
+///   -* hypergraph Generates configuration parameters for the "HYPERGRAPH"
+///            load-balancing method.
+///
+///   -* scotch Generates configuration parameters for the "GRAPH"
+///            load-balancing method, using the "Scotch" graph package.
+///
+/// \param[in] edgeSizeThreshold Low-level Zoltan partitioning control
+/// parameter for when to omit a hyperedge in a hypergraph.  Fraction in the
+/// range [0,1] representing a threshold above which to omit discard
+/// hyperedges.  Used for conf="graph" and conf="hypergraph".  Nullopt to
+/// use the built-in default value.
+///
+/// \return Collection of Zoltan partitioning parameters.
+std::map<std::string, std::string>
+setupZoltanParams(const std::string&           conf,
+                  const std::optional<double>& edgeSizeThreshold = {});
+
+/// Form collection of METIS partitioning parameters from named configuration
+///
+/// \param[in] conf Named METIS configuration. Must either be the name of a
+/// JSON configuration file with the filename extension ".json", or the
+/// known configuration name "default" which uses the built-in default
+/// partitioning parameters.
+///
+/// \return Collection of METIS partitioning parameters.
+std::map<std::string, std::string>
+setupMetisParams(const std::string& conf);
 
 } // namespace Opm