47 #ifndef PACKAGES_MUELU_SRC_REBALANCING_MUELU_REPARTITIONHEURISTICFACTORY_DEF_HPP_
48 #define PACKAGES_MUELU_SRC_REBALANCING_MUELU_REPARTITIONHEURISTICFACTORY_DEF_HPP_
56 #include <Teuchos_CommHelpers.hpp>
61 #include "MueLu_RAPFactory.hpp"
62 #include "MueLu_BlockedRAPFactory.hpp"
63 #include "MueLu_SubBlockAFactory.hpp"
72 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
76 #define SET_VALID_ENTRY(name) validParamList->setEntry(name, MasterList::getEntry(name))
85 #undef SET_VALID_ENTRY
89 validParamList->
set<
RCP<const FactoryBase> >(
"Node Comm", Teuchos::null,
"Generating factory of the node level communicator");
91 return validParamList;
94 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
98 const bool useMap = pL.
get<
bool>(
"repartition: use map");
100 Input(currentLevel,
"Map");
102 Input(currentLevel,
"A");
104 Input(currentLevel,
"A");
105 if (pL.
isParameter(
"repartition: node repartition level")) {
106 const int nodeRepartLevel = pL.
get<
int>(
"repartition: node repartition level");
107 if (currentLevel.
GetLevelID() == nodeRepartLevel) {
108 Input(currentLevel,
"Node Comm");
113 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
120 const int startLevel = pL.
get<
int>(
"repartition: start level");
121 const int nodeRepartLevel = pL.
get<
int>(
"repartition: node repartition level");
122 LO minRowsPerProcess = pL.
get<
LO>(
"repartition: min rows per proc");
123 LO targetRowsPerProcess = pL.
get<
LO>(
"repartition: target rows per proc");
124 LO minRowsPerThread = pL.
get<
LO>(
"repartition: min rows per thread");
125 LO targetRowsPerThread = pL.
get<
LO>(
"repartition: target rows per thread");
126 const double nonzeroImbalance = pL.
get<
double>(
"repartition: max imbalance");
127 const bool useMap = pL.
get<
bool>(
"repartition: use map");
129 int thread_per_mpi_rank = 1;
130 #if defined(KOKKOS_ENABLE_OPENMP)
131 using execution_space =
typename Node::device_type::execution_space;
132 if (std::is_same<execution_space, Kokkos::OpenMP>::value)
133 thread_per_mpi_rank = execution_space().concurrency();
136 if (minRowsPerThread > 0)
138 minRowsPerProcess = minRowsPerThread * thread_per_mpi_rank;
140 if (targetRowsPerThread == 0)
141 targetRowsPerThread = minRowsPerThread;
143 if (targetRowsPerThread > 0)
145 targetRowsPerProcess = targetRowsPerThread * thread_per_mpi_rank;
147 if (targetRowsPerProcess == 0)
148 targetRowsPerProcess = minRowsPerProcess;
151 Set<LO>(currentLevel,
"repartition: heuristic target rows per process", targetRowsPerProcess);
160 Afact = GetFactory(
"A");
161 if (!Afact.
is_null() && Teuchos::rcp_dynamic_cast<
const RAPFactory>(Afact) == Teuchos::null &&
162 Teuchos::rcp_dynamic_cast<const BlockedRAPFactory>(Afact) == Teuchos::null &&
163 Teuchos::rcp_dynamic_cast<
const SubBlockAFactory>(Afact) == Teuchos::null) {
164 GetOStream(
Warnings) <<
"MueLu::RepartitionHeuristicFactory::Build: The generation factory for A must "
165 "be a RAPFactory or a SubBlockAFactory providing the non-rebalanced matrix information! "
166 "It specifically must not be of type Rebalance(Blocked)AcFactory or similar. "
167 "Please check the input. Make also sure that \"number of partitions\" is provided to "
168 "the Interface class and the RepartitionFactory instance. Instead, we have a "
172 A = Get<RCP<Matrix> >(currentLevel,
"A");
173 map = A->getRowMap();
175 map = Get<RCP<const Map> >(currentLevel,
"Map");
185 if (currentLevel.GetLevelID() == nodeRepartLevel && map->getComm()->getSize() > 1) {
190 if (NodeComm()->getSize() != map->getComm()->getSize()) {
191 GetOStream(
Statistics1) <<
"Repartitioning? YES: \n Within node only" << std::endl;
192 int nodeRank = NodeComm->getRank();
195 int isZero = (nodeRank == 0);
197 Teuchos::reduceAll(*map->getComm(),
Teuchos::REDUCE_SUM, isZero, Teuchos::outArg(numNodes));
198 Set(currentLevel,
"number of partitions", numNodes);
204 if (currentLevel.GetLevelID() < startLevel) {
206 <<
"\n current level = " <<
Teuchos::toString(currentLevel.GetLevelID()) <<
", first level where repartitioning can happen is " +
Teuchos::toString(startLevel) << std::endl;
209 Set(currentLevel,
"number of partitions", -1);
223 if (comm->getSize() == 1 && Teuchos::rcp_dynamic_cast<
const RAPFactory>(Afact) != Teuchos::null) {
225 <<
"\n comm size = 1" << std::endl;
227 Set(currentLevel,
"number of partitions", -1);
231 int numActiveProcesses = 0;
232 MueLu_sumAll(comm, Teuchos::as<int>((map->getLocalNumElements() > 0) ? 1 : 0), numActiveProcesses);
234 if (numActiveProcesses == 1) {
236 <<
"\n # processes with rows = " <<
Teuchos::toString(numActiveProcesses) << std::endl;
238 Set(currentLevel,
"number of partitions", 1);
243 bool test3 =
false, test4 =
false;
244 std::string msg3, msg4;
248 if (minRowsPerProcess > 0) {
250 LO haveFewRows = (numMyRows < minRowsPerProcess ? 1 : 0), numWithFewRows = 0;
252 MueLu_minAll(comm, (numMyRows > 0 ? numMyRows : LOMAX), minNumRows);
257 if (numWithFewRows > 0)
268 GO minNnz, maxNnz, numMyNnz = Teuchos::as<GO>(A->getLocalNumEntries());
270 MueLu_minAll(comm, (numMyNnz > 0 ? numMyNnz : maxNnz), minNnz);
271 double imbalance = Teuchos::as<double>(maxNnz) / minNnz;
273 if (imbalance > nonzeroImbalance)
280 if (!test3 && !test4) {
281 GetOStream(
Statistics1) <<
"Repartitioning? NO:" << msg3 + msg4 << std::endl;
284 Set(currentLevel,
"number of partitions", -1);
288 GetOStream(
Statistics1) <<
"Repartitioning? YES:" << msg3 + msg4 << std::endl;
301 const auto globalNumRows = Teuchos::as<GO>(map->getGlobalNumElements());
302 int numPartitions = 1;
303 if (globalNumRows >= targetRowsPerProcess) {
305 numPartitions = std::max(Teuchos::as<int>(globalNumRows / targetRowsPerProcess), 1);
307 numPartitions = std::min(numPartitions, comm->getSize());
309 Set(currentLevel,
"number of partitions", numPartitions);
311 GetOStream(
Statistics1) <<
"Number of partitions to use = " << numPartitions << std::endl;
315 #endif // ifdef HAVE_MPI
#define MueLu_sumAll(rcpComm, in, out)
#define MueLu_maxAll(rcpComm, in, out)
void Build(Level ¤tLevel) const
Build an object with this factory.
T & get(const std::string &name, T def_value)
ParameterList & set(std::string const &name, T const &value, std::string const &docString="", RCP< const ParameterEntryValidator > const &validator=null)
Timer to be used in factories. Similar to Monitor but with additional timers.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
#define SET_VALID_ENTRY(name)
#define MueLu_minAll(rcpComm, in, out)
bool isParameter(const std::string &name) const
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Class that holds all level-specific information.
Factory for building a thresholded operator.
void DeclareInput(Level ¤tLevel) const
Determines the data that RepartitionHeuristicFactory needs, and the factories that generate that data...
int GetLevelID() const
Return level number.
Exception throws to report errors in the internal logical of the program.
Print all warning messages.
Factory for building coarse matrices.
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.
virtual std::string description() const
Return a simple one-line description of this object.
std::string toString(const T &t)