46 #ifndef MUELU_REBALANCEBLOCKINTERPOLATIONFACTORY_DEF_HPP_
47 #define MUELU_REBALANCEBLOCKINTERPOLATIONFACTORY_DEF_HPP_
49 #include <Teuchos_Tuple.hpp>
52 #include "Xpetra_MultiVectorFactory.hpp"
57 #include <Xpetra_MapFactory.hpp>
58 #include <Xpetra_MapExtractor.hpp>
67 #include "MueLu_HierarchyUtils.hpp"
70 #include "MueLu_PerfUtils.hpp"
74 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
78 validParamList->
set<
RCP<const FactoryBase> >(
"P", Teuchos::null,
"Factory of the prolongation operator that need to be rebalanced (only used if type=Interpolation)");
79 validParamList->
set<
RCP<const FactoryBase> >(
"A", Teuchos::null,
"Factory for generating the non-rebalanced coarse level A. We need this to make sure the non-rebalanced coarse A is calculated first before rebalancing takes place.");
81 validParamList->
set<
RCP<const FactoryBase> >(
"Coordinates", Teuchos::null,
"Factory for generating the non-rebalanced Coordinates.");
82 validParamList->
set<
RCP<const FactoryBase> >(
"Importer", Teuchos::null,
"Generating factory of the matrix Importer for rebalancing");
83 validParamList->
set<
RCP<const FactoryBase> >(
"SubImporters", Teuchos::null,
"Generating factory of the matrix sub-Importers for rebalancing");
85 #define SET_VALID_ENTRY(name) validParamList->setEntry(name, MasterList::getEntry(name))
87 #undef SET_VALID_ENTRY
93 return validParamList;
96 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
98 FactManager_.push_back(FactManager);
101 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
103 Input(coarseLevel,
"P");
104 Input(coarseLevel,
"A");
106 std::vector<Teuchos::RCP<const FactoryManagerBase> >::const_iterator it;
107 for (it = FactManager_.begin(); it != FactManager_.end(); ++it) {
113 coarseLevel.
DeclareInput(
"Importer", (*it)->GetFactory(
"Importer").get(),
this);
114 if ((*it)->hasFactory(
"Coordinates") ==
true)
115 coarseLevel.
DeclareInput(
"Coordinates", (*it)->GetFactory(
"Coordinates").get(),
this);
119 if (FactManager_.size() == 0) {
120 Input(coarseLevel,
"Importer");
121 Input(coarseLevel,
"SubImporters");
122 Input(coarseLevel,
"Coordinates");
126 template <
class Scalar,
class LocalOrdinal,
class GlobalOrdinal,
class Node>
132 bool UseSingleSource = FactManager_.size() == 0;
138 originalTransferOp = Get<RCP<Matrix> >(coarseLevel,
"P");
154 bool bThyraRangeGIDs = rangeMapExtractor->getThyraMode();
155 bool bThyraDomainGIDs = domainMapExtractor->getThyraMode();
158 std::vector<GO> fullRangeMapVector;
159 std::vector<GO> fullDomainMapVector;
160 std::vector<RCP<const Map> > subBlockPRangeMaps;
161 std::vector<RCP<const Map> > subBlockPDomainMaps;
162 subBlockPRangeMaps.reserve(bOriginalTransferOp->Rows());
163 subBlockPDomainMaps.reserve(bOriginalTransferOp->Cols());
165 std::vector<Teuchos::RCP<Matrix> > subBlockRebP;
166 subBlockRebP.reserve(bOriginalTransferOp->Rows());
169 std::vector<RCP<const Import> > importers = std::vector<RCP<const Import> >(bOriginalTransferOp->Rows(), Teuchos::null);
170 std::vector<RCP<xdMV> > newCoordinates(bOriginalTransferOp->Rows());
172 if (UseSingleSource) {
173 importers = Get<std::vector<RCP<const Import> > >(coarseLevel,
"SubImporters");
174 oldCoordinates = Get<RCP<xdBV> >(coarseLevel,
"Coordinates");
179 std::vector<Teuchos::RCP<const FactoryManagerBase> >::const_iterator it;
180 for (it = FactManager_.begin(); it != FactManager_.end(); ++it) {
187 rebalanceImporter = importers[curBlockId];
194 TEUCHOS_TEST_FOR_EXCEPTION(Pwii == Teuchos::null,
Xpetra::Exceptions::BadCast,
"MueLu::RebalanceBlockTransferFactory::Build: block " << curBlockId <<
" is not of type CrsMatrixWrap. We need an underlying CsrMatrix to replace domain map and importer!");
198 "MueLu::RebalanceBlockInterpolationFactory::Build: inconsistent Thyra GIDs. Thyra global ids for block range " << curBlockId <<
" start with " << Pii->getRowMap()->getMinAllGlobalIndex() <<
" but should start with 0");
201 "MueLu::RebalanceBlockInterpolationFactory::Build: inconsistent Thyra GIDs. Thyra global ids for block domain " << curBlockId <<
" start with " << Pii->getColMap()->getMinAllGlobalIndex() <<
" but should start with 0");
204 if (rebalanceImporter != Teuchos::null) {
205 std::stringstream ss;
206 ss <<
"Rebalancing prolongator block P(" << curBlockId <<
"," << curBlockId <<
")";
222 SubFactoryMonitor subM(*
this,
"Rebalancing prolongator -- fast map replacement", coarseLevel);
223 newImporter = ImportFactory::Build(rebalanceImporter->getTargetMap(), Pii->getColMap());
224 Pwii->getCrsMatrix()->replaceDomainMapAndImporter(rebalanceImporter->getTargetMap(), newImporter);
228 params->
set(
"printLoadBalancingInfo",
true);
229 std::stringstream ss2;
230 ss2 <<
"P(" << curBlockId <<
"," << curBlockId <<
") rebalanced:";
234 subBlockRebP.push_back(Pii);
239 if (UseSingleSource) {
240 RCP<xdMV> localCoords = oldCoordinates->getMultiVector(curBlockId);
246 permutedLocalCoords->doImport(*localCoords, *coordImporter,
Xpetra::INSERT);
248 newCoordinates[curBlockId] = permutedLocalCoords;
249 }
else if ((*it)->hasFactory(
"Coordinates") ==
true && coarseLevel.IsAvailable(
"Coordinates", (*it)->GetFactory(
"Coordinates").get()) ==
true) {
250 RCP<xdMV> coords = coarseLevel.Get<
RCP<xdMV> >(
"Coordinates", (*it)->GetFactory(
"Coordinates").get());
255 LO nodeNumElts = coords->getMap()->getLocalNumElements();
258 LO myBlkSize = 0, blkSize = 0;
260 if (nodeNumElts > 0) {
263 "MueLu::RebalanceBlockInterpolationFactory::Build: block size. " << rebalanceImporter->getSourceMap()->getLocalNumElements() <<
" not divisable by " << nodeNumElts);
264 myBlkSize = rebalanceImporter->getSourceMap()->getLocalNumElements() / nodeNumElts;
267 MueLu_maxAll(coords->getMap()->getComm(), myBlkSize, blkSize);
271 coordImporter = rebalanceImporter;
277 GO indexBase = origMap->getIndexBase();
280 LO numEntries = OEntries.
size() / blkSize;
282 for (LO i = 0; i < numEntries; i++)
283 Entries[i] = (OEntries[i * blkSize] - indexBase) / blkSize + indexBase;
285 RCP<const Map> targetMap = MapFactory::Build(origMap->lib(), origMap->getGlobalNumElements(), Entries(), indexBase, origMap->getComm());
286 coordImporter = ImportFactory::Build(origMap, targetMap);
290 permutedCoords->doImport(*coords, *coordImporter,
Xpetra::INSERT);
293 if (pL.
isParameter(
"repartition: use subcommunicators") ==
true && pL.
get<
bool>(
"repartition: use subcommunicators") ==
true)
294 permutedCoords->replaceMap(permutedCoords->getMap()->removeEmptyProcesses());
296 Set(coarseLevel,
"Coordinates", permutedCoords);
301 params->
set(
"printLoadBalancingInfo",
true);
302 std::stringstream ss;
303 ss <<
"P(" << curBlockId <<
"," << curBlockId <<
") not rebalanced:";
306 subBlockRebP.push_back(Pii);
311 if ((*it)->hasFactory(
"Coordinates") ==
true && coarseLevel.IsAvailable(
"Coordinates", (*it)->GetFactory(
"Coordinates").get()) ==
true) {
312 coarseLevel.Set(
"Coordinates", coarseLevel.Get<
RCP<xdMV> >(
"Coordinates", (*it)->GetFactory(
"Coordinates").get()),
this);
318 Teuchos::RCP<const StridedMap> orig_stridedRgMap = Teuchos::rcp_dynamic_cast<
const StridedMap>(rangeMapExtractor->getMap(Teuchos::as<size_t>(curBlockId), rangeMapExtractor->getThyraMode()));
320 if (orig_stridedRgMap != Teuchos::null) {
321 std::vector<size_t> stridingData = orig_stridedRgMap->getStridingData();
323 stridedRgMap = StridedMapFactory::Build(
324 originalTransferOp->getRangeMap()->lib(),
327 Pii->getRangeMap()->getIndexBase(),
329 originalTransferOp->getRangeMap()->getComm(),
330 orig_stridedRgMap->getStridedBlockId(),
331 orig_stridedRgMap->getOffset());
333 stridedRgMap = Pii->getRangeMap();
336 Teuchos::RCP<const StridedMap> orig_stridedDoMap = Teuchos::rcp_dynamic_cast<
const StridedMap>(domainMapExtractor->getMap(Teuchos::as<size_t>(curBlockId), domainMapExtractor->getThyraMode()));
339 if (orig_stridedDoMap != Teuchos::null) {
340 std::vector<size_t> stridingData = orig_stridedDoMap->getStridingData();
342 stridedDoMap = StridedMapFactory::Build(originalTransferOp->getDomainMap()->lib(),
345 Pii->getDomainMap()->getIndexBase(),
347 originalTransferOp->getDomainMap()->getComm(),
348 orig_stridedDoMap->getStridedBlockId(),
349 orig_stridedDoMap->getOffset());
351 stridedDoMap = Pii->getDomainMap();
353 TEUCHOS_TEST_FOR_EXCEPTION(stridedRgMap == Teuchos::null,
Exceptions::RuntimeError,
"MueLu::RebalanceBlockInterpolationFactory::Build: failed to generate striding information. error.");
354 TEUCHOS_TEST_FOR_EXCEPTION(stridedDoMap == Teuchos::null,
Exceptions::RuntimeError,
"MueLu::RebalanceBlockInterpolationFactory::Build: failed to generate striding information. error.");
357 if (Pii->IsView(
"stridedMaps")) Pii->RemoveView(
"stridedMaps");
358 Pii->CreateView(
"stridedMaps", stridedRgMap, stridedDoMap);
362 subBlockPRangeMaps.push_back(rangeMapii);
365 fullRangeMapVector.insert(fullRangeMapVector.end(), nodeRangeMapii.
begin(), nodeRangeMapii.
end());
366 if (bThyraRangeGIDs ==
false)
367 sort(fullRangeMapVector.begin(), fullRangeMapVector.end());
371 subBlockPDomainMaps.push_back(domainMapii);
374 fullDomainMapVector.insert(fullDomainMapVector.end(), nodeDomainMapii.
begin(), nodeDomainMapii.
end());
375 if (bThyraDomainGIDs ==
false)
376 sort(fullDomainMapVector.begin(), fullDomainMapVector.end());
383 GO rangeIndexBase = originalTransferOp->getRangeMap()->getIndexBase();
384 GO domainIndexBase = originalTransferOp->getDomainMap()->getIndexBase();
387 Teuchos::ArrayView<GO> fullRangeMapGIDs(fullRangeMapVector.size() ? &fullRangeMapVector[0] : 0, fullRangeMapVector.size());
390 if (stridedRgFullMap != Teuchos::null) {
391 std::vector<size_t> stridedData = stridedRgFullMap->getStridingData();
393 StridedMapFactory::Build(
394 originalTransferOp->getRangeMap()->lib(),
399 originalTransferOp->getRangeMap()->getComm(),
400 stridedRgFullMap->getStridedBlockId(),
401 stridedRgFullMap->getOffset());
405 originalTransferOp->getRangeMap()->lib(),
409 originalTransferOp->getRangeMap()->getComm());
413 Teuchos::ArrayView<GO> fullDomainMapGIDs(fullDomainMapVector.size() ? &fullDomainMapVector[0] : 0, fullDomainMapVector.size());
416 if (stridedDoFullMap != Teuchos::null) {
417 TEUCHOS_TEST_FOR_EXCEPTION(stridedDoFullMap == Teuchos::null,
Exceptions::BadCast,
"MueLu::BlockedPFactory::Build: full map in domain map extractor has no striding information! error.");
418 std::vector<size_t> stridedData2 = stridedDoFullMap->getStridingData();
420 StridedMapFactory::Build(
421 originalTransferOp->getDomainMap()->lib(),
426 originalTransferOp->getDomainMap()->getComm(),
427 stridedDoFullMap->getStridedBlockId(),
428 stridedDoFullMap->getOffset());
432 originalTransferOp->getDomainMap()->lib(),
436 originalTransferOp->getDomainMap()->getComm());
441 MapExtractorFactory::Build(fullRangeMap, subBlockPRangeMaps, bThyraRangeGIDs);
443 MapExtractorFactory::Build(fullDomainMap, subBlockPDomainMaps, bThyraDomainGIDs);
446 for (
size_t i = 0; i < subBlockPRangeMaps.size(); i++) {
448 TEUCHOS_TEST_FOR_EXCEPTION(crsOpii == Teuchos::null,
Xpetra::Exceptions::BadCast,
"MueLu::RebalanceBlockTransferFactory::Build: block P" << i <<
" is not of type CrsMatrixWrap.");
449 bRebP->setMatrix(i, i, crsOpii);
451 bRebP->fillComplete();
453 Set(coarseLevel,
"P", Teuchos::rcp_dynamic_cast<Matrix>(bRebP));
456 if (UseSingleSource) {
457 RCP<xdBV> bcoarseCoords =
rcp(
new xdBV(rebrangeMapExtractor->getBlockedMap(), newCoordinates));
458 Set(coarseLevel,
"Coordinates", bcoarseCoords);
Exception indicating invalid cast attempted.
#define MueLu_maxAll(rcpComm, in, out)
RCP< const ParameterList > GetValidParameterList() const
Return a const parameter list of valid parameters that setParameterList() will accept.
void DeclareInput(Level &fineLevel, Level &coarseLevel) const
Specifies the data that this class needs, and the factories that generate that data.
T & get(const std::string &name, T def_value)
ParameterList & set(std::string const &name, T const &value, std::string const &docString="", RCP< const ParameterEntryValidator > const &validator=null)
Timer to be used in factories. Similar to Monitor but with additional timers.
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
void sort(View &view, const size_t &size)
bool isParameter(const std::string &name) const
Print statistics that do not involve significant additional computation.
TEUCHOS_DEPRECATED RCP< T > rcp(T *p, Dealloc_T dealloc, bool owns_mem)
Class that holds all level-specific information.
Timer to be used in factories. Similar to SubMonitor but adds a timer level by level.
static std::string PrintMatrixInfo(const Matrix &A, const std::string &msgTag, RCP< const Teuchos::ParameterList > params=Teuchos::null)
void Build(Level &fineLevel, Level &coarseLevel) const
Build an object with this factory.
Exception throws to report errors in the internal logical of the program.
#define MUELU_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
An exception safe way to call the method 'Level::SetFactoryManager()'.
void AddFactoryManager(RCP< const FactoryManagerBase > FactManager)
Add a factory manager.
void DeclareInput(const std::string &ename, const FactoryBase *factory, const FactoryBase *requestedBy=NoFactory::get())
Callback from FactoryBase::CallDeclareInput() and FactoryBase::DeclareInput()