Tpetra parallel linear algebra  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Tpetra_FECrsGraph_def.hpp
1 // @HEADER
2 // ***********************************************************************
3 //
4 // Tpetra: Templated Linear Algebra Services Package
5 // Copyright (2008) Sandia Corporation
6 //
7 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
8 // the U.S. Government retains certain rights in this software.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ************************************************************************
40 // @HEADER
41 
42 #ifndef TPETRA_FECRSGRAPH_DEF_HPP
43 #define TPETRA_FECRSGRAPH_DEF_HPP
44 
45 #include <type_traits>
46 #include "Tpetra_CrsGraph.hpp"
49 
50 
51 namespace Tpetra {
52 
53 template<class LocalOrdinal, class GlobalOrdinal, class Node>
55 FECrsGraph(const Teuchos::RCP<const map_type> & ownedRowMap,
56  const Teuchos::RCP<const map_type> & ownedPlusSharedRowMap,
57  const size_t maxNumEntriesPerRow,
58  const Teuchos::RCP<const import_type> & ownedPlusSharedToOwnedimporter,
59  const Teuchos::RCP<const map_type> & domainMap,
60  const Teuchos::RCP<const map_type> & rangeMap,
61  const Teuchos::RCP<Teuchos::ParameterList>& params):
62  crs_graph_type(ownedPlusSharedRowMap, maxNumEntriesPerRow, StaticProfile, params),
63  importer_(ownedPlusSharedToOwnedimporter),
64  domainMap_(domainMap.is_null() ? ownedRowMap : domainMap),
65  rangeMap_(rangeMap.is_null() ? ownedRowMap : rangeMap)
66 {
67  Teuchos::RCP<const map_type> dummy;
68  setup(ownedRowMap,ownedPlusSharedRowMap,dummy,params);
69 }
70 
71 
72 template<class LocalOrdinal, class GlobalOrdinal, class Node>
74 FECrsGraph (const Teuchos::RCP<const map_type> & ownedRowMap,
75  const Teuchos::RCP<const map_type> & ownedPlusSharedRowMap,
76  const Kokkos::DualView<const size_t*, execution_space>& numEntPerRow,
77  const Teuchos::RCP<const import_type> & ownedPlusSharedToOwnedimporter,
78  const Teuchos::RCP<const map_type> & domainMap,
79  const Teuchos::RCP<const map_type> & rangeMap,
80  const Teuchos::RCP<Teuchos::ParameterList>& params):
81  crs_graph_type( ownedPlusSharedRowMap, numEntPerRow, StaticProfile, params),
82  importer_(ownedPlusSharedToOwnedimporter),
83  domainMap_(domainMap.is_null() ? ownedRowMap : domainMap),
84  rangeMap_(rangeMap.is_null() ? ownedRowMap : rangeMap)
85 
86 {
87  Teuchos::RCP<const map_type> dummy;
88  setup(ownedRowMap,ownedPlusSharedRowMap,dummy,params);
89 }
90 
91 
92 template<class LocalOrdinal, class GlobalOrdinal, class Node>
94 FECrsGraph(const Teuchos::RCP<const map_type> & ownedRowMap,
95  const Teuchos::RCP<const map_type> & ownedPlusSharedRowMap,
96  const Teuchos::RCP<const map_type> & ownedPlusSharedColMap,
97  const size_t maxNumEntriesPerRow,
98  const Teuchos::RCP<const import_type> & ownedPlusSharedToOwnedimporter,
99  const Teuchos::RCP<const map_type> & domainMap,
100  const Teuchos::RCP<const map_type> & rangeMap,
101  const Teuchos::RCP<Teuchos::ParameterList>& params):
102  crs_graph_type(ownedPlusSharedRowMap, ownedPlusSharedColMap,maxNumEntriesPerRow, StaticProfile, params),
103  importer_(ownedPlusSharedToOwnedimporter),
104  domainMap_(domainMap.is_null() ? ownedRowMap : domainMap),
105  rangeMap_(rangeMap.is_null() ? ownedRowMap : rangeMap)
106 {
107  setup(ownedRowMap,ownedPlusSharedRowMap, ownedPlusSharedColMap,params);
108 }
109 
110 template<class LocalOrdinal, class GlobalOrdinal, class Node>
112 FECrsGraph (const Teuchos::RCP<const map_type> & ownedRowMap,
113  const Teuchos::RCP<const map_type> & ownedPlusSharedRowMap,
114  const Teuchos::RCP<const map_type> & ownedPlusSharedColMap,
115  const Kokkos::DualView<const size_t*, execution_space>& numEntPerRow,
116  const Teuchos::RCP<const import_type> & ownedPlusSharedToOwnedimporter,
117  const Teuchos::RCP<const map_type> & domainMap,
118  const Teuchos::RCP<const map_type> & rangeMap,
119  const Teuchos::RCP<Teuchos::ParameterList>& params):
120  crs_graph_type(ownedPlusSharedRowMap, ownedPlusSharedColMap, numEntPerRow, StaticProfile, params),
121  importer_(ownedPlusSharedToOwnedimporter),
122  domainMap_(domainMap.is_null() ? ownedRowMap : domainMap),
123  rangeMap_(rangeMap.is_null() ? ownedRowMap : rangeMap)
124 
125 {
126  setup(ownedRowMap,ownedPlusSharedRowMap, ownedPlusSharedColMap,params);
127 }
128 
129 
130 template<class LocalOrdinal, class GlobalOrdinal, class Node>
131 void FECrsGraph<LocalOrdinal, GlobalOrdinal, Node>::setup(const Teuchos::RCP<const map_type> & ownedRowMap, const Teuchos::RCP<const map_type> & ownedPlusSharedRowMap, const Teuchos::RCP<const map_type> & ownedPlusSharedColMap,const Teuchos::RCP<Teuchos::ParameterList>& params) {
132  const char tfecfFuncName[] = "FECrsGraph::setup(): ";
133 
134  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(ownedRowMap.is_null (), std::runtime_error, "ownedRowMap is null.");
135  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(ownedPlusSharedRowMap.is_null (), std::runtime_error, "ownedPlusSharedRowMap is null.");
136 
137  // If we have a colMap, we're local, otherwise global
138  if(ownedPlusSharedColMap.is_null()) this->allocateIndices(GlobalIndices);
139  else this->allocateIndices(LocalIndices);
140 
141  activeCrsGraph_ = Teuchos::rcp(new FEWhichActive(FE_ACTIVE_OWNED_PLUS_SHARED));
142 
143  // Use a very strong map equivalence check
144  bool maps_are_the_same = ownedRowMap->isSameAs(*ownedPlusSharedRowMap);
145  if(!maps_are_the_same) {
146  // Make an importer if we need to, check map compatability if we don't
147  if(importer_.is_null()) {
148  importer_ = Teuchos::rcp(new import_type(ownedRowMap,ownedPlusSharedRowMap));
149  }
150  else {
151  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!ownedRowMap->isSameAs(*importer_->getSourceMap()), std::runtime_error, "ownedRowMap does not match importer source map.");
152  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(!ownedPlusSharedRowMap->isSameAs(*importer_->getTargetMap()), std::runtime_error, "ownedPlusSharedRowMap does not match importer target map.");
153  }
154 
155  // Make sure the ownedPlusSharedRowMap has at least as many entries at the ownedRowMap (due to our superset requriement)
156  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( importer_->getNumSameIDs() != importer_->getSourceMap()->getNodeNumElements(),
157  std::runtime_error,"ownedRowMap contains entries which are not in the ownedPlusSharedRowMap.");
158 
159  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( ownedRowMap->getNodeNumElements() > ownedPlusSharedRowMap->getNodeNumElements(),
160  std::runtime_error,"ownedRowMap more entries than the ownedPlusSharedRowMap.");
161 
162  // The locallyFitted check is debug mode only since it is more expensive
163  const bool debug = ::Tpetra::Details::Behavior::debug ();
164  if(debug) {
165  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC( !ownedPlusSharedRowMap->isLocallyFitted(*ownedRowMap),
166  std::runtime_error,"ownedPlusSharedRowMap must be locally fitted to the ownedRowMap");
167 
168  }
169 
170  // For FECrsGraph, we do all the aliasing AFTER import. All we need here is a constructor
171  if(ownedPlusSharedColMap.is_null())
172  inactiveCrsGraph_ = Teuchos::rcp(new crs_graph_type(ownedRowMap,0,StaticProfile,params));
173  else
174  inactiveCrsGraph_ = Teuchos::rcp(new crs_graph_type(ownedRowMap,ownedPlusSharedColMap,0,StaticProfile,params));
175  }
176 
177 }
178 
179 template<class LocalOrdinal, class GlobalOrdinal, class Node>
180 void FECrsGraph<LocalOrdinal, GlobalOrdinal, Node>::doOwnedPlusSharedToOwned(const CombineMode CM) {
181  const char tfecfFuncName[] = "FECrsGraph::doOwnedPlusSharedToOwned(CombineMode): ";
182  if(!inactiveCrsGraph_.is_null() && *activeCrsGraph_ == FE_ACTIVE_OWNED_PLUS_SHARED) {
183  Teuchos::RCP<const map_type> ownedRowMap = inactiveCrsGraph_->getRowMap();
184 
185  // Do a self-export in "restricted mode"
186  this->doExport(*this,*importer_,CM,true);
187 
188  // Under the "if you own an element, you own at least one of its nodes" assumption,
189  // we can start by making a columnmap for ownedPlusShared
190  if(!this->hasColMap()) {
191  Teuchos::Array<int> remotePIDs (0);
192  this->makeColMap(remotePIDs);
193  }
194 
195  // Now run CrsGraph's fillComplete to get the final importer
196  crs_graph_type::fillComplete(domainMap_,this->getRowMap());
197 
198  // In debug mode, we check to make sure the "if you own an element, you own at least one of its nodes"
199  const bool debug = ::Tpetra::Details::Behavior::debug ();
200  if (debug) {
201  Teuchos::RCP<const map_type> colmap = this->getColMap();
202  Teuchos::Array<bool> flag(colmap->getNodeNumElements(),false);
203  Teuchos::Array<LocalOrdinal> indices(this->getNodeMaxNumRowEntries());
204  for(size_t i=0; i<ownedRowMap->getNodeNumElements(); i++) {
205  size_t NumEntries=0;
206  this->getLocalRowCopy(i,indices,NumEntries);
207  for(size_t j=0; j<NumEntries; j++)
208  flag[indices[j]] = true;
209  }
210 
211  bool lclSuccess = true;
212  for(size_t i=0; i<(size_t)flag.size(); i++)
213  if(!flag[i]) {lclSuccess=false; break;}
214  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC
215  (lclSuccess ==false,
216  std::invalid_argument, "if you own an element (in the finite element sense) you "
217  "must also own one of the attached nodes. This assumption has been violated in "
218  "your matrix fill.");
219  }
220 
221  // Time to build an owned localGraph via subviews
222  local_graph_type ownedPlusSharedGraph = this->getLocalGraph();
223  size_t numOwnedRows = ownedRowMap->getNodeNumElements();
224  size_t numOwnedNonZeros = Tpetra::Details::getEntryOnHost(ownedPlusSharedGraph.row_map,numOwnedRows);
225 
226  // Build the inactive guy
227  // NOTE: We can't use the local_graph_type constructor, because it does not allow us to provide an importer
228  if(!inactiveCrsGraph_->hasColMap()) inactiveCrsGraph_->replaceColMap(this->getColMap());
229  inactiveCrsGraph_->setAllIndices(Kokkos::subview(ownedPlusSharedGraph.row_map,Kokkos::pair<size_t,size_t>(0,numOwnedRows+1)),
230  Kokkos::subview(ownedPlusSharedGraph.entries,Kokkos::pair<size_t,size_t>(0,numOwnedNonZeros)));
231  // This will generate an exporter if we need one.
232  inactiveCrsGraph_->expertStaticFillComplete(domainMap_,rangeMap_,this->getImporter(),Teuchos::null);
233  }
234 }//end doOverlapToLocal
235 
236 
237 template<class LocalOrdinal, class GlobalOrdinal, class Node>
238 void FECrsGraph<LocalOrdinal, GlobalOrdinal, Node>::doOwnedToOwnedPlusShared(const CombineMode /* CM */) {
239  // This should be a no-op for all of our purposes
240 }//end doLocalToOverlap
241 
242 template<class LocalOrdinal, class GlobalOrdinal, class Node>
244  if(*activeCrsGraph_ == FE_ACTIVE_OWNED_PLUS_SHARED)
245  *activeCrsGraph_ = FE_ACTIVE_OWNED;
246  else
247  *activeCrsGraph_ = FE_ACTIVE_OWNED_PLUS_SHARED;
248 
249  if(inactiveCrsGraph_.is_null()) return;
250 
251  this->swap(*inactiveCrsGraph_);
252 
253 }//end switchActiveCrsGraph
254 
255 
256 
257 template<class LocalOrdinal, class GlobalOrdinal, class Node>
259  const char tfecfFuncName[] = "FECrsGraph::endFill(): ";
260 
261  /* What has to go on here is complicated.
262  First off, if we don't really have two graphs (e.g. the rowMaps are the same, because we're in serial or
263  doing finite differences, things are easy --- just call fillComplete().
264 
265  If, we are in the parallel FE case, then:
266  Precondition: FE_ACTIVE_OWNED_PLUS_SHARED mode
267 
268  Postconditions:
269  1) FE_ACTIVE_OWNED mode
270  2) The OWNED graph has been fillCompleted with an Aztec-compatible column map
271  3) rowptr & (local) colinds are aliased between the two graphs
272  4) The OWNED_PLUS_SHARED graph has been fillCompleted with a column map whose first chunk
273  is the column map for the OWNED graph.
274  If we assume that (a) if you own an element, you also own at least one of the connected nodes and (b) elements are cliques, then
275  the columnMap is the same for both graphs!!!
276 
277  5) The OWNED_PLUS_SHARED graph has neither an importer nor exporter. Making these is expensive and we don't need them.
278  */
279  // Precondition
280  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(*activeCrsGraph_ != FE_ACTIVE_OWNED_PLUS_SHARED,std::runtime_error, "must be in owned+shared mode.");
281 
282  if(inactiveCrsGraph_.is_null()) {
283  // The easy case: One graph
284  switchActiveCrsGraph();
285  crs_graph_type::fillComplete(domainMap_,rangeMap_);
286  }
287  else {
288  // The hard case: Two graphs
289 
290  // fillComplete the owned+shared graph in a way that generates the owned+shared grep w/o an importer or exporter
291  // Migrate data to the owned graph
292  doOwnedPlusSharedToOwned(Tpetra::ADD);
293 
294  // Load up the owned graph
295  switchActiveCrsGraph();
296 
297  }
298 }
299 
300 
301 template<class LocalOrdinal, class GlobalOrdinal, class Node>
303  const char tfecfFuncName[] = "FECrsGraph::beginFill(): ";
304 
305  // Unlike FECrsMatrix and FEMultiVector, we do not allow you to call beginFill() after calling endFill()
306  // So we throw an exception if you're in owned mode
307  TEUCHOS_TEST_FOR_EXCEPTION_CLASS_FUNC(*activeCrsGraph_ == FE_ACTIVE_OWNED,std::runtime_error, "can only be called once.");
308 
309 }
310 
311 
312 } // end namespace Tpetra
313 
314 
315 //
316 // Explicit instantiation macro
317 //
318 // Must be expanded from within the Tpetra namespace!
319 //
320 #define TPETRA_FECRSGRAPH_GRAPH_INSTANT(LO,GO,NODE) \
321  template class FECrsGraph<LO, GO, NODE>;
322 
323 
324 
325 #endif // TPETRA_FECRSGRAPH_DEF
A distributed graph accessed by rows (adjacency lists) and stored sparsely.
void endFill()
Migrates data to the owned mode.
static bool debug()
Whether Tpetra is in debug mode.
void beginFill()
Activates the owned+shared mode for assembly. This can only be called once.
CombineMode
Rule for combining data in an Import or Export.
Sum new values into existing values.
void switchActiveCrsGraph()
Switches which CrsGraph is active (without migrating data)
A distributed graph accessed by rows (adjacency lists) and stored sparsely.
int makeColMap(Teuchos::RCP< const Tpetra::Map< LO, GO, NT > > &colMap, Teuchos::Array< int > &remotePIDs, const Teuchos::RCP< const Tpetra::Map< LO, GO, NT > > &domMap, const RowGraph< LO, GO, NT > &graph, const bool sortEachProcsGids=true, std::ostream *errStrm=NULL)
Make the graph&#39;s column Map.
Declaration and definition of Tpetra::Details::getEntryOnHost.
FECrsGraph(const Teuchos::RCP< const map_type > &ownedRowMap, const Teuchos::RCP< const map_type > &ownedPlusSharedRowMap, const size_t maxNumEntriesPerRow, const Teuchos::RCP< const import_type > &ownedPlusSharedToOwnedimporter=Teuchos::null, const Teuchos::RCP< const map_type > &domainMap=Teuchos::null, const Teuchos::RCP< const map_type > &rangeMap=Teuchos::null, const Teuchos::RCP< Teuchos::ParameterList > &params=Teuchos::null)
Constructorfor globally-indexed assembly specifying a single upper bound for the number of entries in...
Declaration of Tpetra::Details::Behavior, a class that describes Tpetra&#39;s behavior.