Zoltan2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Zoltan2_AlgParMA.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // Zoltan2: A package of combinatorial algorithms for scientific computing
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact Karen Devine (kddevin@sandia.gov)
39 // Erik Boman (egboman@sandia.gov)
40 // Siva Rajamanickam (srajama@sandia.gov)
41 //
42 // ***********************************************************************
43 //
44 // @HEADER
45 #ifndef _ZOLTAN2_ALGPARMA_HPP_
46 #define _ZOLTAN2_ALGPARMA_HPP_
47 
48 #include <Zoltan2_Algorithm.hpp>
50 #include <Zoltan2_Util.hpp>
51 #include <Zoltan2_TPLTraits.hpp>
52 
56 //
57 // This design creates an apf mesh to run the ParMA algorithms on. The
58 // final solution is determined by changes from beginning to end of the mesh.
59 // This approach allows development closer to that of PUMI setup but at the
60 // cost of creating an extra mesh representation.
61 //
62 // Available ParMA algorithms are given by setting the parma_method parameter
63 // of the sublist parma_paramaters to one of the following:
64 // Vertex - Balances targeting vertex imbalance
65 // Element - Balances targeting element imbalance
66 // VtxElm - Balances targeting vertex and element imbalance
67 // VtxEdgeElm - Balances targeting vertex, edge, and element imbalance
68 // Ghost - Balances using ghost element aware diffusion
69 // Shape - Optimizes shape of parts by increasing the size of small part boundaries
70 // Centroid - Balances using centroid diffusion
72 
73 #ifndef HAVE_ZOLTAN2_PARMA
74 
75 // Error handling for when ParMA is requested
76 // but Zoltan2 not built with ParMA.
77 
78 namespace Zoltan2 {
79 template <typename Adapter>
80 class AlgParMA : public Algorithm<Adapter>
81 {
82 public:
83  typedef typename Adapter::user_t user_t;
84 
85  AlgParMA(const RCP<const Environment> &/* env */,
86  const RCP<const Comm<int> > &/* problemComm */,
87  const RCP<const BaseAdapter<user_t> > &/* adapter */)
88  {
89  throw std::runtime_error(
90  "BUILD ERROR: ParMA requested but not compiled into Zoltan2.\n"
91  "Please set CMake flag Zoltan2_ENABLE_ParMA:BOOL=ON.");
92  }
93 };
94 }
95 
96 #endif
97 
98 #ifdef HAVE_ZOLTAN2_PARMA
99 
100 
101 #include <apf.h>
102 #include <gmi_null.h>
103 #include <apfMDS.h>
104 #include <apfMesh2.h>
105 #include <apfNumbering.h>
106 #include <PCU.h>
107 #include <parma.h>
108 #include <apfConvert.h>
109 #include <apfShape.h>
110 #include <map>
111 #include <cassert>
112 
113 namespace Zoltan2 {
114 
115 template <typename Adapter>
116 class AlgParMA : public Algorithm<Adapter>
117 {
118 
119 private:
120 
121  typedef typename Adapter::lno_t lno_t;
122  typedef typename Adapter::gno_t gno_t;
123  typedef typename Adapter::scalar_t scalar_t;
124  typedef typename Adapter::offset_t offset_t;
125  typedef typename Adapter::part_t part_t;
126  typedef typename Adapter::user_t user_t;
127  typedef typename Adapter::userCoord_t userCoord_t;
128 
129  const RCP<const Environment> env;
130  const RCP<const Comm<int> > problemComm;
131  const RCP<const MeshAdapter<user_t> > adapter;
132 
133  apf::Mesh2* m;
134  apf::Numbering* gids;
135  apf::Numbering* origin_part_ids;
136  std::map<gno_t, lno_t> mapping_elm_gids_index;
137 
138  MPI_Comm mpicomm;
139  bool pcu_outside;
140 
141  void setMPIComm(const RCP<const Comm<int> > &problemComm__) {
142 # ifdef HAVE_ZOLTAN2_MPI
143  mpicomm = Teuchos::getRawMpiComm(*problemComm__);
144 # else
145  mpicomm = MPI_COMM_WORLD; // taken from siMPI
146 # endif
147  }
148  // provides conversion from an APF entity dimension to a Zoltan2 entity type
149  enum MeshEntityType entityAPFtoZ2(int dimension) const {return static_cast<MeshEntityType>(dimension);}
150 
151  //provides a conversion from the Zoltan2 topology type to and APF type
152  // throws an error on topology types not supported by APF
153  enum apf::Mesh::Type topologyZ2toAPF(enum EntityTopologyType ttype) const {
154  if (ttype==POINT)
155  return apf::Mesh::VERTEX;
156  else if (ttype==LINE_SEGMENT)
157  return apf::Mesh::EDGE;
158  else if (ttype==TRIANGLE)
159  return apf::Mesh::TRIANGLE;
160  else if (ttype==QUADRILATERAL)
161  return apf::Mesh::QUAD;
162  else if (ttype==TETRAHEDRON)
163  return apf::Mesh::TET;
164  else if (ttype==HEXAHEDRON)
165  return apf::Mesh::HEX;
166  else if (ttype==PRISM)
167  return apf::Mesh::PRISM;
168  else if (ttype==PYRAMID)
169  return apf::Mesh::PYRAMID;
170  else
171  throw std::runtime_error("APF does not support this topology type");
172 
173  }
174 
175  //Sets the weights of each entity in dimension 'dim' to those provided by the mesh adapter
176  //sets all weights in the mesh adapter but currently only one is considered by ParMA
177  void setEntWeights(int dim, apf::MeshTag* tag) {
178  MeshEntityType etype = entityAPFtoZ2(dim);
179  for (int i=0;i<m->getTagSize(tag);i++) {
180  apf::MeshIterator* itr = m->begin(dim);
181  apf::MeshEntity* ent;
182  const scalar_t* ws=NULL;
183  int stride;
184  if (i<adapter->getNumWeightsPerOf(etype))
185  adapter->getWeightsViewOf(etype,ws,stride,i);
186  int j=0;
187  while ((ent= m->iterate(itr))) {
188  double w = 1.0;
189  if (ws!=NULL)
190  w = static_cast<double>(ws[j]);
191  m->setDoubleTag(ent,tag,&w);
192  j++;
193  }
194  m->end(itr);
195  }
196  }
197 
198  //Helper function to set the weights of each dimension needed by the specific parma algorithm
199  apf::MeshTag* setWeights(bool vtx, bool edge, bool face, bool elm) {
200  int num_ws=1;
201  if (vtx)
202  num_ws = std::max(num_ws,adapter->getNumWeightsPerOf(MESH_VERTEX));
203  if (edge)
204  num_ws = std::max(num_ws,adapter->getNumWeightsPerOf(MESH_EDGE));
205  if (face)
206  num_ws = std::max(num_ws,adapter->getNumWeightsPerOf(MESH_FACE));
207  if (elm)
208  num_ws = std::max(num_ws,adapter->getNumWeightsPerOf(entityAPFtoZ2(m->getDimension())));
209  apf::MeshTag* tag = m->createDoubleTag("parma_weight",num_ws);
210  if (vtx)
211  setEntWeights(0,tag);
212  if (edge)
213  setEntWeights(1,tag);
214  if (face)
215  setEntWeights(2,tag);
216  if (elm) {
217  setEntWeights(m->getDimension(),tag);
218  }
219  return tag;
220  }
221 
222 
223  //APF Mesh construction helper functions modified and placed here to support arbitrary entity types
224  void constructElements(const gno_t* conn, lno_t nelem, const offset_t* offsets,
225  const EntityTopologyType* tops, apf::GlobalToVert& globalToVert)
226  {
227  apf::ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
228  for (lno_t i = 0; i < nelem; ++i) {
229  apf::Mesh::Type etype = topologyZ2toAPF(tops[i]);
230  apf::Downward verts;
231  for (offset_t j = offsets[i]; j < offsets[i+1]; ++j)
232  verts[j-offsets[i]] = globalToVert[conn[j]];
233  buildElement(m, interior, etype, verts);
234  }
235  }
236  int getMax(const apf::GlobalToVert& globalToVert)
237  {
238  int max = -1;
239  APF_CONST_ITERATE(apf::GlobalToVert, globalToVert, it)
240  max = std::max(max, it->first);
241  PCU_Max_Ints(&max, 1); // this is type-dependent
242  return max;
243  }
244  void constructResidence(apf::GlobalToVert& globalToVert)
245  {
246  int max = getMax(globalToVert);
247  int total = max + 1;
248  int peers = PCU_Comm_Peers();
249  int quotient = total / peers;
250  int remainder = total % peers;
251  int mySize = quotient;
252  int self = PCU_Comm_Self();
253  if (self == (peers - 1))
254  mySize += remainder;
255  typedef std::vector< std::vector<int> > TmpParts;
256  TmpParts tmpParts(mySize);
257  /* if we have a vertex, send its global id to the
258  broker for that global id */
259  PCU_Comm_Begin();
260  APF_ITERATE(apf::GlobalToVert, globalToVert, it) {
261  int gid = it->first;
262  int to = std::min(peers - 1, gid / quotient);
263  PCU_COMM_PACK(to, gid);
264  }
265  PCU_Comm_Send();
266  int myOffset = self * quotient;
267  /* brokers store all the part ids that sent messages
268  for each global id */
269  while (PCU_Comm_Receive()) {
270  int gid;
271  PCU_COMM_UNPACK(gid);
272  int from = PCU_Comm_Sender();
273  tmpParts.at(gid - myOffset).push_back(from);
274  }
275  /* for each global id, send all associated part ids
276  to all associated parts */
277  PCU_Comm_Begin();
278  for (int i = 0; i < mySize; ++i) {
279  std::vector<int>& parts = tmpParts[i];
280  for (size_t j = 0; j < parts.size(); ++j) {
281  int to = parts[j];
282  int gid = i + myOffset;
283  int nparts = parts.size();
284  PCU_COMM_PACK(to, gid);
285  PCU_COMM_PACK(to, nparts);
286  for (size_t k = 0; k < parts.size(); ++k)
287  PCU_COMM_PACK(to, parts[k]);
288  }
289  }
290  PCU_Comm_Send();
291  /* receiving a global id and associated parts,
292  lookup the vertex and classify it on the partition
293  model entity for that set of parts */
294  while (PCU_Comm_Receive()) {
295  int gid;
296  PCU_COMM_UNPACK(gid);
297  int nparts;
298  PCU_COMM_UNPACK(nparts);
299  apf::Parts residence;
300  for (int i = 0; i < nparts; ++i) {
301  int part;
302  PCU_COMM_UNPACK(part);
303  residence.insert(part);
304  }
305  apf::MeshEntity* vert = globalToVert[gid];
306  m->setResidence(vert, residence);
307  }
308  }
309 
310  /* given correct residence from the above algorithm,
311  negotiate remote copies by exchanging (gid,pointer)
312  pairs with parts in the residence of the vertex */
313  void constructRemotes(apf::GlobalToVert& globalToVert)
314  {
315  int self = PCU_Comm_Self();
316  PCU_Comm_Begin();
317  APF_ITERATE(apf::GlobalToVert, globalToVert, it) {
318  int gid = it->first;
319  apf::MeshEntity* vert = it->second;
320  apf::Parts residence;
321  m->getResidence(vert, residence);
322  APF_ITERATE(apf::Parts, residence, rit)
323  if (*rit != self) {
324  PCU_COMM_PACK(*rit, gid);
325  PCU_COMM_PACK(*rit, vert);
326  }
327  }
328  PCU_Comm_Send();
329  while (PCU_Comm_Receive()) {
330  int gid;
331  PCU_COMM_UNPACK(gid);
332  apf::MeshEntity* remote;
333  PCU_COMM_UNPACK(remote);
334  int from = PCU_Comm_Sender();
335  apf::MeshEntity* vert = globalToVert[gid];
336  m->addRemote(vert, from, remote);
337  }
338  }
339 
340 public:
341 
347  AlgParMA(const RCP<const Environment> &env__,
348  const RCP<const Comm<int> > &problemComm__,
349  const RCP<const IdentifierAdapter<user_t> > &adapter__)
350  {
351  throw std::runtime_error("ParMA needs a MeshAdapter but you haven't given it one");
352  }
353 
354  AlgParMA(const RCP<const Environment> &env__,
355  const RCP<const Comm<int> > &problemComm__,
356  const RCP<const VectorAdapter<user_t> > &adapter__)
357  {
358  throw std::runtime_error("ParMA needs a MeshAdapter but you haven't given it one");
359  }
360 
361  AlgParMA(const RCP<const Environment> &env__,
362  const RCP<const Comm<int> > &problemComm__,
363  const RCP<const GraphAdapter<user_t,userCoord_t> > &adapter__)
364  {
365  throw std::runtime_error("ParMA needs a MeshAdapter but you haven't given it one");
366  }
367 
368  AlgParMA(const RCP<const Environment> &env__,
369  const RCP<const Comm<int> > &problemComm__,
370  const RCP<const MatrixAdapter<user_t,userCoord_t> > &adapter__)
371  {
372  throw std::runtime_error("ParMA needs a MeshAdapter but you haven't given it one");
373 
374  }
375 
376  AlgParMA(const RCP<const Environment> &env__,
377  const RCP<const Comm<int> > &problemComm__,
378  const RCP<const MeshAdapter<user_t> > &adapter__) :
379  env(env__), problemComm(problemComm__), adapter(adapter__)
380  {
381  setMPIComm(problemComm__);
382 
383  //Setup PCU communications
384  //If PCU was already initialized outside (EX: for the APFMeshAdapter)
385  // we don't initialize it again.
386  pcu_outside=false;
387  if (!PCU_Comm_Initialized())
388  PCU_Comm_Init();
389  else
390  pcu_outside=true;
391  PCU_Switch_Comm(mpicomm);
392 
393  //Find the mesh dimension based on if there are any regions or faces in the part
394  // an all reduce is needed in case one part is empty (Ex: after hypergraph partitioning)
395  int dim;
396  if (adapter->getLocalNumOf(MESH_REGION)>0)
397  dim=3;
398  else if (adapter->getLocalNumOf(MESH_FACE)>0)
399  dim=2;
400  else
401  dim=0;
402  PCU_Max_Ints(&dim,1);
403  if (dim<2)
404  throw std::runtime_error("ParMA neeeds faces or region information");
405 
406  //GFD Currently not allowing ParMA to balance non element primary types
407  if (dim!=adapter->getPrimaryEntityType())
408  throw std::runtime_error("ParMA only supports balancing primary type==mesh element");
409 
410  //Create empty apf mesh
411  gmi_register_null();
412  gmi_model* g = gmi_load(".null");
413  enum MeshEntityType primary_type = entityAPFtoZ2(dim);
414  m = apf::makeEmptyMdsMesh(g,dim,false);
415 
416  //Get entity topology types
417  const EntityTopologyType* tops;
418  try {
419  adapter->getTopologyViewOf(primary_type,tops);
420  }
422 
423  //Get element global ids and part ids
424  const gno_t* element_gids;
425  const part_t* part_ids;
426  adapter->getIDsViewOf(primary_type,element_gids);
427  adapter->getPartsView(part_ids);
428  for (size_t i =0;i<adapter->getLocalNumOf(primary_type);i++)
429  mapping_elm_gids_index[element_gids[i]] = i;
430 
431  //get vertex global ids
432  const gno_t* vertex_gids;
433  adapter->getIDsViewOf(MESH_VERTEX,vertex_gids);
434 
435  //Get vertex coordinates
436  int c_dim = adapter->getDimension();
437  const scalar_t ** vertex_coords = new const scalar_t*[c_dim];
438  int* strides = new int[c_dim];
439  for (int i=0;i<c_dim;i++)
440  adapter->getCoordinatesViewOf(MESH_VERTEX,vertex_coords[i],strides[i],i);
441 
442  //Get first adjacencies from elements to vertices
443  if (!adapter->availAdjs(primary_type,MESH_VERTEX))
444  throw "APF needs adjacency information from elements to vertices";
445  const offset_t* offsets;
446  const gno_t* adjacent_vertex_gids;
447  adapter->getAdjsView(primary_type, MESH_VERTEX,offsets,adjacent_vertex_gids);
448 
449  //build the apf mesh
450  apf::GlobalToVert vertex_mapping;
451  apf::ModelEntity* interior = m->findModelEntity(m->getDimension(), 0);
452  for (size_t i=0;i<adapter->getLocalNumOf(MESH_VERTEX);i++) {
453  apf::MeshEntity* vtx = m->createVert_(interior);
454  scalar_t temp_coords[3];
455  for (int k=0;k<c_dim&&k<3;k++)
456  temp_coords[k] = vertex_coords[k][i*strides[k]];
457 
458  for (int k=c_dim;k<3;k++)
459  temp_coords[k] = 0;
460  apf::Vector3 point(temp_coords[0],temp_coords[1],temp_coords[2]);
461  m->setPoint(vtx,0,point);
462  vertex_mapping[vertex_gids[i]] = vtx;
463  }
464  //Call modified helper functions to build the mesh from element to vertex adjacency
465  constructElements(adjacent_vertex_gids, adapter->getLocalNumOf(primary_type), offsets, tops, vertex_mapping);
466  constructResidence(vertex_mapping);
467  constructRemotes(vertex_mapping);
468  stitchMesh(m);
469  m->acceptChanges();
470 
471 
472  //Setup numberings of global ids and original part ids
473  // for use after ParMA is run
474  apf::FieldShape* s = apf::getConstant(dim);
475  gids = apf::createNumbering(m,"global_ids",s,1);
476  origin_part_ids = apf::createNumbering(m,"origin",s,1);
477 
478  //number the global ids and original part ids
479  apf::MeshIterator* itr = m->begin(dim);
480  apf::MeshEntity* ent;
481  int i = 0;
482  while ((ent = m->iterate(itr))) {
483  apf::number(gids,ent,0,0,element_gids[i]);
484  apf::number(origin_part_ids,ent,0,0,PCU_Comm_Self());
485  i++;
486  }
487  m->end(itr);
488 
489  //final setup for apf mesh
490  apf::alignMdsRemotes(m);
491  apf::deriveMdsModel(m);
492  m->acceptChanges();
493  m->verify();
494 
495  //cleanup temp storage
496  delete [] vertex_coords;
497  delete [] strides;
498  }
499  void partition(const RCP<PartitioningSolution<Adapter> > &solution);
500 
501 };
502 
504 template <typename Adapter>
506  const RCP<PartitioningSolution<Adapter> > &solution
507 )
508 {
509  //Get parameters
510  std::string alg_name = "VtxElm";
511  double imbalance = 1.1;
512  double step = .5;
513  int ghost_layers=3;
514  int ghost_bridge=m->getDimension()-1;
515 
516  //Get the parameters for ParMA
517  const Teuchos::ParameterList &pl = env->getParameters();
518  try {
519  const Teuchos::ParameterList &ppl = pl.sublist("parma_parameters");
520  for (ParameterList::ConstIterator iter = ppl.begin();
521  iter != ppl.end(); iter++) {
522  const std::string &zname = pl.name(iter);
523  if (zname == "parma_method") {
524  std::string zval = pl.entry(iter).getValue(&zval);
525  alg_name = zval;
526  }
527  else if (zname == "step_size") {
528  double zval = pl.entry(iter).getValue(&zval);
529  step = zval;
530  }
531  else if (zname=="ghost_layers" || zname=="ghost_bridge") {
532  int zval = pl.entry(iter).getValue(&zval);
533  if (zname=="ghost_layers")
534  ghost_layers = zval;
535  else
536  ghost_bridge = zval;
537  }
538  }
539  }
540  catch (std::exception &e) {
541  //No parma_parameters sublist found
542  }
543 
544  const Teuchos::ParameterEntry *pe2 = pl.getEntryPtr("imbalance_tolerance");
545  if (pe2){
546  imbalance = pe2->getValue<double>(&imbalance);
547  }
548 
549  //booleans for which dimensions need weights
550  bool weightVertex,weightEdge,weightFace,weightElement;
551  weightVertex=weightEdge=weightFace=weightElement=false;
552 
553  //Build the selected balancer
554  apf::Balancer* balancer;
555  const int verbose = 1;
556  if (alg_name=="Vertex") {
557  balancer = Parma_MakeVtxBalancer(m, step, verbose);
558  weightVertex = true;
559  }
560  else if (alg_name=="Element") {
561  balancer = Parma_MakeElmBalancer(m, step, verbose);
562  weightElement=true;
563  }
564  else if (alg_name=="VtxElm") {
565  balancer = Parma_MakeVtxElmBalancer(m,step,verbose);
566  weightVertex = weightElement=true;
567  }
568  else if (alg_name=="VtxEdgeElm") {
569  balancer = Parma_MakeVtxEdgeElmBalancer(m, step, verbose);
570  weightVertex=weightEdge=weightElement=true;
571  }
572  else if (alg_name=="Ghost") {
573  balancer = Parma_MakeGhostDiffuser(m, ghost_layers, ghost_bridge, step, verbose);
574  weightVertex=weightEdge=weightFace=true;
575  if (3 == m->getDimension()) {
576  weightElement=true;
577  }
578  }
579  else if (alg_name=="Shape") {
580  balancer = Parma_MakeShapeOptimizer(m,step,verbose);
581  weightElement=true;
582  }
583  else if (alg_name=="Centroid") {
584  balancer = Parma_MakeCentroidDiffuser(m,step,verbose);
585  weightElement=true;
586  }
587  else {
588  throw std::runtime_error("No such parma method defined");
589  }
590 
591  //build the weights
592  apf::MeshTag* weights = setWeights(weightVertex,weightEdge,weightFace,weightElement);
593 
594  //balance the apf mesh
595  balancer->balance(weights, imbalance);
596  delete balancer;
597 
598  // Load answer into the solution.
599  int num_local = adapter->getLocalNumOf(adapter->getPrimaryEntityType());
600  ArrayRCP<part_t> partList(new part_t[num_local], 0, num_local, true);
601 
602  //Setup for communication
603  PCU_Comm_Begin();
604  apf::MeshEntity* ent;
605  apf::MeshIterator* itr = m->begin(m->getDimension());
606  //Pack information back to each elements original owner
607  while ((ent=m->iterate(itr))) {
608  if (m->isOwned(ent)) {
609  part_t target_part_id = apf::getNumber(origin_part_ids,ent,0,0);
610  gno_t element_gid = apf::getNumber(gids,ent,0,0);
611  PCU_COMM_PACK(target_part_id,element_gid);
612  }
613  }
614  m->end(itr);
615 
616  //Send information off
617  PCU_Comm_Send();
618 
619  //Unpack information and set new part ids
620  while (PCU_Comm_Receive()) {
621  gno_t global_id;
622  PCU_COMM_UNPACK(global_id);
623  lno_t local_id = mapping_elm_gids_index[global_id];
624  part_t new_part_id = PCU_Comm_Sender();
625  partList[local_id] = new_part_id;
626  }
627  //construct partition solution
628  solution->setParts(partList);
629 
630  // Clean up
631  apf::destroyNumbering(gids);
632  apf::destroyNumbering(origin_part_ids);
633  apf::removeTagFromDimension(m, weights, m->getDimension());
634  m->destroyTag(weights);
635  m->destroyNative();
636  apf::destroyMesh(m);
637  //only free PCU if it isn't being used outside
638  if (!pcu_outside)
639  PCU_Comm_Free();
640 }
641 
642 } // namespace Zoltan2
643 
644 #endif // HAVE_ZOLTAN2_PARMA
645 
646 #endif
virtual void partition(const RCP< PartitioningSolution< Adapter > > &)
Partitioning method.
AlgParMA(const RCP< const Environment > &, const RCP< const Comm< int > > &, const RCP< const BaseAdapter< user_t > > &)
#define Z2_FORWARD_EXCEPTIONS
Forward an exception back through call stack.
static ArrayRCP< ArrayRCP< zscalar_t > > weights
Defines the PartitioningSolution class.
Adapter::user_t user_t
SparseMatrixAdapter_t::part_t part_t
Adapter::scalar_t scalar_t
Adapter::part_t part_t
Algorithm defines the base class for all algorithms.
EntityTopologyType
Enumerate entity topology types for meshes: points,lines,polygons,triangles,quadrilaterals, polyhedrons, tetrahedrons, hexhedrons, prisms, or pyramids.
Traits class to handle conversions between gno_t/lno_t and TPL data types (e.g., ParMETIS&#39;s idx_t...
MeshEntityType
Enumerate entity types for meshes: Regions, Faces, Edges, or Vertices.
A gathering of useful namespace methods.
Zoltan2::BasicUserTypes< zscalar_t, zlno_t, zgno_t > user_t
Definition: Metric.cpp:74