Zoltan2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Zoltan2_MachineTopoMgr.hpp
Go to the documentation of this file.
1 #ifndef _ZOLTAN2_MACHINE_TOPOMANAGER_HPP_
2 #define _ZOLTAN2_MACHINE_TOPOMANAGER_HPP_
3 
4 #include <Teuchos_Comm.hpp>
5 #include <Teuchos_CommHelpers.hpp>
6 #include <Zoltan2_Machine.hpp>
7 
8 #ifdef HAVE_ZOLTAN2_TOPOMANAGER
9 #include <TopoManager.h>
10 #endif
11 
12 namespace Zoltan2{
13 
18 template <typename pcoord_t, typename part_t>
19 class MachineTopoMgr : public Machine <pcoord_t, part_t> {
20 
21 public:
26  MachineTopoMgr(const Teuchos::Comm<int> &comm):
27  Machine<pcoord_t,part_t>(comm),
28 #if defined (CMK_BLUEGENEQ)
29  networkDim(6), tmgr(comm.getSize()),
30 #elif defined (CMK_BLUEGENEP)
31  networkDim(4), tmgr(comm.getSize()),
32 #else
33  networkDim(3),
34 #endif
35  procCoords(NULL), machine_extent(NULL),
36  delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL), pl(NULL)
37  {
38  transformed_network_dim = networkDim - 1;
39  transformed_coordinates = procCoords;
40  machine_extent = new int[networkDim];
41  this->getMachineExtent(this->machine_extent);
42  //allocate memory for processor coordinates.
43  procCoords = new pcoord_t *[networkDim];
44  for (int i = 0; i < networkDim; ++i){
45  procCoords[i] = new pcoord_t[this->numRanks];
46  memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
47  }
48 
49  //obtain the coordinate of the processor.
50  pcoord_t *xyz = new pcoord_t[networkDim];
52  for (int i = 0; i < networkDim; i++)
53  procCoords[i][this->myRank] = xyz[i];
54  delete [] xyz;
55 
56  //reduceAll the coordinates of each processor.
57  gatherMachineCoordinates(comm);
58 
59  }
60 
61  MachineTopoMgr(const Teuchos::Comm<int> &comm, const Teuchos::ParameterList &pl_ ):
62  Machine<pcoord_t,part_t>(comm),
63 #if defined (CMK_BLUEGENEQ)
64  networkDim(6), tmgr(comm.getSize()),
65 #elif defined (CMK_BLUEGENEP)
66  networkDim(4), tmgr(comm.getSize()),
67 #else
68  networkDim(3),
69 #endif
70  procCoords(NULL), machine_extent(NULL),
71  delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL),
72  pl(&pl_)
73  {
74  transformed_network_dim = networkDim - 1;
75  transformed_coordinates = procCoords;
76  machine_extent = new int[networkDim];
77  this->getMachineExtent(this->machine_extent);
78  //allocate memory for processor coordinates.
79  procCoords = new pcoord_t *[networkDim];
80  for (int i = 0; i < networkDim; ++i){
81  procCoords[i] = new pcoord_t[this->numRanks];
82  memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
83  }
84 
85  //obtain the coordinate of the processor.
86  pcoord_t *xyz = new pcoord_t[networkDim];
88  for (int i = 0; i < networkDim; i++)
89  procCoords[i][this->myRank] = xyz[i];
90  delete [] xyz;
91 
92  //reduceAll the coordinates of each processor.
93  gatherMachineCoordinates(comm);
94 
95  const Teuchos::ParameterEntry *pe = this->pl->getEntryPtr("Machine_Optimization_Level");
96  if (pe){
97 
98  int optimization_level = 0;
99 
100  optimization_level = pe->getValue<int>(&optimization_level);
101 
102  if (optimization_level == 0){
103  transformed_network_dim = networkDim - 1;
104  transformed_coordinates = procCoords;
105  }
106 
107  else if (optimization_level >= 1){
108  transformed_network_dim = networkDim - 2;
109  transformed_coordinates = procCoords;
110  }
111  }
112 
113  }
114 
115  virtual ~MachineTopoMgr() {
116  for (int i = 0; i < networkDim; i++){
117  delete [] procCoords[i];
118  }
119  delete [] procCoords;
120  delete [] machine_extent;
121 
122  if (delete_transformed_coords){
123  for (int i = 0; i < transformed_network_dim; i++){
124  delete [] transformed_coordinates[i];
125  }
126  delete [] transformed_coordinates;
127  }
128 
129  }
130 
131  bool hasMachineCoordinates() const { return true; }
132 
133  int getMachineDim() const { return transformed_network_dim; }
134 
135  int getRealMachineDim() const { return networkDim; }
136 
137  bool getMachineExtent(int *nxyz) const {
138 #if defined (CMK_BLUEGENEQ)
139  int dim = 0;
140  if (dim < transformed_network_dim)
141  nxyz[dim++] = tmgr.getDimNA();
142  if (dim < transformed_network_dim)
143  nxyz[dim++] = tmgr.getDimNB();
144  if (dim < transformed_network_dim)
145  nxyz[dim++] = tmgr.getDimNC();
146  if (dim < transformed_network_dim)
147  nxyz[dim++] = tmgr.getDimND();
148  if (dim < transformed_network_dim)
149  nxyz[dim++] = tmgr.getDimNE();
150  if (dim < transformed_network_dim)
151  nxyz[dim++] = tmgr.getDimNT();
152  return true;
153 #elif defined (CMK_BLUEGENEP)
154  int dim = 0;
155  if (dim < transformed_network_dim)
156  nxyz[dim++] = tmgr.getDimNX();
157  if (dim < transformed_network_dim)
158  nxyz[dim++] = tmgr.getDimNY();
159  if (dim < transformed_network_dim)
160  nxyz[dim++] = tmgr.getDimNZ();
161  if (dim < transformed_network_dim)
162  nxyz[dim++] = tmgr.getDimNT();
163  return true;
164 #else
165  return false;
166 #endif
167  }
168 
169  //MD TODO: Not always it has wrap-around links.
170  bool getMachineExtentWrapArounds(bool *wrap_around) const {
171 #if defined (CMK_BLUEGENEQ)
172  //leave it as this for now, figure out if there is a way to determine tourus from topomanager.
173  int dim = 0;
174  if (dim < transformed_network_dim)
175  wrap_around[dim++] = true;
176  if (dim < transformed_network_dim)
177  wrap_around[dim++] = true;
178  if (dim < transformed_network_dim)
179  wrap_around[dim++] = true;
180  if (dim < transformed_network_dim)
181  wrap_around[dim++] = true;
182  if (dim < transformed_network_dim)
183  wrap_around[dim++] = true;
184  if (dim < transformed_network_dim)
185  wrap_around[dim++] = true;
186 #elif defined (CMK_BLUEGENEP)
187  int dim = 0;
188  if (dim < transformed_network_dim)
189  wrap_around[dim++] = true;
190  if (dim < transformed_network_dim)
191  wrap_around[dim++] = true;
192  if (dim < transformed_network_dim)
193  wrap_around[dim++] = true;
194  if (dim < transformed_network_dim)
195  wrap_around[dim++] = true;
196 #else
197 #endif
198  return true;
199  }
200 
201  bool getMyMachineCoordinate(pcoord_t *xyz) {
202  for (int i = 0; i < this->transformed_network_dim; ++i){
203  xyz[i] = transformed_coordinates[i][this->myRank];
204  }
205  return true;
206  }
207 
208  bool getMyActualMachineCoordinate(pcoord_t *xyz) {
209 #if defined (CMK_BLUEGENEQ)
210  int a,b,c,d,e,t;
211  tmgr.rankToCoordinates(this->myRank, a,b,c,d,e,t);
212  xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = d; xyz[4] = e; xyz[5] = t;
213  //std::cout << "me:" << this->myRank << " " << a << " " << b << " " << c << " " << d << " " << e << " " << t << std::endl;
214  return true;
215 #elif defined (CMK_BLUEGENEP)
216  int a,b,c,t;
217  tmgr.rankToCoordinates(this->myRank, a,b,c,t);
218  xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = t;
219  return true;
220 #else
221  return false;
222 #endif
223  }
224 
225  bool getMachineExtentWrapArounds(part_t *wrap_around) const {
226 
227  int dim = 0;
228  if (dim < transformed_network_dim)
229  wrap_around[dim++] = true;
230 
231  if (dim < transformed_network_dim)
232  wrap_around[dim++] = true;
233 
234  if (dim < transformed_network_dim)
235  wrap_around[dim++] = true;
236 
237  if (dim < transformed_network_dim)
238  wrap_around[dim++] = true;
239 
240  if (dim < transformed_network_dim)
241  wrap_around[dim++] = true;
242 
243  if (dim < transformed_network_dim)
244  wrap_around[dim++] = true;
245  return true;
246  }
247  inline bool getMachineCoordinate(const int rank,
248  pcoord_t *xyz) const {
249  return false;
250  }
251 
252 
253  bool getMachineCoordinate(const char *nodename, pcoord_t *xyz) {
254  return false; // cannot yet return from nodename
255  }
256 
257  bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const {
258  allCoords = procCoords;
259  return true;
260  }
261 
262  virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops){
263  hops = 0;
264  for (int i = 0; i < networkDim - 1; ++i){
265  pcoord_t distance = procCoords[i][rank1] - procCoords[i][rank2];
266  if (distance < 0 ) distance = -distance;
267  if (machine_extent[i] - distance < distance) distance = machine_extent[i] - distance;
268  hops += distance;
269  }
270  return true;
271  }
272 
273 
274 private:
275 
276  int networkDim;
277 
278 #ifdef HAVE_ZOLTAN2_TOPOMANAGER
279  TopoManager tmgr;
280 #endif
281  pcoord_t **procCoords; // KDD Maybe should be RCP?
282  part_t *machine_extent;
283  const Teuchos::ParameterList *pl;
284 
285 
286  bool delete_transformed_coords;
287  int transformed_network_dim;
288  pcoord_t **transformed_coordinates;
289 
290  void gatherMachineCoordinates(const Teuchos::Comm<int> &comm) {
291  // reduces and stores all machine coordinates.
292  pcoord_t *tmpVect = new pcoord_t [this->numRanks];
293 
294  for (int i = 0; i < networkDim; i++) {
295  Teuchos::reduceAll<int, pcoord_t>(comm, Teuchos::REDUCE_SUM,
296  this->numRanks, procCoords[i], tmpVect);
297  pcoord_t *tmp = tmpVect;
298  tmpVect = procCoords[i];
299  procCoords[i] = tmp;
300  }
301  delete [] tmpVect;
302  }
303 };
304 }
305 #endif
bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const
MachineTopoMgr(const Teuchos::Comm< int > &comm)
Constructor: A BlueGeneQ network machine description;.
bool getMachineExtentWrapArounds(bool *wrap_around) const
bool getMachineCoordinate(const int rank, pcoord_t *xyz) const
MachineClass Base class for representing machine coordinates, networks, etc.
bool getMachineExtent(int *nxyz) const
bool getMyActualMachineCoordinate(pcoord_t *xyz)
SparseMatrixAdapter_t::part_t part_t
bool getMyMachineCoordinate(pcoord_t *xyz)
MachineTopoMgr(const Teuchos::Comm< int > &comm, const Teuchos::ParameterList &pl_)
bool getMachineExtentWrapArounds(part_t *wrap_around) const
A Machine Class for testing only A more realistic machine should be used for task mapping...
virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops)
bool getMachineCoordinate(const char *nodename, pcoord_t *xyz)