Zoltan2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Zoltan2_MachineTopoMgrForTest.hpp
Go to the documentation of this file.
1 #ifndef _ZOLTAN2_MACHINE_TOPOMANAGERTEST_HPP_
2 #define _ZOLTAN2_MACHINE_TOPOMANAGERTEST_HPP_
3 
4 #include <Teuchos_Comm.hpp>
5 #include <Teuchos_CommHelpers.hpp>
6 #include <Zoltan2_Machine.hpp>
7 
8 
9 namespace Zoltan2{
10 
15 template <typename pcoord_t, typename part_t>
16 class MachineBGQTest : public Machine <pcoord_t, part_t> {
17 
18 public:
23  MachineBGQTest(const Teuchos::Comm<int> &comm ):
24  Machine<pcoord_t,part_t>(comm),
25  networkDim(6),
26  procCoords(NULL),machine_extent(NULL),
27  delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL), pl(NULL){
28  transformed_network_dim = networkDim - 1;
29  transformed_coordinates = procCoords;
30 
31  machine_extent = new int[networkDim];
32  this->getMachineExtent(this->machine_extent);
33  machine_extent[5] = 1;
34  //allocate memory for processor coordinates.
35  procCoords = new pcoord_t *[networkDim];
36  for (int i = 0; i < networkDim; ++i){
37  procCoords[i] = new pcoord_t[this->numRanks];
38  memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
39  }
40 
41  //obtain the coordinate of the processor.
42  pcoord_t *xyz = new pcoord_t[networkDim];
44  for (int i = 0; i < networkDim; i++)
45  procCoords[i][this->myRank] = xyz[i];
46  delete [] xyz;
47 
48  //reduceAll the coordinates of each processor.
49  gatherMachineCoordinates(comm);
50 
51  }
52 
53  MachineBGQTest(const Teuchos::Comm<int> &comm, const Teuchos::ParameterList &pl_):
54  Machine<pcoord_t,part_t>(comm),
55  networkDim(6),
56  procCoords(NULL),machine_extent(NULL),
57  delete_transformed_coords(false), transformed_network_dim(0),transformed_coordinates (NULL),
58  pl(&pl_)
59  {
60  transformed_network_dim = networkDim - 1;
61  transformed_coordinates = procCoords;
62  machine_extent = new int[networkDim];
63 
64  this->getMachineExtent(this->machine_extent);
65  machine_extent[5] = 1;
66  //allocate memory for processor coordinates.
67  procCoords = new pcoord_t *[networkDim];
68  for (int i = 0; i < networkDim; ++i){
69  procCoords[i] = new pcoord_t[this->numRanks];
70  memset(procCoords[i], 0, sizeof(pcoord_t) * this->numRanks);
71  }
72 
73  //obtain the coordinate of the processor.
74  pcoord_t *xyz = new pcoord_t[networkDim];
76  for (int i = 0; i < networkDim; i++)
77  procCoords[i][this->myRank] = xyz[i];
78  delete [] xyz;
79 
80  //reduceAll the coordinates of each processor.
81  gatherMachineCoordinates(comm);
82 
83  const Teuchos::ParameterEntry *pe = this->pl->getEntryPtr("Machine_Optimization_Level");
84 
85  if (pe){
86 
87  int optimization_level = 0;
88 
89  optimization_level = pe->getValue<int>(&optimization_level);
90 
91  if (optimization_level == 0){
92  transformed_network_dim = networkDim - 1;
93  transformed_coordinates = procCoords;
94  }
95 
96  else if (optimization_level >= 1){
97  transformed_network_dim = networkDim - 2;
98  transformed_coordinates = procCoords;
99  }
100  }
101  }
102 
103 
104  virtual ~MachineBGQTest() {
105  for (int i = 0; i < networkDim; i++){
106  delete [] procCoords[i];
107  }
108  delete [] procCoords;
109 
110  delete [] machine_extent;
111  if (delete_transformed_coords){
112  for (int i = 0; i < transformed_network_dim; i++){
113  delete [] transformed_coordinates[i];
114  }
115  delete [] transformed_coordinates;
116  }
117  }
118 
119  bool hasMachineCoordinates() const { return true; }
120 
121  int getMachineDim() const { return transformed_network_dim; }
122 
123  bool getMachineExtentWrapArounds(bool *wrap_around) const {
124  int dim = 0;
125  if (dim < transformed_network_dim)
126  wrap_around[dim++] = true;
127  if (dim < transformed_network_dim)
128  wrap_around[dim++] = true;
129  if (dim < transformed_network_dim)
130  wrap_around[dim++] = true;
131  if (dim < transformed_network_dim)
132  wrap_around[dim++] = true;
133  if (dim < transformed_network_dim)
134  wrap_around[dim++] = true;
135  if (dim < transformed_network_dim)
136  wrap_around[dim++] = true;
137  return true;
138  }
139 
140 
141  bool getMachineExtentWrapArounds(part_t *wrap_around) const {
142 
143  int dim = 0;
144  if (dim < transformed_network_dim)
145  wrap_around[dim++] = true;
146 
147  if (dim < transformed_network_dim)
148  wrap_around[dim++] = true;
149 
150  if (dim < transformed_network_dim)
151  wrap_around[dim++] = true;
152 
153  if (dim < transformed_network_dim)
154  wrap_around[dim++] = true;
155 
156  if (dim < transformed_network_dim)
157  wrap_around[dim++] = true;
158 
159  if (dim < transformed_network_dim)
160  wrap_around[dim++] = true;
161  return true;
162  }
163 
164  bool getMachineExtent(part_t *extent) const {
165  part_t nxyz[6];
166 
167  nxyz[0] = 1;
168  nxyz[1] = 1;
169  nxyz[2] = 1;
170  nxyz[3] = 1;
171  nxyz[4] = 1;
172  nxyz[5] = 1;
173  const int rank_per_node = 1;
174  const int num_nodes = this->numRanks / rank_per_node;
175  if (num_nodes <= 1){
176  nxyz[0] = nxyz[1] = nxyz[2] = nxyz[3] = 1;
177  nxyz[4] = 1;
178  nxyz[5] = 1;
179  }
180  else if (num_nodes <= 2){
181 
182  nxyz[0] = nxyz[1] = nxyz[2] = nxyz[3] = 1;
183  nxyz[4] = 2;
184  nxyz[5] = 1;
185  }
186  else if (num_nodes <= 4){
187  nxyz[0] = nxyz[1] = nxyz[2] = 1;
188  nxyz[3] = 2;
189  nxyz[4] = 2;
190  nxyz[5] = 1;
191 
192  }
193  else if (num_nodes <= 8){
194  nxyz[0] = nxyz[1] = 1;
195  nxyz[2] = 2;
196  nxyz[3] = 2;
197  nxyz[4] = 2;
198  nxyz[5] = 1;
199  }
200 
201  else if (num_nodes <= 16){
202 
203  nxyz[0] = 1;
204  nxyz[1] = 2;
205  nxyz[2] = 2;
206  nxyz[3] = 2;
207  nxyz[4] = 2;
208  nxyz[5] = 1;
209  }
210 
211  else if (num_nodes <= 32){
212 
213  nxyz[0] = 1;
214  nxyz[1] = 2;
215  nxyz[2] = 2;
216  nxyz[3] = 4;
217  nxyz[4] = 2;
218  nxyz[5] = 1;
219  }
220 
221  else if (num_nodes <= 64){
222 
223  nxyz[0] = 1;
224  nxyz[1] = 2;
225  nxyz[2] = 4;
226  nxyz[3] = 4;
227  nxyz[4] = 2;
228  nxyz[5] = 1;
229  }
230 
231  else if (num_nodes <= 128){
232 
233  nxyz[0] = 2;
234  nxyz[1] = 2;
235  nxyz[2] = 4;
236  nxyz[3] = 4;
237  nxyz[4] = 2;
238  nxyz[5] = 1;
239  }
240  else if (num_nodes <= 256){
241 
242  nxyz[0] = 4;
243  nxyz[1] = 2;
244  nxyz[2] = 4;
245  nxyz[3] = 4;
246  nxyz[4] = 2;
247  nxyz[5] = 1;
248  }
249  else if (num_nodes <= 512){
250 
251  nxyz[0] = 4;
252  nxyz[1] = 4;
253  nxyz[2] = 4;
254  nxyz[3] = 4;
255  nxyz[4] = 2;
256  nxyz[5] = 1;
257  }
258  else if (num_nodes <= 1024){
259 
260  nxyz[0] = 4;
261  nxyz[1] = 4;
262  nxyz[2] = 4;
263  nxyz[3] = 8;
264  nxyz[4] = 2;
265  nxyz[5] = 1;
266  }
267  else if (num_nodes <= 2048){
268 
269  nxyz[0] = 4;
270  nxyz[1] = 4;
271  nxyz[2] = 4;
272  nxyz[3] = 16;
273  nxyz[4] = 2;
274  nxyz[5] = 1;
275  }
276  else if (num_nodes <= 4096){
277 
278  nxyz[0] = 8;
279  nxyz[1] = 4;
280  nxyz[2] = 4;
281  nxyz[3] = 16;
282  nxyz[4] = 2;
283  nxyz[5] = 1;
284  }
285  else {
286 
287  std::cerr << "Too many ranks to test" << std::endl;
288  }
289  nxyz[5] = rank_per_node;
290 
291 
292  int dim = 0;
293  if (dim < transformed_network_dim)
294  extent[dim] = nxyz[dim];
295  ++dim;
296  if (dim < transformed_network_dim)
297  extent[dim] = nxyz[dim];
298  ++dim;
299  if (dim < transformed_network_dim)
300  extent[dim] = nxyz[dim];
301  ++dim;
302  if (dim < transformed_network_dim)
303  extent[dim] = nxyz[dim];
304  ++dim;
305  if (dim < transformed_network_dim)
306  extent[dim] = nxyz[dim];
307  ++dim;
308  if (dim < transformed_network_dim)
309  extent[dim] = nxyz[dim];
310 
311  return true;
312  }
313 
314 
315  bool getMyMachineCoordinate(pcoord_t *xyz) {
316  for (int i = 0; i < this->transformed_network_dim; ++i){
317  xyz[i] = transformed_coordinates[i][this->myRank];
318  }
319  return true;
320  }
321 
322  bool getMyActualMachineCoordinate(pcoord_t *xyz) {
323  int a,b,c,d,e,t;
324 
325  int me = this->myRank;
326  t = me % machine_extent[5];
327 
328  me = me / machine_extent[5];
329  e = me % machine_extent[4];
330 
331  me = me / machine_extent[4];
332  d = me % machine_extent[3];
333 
334  me = me / machine_extent[3];
335  c = me % machine_extent[2];
336 
337  me = me / machine_extent[2];
338  b = me % machine_extent[1];
339 
340  me = me / machine_extent[1];
341  a = me % machine_extent[0];
342 
343  xyz[0] = a; xyz[1] = b; xyz[2] = c; xyz[3] = d; xyz[4] = e; xyz[5] = t;
344 
345  //std::cout << "me:" << this->myRank << " " << a << " " << b << " " << c << " " << d << " " << e << " " << t << std::endl;
346  return true;
347  }
348 
349 
350  inline bool getMachineCoordinate(const int rank,
351  pcoord_t *xyz) const {
352  return false;
353  }
354  bool getMachineCoordinate(const char *nodename, pcoord_t *xyz) {
355  return false; // cannot yet return from nodename
356  }
357 
358  bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const {
359  allCoords = procCoords;
360  return true;
361  }
362 
363  virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops){
364 
365  hops = 0;
366  for (int i = 0; i < networkDim - 1; ++i){
367  pcoord_t distance = procCoords[i][rank1] - procCoords[i][rank2];
368  if (distance < 0 ) distance = -distance;
369  if (machine_extent[i] - distance < distance) distance = machine_extent[i] - distance;
370  hops += distance;
371  }
372 
373  /*
374  if (this->myRank == 0){
375  std::cout << "rank1:" << rank1 << " rank2:" << rank2 << " hops:" << hops << std::endl;
376  }
377  */
378 
379  return true;
380  }
381 
382 
383 private:
384 
385  int networkDim;
386  pcoord_t **procCoords; // KDD Maybe should be RCP?
387  part_t *machine_extent;
388 
389 
390 
391 
392  bool delete_transformed_coords;
393  int transformed_network_dim;
394  pcoord_t **transformed_coordinates;
395  const Teuchos::ParameterList *pl;
396 
397  void gatherMachineCoordinates(const Teuchos::Comm<int> &comm) {
398  // reduces and stores all machine coordinates.
399  pcoord_t *tmpVect = new pcoord_t [this->numRanks];
400 
401  for (int i = 0; i < networkDim; i++) {
402  Teuchos::reduceAll<int, pcoord_t>(comm, Teuchos::REDUCE_SUM,
403  this->numRanks, procCoords[i], tmpVect);
404  pcoord_t *tmp = tmpVect;
405  tmpVect = procCoords[i];
406  procCoords[i] = tmp;
407  }
408  delete [] tmpVect;
409  }
410 };
411 }
412 #endif
virtual bool getHopCount(int rank1, int rank2, pcoord_t &hops)
bool getMyActualMachineCoordinate(pcoord_t *xyz)
MachineClass Base class for representing machine coordinates, networks, etc.
bool getMachineExtent(part_t *extent) const
MachineBGQTest(const Teuchos::Comm< int > &comm, const Teuchos::ParameterList &pl_)
bool getMachineExtentWrapArounds(part_t *wrap_around) const
SparseMatrixAdapter_t::part_t part_t
bool getMachineCoordinate(const char *nodename, pcoord_t *xyz)
bool getAllMachineCoordinatesView(pcoord_t **&allCoords) const
bool getMachineExtentWrapArounds(bool *wrap_around) const
bool getMachineCoordinate(const int rank, pcoord_t *xyz) const
A Machine Class for testing only A more realistic machine should be used for task mapping...
MachineBGQTest(const Teuchos::Comm< int > &comm)
Constructor: A BlueGeneQ network machine description;.