Zoltan2
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Macros Pages
Zoltan2_MachineTorusLDMS.hpp
Go to the documentation of this file.
1 #ifndef _ZOLTAN2_MACHINEDEFAULT_HPP_
2 #define _ZOLTAN2_MACHINEDEFAULT_HPP_
3 
4 #include <Teuchos_Comm.hpp>
5 #include <Teuchos_CommHelpers.hpp>
6 
7 namespace Zoltan2{
8 
14 template <typename nNo_t, typename nCoord_t>
15 class DefaultMachine : public MachineRepresentation<nNo_t, nCoord_t> {
16 
17 private:
18  int networkDim;
19  int numProcs;
20  int myRank;
21 
22  nCoord_t **procCoords; // KDD Maybe should be RCP?
23 
24 public:
29  MachineRepresentation(const Comm<int> &comm):
30  networkDim(0),
31  numProcs(comm.getSize()),
32  myRank(comm.getRank()),
33  procCoords(NULL)
34  {
35  // Will need this constructor to be specific to RAAMP (MD).
36  // Will need a default constructor using, e.g., GeometricGenerator
37  // or nothing at all, for when RAAMP is not available as TPL.
38  //
39  // (AG) In addition, need to be able to run without special
40  // privileges in system (e.g., on hopper).
41  // Notes: For now, all cores connected to same NIC will get the
42  // same coordinates; later, we could add extra coordinate dimensions
43  // to represent nodes or dies (using hwloc info through RAAMP
44  // data object).
45 
46  // (MD) will modify mapping test to use machine representation
47  // #ifdef HAVE_ZOLTAN2_OVIS
48 
49  // Call initializer for RAAMP data object (AG)
50 
51  // get network dimension.
52  // TODO change.
53  // Call RAAMP Data Object to get the network dimension (AG)
54  networkDim = 3;
55 
56  //allocate memory for processor coordinates.
57  procCoords = new nCoord_t *[networkDim];
58  for (int i = 0; i < networkDim; ++i) {
59  procCoords[i] = new nCoord_t [numProcs];
60  memset (procCoords[i], 0, sizeof(nCoord_t) * numProcs);
61  }
62  // Obtain the coordinate of the processor.
63  this->getMyCoordinate(/*nCoord_t &xyz[networkDim]*/);
64  // Copy xyz into appropriate spot in procCoords. (MD)
65  // KDD I agree with this
66 
67  // reduceAll the coordinates of each processor.
69  }
70 
71 
75  MachineRepresentation(const RCP<Comm<int> > &comm_):
76  networkDim(0),
77  numProcs(comm_->getSize()),
78  procCoords(0),
79  comm(comm_)
80  {
81  // Will need this constructor to be specific to RAAMP (MD).
82  // Will need a default constructor using, e.g., GeometricGenerator
83  // or nothing at all, for when RAAMP is not available as TPL.
84  //
85  // (AG) In addition, need to be able to run without special
86  // privileges in system (e.g., on hopper).
87  // Notes: For now, all cores connected to same NIC will get the
88  // same coordinates; later, we could add extra coordinate dimensions
89  // to represent nodes or dies (using hwloc info through RAAMP
90  // data object).
91 
92  // (MD) will modify mapping test to use machine representation
93  // #ifdef HAVE_ZOLTAN2_OVIS
94 
95  // Call initializer for RAAMP data object (AG)
96 
97  // get network dimension.
98  // TODO change.
99  // Call RAAMP Data Object to get the network dimension (AG)
100  networkDim = 3;
101 
102  // Allocate memory for processor coordinates.
103  procCoords = new nCoord_t *[networkDim];
104  for (int i = 0; i < networkDim; ++i) {
105  procCoords[i] = new nCoord_t [numProcs];
106  memset (procCoords[i], 0, sizeof(nCoord_t) * numProcs);
107  }
108  // Obtain the coordinate of the processor.
109  this->getMyCoordinate(/*nCoord_t &xyz[networkDim]*/);
110  // Copy xyz into appropriate spot in procCoords. (MD) // KDD I Agree.
111 
112  // reduceAll the coordinates of each processor.
113  this->gatherMachineCoordinates();
114  }
115 
116 
120  void getMyCoordinate(/* nCoord_t &xyz[networkDim]*/) {
121  // KDD Enable the argument rather
122  // KDD than writing into array here
123 
124  // Call RAAMP system to get coordinates and store in xyz (MD)
125  // What is the RAAMP call? (AG)
126  // AG will return a view (pointer) to RAAMP's data.
127  // We will copy it into xyz.
128 
129 //KDD #if defined(HAVE_ZOLTAN2_LDMS)
130 //KDD #elif defined(HAVE_ZOLTAN2_TOPOMGR)
131 //KDD #elif defined(HAVE_ZOLTAN2_RCA)
132 //KDD #else
133 
134  // The code below may be good for the default constructor, perhaps,
135  // but it should copy the data into xyz instead of the procCoords.
136  int myRank = comm->getRank();
137 
138  int slice = int (pow( double(numProcs), double(1.0 / networkDim)) + 0.5 );
139 
140  int m = myRank;
141  for (int i = 0; i < networkDim; ++i) {
142  procCoords[i][myRank] = m / int(pow(slice, double(networkDim - i - 1)));
143  m = m % int(pow(double(slice), double(networkDim - i - 1)));
144  }
145 //KDD #endif
146  }
147 
148  // KDD Need to return coordinate of any rank?
149  // void getCoordinate(partId_t rank, nCoord_t &xyz[networkDim]) { }
150 
154  void gatherMachineCoordinates() { // KDD Should be private
155  nCoord_t *tmpVect = new nCoord_t [numProcs];
156 
157  for (int i = 0; i < networkDim; ++i) {
158  reduceAll<int, nCoord_t>(
159  *comm,
160  Teuchos::REDUCE_SUM,
161  numProcs,
162  procCoords[i],
163  tmpVect);
164  nCoord_t *tmp = tmpVect;
165  tmpVect = procCoords[i];
166  procCoords[i] = tmp;
167  }
168  delete [] tmpVect;
169  }
170 
175  for (int i = 0; i < networkDim; ++i) {
176  delete [] procCoords[i];
177  }
178  delete [] procCoords;
179  // Free/release THE RAAMP Data Object.
180  // Deinitialize/finalize/whatever (AG)
181  }
182 
186  int getProcDim() const{ // KDD Maybe getNetworkDim or getProcCoordDim
187  return networkDim;
188  }
189 
193  nCoord_t** getProcCoords() const{
194  // KDD Make clear that returning a View; maybe return ArrayView
195  return procCoords;
196  }
197 
201  int getNumProcs() const{
202  return numProcs;
203  }
204 
205  // KDD TODO: Need more for full LDMS interface.
206 
207 };
208 }
209 #endif
A Default MachineRepresentation Class.
int getProcDim() const
getProcDim function returns the dimension of the physical processor layout.
nCoord_t ** getProcCoords() const
getProcDim function returns the coordinates of processors in two dimensional array.
void getMyCoordinate()
getMyCoordinate function stores the coordinate of the current processor in procCoords[*][rank] ...
virtual ~MachineRepresentation()
destructor of the class free memory in procCoords.
int getNumProcs() const
getNumProcs function returns the number of processors.
MachineRepresentation(const RCP< Comm< int > > &comm_)
Constructor MachineRepresentation Class.
MachineRepresentation Class Base class for representing machine coordinates, networks, etc.
MachineRepresentation(const Comm< int > &comm)
Constructor MachineRepresentation Class.
void gatherMachineCoordinates()
gatherMachineCoordinates function reduces and stores all machine coordinates.