Zoltan2_MachineTorusLDMS.hpp

Go to the documentation of this file.

// @HEADER
// *****************************************************************************
//   Zoltan2: A package of combinatorial algorithms for scientific computing
//
// Copyright 2012 NTESS and the Zoltan2 contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER

#ifndef _ZOLTAN2_MACHINEDEFAULT_HPP_
#define _ZOLTAN2_MACHINEDEFAULT_HPP_

#include <Teuchos_Comm.hpp>
#include <Teuchos_CommHelpers.hpp>

namespace Zoltan2{

template <typename nNo_t, typename nCoord_t>
class DefaultMachine : public MachineRepresentation<nNo_t, nCoord_t> {

private:
  int networkDim;
  int numProcs;
  int myRank;

  nCoord_t **procCoords;   // KDD Maybe should be RCP?

public:
  MachineRepresentation(const Comm<int> &comm):
    networkDim(0), 
    numProcs(comm.getSize()), 
    myRank(comm.getRank()),
    procCoords(NULL)
  {
    // Will need this constructor to be specific to RAAMP (MD).
    // Will need a default constructor using, e.g., GeometricGenerator
    // or nothing at all, for when RAAMP is not available as TPL.
    //
    // (AG) In addition, need to be able to run without special
    // privileges in system (e.g., on hopper).
    // Notes:  For now, all cores connected to same NIC will get the
    // same coordinates; later, we could add extra coordinate dimensions
    // to represent nodes or dies (using hwloc info through RAAMP
    // data object).

    // (MD) will modify mapping test to use machine representation
    // #ifdef HAVE_ZOLTAN2_OVIS

    // Call initializer for RAAMP data object (AG)

    // get network dimension.
    // TODO change.
    // Call RAAMP Data Object to get the network dimension (AG)
    networkDim = 3;

    //allocate memory for processor coordinates.
    procCoords = new nCoord_t *[networkDim];
    for (int i = 0; i < networkDim; ++i) {
      procCoords[i] = new nCoord_t [numProcs];
      memset (procCoords[i], 0, sizeof(nCoord_t) * numProcs);
    }
    // Obtain the coordinate of the processor.
    this->getMyCoordinate(/*nCoord_t &xyz[networkDim]*/);
    // Copy xyz into appropriate spot in procCoords. (MD)  
    // KDD I agree with this

    // reduceAll the coordinates of each processor.
    this->gatherMachineCoordinates();
  }


  MachineRepresentation(const RCP<Comm<int> > &comm_):
    networkDim(0), 
    numProcs(comm_->getSize()), 
    procCoords(0), 
    comm(comm_) 
  {
    // Will need this constructor to be specific to RAAMP (MD).
    // Will need a default constructor using, e.g., GeometricGenerator
    // or nothing at all, for when RAAMP is not available as TPL.
    //
    // (AG) In addition, need to be able to run without special
    // privileges in system (e.g., on hopper).  
    // Notes:  For now, all cores connected to same NIC will get the
    // same coordinates; later, we could add extra coordinate dimensions
    // to represent nodes or dies (using hwloc info through RAAMP
    // data object).

    // (MD) will modify mapping test to use machine representation
    // #ifdef HAVE_ZOLTAN2_OVIS

    // Call initializer for RAAMP data object (AG)

    // get network dimension.
    // TODO change.
    // Call RAAMP Data Object to get the network dimension (AG)
    networkDim = 3;

    // Allocate memory for processor coordinates.
    procCoords = new nCoord_t *[networkDim];
    for (int i = 0; i < networkDim; ++i) {
      procCoords[i] = new nCoord_t [numProcs];
      memset (procCoords[i], 0, sizeof(nCoord_t) * numProcs);
    }
    // Obtain the coordinate of the processor.
    this->getMyCoordinate(/*nCoord_t &xyz[networkDim]*/);
    // Copy xyz into appropriate spot in procCoords. (MD)  // KDD I Agree.

    // reduceAll the coordinates of each processor.
    this->gatherMachineCoordinates();
  }


  void getMyCoordinate(/* nCoord_t &xyz[networkDim]*/) {  
    // KDD Enable the argument rather
    // KDD than writing into array here

    // Call RAAMP system to get coordinates and store in xyz (MD)
    // What is the RAAMP call?  (AG)
    // AG will return a view (pointer) to RAAMP's data.
    // We will copy it into xyz.

//KDD #if defined(HAVE_ZOLTAN2_LDMS)
//KDD #elif defined(HAVE_ZOLTAN2_TOPOMGR)
//KDD #elif defined(HAVE_ZOLTAN2_RCA)
//KDD #else

    // The code below may be good for the default constructor, perhaps,
    // but it should copy the data into xyz instead of the procCoords.
    int myRank = comm->getRank();

    int slice = int (pow( double(numProcs), double(1.0 / networkDim)) + 0.5 );

    int m = myRank;
    for (int i = 0; i < networkDim; ++i) {
      procCoords[i][myRank] = m / int(pow(slice, double(networkDim - i - 1)));
      m = m % int(pow(double(slice), double(networkDim - i - 1)));
    }
//KDD #endif
  }

  // KDD Need to return coordinate of any rank?
  // void getCoordinate(partId_t rank, nCoord_t &xyz[networkDim]) { }

  void gatherMachineCoordinates() {  // KDD Should be private
    nCoord_t *tmpVect = new nCoord_t [numProcs];

    for (int i = 0; i < networkDim; ++i) {
      reduceAll<int, nCoord_t>(
          *comm,
          Teuchos::REDUCE_SUM,
          numProcs,
          procCoords[i],
          tmpVect);
      nCoord_t *tmp = tmpVect;
      tmpVect = procCoords[i];
      procCoords[i] = tmp;
    }
    delete [] tmpVect;
  }

  virtual ~MachineRepresentation() {
    for (int i = 0; i < networkDim; ++i) {
      delete [] procCoords[i];
    }
    delete [] procCoords;
    // Free/release THE RAAMP Data Object.
    // Deinitialize/finalize/whatever (AG)
  }

  int getProcDim() const{  // KDD Maybe getNetworkDim or getProcCoordDim
    return networkDim;
  }

  nCoord_t** getProcCoords() const{  
    // KDD Make clear that returning a View; maybe return ArrayView
    return procCoords;
  }

  int getNumProcs() const{
    return numProcs;
  }

  // KDD TODO:  Need more for full LDMS interface.

};
}
#endif