Domi_MDComm.hpp

// @HEADER
// ***********************************************************************
//
//     Domi: Multi-dimensional Distributed Linear Algebra Services
//                 Copyright (2014) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia
// Corporation, the U.S. Government retains certain rights in this
// software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact William F. Spotz (wfspotz@sandia.gov)
//
// ***********************************************************************
// @HEADER

#ifndef DOMI_MDCOMM_HPP
#define DOMI_MDCOMM_HPP

// Teuchos includes
#include "Teuchos_Comm.hpp"
#include "Teuchos_Array.hpp"
#include "Teuchos_ArrayView.hpp"
#include "Teuchos_ParameterList.hpp"

// Domi includes
#include "Domi_ConfigDefs.hpp"
#include "Domi_Slice.hpp"

#ifdef HAVE_EPETRA
#ifdef HAVE_MPI
#include "Epetra_MpiComm.h"
#else
#include "Epetra_SerialComm.h"
#endif
#endif

namespace Domi
{

class MDComm
{
public:


  typedef Teuchos::Array< int >::size_type size_type;



  static const Layout commLayout;



  MDComm(const Teuchos::ArrayView< const int > & commDims,
         const Teuchos::ArrayView< const int > & periodic =
           Teuchos::ArrayView< const int >());

  MDComm(const Teuchos::RCP< const Teuchos::Comm< int > > teuchosComm,
         const Teuchos::ArrayView< const int > & commDims,
         const Teuchos::ArrayView< const int > & periodic =
           Teuchos::ArrayView< const int >());

  MDComm(Teuchos::ParameterList & plist);

  MDComm(const Teuchos::RCP< const Teuchos::Comm< int > > teuchosComm,
         Teuchos::ParameterList & plist);

  MDComm(int numDims);

  MDComm(const Teuchos::RCP< const Teuchos::Comm< int > > teuchosComm,
         int numDims);

  MDComm(int numDims,
         const Teuchos::ArrayView< const int > & commDims,
         const Teuchos::ArrayView< const int > & periodic =
           Teuchos::ArrayView< const int >());

  MDComm(const Teuchos::RCP< const Teuchos::Comm< int > > teuchosComm,
         int numDims,
         const Teuchos::ArrayView< const int > & commDims,
         const Teuchos::ArrayView< const int > & periodic =
           Teuchos::ArrayView< const int >());

   MDComm(const MDComm & parent,
         int axis,
         int axisRank);

  MDComm(const MDComm & parent,
         int axis,
         const Slice & slice);

  MDComm(const MDComm & parent,
         const Teuchos::ArrayView< Slice > & slices);

  MDComm(const MDComm & source);

  ~MDComm();

  MDComm & operator=(const MDComm & source);



  bool onSubcommunicator() const;

  Teuchos::RCP< const Teuchos::Comm< int > > getTeuchosComm() const;

#ifdef HAVE_EPETRA

  Teuchos::RCP< const Epetra_Comm > getEpetraComm() const;
#endif

  int numDims() const;

  Teuchos::Array< int > getCommDims() const;

  int getCommDim(int axis) const;

  bool isPeriodic(int axis) const;

  int getCommIndex(int axis) const;

  int getLowerNeighbor(int axis) const;

  int getUpperNeighbor(int axis) const;



  Teuchos::ArrayView< Teuchos::RCP< const MDComm > > getAxisComms() const;

  Teuchos::RCP< const MDComm > getAxisComm(int axis) const;


protected:

  // Not implemented
  MDComm();

private:

  // The Teuchos communicator
  Teuchos::RCP< const Teuchos::Comm< int > > _teuchosComm;

#ifdef HAVE_EPETRA
  // An equivalent Epetra communicator.  This is mutable because we
  // only compute it if requested by a get method that is const.
  mutable Teuchos::RCP< const Epetra_Comm > _epetraComm;
#endif

  // An array of the sizes of the communicator along each axis
  Teuchos::Array< int > _commDims;

  // An array of the strides between processor ranks along each axis.
  Teuchos::Array< int > _commStrides;

  // The comm index for this processor along each axis
  Teuchos::Array< int > _commIndex;

  // An array of flags denoting periodic axes
  Teuchos::Array< int > _periodic;

  // An array of 1D axis communicators, that represent the comm along
  // each axis of this MDComm.  These comms are technically
  // sub-communicators, but each one is guaranteed to exist on the
  // processor that constructs it. This array is mutable so that its
  // construction can be delayed until it is requested.
  mutable Teuchos::Array< Teuchos::RCP< const MDComm > > _axisComms;

};

}  // namespace Domi

#endif