Tpetra_Details_DistributorPlan.hpp

Go to the documentation of this file.

// @HEADER
// ***********************************************************************
//
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2008) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// ************************************************************************
// @HEADER

#ifndef TPETRA_DETAILS_DISTRIBUTOR_PLAN_HPP
#define TPETRA_DETAILS_DISTRIBUTOR_PLAN_HPP

#include "Teuchos_ParameterListAcceptorDefaultBase.hpp"
#include "Teuchos_Array.hpp"
#include "Teuchos_Comm.hpp"
#include "Teuchos_RCP.hpp"
#include "TpetraCore_config.h"

#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
#include <mpi_advance.h>
#endif

namespace Tpetra {
namespace Details {

enum EDistributorSendType {
  DISTRIBUTOR_ISEND, // Use MPI_Isend (Teuchos::isend)
  DISTRIBUTOR_SEND,  // Use MPI_Send (Teuchos::send)
  DISTRIBUTOR_ALLTOALL // Use MPI_Alltoall
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  ,
  DISTRIBUTOR_MPIADVANCE_ALLTOALL,
  DISTRIBUTOR_MPIADVANCE_NBRALLTOALLV
#endif
};

std::string
DistributorSendTypeEnumToString (EDistributorSendType sendType);

enum EDistributorHowInitialized {
  DISTRIBUTOR_NOT_INITIALIZED, // Not initialized yet
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS, // By createFromSends
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_RECVS, // By createFromRecvs
  DISTRIBUTOR_INITIALIZED_BY_CREATE_FROM_SENDS_N_RECVS, // By createFromSendsAndRecvs
  DISTRIBUTOR_INITIALIZED_BY_REVERSE, // By createReverseDistributor
  DISTRIBUTOR_INITIALIZED_BY_COPY, // By copy constructor
};

std::string
DistributorHowInitializedEnumToString (EDistributorHowInitialized how);

class DistributorPlan : public Teuchos::ParameterListAcceptorDefaultBase {
  static constexpr int DEFAULT_MPI_TAG = 0;

public:
  DistributorPlan(Teuchos::RCP<const Teuchos::Comm<int>> comm);
  DistributorPlan(const DistributorPlan& otherPlan);

  size_t createFromSends(const Teuchos::ArrayView<const int>& exportProcIDs);
  void createFromRecvs(const Teuchos::ArrayView<const int>& remoteProcIDs);
  void createFromSendsAndRecvs(const Teuchos::ArrayView<const int>& exportProcIDs,
                               const Teuchos::ArrayView<const int>& remoteProcIDs);

  void setParameterList(const Teuchos::RCP<Teuchos::ParameterList>& plist);

  Teuchos::RCP<DistributorPlan> getReversePlan() const;

  Teuchos::RCP<const Teuchos::Comm<int>> getComm() const { return comm_; }
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  Teuchos::RCP<MPIX_Comm*> getMPIXComm() const { return mpixComm_; }
#endif
  EDistributorSendType getSendType() const { return sendType_; }
  size_t getNumReceives() const { return numReceives_; }
  size_t getNumSends() const { return numSendsToOtherProcs_; }
  bool hasSelfMessage() const { return sendMessageToSelf_; }
  size_t getMaxSendLength() const { return maxSendLength_; }
  size_t getTotalReceiveLength() const { return totalReceiveLength_; }
  Teuchos::ArrayView<const int> getProcsFrom() const { return procsFrom_; }
  Teuchos::ArrayView<const int> getProcsTo() const { return procIdsToSendTo_; }
  Teuchos::ArrayView<const size_t> getLengthsFrom() const { return lengthsFrom_; }
  Teuchos::ArrayView<const size_t> getLengthsTo() const { return lengthsTo_; }
  Teuchos::ArrayView<const size_t> getStartsTo() const { return startsTo_; }
  Teuchos::ArrayView<const size_t> getIndicesTo() const { return indicesTo_; }
  Details::EDistributorHowInitialized howInitialized() const { return howInitialized_; }

private:

  // after the plan has been created we have the info we need to initialize the MPI advance communicator
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  void initializeMpiAdvance();
#endif

  Teuchos::RCP<const Teuchos::ParameterList> getValidParameters() const;

  void createReversePlan() const;

  void computeReceives();

  Teuchos::RCP<const Teuchos::Comm<int>> comm_;
#if defined(HAVE_TPETRACORE_MPI_ADVANCE)
  Teuchos::RCP<MPIX_Comm*> mpixComm_;
#endif

  Details::EDistributorHowInitialized howInitialized_;
  mutable Teuchos::RCP<DistributorPlan> reversePlan_;


  EDistributorSendType sendType_;

  bool sendMessageToSelf_;
  size_t numSendsToOtherProcs_;
  Teuchos::Array<int> procIdsToSendTo_;

  Teuchos::Array<size_t> startsTo_;

  Teuchos::Array<size_t> lengthsTo_;

  size_t maxSendLength_;

  Teuchos::Array<size_t> indicesTo_;

  size_t numReceives_;

  size_t totalReceiveLength_;

  Teuchos::Array<size_t> lengthsFrom_;

  Teuchos::Array<int> procsFrom_;

  Teuchos::Array<size_t> startsFrom_;

  Teuchos::Array<size_t> indicesFrom_;
};

}
}

#endif