Tpetra_Details_Ialltofewv.hpp

// @HEADER
// *****************************************************************************
//          Tpetra: Templated Linear Algebra Services Package
//
// Copyright 2008 NTESS and the Tpetra contributors.
// SPDX-License-Identifier: BSD-3-Clause
// *****************************************************************************
// @HEADER

#pragma once

#include <memory>

#include <mpi.h>

namespace Tpetra::Details {

struct Ialltofewv {
  struct Req {
    const void *sendbuf;
    const int *sendcounts;
    const int *sdispls;
    MPI_Datatype sendtype;
    void *recvbuf;
    const int *recvcounts;
    const int *rdispls;
    const int *roots;
    int nroots;
    MPI_Datatype recvtype;
    int tag;
    MPI_Comm comm;

    bool devAccess;
    bool completed;
  };

  template <bool DevAccess>
  int post(const void *sendbuf,
           const int *sendcounts,  // how much to each root (length nroots)
           const int *sdispls,     // where data for each root starts (length nroots)
           MPI_Datatype sendtype,
           void *recvbuf,          // address of recv buffer (significant only at root)
           const int *recvcounts,  // the number of elements recvd from each process
                                   // (signficant only at roots)
           const int *rdispls,     // where in `recvbuf` to place incoming data from
                                   // process i (signficant only at roots)
           const int *roots,       // list of root ranks (must be same on all procs)
           int nroots,             // size of list of root ranks
           MPI_Datatype recvtype,
           int tag,
           MPI_Comm comm,
           Req *req) {
    req->sendbuf    = sendbuf;
    req->sendcounts = sendcounts;
    req->sdispls    = sdispls;
    req->sendtype   = sendtype;
    req->recvbuf    = recvbuf;
    req->recvcounts = recvcounts;
    req->rdispls    = rdispls;
    req->roots      = roots;
    req->nroots     = nroots;
    req->recvtype   = recvtype;
    req->tag        = tag;
    req->comm       = comm;

    req->devAccess = DevAccess;
    req->completed = false;
#ifndef NDEBUG
    // {
    //   std::stringstream ss;
    //   ss << __FILE__ << ":" << __LINE__ << "\n";
    //   std::cerr << ss.str();
    // }
#endif
    return MPI_SUCCESS;
  }

  int wait(Req &req);

  int get_status(const Req &req, int *flag, MPI_Status *status) const;

  struct Cache {
    struct impl;
    std::shared_ptr<impl> pimpl;

    Cache();
    ~Cache();
  };

 private:
  Cache cache_;

};  // struct Ialltofewv
}  // namespace Tpetra::Details