Epetra_MpiDistributor.h

Go to the documentation of this file.

/*
//@HEADER
// ************************************************************************
//
//               Epetra: Linear Algebra Services Package
//                 Copyright 2011 Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ************************************************************************
//@HEADER
*/

#ifndef EPETRA_MPIDISTRIBUTOR_H
#define EPETRA_MPIDISTRIBUTOR_H
#include "Epetra_Object.h"
#include "Epetra_Distributor.h"
class Epetra_MpiComm;
#include <mpi.h>

class Epetra_MpiDistributor: public Epetra_Object, public virtual Epetra_Distributor {

  public:



  Epetra_MpiDistributor(const Epetra_MpiComm & Comm);

  Epetra_MpiDistributor(const Epetra_MpiDistributor & Distributor);

  Epetra_Distributor * Clone(){return(dynamic_cast<Epetra_Distributor *>(new Epetra_MpiDistributor(*this)));};


  Epetra_Distributor * ReverseClone();

  virtual ~Epetra_MpiDistributor();




  int CreateFromSends( const int & NumExportIDs,
                       const int * ExportPIDs,
           bool Deterministic,
                       int & NumRemoteIDs );

  int CreateFromRecvs( const int & NumRemoteIDs,
                       const int * RemoteGIDs,
                       const int * RemotePIDs,
           bool Deterministic,
                       int & NumExportIDs,
                       int *& ExportGIDs,
                       int *& ExportPIDs);

#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
  int CreateFromRecvs( const int & NumRemoteIDs,
                       const long long * RemoteGIDs,
                       const int * RemotePIDs,
           bool Deterministic,
                       int & NumExportIDs,
                       long long *& ExportGIDs,
                       int *& ExportPIDs);
#endif






 int CreateFromSendsAndRecvs( const int & NumExportIDs,
             const int * ExportPIDs,
             const int & NumRemoteIDs,
             const int * RemoteGIDs,
             const int * RemotePIDs,
             bool Deterministic);

#ifndef EPETRA_NO_64BIT_GLOBAL_INDICES
  int CreateFromSendsAndRecvs( const int & NumExportIDs,
             const int * ExportPIDs,
             const int & NumRemoteIDs,
             const long long * RemoteGIDs,
             const int * RemotePIDs,
             bool Deterministic);
#endif
      




  int Do( char * export_objs,
          int obj_size,
          int & len_import_objs,
          char *& import_objs );

  int DoReverse( char * export_objs,
                 int obj_size,
                 int & len_import_objs,
                 char *& import_objs );

  int DoPosts( char * export_objs,
               int obj_size,
               int & len_import_objs,
               char *& import_objs );
  int DoWaits();

  int DoReversePosts( char * export_objs,
                      int obj_size,
                      int & len_import_objs,
                      char *& import_objs );

  int DoReverseWaits();



  int Do( char * export_objs,
          int obj_size,
          int *& sizes,
          int & len_import_objs,
          char *& import_objs );

  int DoReverse( char * export_objs,
                 int obj_size,
                 int *& sizes,
                 int & len_import_objs,
                 char *& import_objs );

  int DoPosts( char * export_objs,
               int obj_size,
               int *& sizes,
               int & len_import_objs,
               char *& import_objs);

  int DoReversePosts( char * export_objs,
                      int obj_size,
                      int *& sizes,
                      int & len_import_objs,
                      char *& import_objs );



  int NumReceives() const {return nrecvs_;}

  int NumSends() const {return nsends_;}

  int MaxSendLength() const {return max_send_length_;}

  int TotalReceiveLength() const { return total_recv_length_;}

  const int * ProcsFrom() const {return procs_from_;}

  const int * ProcsTo() const {return procs_to_;}


  const int * LengthsFrom() const {return lengths_from_;}


  const int * LengthsTo() const {return lengths_to_;}

  void GetLastDoStatistics(int & bytes_sent, int & bytes_recvd) const {
    bytes_sent  = lastRoundBytesSend_;
    bytes_recvd = lastRoundBytesRecv_;
  }



  void Print(std::ostream & os) const;
  private:
  int CreateSendStructures_(int my_proc,
          int nprocs,
          const int & NumExportIDs,
          const int * ExportPIDs);


  int CreateRecvStructures_(const int & NumRemoteIDs,
          const int * RemotePIDs);


  int ComputeRecvs_( int my_proc,
         int nprocs );

  template<typename id_type>
    int ComputeSends_( int num_imports,
           const id_type *& import_ids,
           const int *& import_procs,
           int & num_exports,
           id_type *& export_ids,
           int *& export_procs,
           int my_proc );


    int Resize_(int *sizes);

    int Sort_ints_( int *vals, int *other, int nvals );

  private:
    Epetra_MpiDistributor& operator=(const Epetra_MpiDistributor& src);

    void CreateReverseDistributor();

    int * lengths_to_;
    int * procs_to_;
    int * indices_to_;
    int   size_indices_to_;

    int * lengths_from_;
    int * procs_from_;
    int * indices_from_;
    int   size_indices_from_;

    bool  resized_;
    int * sizes_;

    int * sizes_to_;
    int * starts_to_;
    int * starts_to_ptr_;
    int * indices_to_ptr_;

    int * sizes_from_;
    int * starts_from_;
    int * starts_from_ptr_;
    int * indices_from_ptr_;

    int   nrecvs_;
    int   nsends_;
    int   nexports_;

    int   self_msg_;

    int   max_send_length_;
    int   total_recv_length_;

    int   tag_;

    const Epetra_MpiComm * epComm_;
    const MPI_Comm      comm_;

    MPI_Request * request_;
    MPI_Status  * status_;

    bool no_delete_;

    char * send_array_;
    int send_array_size_;

    Epetra_MpiDistributor * comm_plan_reverse_;
    
    int lastRoundBytesSend_;
    int lastRoundBytesRecv_;
};
#endif /* EPETRA_MPIDISTRIBUTOR_H */