60 for (
int i = 0; i < from->
nrecvs; i++) {
87 #define PRINT_VECTOR(v) \
88 if(v != Teuchos::null) { \
89 std::cout << " " << #v << " "; \
90 for(Teuchos::ArrayRCP<int>::size_type n = 0; n < v.size(); ++n) { \
91 std::cout << v[n] << " "; \
93 std::cout << std::endl; \
96 #define PRINT_VAL(val) std::cout << " " << #val << ": " << val << std::endl;
98 for(
int proc = 0; proc <
comm->getSize(); ++proc) {
100 if(proc ==
comm->getRank()) {
102 std::cout <<
"Rank " << proc <<
" " << headerMessage << std::endl;
135 const Teuchos::ArrayRCP<int> &assign,
136 Teuchos::RCP<
const Teuchos::Comm<int> > comm,
141 if (comm == Teuchos::null){
142 throw std::logic_error(
"Invalid communicator: MPI_COMM_NULL.");
145 int my_proc = comm->getRank();
146 int nprocs = comm->getSize();
153 Teuchos::ArrayRCP<int> starts(
new int[nprocs + 1], 0, nprocs + 1,
true);
154 for(
int n = 0; n < starts.size(); ++n) {
163 int no_send_buff = 1;
165 int prev_proc = nprocs;
167 for (
int i = 0; i < nvals; i++) {
168 int proc = assign[i];
169 if (no_send_buff && proc != prev_proc) {
170 if (proc >= 0 && (starts[proc] || prev_proc < 0)) {
183 int self_msg = (starts[my_proc] != 0);
185 Teuchos::ArrayRCP<int> lengths_to;
186 Teuchos::ArrayRCP<int> procs_to;
187 Teuchos::ArrayRCP<int> starts_to;
188 Teuchos::ArrayRCP<int> indices_to;
190 int max_send_size = 0;
197 for (
int i = 0; i < nprocs; i++) {
198 if (starts[i] != 0) ++nsends;
201 lengths_to.resize(nsends);
202 starts_to.resize(nsends);
203 procs_to.resize(nsends);
207 for (
int i = 0; i < nsends; i++) {
208 starts_to[i] = index;
209 int proc = assign[index];
211 index += starts[proc];
217 sort_ints(procs_to, starts_to);
220 for (
int i = 0; i < nsends; i++) {
221 int proc = procs_to[i];
222 lengths_to[i] = starts[proc];
223 if (proc != my_proc && lengths_to[i] > max_send_size) {
224 max_send_size = lengths_to[i];
230 nsends = (starts[0] != 0);
231 for (
int i = 1; i < nprocs; i++) {
234 starts[i] += starts[i - 1];
237 for (
int i = nprocs - 1; i; i--) {
238 starts[i] = starts[i - 1];
243 indices_to = (nactive > 0) ?
244 Teuchos::arcp(
new int[nactive], 0, nactive,
true) : Teuchos::null;
246 for (
int i = 0; i < nvals; i++) {
247 int proc = assign[i];
249 indices_to[starts[proc]] = i;
256 for (
int i = nprocs - 1; i; i--) {
257 starts[i] = starts[i - 1];
260 starts[nprocs] = nactive;
264 lengths_to.resize(nsends);
265 starts_to.resize(nsends);
266 procs_to.resize(nsends);
270 for (
int i = 0; i < nprocs; i++) {
271 if (starts[i + 1] != starts[i]) {
272 starts_to[j] = starts[i];
273 lengths_to[j] = starts[i + 1] - starts[i];
274 if (i != my_proc && lengths_to[j] > max_send_size) {
275 max_send_size = lengths_to[j];
287 Teuchos::ArrayRCP<int> lengths_from;
288 Teuchos::ArrayRCP<int> procs_from;
291 int comm_flag = invert_map(lengths_to, procs_to, nsends, self_msg,
292 lengths_from, procs_from, &nrecvs, my_proc, nprocs,
293 out_of_mem, tag, comm);
296 Teuchos::ArrayRCP<int> starts_from(
297 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
299 for (
int i = 0; i < nrecvs + self_msg; i++) {
301 j += lengths_from[i];
304 if (comm_flag != 0) {
305 throw std::logic_error(
"Failed to construct Zoltan2_Directory_Comm");
308 int total_recv_size = 0;
309 for (
int i = 0; i < nrecvs + self_msg; i++) {
310 total_recv_size += lengths_from[i];
321 plan_forward->
nvals = nvals;
323 plan_forward->
nrecvs = nrecvs;
324 plan_forward->
nsends = nsends;
329 plan_forward->
comm = comm;
333 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (1)");
340 Teuchos::reduceAll(*comm, Teuchos::REDUCE_MAX, 1, &nrecvs, &global_nrecvs);
351 nrec = total_recv_size;
359 int Zoltan2_Directory_Comm::invert_map(
360 const Teuchos::ArrayRCP<int> &lengths_to,
361 const Teuchos::ArrayRCP<int> &procs_to,
364 Teuchos::ArrayRCP<int> &lengths_from,
365 Teuchos::ArrayRCP<int> &procs_from,
371 Teuchos::RCP<
const Teuchos::Comm<int> > comm)
373 Teuchos::ArrayRCP<int> msg_count(
new int[nprocs], 0, nprocs,
true);
374 Teuchos::ArrayRCP<int> counts(
new int[nprocs], 0, nprocs,
true);
375 for(
int i = 0; i < nprocs; ++i) {
380 for (
int i = 0; i < nsends + self_msg; i++) {
381 msg_count[procs_to[i]] = 1;
394 Teuchos::reduceAll<int>(*comm, Teuchos::REDUCE_SUM, nprocs,
395 msg_count.getRawPtr(), counts.getRawPtr());
399 Teuchos::scatter<int, int>(&(counts[0]), 1, &nrecvs, 1, 0, *comm);
403 for (
int i=0; i < nprocs; i++) {
404 if (counts[i] > max_nrecvs) {
405 max_nrecvs = counts[i];
410 Teuchos::broadcast(*comm, 0, &max_nrecvs);
413 lengths_from.resize(nrecvs);
414 procs_from.resize(nrecvs);
419 Teuchos::ArrayRCP<Teuchos::RCP<Teuchos::CommRequest<int> > > requests(nrecvs);
423 for (
int i=0; i < nrecvs; i++) {
424 #ifdef HAVE_MPI // Teuchos::ireceive not implemented for Serial - Serial is just for debugging
425 Teuchos::ArrayRCP<int> single_elem(&lengths_from[i], 0, 1,
false);
426 requests[i] = Teuchos::ireceive(single_elem, MPI_ANY_SOURCE, tag, *comm);
430 for (
int i=0; i < nsends+self_msg; i++) {
431 #ifdef HAVE_MPI // Teuchos::send not implemented for Serial - Serial is just for debugging
432 Teuchos::send(&lengths_to[i], 1, procs_to[i], tag, *comm);
436 for (
int i=0; i < nrecvs; i++) {
438 procs_from[i] = requests[i]->wait()->getSourceRank();
442 lengths_from[i] = lengths_to[i];
448 Teuchos::ArrayRCP<int> sendbuf(
new int[nprocs], 0, nprocs,
true);
449 Teuchos::ArrayRCP<int> recvbuf(
new int[nprocs], 0, nprocs,
true);
451 for (
int i=0; i < nsends + self_msg; i++) {
452 sendbuf[procs_to[i]] = lengths_to[i];
455 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (2)");
459 for (
int i=0, j=0; i < nprocs; i++) {
461 lengths_from[j] = recvbuf[i];
471 sort_ints(procs_from, lengths_from);
473 *pnrecvs = nrecvs - self_msg;
478 int Zoltan2_Directory_Comm::sort_ints(
479 Teuchos::ArrayRCP<int> &vals_sort,
480 Teuchos::ArrayRCP<int> &vals_other)
483 if (vals_sort == Teuchos::null || vals_sort.size() == 0) {
486 if (vals_other == Teuchos::null || vals_other.size() == 0) {
489 if (vals_sort == Teuchos::null || vals_sort.size() == 1) {
494 int already_sorted = 1;
497 int top = vals_sort[0];
498 for (Teuchos::ArrayRCP<int>::size_type i = 1; i < vals_sort.size(); i++) {
499 if (vals_sort[i-1] > vals_sort[i]) {
502 if (top < vals_sort[i]) {
507 if (already_sorted) {
511 Teuchos::ArrayRCP<int> store(
new int[top+2], 0, top+2,
true);
512 for(
int n = 0; n < store.size(); ++n) {
516 Teuchos::ArrayRCP<int> copy_sort(
new int[vals_sort.size()], 0, vals_sort.size(),
true);
517 for(Teuchos::ArrayRCP<int>::size_type n = 0; n < copy_sort.size(); ++n) {
518 copy_sort[n] = vals_sort[n];
521 Teuchos::ArrayRCP<int> copy_other(
new int[vals_other.size()], 0, vals_other.size(),
true);
522 for(Teuchos::ArrayRCP<int>::size_type n = 0; n < copy_other.size(); ++n) {
523 copy_other[n] = vals_other[n];
528 int *p = &(store[1]);
529 for (Teuchos::ArrayRCP<int>::size_type i = 0; i < vals_sort.size(); i++) {
533 for (
int i = 1; i < top+1; i++) {
538 for (Teuchos::ArrayRCP<int>::size_type i = 0; i < vals_sort.size(); i++) {
539 vals_sort[p[copy_sort[i]]] = copy_sort[i];
540 vals_other[p[copy_sort[i]]] = copy_other[i];
549 const Teuchos::ArrayRCP<char> &send_data,
551 Teuchos::ArrayRCP<char> &recv_data)
556 status = do_post (plan_forward, tag, send_data, nbytes, recv_data);
558 status = do_wait (plan_forward, tag, send_data, nbytes, recv_data);
562 status = do_all_to_all(plan_forward, send_data, nbytes, recv_data);
568 int Zoltan2_Directory_Comm::do_post(
571 const Teuchos::ArrayRCP<char> &send_data,
573 Teuchos::ArrayRCP<char> &recv_data)
577 throw std::logic_error(
"Communication plan = NULL");
582 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (3)");
583 return do_all_to_all(plan, send_data, nbytes, recv_data);
586 int my_proc = plan->
comm->getRank();
588 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (4)");
596 throw std::logic_error(
"nsends not zero, but send_data = NULL");
599 if ((plan->
nrecvs + plan->
self_msg) && recv_data == Teuchos::null) {
600 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (5)");
606 throw std::logic_error(
"nrecvs not zero, but recv_data = NULL");
617 plan->
recv_buff = Teuchos::arcp(
new char[rsize], 0, rsize,
true);
620 size_t self_recv_address = 0;
625 Teuchos::ArrayRCP<char> subview(
632 self_recv_address = (size_t)(plan->
starts_from[i]) * (size_t)nbytes;
641 Teuchos::ArrayRCP<char> subview(
648 plan->
request[k] = Teuchos::null;
659 Teuchos::ArrayRCP<char> send_buff;
673 int global_out_of_mem;
674 Teuchos::reduceAll(*plan->
comm, Teuchos::REDUCE_SUM, 1, &out_of_mem,
684 while (proc_index < nblocks && plan->procs_to[proc_index] < my_proc) {
687 if (proc_index == nblocks) {
694 for (
int i = proc_index, j = 0; j < nblocks; j++) {
696 Teuchos::ArrayRCP<char> subview(
699 Teuchos::readySend(subview.getRawPtr(),
static_cast<int>(subview.size()), plan->
procs_to[i], tag, *comm_);
704 if (++i == nblocks) {
718 send_data.getRawPtr()+(size_t)(plan->
starts_to[self_num])*(size_t)nbytes,
719 (
size_t) (plan->
lengths_to[self_num]) * (
size_t) nbytes);
725 for (
int i = proc_index, jj = 0; jj < nblocks; jj++) {
730 for (
int k = 0; k < plan->
lengths_to[i]; k++) {
731 memcpy(&send_buff[offset],
732 &send_data[(
size_t)(plan->
indices_to[j++]) * (
size_t)nbytes], nbytes);
735 Teuchos::readySend(&send_buff[0], plan->
lengths_to[i] * nbytes,
747 for (
int k = 0; k < plan->
lengths_to[self_num]; k++) {
750 (
size_t)(plan->
indices_to[self_index++]) * (
size_t)nbytes],
752 self_recv_address += nbytes;
760 for (
int i = proc_index, j = 0; j < nblocks; j++) {
775 char* lrecv = &plan->
getRecvBuff().getRawPtr()[self_recv_address];
777 &send_data.getRawPtr()[(size_t)(plan->
starts_to_ptr[self_num]) * (size_t)nbytes];
778 int sindex = plan->
sizes_to[self_num], idx;
779 for (idx=0; idx<nbytes; idx++) {
780 memcpy(lrecv, lsend, sindex);
789 for (
int i = proc_index, jj = 0; jj < nblocks; jj++) {
793 for (
int k = 0; k < plan->
lengths_to[i]; k++) {
795 memcpy(&send_buff[offset],
804 Teuchos::readySend(&send_buff[0], plan->
sizes_to[i] * nbytes,
817 for (
int k = 0; k < plan->
lengths_to[self_num]; k++) {
819 char* lrecv = &(plan->
getRecvBuff())[self_recv_address];
820 size_t send_idx = (size_t)kk * (
size_t)nbytes;
821 const char* lsend = &send_data[send_idx];
823 for (idx=0; idx<nbytes; idx++) {
824 memcpy(lrecv, lsend, sindex);
840 int Zoltan2_Directory_Comm::do_wait(
841 Zoltan2_Directory_Plan *plan,
843 const Teuchos::ArrayRCP<char> &,
845 Teuchos::ArrayRCP<char> &recv_data)
848 if (plan->maxed_recvs){
853 int my_proc = plan->comm->getRank();
857 if (plan->indices_from == Teuchos::null) {
858 if (plan->nrecvs > 0) {
859 Teuchos::waitAll(*comm_, plan->request());
864 size_t offsetDst = 0;
865 if (plan->self_msg) {
866 for (self_num = 0; self_num < plan->nrecvs + plan->self_msg;
868 if (plan->procs_from[self_num] == my_proc) {
873 if(plan->sizes_from.size()) {
880 memcpy(&recv_data[offsetDst * (
size_t)nbytes],
881 &(plan->getRecvBuff())[plan->starts_from_ptr[self_num] * (
size_t)nbytes],
882 plan->sizes_from[self_num] * (size_t)nbytes);
883 offsetDst += plan->sizes_from[self_num];
886 int k = plan->starts_from[self_num];
887 for (
int j = plan->lengths_from[self_num]; j; j--) {
888 memcpy(&recv_data[(
size_t)(plan->indices_from[k]) * (
size_t)nbytes],
889 &(plan->getRecvBuff())[(
size_t)k * (
size_t)nbytes], nbytes);
895 self_num = plan->nrecvs;
898 for (
int jj = 0; jj < plan->nrecvs; jj++) {
903 plan->request[jj]->wait();
911 if (index >= self_num) {
915 if(plan->sizes_from.size()) {
922 memcpy(&recv_data[offsetDst * (
size_t)nbytes],
923 &plan->getRecvBuff().getRawPtr()[plan->starts_from_ptr[index] * (size_t)nbytes],
924 plan->sizes_from[index] * (
size_t)nbytes);
925 offsetDst += plan->sizes_from[index];
928 int k = plan->starts_from[index];
929 for (
int j = plan->lengths_from[index]; j; j--) {
930 memcpy(&recv_data.getRawPtr()[(size_t)(plan->indices_from[k]) * (size_t)nbytes],
931 &plan->getRecvBuff().getRawPtr()[(size_t)k * (
size_t)nbytes], nbytes);
946 int Zoltan2_Directory_Comm::do_all_to_all(
947 Zoltan2_Directory_Plan *plan,
948 const Teuchos::ArrayRCP<char> &send_data,
950 Teuchos::ArrayRCP<char> &recv_data)
952 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (6)");
954 int sm = (plan->self_msg > 0) ? 1 : 0;
956 int nSendMsgs = plan->nsends + sm;
957 int nRecvMsgs = plan->nrecvs + sm;
960 for (
int i=0; i <nSendMsgs; i++) {
961 nSendItems += plan->lengths_to[i];
964 for (
int i=0; i <nRecvMsgs; i++) {
965 nRecvItems += plan->lengths_from[i];
968 int nprocs = plan->comm->getSize();
970 Teuchos::ArrayRCP<int> outbufCounts(
new int[nprocs], 0, nprocs,
true);
971 Teuchos::ArrayRCP<int> outbufOffsets(
new int[nprocs], 0, nprocs,
true);
972 Teuchos::ArrayRCP<int> inbufCounts(
new int[nprocs], 0, nprocs,
true);
973 Teuchos::ArrayRCP<int> inbufOffsets(
new int[nprocs], 0, nprocs,
true);
985 if (plan->indices_to == Teuchos::null){
987 for (
int i=1; i< nSendMsgs; i++){
988 if (plan->starts_to[i] < plan->starts_to[i-1]){
995 Teuchos::ArrayRCP<char> outbuf;
996 Teuchos::ArrayRCP<char> inbuf;
997 Teuchos::ArrayRCP<char> buf;
999 if (plan->sizes_to.size()){
1006 for (
int i = 0; i < nSendMsgs; i++){
1007 outbufLen += plan->sizes_to[i];
1010 if (plan->indices_to != Teuchos::null) {
1014 buf.resize(outbufLen*nbytes);
1015 outbuf.resize(outbufLen*nbytes);
1016 char * pBufPtr = &(outbuf[0]);
1019 for (
int p = 0; p < nprocs; p++) {
1024 if (plan->procs_to[i] == p){
1026 for (
int j=0; j < plan->lengths_to[i]; j++,k++){
1027 int itemSize = plan->sizes[plan->indices_to[k]] * nbytes;
1028 int offset = plan->indices_to_ptr[k] * nbytes;
1030 memcpy(pBufPtr, &(send_data[0]) + offset, itemSize);
1032 pBufPtr += itemSize;
1039 outbufCounts[p] = length;
1041 outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
1050 if (!sorted || (plan->nvals > nSendItems) ){
1051 buf.resize(outbufLen*nbytes);
1052 outbuf.resize(outbufLen*nbytes);
1059 for(
int n = 0; n < outbufLen*nbytes; ++n) {
1060 outbuf[n] = send_data[n];
1064 char * pBufPtr = &(outbuf[0]);
1067 for (
int p = 0; p < nprocs; p++) {
1072 if (plan->procs_to[i] == p){
1073 length = plan->sizes_to[i] * nbytes;
1074 int offset = plan->starts_to_ptr[i] * nbytes;
1076 if ((!sorted || (plan->nvals > nSendItems)) && length){
1077 memcpy(pBufPtr, &(send_data[0]) + offset, length);
1084 outbufCounts[p] = length;
1086 outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
1091 else if (plan->indices_to != Teuchos::null) {
1096 buf.resize(nSendItems*nbytes);
1097 outbuf.resize(nSendItems*nbytes);
1098 char * pBufPtr = &(outbuf[0]);
1101 for (
int p = 0; p < nprocs; p++){
1106 if (plan->procs_to[i] == p){
1107 for (
int j=0; j < plan->lengths_to[i]; j++,k++) {
1108 int offset = plan->indices_to[k] * nbytes;
1109 memcpy(pBufPtr, &(send_data[0]) + offset, nbytes);
1112 length = plan->lengths_to[i] * nbytes;
1117 outbufCounts[p] = length;
1119 outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
1129 if (!sorted || (plan->nvals > nSendItems)){
1130 buf.resize(nSendItems*nbytes);
1131 outbuf.resize(nSendItems*nbytes);
1142 char * pBufPtr = &(outbuf[0]);
1145 for (
int p=0; p < nprocs; p++) {
1150 if (plan->procs_to[i] == p){
1151 int offset = plan->starts_to[i] * nbytes;
1152 length = plan->lengths_to[i] * nbytes;
1154 if ((!sorted || (plan->nvals > nSendItems)) && length){
1155 memcpy(pBufPtr, &(send_data[0]) + offset, length);
1162 outbufCounts[p] = length;
1164 outbufOffsets[p] = outbufOffsets[p-1] + outbufCounts[p-1];
1173 if (plan->indices_from == Teuchos::null) {
1175 for (i=1; i< nRecvMsgs; i++) {
1176 if (plan->starts_from[i] < plan->starts_from[i-1]){
1193 inbuf.resize(plan->total_recv_size * nbytes);
1196 for (
int p = 0; p < nprocs; p++) {
1200 if (plan->procs_from[i] == p){
1202 if (!plan->using_sizes){
1203 length = plan->lengths_from[i] * nbytes;
1206 length = plan->sizes_from[i] * nbytes;
1212 inbufCounts[p] = length;
1214 inbufOffsets[p] = inbufOffsets[p-1] + inbufCounts[p-1];
1227 char * pBufPtr = &(inbuf[0]);
1229 if (!plan->using_sizes){
1233 if (plan->indices_from == Teuchos::null) {
1234 for (i=0; i < nRecvMsgs; i++){
1235 int offset = plan->starts_from[i] * nbytes;
1236 int length = plan->lengths_from[i] * nbytes;
1237 memcpy(&(recv_data[0]) + offset, pBufPtr, length);
1243 for (i=0; i < nRecvMsgs; i++) {
1245 for (
int j=0; j < plan->lengths_from[i]; j++,k++){
1246 int offset = plan->indices_from[k] * nbytes;
1247 memcpy(&(recv_data[0]) + offset, pBufPtr, nbytes);
1257 for (i=0; i < nRecvMsgs; i++){
1258 int offset = plan->starts_from_ptr[i] * nbytes;
1259 int length = plan->sizes_from[i] * nbytes;
1260 memcpy(&(recv_data[0]) + offset, pBufPtr, length);
1271 const Teuchos::ArrayRCP<char> &send_data,
1273 const Teuchos::ArrayRCP<int> &sizes,
1274 Teuchos::ArrayRCP<char> &recv_data)
1278 int status = create_reverse_plan(tag, sizes);
1289 recv_data = Teuchos::arcp(
new char[new_size], 0, new_size,
true);
1296 throw std::logic_error(
"Zoltan2_Directory_Comm.coom untested refactored code (7)");
1302 status = do_all_to_all(plan_forward->
plan_reverse, send_data,
1308 status = do_post(plan_forward->
plan_reverse, tag, send_data,
1312 status = do_wait (plan_forward->
plan_reverse, tag, send_data,
1318 free_reverse_plan(plan_forward);
1326 throw std::logic_error(
"Plan is NULL!");
1332 int Zoltan2_Directory_Comm::create_reverse_plan(
1334 const Teuchos::ArrayRCP<int> &sizes)
1338 throw std::logic_error(
"memory error");
1342 plan_forward->
plan_reverse =
new Zoltan2_Directory_Plan;
1351 Teuchos::reduceAll<int>(*plan_forward->
comm, Teuchos::REDUCE_SUM, 1,
1352 &plan_forward->
nsends, &global_nsends);
1367 sizes, tag, &sum_recv_sizes);
1369 if (comm_flag != 0) {
1382 const Teuchos::ArrayRCP<int> &sizes,
1384 int *sum_recv_sizes)
1386 return resize(plan_forward, sizes, tag, sum_recv_sizes);
1391 const Teuchos::ArrayRCP<int> &sizes,
1393 int *sum_recv_sizes)
1397 int my_proc = plan->
comm->getRank();
1398 int has_sizes = (sizes.size() != 0);
1401 Teuchos::reduceAll(*comm_, Teuchos::REDUCE_BOR, 1, &has_sizes, &var_sizes);
1403 if (var_sizes && plan->
indices_from != Teuchos::null) {
1410 int nsends = plan->
nsends;
1411 int nrecvs = plan->
nrecvs;
1414 Teuchos::ArrayRCP<int> sizes_to;
1415 Teuchos::ArrayRCP<int> sizes_from;
1416 Teuchos::ArrayRCP<int> starts_to_ptr;
1417 Teuchos::ArrayRCP<int> starts_from_ptr;
1418 Teuchos::ArrayRCP<int> indices_to_ptr;
1419 Teuchos::ArrayRCP<int> indices_from_ptr;
1423 for (
int i = 0; i < nrecvs + self_msg; i++) {
1428 for (
int i = 0; i < nsends + self_msg; i++) {
1429 if (plan->
procs_to[i] != my_proc &&
1446 plan->
sizes = sizes;
1449 if(nsends + self_msg > 0) {
1450 sizes_to = Teuchos::arcp(
1451 new int[nsends + self_msg], 0, nsends + self_msg,
true);
1452 for(
int n = 0; n < sizes_to.size(); ++n) {
1456 if(nrecvs + self_msg > 0) {
1457 sizes_from = Teuchos::arcp(
1458 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
1468 if(nsends + self_msg > 0) {
1469 starts_to_ptr = Teuchos::arcp(
1470 new int[nsends + self_msg], 0, nsends + self_msg,
true);
1474 Teuchos::ArrayRCP<int> index;
1475 Teuchos::ArrayRCP<int> sort_val;
1476 if(nsends + self_msg > 0) {
1477 index = Teuchos::arcp(
1478 new int[nsends + self_msg], 0, nsends + self_msg,
true);
1479 sort_val = Teuchos::arcp(
1480 new int[nsends + self_msg], 0, nsends + self_msg,
true);
1482 for (
int i = 0; i < nsends + self_msg; i++) {
1485 for (
int k = 0; k < plan->
lengths_to[i]; k++) {
1486 sizes_to[i] += sizes[j++];
1492 for (
int i = 0; i < nsends + self_msg; i++) {
1496 sort_ints(sort_val, index);
1498 for (
int i = 0; i < nsends + self_msg; i++) {
1499 starts_to_ptr[index[i]] = sum;
1500 sum += sizes_to[index[i]];
1504 Teuchos::ArrayRCP<int> offset;
1505 if(plan->
nvals > 0) {
1506 offset = Teuchos::arcp(
new int[plan->
nvals], 0, plan->
nvals,
true);
1508 indices_to_ptr.resize(plan->
nvals);
1512 for (
int i = 0; i < plan->
nvals; i++) {
1519 for (
int i = 0; i < nsends + self_msg; i++) {
1520 starts_to_ptr[i] = sum;
1522 for (
int k = 0; k < plan->
lengths_to[i]; k++) {
1523 indices_to_ptr[j] = offset[plan->
indices_to[j]];
1536 exchange_sizes(sizes_to, plan->
procs_to, nsends, self_msg,
1540 if(nrecvs + self_msg > 0) {
1541 starts_from_ptr = Teuchos::arcp(
1542 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
1547 Teuchos::ArrayRCP<int> index;
1548 Teuchos::ArrayRCP<int> sort_val;
1549 if(nrecvs + self_msg > 0) {
1550 index = Teuchos::arcp(
1551 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
1552 sort_val = Teuchos::arcp<int>(
1553 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
1556 for (
int i = 0; i < nrecvs + self_msg; i++) {
1560 sort_ints(sort_val, index);
1563 for (
int i = 0; i < nrecvs + self_msg; i++) {
1564 starts_from_ptr[index[i]] = sum;
1565 sum += sizes_from[index[i]];
1585 Teuchos::ArrayRCP<int> index;
1586 Teuchos::ArrayRCP<int> sort_val;
1587 if(nrecvs + self_msg > 0) {
1588 index = Teuchos::arcp(
1589 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
1590 sort_val = Teuchos::arcp(
1591 new int[nrecvs + self_msg], 0, nrecvs + self_msg,
true);
1594 for (
int i = 0; i < nrecvs + self_msg; i++) {
1598 sort_ints(sort_val, index);
1601 for (
int i = 0; i < nrecvs + self_msg; i++) {
1602 starts_from_ptr[index[i]] = sum;
1603 sum += sizes_from[index[i]];
1614 if (sum_recv_sizes) {
1621 int Zoltan2_Directory_Comm::exchange_sizes(
1622 const Teuchos::ArrayRCP<int> &sizes_to,
1623 const Teuchos::ArrayRCP<int> &procs_to,
1626 Teuchos::ArrayRCP<int> &sizes_from,
1627 const Teuchos::ArrayRCP<int> &procs_from,
1629 int *total_recv_size,
1632 Teuchos::RCP<
const Teuchos::Comm<int> > ) {
1635 int self_index_to = -1;
1636 for (
int i = 0; i < nsends + self_msg; i++) {
1637 if (procs_to[i] != my_proc) {
1638 #ifdef HAVE_MPI // Teuchos::send not implemented for Serial - Serial is just for debugging
1639 Teuchos::send(*comm_, 1, &sizes_to[i], procs_to[i]);
1647 *total_recv_size = 0;
1649 for (
int i = 0; i < nrecvs + self_msg; i++) {
1650 if (procs_from[i] != my_proc) {
1651 #ifdef HAVE_MPI // Teuchos::receive not implemented for Serial - Serial is just for debugging
1652 Teuchos::receive(*comm_, procs_from[i], 1, &sizes_from[i]);
1656 sizes_from[i] = sizes_to[self_index_to];
1658 *total_recv_size += sizes_from[i];
Teuchos::ArrayRCP< int > procs_from
Teuchos::ArrayRCP< int > indices_to
Teuchos::ArrayRCP< int > lengths_to
Teuchos::ArrayRCP< char > recv_buff
Teuchos::ArrayRCP< int > indices_from_ptr
Teuchos::ArrayRCP< int > starts_from
void print(const std::string &headerMessage) const
Teuchos::ArrayRCP< int > procs_to
int do_reverse(int tag, const Teuchos::ArrayRCP< char > &send_data, int nbytes, const Teuchos::ArrayRCP< int > &sizes, Teuchos::ArrayRCP< char > &recv_data)
Teuchos::ArrayRCP< char > getRecvBuff() const
~Zoltan2_Directory_Comm()
Teuchos::ArrayRCP< int > starts_to
Teuchos::ArrayRCP< int > indices_to_ptr
Teuchos::ArrayRCP< int > indices_from
Teuchos::ArrayRCP< int > starts_from_ptr
Teuchos::ArrayRCP< int > sizes_from
Teuchos::ArrayRCP< int > sizes_to
int do_forward(int tag, const Teuchos::ArrayRCP< char > &send_data, int nbytes, Teuchos::ArrayRCP< char > &recv_data)
Teuchos::ArrayRCP< Teuchos::RCP< Teuchos::CommRequest< int > > > request
int resize(const Teuchos::ArrayRCP< int > &sizes, int tag, int *sum_recv_sizes)
Teuchos::ArrayRCP< int > lengths_from
Teuchos::ArrayRCP< int > sizes
Zoltan2_Directory_Plan * plan_reverse
Teuchos::RCP< const Teuchos::Comm< int > > comm
Zoltan2_Directory_Comm(int nvals, const Teuchos::ArrayRCP< int > &assign, Teuchos::RCP< const Teuchos::Comm< int > > comm, int tag)
Teuchos::ArrayRCP< int > starts_to_ptr
void getInvertedValues(Zoltan2_Directory_Plan *from)