IFPACK  Development
 All Classes Namespaces Files Functions Variables Enumerations Friends Pages
ExternalRows_dh.c
1 /*@HEADER
2 // ***********************************************************************
3 //
4 // Ifpack: Object-Oriented Algebraic Preconditioner Package
5 // Copyright (2002) Sandia Corporation
6 //
7 // Under terms of Contract DE-AC04-94AL85000, there is a non-exclusive
8 // license for use of this work by or on behalf of the U.S. Government.
9 //
10 // Redistribution and use in source and binary forms, with or without
11 // modification, are permitted provided that the following conditions are
12 // met:
13 //
14 // 1. Redistributions of source code must retain the above copyright
15 // notice, this list of conditions and the following disclaimer.
16 //
17 // 2. Redistributions in binary form must reproduce the above copyright
18 // notice, this list of conditions and the following disclaimer in the
19 // documentation and/or other materials provided with the distribution.
20 //
21 // 3. Neither the name of the Corporation nor the names of the
22 // contributors may be used to endorse or promote products derived from
23 // this software without specific prior written permission.
24 //
25 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
26 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
28 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
29 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
30 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
31 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
32 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
33 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
34 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
35 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
36 //
37 // Questions? Contact Michael A. Heroux (maherou@sandia.gov)
38 //
39 // ***********************************************************************
40 //@HEADER
41 */
42 
43 #include "ExternalRows_dh.h"
44 #include "Factor_dh.h"
45 #include "Euclid_dh.h"
46 #include "SubdomainGraph_dh.h"
47 #include "Mem_dh.h"
48 #include "Parser_dh.h"
49 #include "Hash_dh.h"
50 
51  /* tags for MPI comms */
52 enum
53 { ROW_CT_TAG, NZ_CT_TAG, ROW_LENGTH_TAG, ROW_NUMBER_TAG,
54  CVAL_TAG, FILL_TAG, AVAL_TAG
55 };
56 
57 #undef __FUNC__
58 #define __FUNC__ "ExternalRows_dhCreate"
59 void
60 ExternalRows_dhCreate (ExternalRows_dh * er)
61 {
62  START_FUNC_DH
63  struct _extrows_dh *tmp =
64  (struct _extrows_dh *) MALLOC_DH (sizeof (struct _extrows_dh));
65  CHECK_V_ERROR;
66  *er = tmp;
67 
68  if (MAX_MPI_TASKS < np_dh)
69  {
70  SET_V_ERROR ("MAX_MPI_TASKS is too small; change, then recompile!");
71  }
72 
73  {
74  int i;
75  for (i = 0; i < MAX_MPI_TASKS; ++i)
76  {
77  tmp->rcv_row_lengths[i] = NULL;
78  tmp->rcv_row_numbers[i] = NULL;
79  }
80  }
81 
82  tmp->cvalExt = NULL;
83  tmp->fillExt = NULL;
84  tmp->avalExt = NULL;
85  tmp->my_row_counts = NULL;
86  tmp->my_row_numbers = NULL;
87  tmp->cvalSend = NULL;
88  tmp->fillSend = NULL;
89  tmp->avalSend = NULL;
90  tmp->rowLookup = NULL;
91  tmp->sg = NULL;
92  tmp->F = NULL;
93  tmp->debug = Parser_dhHasSwitch (parser_dh, "-debug_ExtRows");
94 END_FUNC_DH}
95 
96 #undef __FUNC__
97 #define __FUNC__ "ExternalRows_dhDestroy"
98 void
99 ExternalRows_dhDestroy (ExternalRows_dh er)
100 {
101  START_FUNC_DH int i;
102 
103  for (i = 0; i < MAX_MPI_TASKS; ++i)
104  {
105  if (er->rcv_row_lengths[i] != NULL)
106  {
107  FREE_DH (er->rcv_row_lengths[i]);
108  CHECK_V_ERROR;
109  }
110  if (er->rcv_row_numbers[i] != NULL)
111  {
112  FREE_DH (er->rcv_row_numbers[i]);
113  CHECK_V_ERROR;
114  }
115  }
116 
117  if (er->cvalExt != NULL)
118  {
119  FREE_DH (er->cvalExt);
120  CHECK_V_ERROR;
121  }
122  if (er->fillExt != NULL)
123  {
124  FREE_DH (er->fillExt);
125  CHECK_V_ERROR;
126  }
127  if (er->avalExt != NULL)
128  {
129  FREE_DH (er->avalExt);
130  CHECK_V_ERROR;
131  }
132 
133  if (er->my_row_counts != NULL)
134  {
135  FREE_DH (er->my_row_counts);
136  CHECK_V_ERROR;
137  }
138  if (er->my_row_numbers != NULL)
139  {
140  FREE_DH (er->my_row_numbers);
141  CHECK_V_ERROR;
142  }
143 
144  if (er->cvalSend != NULL)
145  {
146  FREE_DH (er->cvalSend);
147  CHECK_V_ERROR;
148  }
149  if (er->fillSend != NULL)
150  {
151  FREE_DH (er->fillSend);
152  CHECK_V_ERROR;
153  }
154  if (er->avalSend != NULL)
155  {
156  FREE_DH (er->avalSend);
157  CHECK_V_ERROR;
158  }
159 
160  if (er->rowLookup != NULL)
161  {
162  Hash_dhDestroy (er->rowLookup);
163  CHECK_V_ERROR;
164  }
165  FREE_DH (er);
166  CHECK_V_ERROR;
167 END_FUNC_DH}
168 
169 #undef __FUNC__
170 #define __FUNC__ "ExternalRows_dhInit"
171 void
172 ExternalRows_dhInit (ExternalRows_dh er, Euclid_dh ctx)
173 {
174  START_FUNC_DH er->sg = ctx->sg;
175  er->F = ctx->F;
176 END_FUNC_DH}
177 
178 /*=====================================================================
179  * method for accessing external rows
180  *=====================================================================*/
181 
182 #undef __FUNC__
183 #define __FUNC__ "ExternalRows_dhGetRow"
184 void
185 ExternalRows_dhGetRow (ExternalRows_dh er, int globalRow,
186  int *len, int **cval, int **fill, REAL_DH ** aval)
187 {
188  START_FUNC_DH if (er->rowLookup == NULL)
189  {
190  *len = 0;
191  }
192 
193  else
194  {
195  HashData *r = NULL;
196  r = Hash_dhLookup (er->rowLookup, globalRow);
197  CHECK_V_ERROR;
198  if (r != NULL)
199  {
200  *len = r->iData;
201  if (cval != NULL)
202  *cval = r->iDataPtr;
203  if (fill != NULL)
204  *fill = r->iDataPtr2;
205  if (aval != NULL)
206  *aval = r->fDataPtr;
207  }
208  else
209  {
210  *len = 0;
211  }
212  }
213 END_FUNC_DH}
214 
215 /*=====================================================================
216  * methods for receiving external rows from lower-ordered subdomains
217  *=====================================================================*/
218 static void rcv_ext_storage_private (ExternalRows_dh extRows);
219 static void build_hash_table_private (ExternalRows_dh er);
220 static void rcv_external_rows_private (ExternalRows_dh er);
221 static void allocate_ext_row_storage_private (ExternalRows_dh er);
222 static void print_received_rows_private (ExternalRows_dh er);
223 
224 #undef __FUNC__
225 #define __FUNC__ "ExternalRows_dhRecvRows"
226 void
227 ExternalRows_dhRecvRows (ExternalRows_dh er)
228 {
229  START_FUNC_DH bool debug = false;
230  if (logFile != NULL && er->debug)
231  debug = true;
232 
233  if (er->sg->loCount > 0)
234  {
235  /* get number of rows and length of each row to be received
236  from each lower ordered nabor.
237  (allocates: *rcv_row_lengths[], *rcv_row_numbers[])
238  */
239  rcv_ext_storage_private (er);
240  CHECK_V_ERROR;
241 
242 
243  /* allocate data structures for receiving the rows (no comms)
244  (allocates: cvalExt, fillExt, avalExt)
245  (no communications)
246  */
247  allocate_ext_row_storage_private (er);
248  CHECK_V_ERROR;
249 
250 
251  /* construct hash table for external row lookup (no comms)
252  (Creates/allocates: rowLookup)
253  (no communications)
254  */
255  build_hash_table_private (er);
256  CHECK_V_ERROR;
257 
258  /* receive the actual row structures and values
259  from lower ordered neighbors
260  */
261  rcv_external_rows_private (er);
262  CHECK_V_ERROR;
263 
264  if (debug)
265  {
266  print_received_rows_private (er);
267  CHECK_V_ERROR;
268  }
269  }
270 END_FUNC_DH}
271 
272 #undef __FUNC__
273 #define __FUNC__ "rcv_ext_storage_private"
274 void
275 rcv_ext_storage_private (ExternalRows_dh er)
276 {
277  START_FUNC_DH int i;
278  int loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
279  int *rcv_row_counts = er->rcv_row_counts;
280  int *rcv_nz_counts = er->rcv_nz_counts;
281  int **lengths = er->rcv_row_lengths, **numbers = er->rcv_row_numbers;
282  bool debug = false;
283 
284  if (logFile != NULL && er->debug)
285  debug = true;
286 
287  /* get number of rows, and total nonzeros, that each lo-nabor will send */
288  for (i = 0; i < loCount; ++i)
289  {
290  int nabor = loNabors[i];
291  MPI_Irecv (rcv_row_counts + i, 1, MPI_INT, nabor, ROW_CT_TAG, comm_dh,
292  er->req1 + i);
293  MPI_Irecv (rcv_nz_counts + i, 1, MPI_INT, nabor, NZ_CT_TAG, comm_dh,
294  er->req2 + i);
295  }
296  MPI_Waitall (loCount, er->req1, er->status);
297  MPI_Waitall (loCount, er->req2, er->status);
298 
299  if (debug)
300  {
301  fprintf (logFile,
302  "\nEXR rcv_ext_storage_private:: <nabor,rowCount,nzCount>\nEXR ");
303  for (i = 0; i < loCount; ++i)
304  {
305  fprintf (logFile, "<%i,%i,%i> ", loNabors[i], rcv_row_counts[i],
306  rcv_nz_counts[i]);
307  }
308  }
309 
310  /* get lengths and global number of each row to be received */
311  for (i = 0; i < loCount; ++i)
312  {
313  int nz = rcv_nz_counts[i];
314  int nabor = loNabors[i];
315  lengths[i] = (int *) MALLOC_DH (nz * sizeof (int));
316  CHECK_V_ERROR;
317  numbers[i] = (int *) MALLOC_DH (nz * sizeof (int));
318  CHECK_V_ERROR;
319  MPI_Irecv (lengths[i], nz, MPI_INT, nabor, ROW_LENGTH_TAG, comm_dh,
320  er->req1 + i);
321  MPI_Irecv (numbers[i], nz, MPI_INT, nabor, ROW_NUMBER_TAG, comm_dh,
322  er->req2 + i);
323  }
324  MPI_Waitall (loCount, er->req1, er->status);
325  MPI_Waitall (loCount, er->req2, er->status);
326 
327  if (debug)
328  {
329  int j, nz;
330  for (i = 0; i < loCount; ++i)
331  {
332  fprintf (logFile,
333  "\nEXR rows <number,length> to be received from P_%i\nEXR ",
334  loNabors[i]);
335  nz = rcv_row_counts[i];
336  for (j = 0; j < nz; ++j)
337  fprintf (logFile, "<%i,%i> ", numbers[i][j], lengths[i][j]);
338  fprintf (logFile, "\n");
339  }
340  }
341 
342 END_FUNC_DH}
343 
344 /* allocates: cvalExt, fillExt, avalExt */
345 #undef __FUNC__
346 #define __FUNC__ "allocate_ext_row_storage_private"
347 void
348 allocate_ext_row_storage_private (ExternalRows_dh er)
349 {
350  START_FUNC_DH int i, nz = 0;
351  int loCount = er->sg->loCount;
352  int *rcv_nz_counts = er->rcv_nz_counts;
353 
354  /* count total number of nonzeros to be received */
355  for (i = 0; i < loCount; ++i)
356  nz += rcv_nz_counts[i];
357 
358  /* allocate buffers */
359  er->cvalExt = (int *) MALLOC_DH (nz * sizeof (int));
360  CHECK_V_ERROR;
361  er->fillExt = (int *) MALLOC_DH (nz * sizeof (int));
362  CHECK_V_ERROR;
363  er->avalExt = (REAL_DH *) MALLOC_DH (nz * sizeof (REAL_DH));
364  CHECK_V_ERROR;
365 END_FUNC_DH}
366 
367 #undef __FUNC__
368 #define __FUNC__ "build_hash_table_private"
369 void
370 build_hash_table_private (ExternalRows_dh er)
371 {
372  START_FUNC_DH int loCount = er->sg->loCount;
373  int i, j, offset, rowCt = 0;
374  Hash_dh table;
375  HashData record;
376  int *extRowCval = er->cvalExt, *extRowFill = er->fillExt;
377  REAL_DH *extRowAval = er->avalExt;
378  int *rcv_row_counts = er->rcv_row_counts;
379  int **rcv_row_numbers = er->rcv_row_numbers;
380  int **rcv_row_lengths = er->rcv_row_lengths;
381 
382  /* count total number of rows to be received */
383  for (i = 0; i < loCount; ++i)
384  rowCt += rcv_row_counts[i];
385 
386  /* build table for looking up external rows */
387  Hash_dhCreate (&table, rowCt);
388  CHECK_V_ERROR;
389  er->rowLookup = table;
390  offset = 0;
391 
392  /* loop over lower ordered nabors in subdomain graph */
393  for (i = 0; i < loCount; ++i)
394  {
395 
396  /* number of rows to be received from nabor(i) */
397  int rowCount = rcv_row_counts[i];
398 
399  /* loop over rows to be received from nabor(i) */
400  for (j = 0; j < rowCount; ++j)
401  {
402 
403  /* insert a record to locate row(j) in the hash table */
404  int row = rcv_row_numbers[i][j];
405  int rowLength = rcv_row_lengths[i][j];
406  record.iData = rowLength;
407  record.iDataPtr = extRowCval + offset;
408  record.iDataPtr2 = extRowFill + offset;
409  record.fDataPtr = extRowAval + offset;
410  Hash_dhInsert (table, row, &record);
411  CHECK_V_ERROR;
412  offset += rowLength;
413  }
414  }
415 END_FUNC_DH}
416 
417 #undef __FUNC__
418 #define __FUNC__ "rcv_external_rows_private"
419 void
420 rcv_external_rows_private (ExternalRows_dh er)
421 {
422  START_FUNC_DH int *rcv_nz_counts = er->rcv_nz_counts;
423  int i, loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
424  int nabor, nz = 0, offset = 0;
425  int *extRowCval = er->cvalExt, *extRowFill = er->fillExt;
426  double *extRowAval = er->avalExt;
427 
428  /* start receives of external rows */
429  nz = 0;
430  for (i = 0; i < loCount; ++i)
431  {
432  nabor = loNabors[i];
433  nz = rcv_nz_counts[i];
434  MPI_Irecv (extRowCval + offset, nz, MPI_INT, nabor, CVAL_TAG, comm_dh,
435  er->req1 + i);
436  MPI_Irecv (extRowFill + offset, nz, MPI_INT, nabor, FILL_TAG, comm_dh,
437  er->req2 + i);
438  MPI_Irecv (extRowAval + offset, nz, MPI_DOUBLE, nabor, AVAL_TAG,
439  comm_dh, er->req3 + i);
440  offset += nz;
441  }
442 
443  /* wait for external rows to arrive */
444  MPI_Waitall (loCount, er->req1, er->status);
445  MPI_Waitall (loCount, er->req2, er->status);
446  MPI_Waitall (loCount, er->req3, er->status);
447 END_FUNC_DH}
448 
449 
450 #undef __FUNC__
451 #define __FUNC__ "print_received_rows_private"
452 void
453 print_received_rows_private (ExternalRows_dh er)
454 {
455  START_FUNC_DH bool noValues = (Parser_dhHasSwitch (parser_dh, "-noValues"));
456  int i, j, k, rwCt, idx = 0, nabor;
457  int loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
458  int n = er->F->n;
459 
460  fprintf (logFile,
461  "\nEXR ================= received rows, printed from buffers ==============\n");
462 
463  /* loop over nabors from whom we received rows */
464  for (i = 0; i < loCount; ++i)
465  {
466  rwCt = er->rcv_row_counts[i];
467  nabor = loNabors[i];
468  fprintf (logFile, "\nEXR Rows received from P_%i:\n", nabor);
469 
470  /* loop over each row to be received from this nabor */
471  for (j = 0; j < rwCt; ++j)
472  {
473  int rowNum = er->rcv_row_numbers[i][j];
474  int rowLen = er->rcv_row_lengths[i][j];
475  fprintf (logFile, "EXR %i :: ", 1 + rowNum);
476  for (k = 0; k < rowLen; ++k)
477  {
478  if (noValues)
479  {
480  fprintf (logFile, "%i,%i ; ", er->cvalExt[idx],
481  er->fillExt[idx]);
482  }
483  else
484  {
485  fprintf (logFile, "%i,%i,%g ; ", er->cvalExt[idx],
486  er->fillExt[idx], er->avalExt[idx]);
487  }
488  ++idx;
489  }
490  fprintf (logFile, "\n");
491  }
492  }
493 
494  fprintf (logFile,
495  "\nEXR =============== received rows, printed from hash table =============\n");
496  for (i = 0; i < n; ++i)
497  {
498  int len, *cval, *fill;
499  REAL_DH *aval;
500  ExternalRows_dhGetRow (er, i, &len, &cval, &fill, &aval);
501  CHECK_V_ERROR;
502  if (len > 0)
503  {
504  fprintf (logFile, "EXR %i :: ", i + 1);
505  for (j = 0; j < len; ++j)
506  {
507  if (noValues)
508  {
509  fprintf (logFile, "%i,%i ; ", cval[j], fill[j]);
510  }
511  else
512  {
513  fprintf (logFile, "%i,%i,%g ; ", cval[j], fill[j], aval[j]);
514  }
515  }
516  fprintf (logFile, "\n");
517  }
518  }
519 
520 END_FUNC_DH}
521 
522 /*=====================================================================
523  * methods for sending rows to higher ordered nabors in subdomain graph
524  *=====================================================================*/
525 
526 static void send_ext_storage_private (ExternalRows_dh er);
527 static void send_external_rows_private (ExternalRows_dh er);
528 static void waitfor_sends_private (ExternalRows_dh er);
529 
530 #undef __FUNC__
531 #define __FUNC__ "ExternalRows_dhSendRows"
532 void
533 ExternalRows_dhSendRows (ExternalRows_dh er)
534 {
535  START_FUNC_DH if (er->sg->hiCount > 0)
536  {
537  /* send number of rows and length of each row to be sent
538  to each higher ordered nabor.
539  */
540  send_ext_storage_private (er);
541  CHECK_V_ERROR;
542 
543  /* send the row's colum indices, fill levels, and values */
544  send_external_rows_private (er);
545  CHECK_V_ERROR;
546 
547  waitfor_sends_private (er);
548  CHECK_V_ERROR;
549  }
550 END_FUNC_DH}
551 
552 
553 #undef __FUNC__
554 #define __FUNC__ "send_ext_storage_private"
555 void
556 send_ext_storage_private (ExternalRows_dh er)
557 {
558  START_FUNC_DH int nz, i, j;
559  int *nzCounts, *nzNumbers;
560  int hiCount = er->sg->hiCount, *hiNabors = er->sg->hiNabors;
561  int *rp = er->F->rp, *diag = er->F->diag;
562  int m = er->F->m;
563  int beg_row = er->F->beg_row;
564  int rowCount = er->F->bdry_count; /* number of boundary rows */
565  int first_bdry = er->F->first_bdry;
566  bool debug = false;
567 
568  if (logFile != NULL && er->debug)
569  debug = true;
570 
571  /* allocate storage to hold nz counts for each row */
572  nzCounts = er->my_row_counts = (int *) MALLOC_DH (rowCount * sizeof (int));
573  CHECK_V_ERROR;
574  nzNumbers = er->my_row_numbers =
575  (int *) MALLOC_DH (rowCount * sizeof (int));
576  CHECK_V_ERROR;
577 
578  /* count nonzeros in upper triangular portion of each boundary row */
579  nz = 0;
580  for (i = first_bdry, j = 0; i < m; ++i, ++j)
581  {
582  int tmp = (rp[i + 1] - diag[i]);
583  nz += tmp;
584  nzCounts[j] = tmp;
585  }
586  er->nzSend = nz;
587 
588  if (debug)
589  {
590  fprintf (logFile, "EXR send_ext_storage_private:: rowCount = %i\n",
591  rowCount);
592  fprintf (logFile, "EXR send_ext_storage_private:: nz Count = %i\n", nz);
593  }
594 
595  /* send number of rows, and total nonzeros, to higher ordered nabors */
596  for (i = 0; i < hiCount; ++i)
597  {
598  int nabor = hiNabors[i];
599  MPI_Isend (&rowCount, 1, MPI_INT, nabor, ROW_CT_TAG, comm_dh,
600  er->req1 + i);
601  MPI_Isend (&nz, 1, MPI_INT, nabor, NZ_CT_TAG, comm_dh, er->req2 + i);
602  }
603 
604  /* set up array for global row numbers */
605  for (i = 0, j = first_bdry; j < m; ++i, ++j)
606  {
607  nzNumbers[i] = j + beg_row;
608  }
609 
610  /* start sends of length and global number of each of this processor's
611  boundary row to higher ordered nabors; the receiving processor will
612  use this information to allocate storage buffers for the actual
613  row structures and values.
614  */
615  for (i = 0; i < hiCount; ++i)
616  {
617  int nabor = hiNabors[i];
618  MPI_Isend (nzNumbers, rowCount, MPI_INT, nabor, ROW_NUMBER_TAG, comm_dh,
619  er->req3 + i);
620  MPI_Isend (nzCounts, rowCount, MPI_INT, nabor, ROW_LENGTH_TAG, comm_dh,
621  er->req4 + i);
622  }
623 
624 END_FUNC_DH}
625 
626 
627 #undef __FUNC__
628 #define __FUNC__ "send_external_rows_private"
629 void
630 send_external_rows_private (ExternalRows_dh er)
631 {
632  START_FUNC_DH
633  int i, j, hiCount = er->sg->hiCount, *hiNabors = er->sg->hiNabors;
634  int offset, nz = er->nzSend;
635  int *cvalSend, *fillSend;
636  REAL_DH *avalSend;
637  int *cval = er->F->cval, *fill = er->F->fill;
638  int m = er->F->m;
639  int *rp = er->F->rp, *diag = er->F->diag;
640  int first_bdry = er->F->first_bdry;
641  REAL_DH *aval = er->F->aval;
642  bool debug = false;
643 
644  if (logFile != NULL && er->debug)
645  debug = true;
646 
647  /* allocate buffers to hold upper triangular portion of boundary rows */
648  cvalSend = er->cvalSend = (int *) MALLOC_DH (nz * sizeof (int));
649  CHECK_V_ERROR;
650  fillSend = er->fillSend = (int *) MALLOC_DH (nz * sizeof (int));
651  CHECK_V_ERROR;
652  avalSend = er->avalSend = (double *) MALLOC_DH (nz * sizeof (double));
653  CHECK_V_ERROR;
654 
655  /* copy upper triangular portion of boundary rows int send buffers */
656  offset = 0;
657  for (i = first_bdry, j = 0; i < m; ++i, ++j)
658  {
659  int tmp = (rp[i + 1] - diag[i]);
660 
661  memcpy (cvalSend + offset, cval + diag[i], tmp * sizeof (int));
662  memcpy (fillSend + offset, fill + diag[i], tmp * sizeof (int));
663  memcpy (avalSend + offset, aval + diag[i], tmp * sizeof (double));
664  offset += tmp;
665  }
666 
667  if (debug)
668  {
669  int beg_row = er->F->beg_row;
670  int idx = 0;
671  bool noValues = (Parser_dhHasSwitch (parser_dh, "-noValues"));
672 
673  fprintf (logFile,
674  "\nEXR ======================= send buffers ======================\n");
675 
676  for (i = first_bdry, j = 0; i < m; ++i, ++j)
677  {
678  int tmp = (rp[i + 1] - diag[i]);
679  fprintf (logFile, "EXR %i :: ", i + beg_row);
680 
681  for (j = 0; j < tmp; ++j)
682  {
683  if (noValues)
684  {
685  fprintf (logFile, "%i,%i ; ", cvalSend[idx], fillSend[idx]);
686  }
687  else
688  {
689  fprintf (logFile, "%i,%i,%g ; ", cvalSend[idx],
690  fillSend[idx], avalSend[idx]);
691  }
692  ++idx;
693  }
694  fprintf (logFile, "\n");
695  }
696  }
697 
698  /* start sends to higher-ordred nabors */
699  for (i = 0; i < hiCount; ++i)
700  {
701  int nabor = hiNabors[i];
702  MPI_Isend (cvalSend, nz, MPI_INT, nabor, CVAL_TAG, comm_dh,
703  er->cval_req + i);
704  MPI_Isend (fillSend, nz, MPI_INT, nabor, FILL_TAG, comm_dh,
705  er->fill_req + i);
706  MPI_Isend (avalSend, nz, MPI_DOUBLE, nabor, AVAL_TAG, comm_dh,
707  er->aval_req + i);
708  }
709 END_FUNC_DH}
710 
711 
712 #undef __FUNC__
713 #define __FUNC__ "waitfor_sends_private"
714 void
715 waitfor_sends_private (ExternalRows_dh er)
716 {
717  START_FUNC_DH MPI_Status * status = er->status;
718  int hiCount = er->sg->hiCount;
719 
720  if (hiCount)
721  {
722  MPI_Waitall (hiCount, er->req1, status);
723  MPI_Waitall (hiCount, er->req2, status);
724  MPI_Waitall (hiCount, er->req3, status);
725  MPI_Waitall (hiCount, er->req4, status);
726  MPI_Waitall (hiCount, er->cval_req, status);
727  MPI_Waitall (hiCount, er->fill_req, status);
728  MPI_Waitall (hiCount, er->aval_req, status);
729  }
730 END_FUNC_DH}