43 #include "ExternalRows_dh.h"
44 #include "Factor_dh.h"
45 #include "Euclid_dh.h"
46 #include "SubdomainGraph_dh.h"
48 #include "Parser_dh.h"
53 { ROW_CT_TAG, NZ_CT_TAG, ROW_LENGTH_TAG, ROW_NUMBER_TAG,
54 CVAL_TAG, FILL_TAG, AVAL_TAG
58 #define __FUNC__ "ExternalRows_dhCreate"
68 if (MAX_MPI_TASKS < np_dh)
70 SET_V_ERROR (
"MAX_MPI_TASKS is too small; change, then recompile!");
75 for (i = 0; i < MAX_MPI_TASKS; ++i)
77 tmp->rcv_row_lengths[i] = NULL;
78 tmp->rcv_row_numbers[i] = NULL;
85 tmp->my_row_counts = NULL;
86 tmp->my_row_numbers = NULL;
90 tmp->rowLookup = NULL;
93 tmp->debug = Parser_dhHasSwitch (parser_dh,
"-debug_ExtRows");
97 #define __FUNC__ "ExternalRows_dhDestroy"
103 for (i = 0; i < MAX_MPI_TASKS; ++i)
105 if (er->rcv_row_lengths[i] != NULL)
107 FREE_DH (er->rcv_row_lengths[i]);
110 if (er->rcv_row_numbers[i] != NULL)
112 FREE_DH (er->rcv_row_numbers[i]);
117 if (er->cvalExt != NULL)
119 FREE_DH (er->cvalExt);
122 if (er->fillExt != NULL)
124 FREE_DH (er->fillExt);
127 if (er->avalExt != NULL)
129 FREE_DH (er->avalExt);
133 if (er->my_row_counts != NULL)
135 FREE_DH (er->my_row_counts);
138 if (er->my_row_numbers != NULL)
140 FREE_DH (er->my_row_numbers);
144 if (er->cvalSend != NULL)
146 FREE_DH (er->cvalSend);
149 if (er->fillSend != NULL)
151 FREE_DH (er->fillSend);
154 if (er->avalSend != NULL)
156 FREE_DH (er->avalSend);
160 if (er->rowLookup != NULL)
162 Hash_dhDestroy (er->rowLookup);
170 #define __FUNC__ "ExternalRows_dhInit"
174 START_FUNC_DH er->sg = ctx->sg;
183 #define __FUNC__ "ExternalRows_dhGetRow"
186 int *len,
int **cval,
int **fill, REAL_DH ** aval)
188 START_FUNC_DH
if (er->rowLookup == NULL)
196 r = Hash_dhLookup (er->rowLookup, globalRow);
204 *fill = r->iDataPtr2;
225 #define __FUNC__ "ExternalRows_dhRecvRows"
229 START_FUNC_DH
bool debug =
false;
230 if (logFile != NULL && er->debug)
233 if (er->sg->loCount > 0)
239 rcv_ext_storage_private (er);
247 allocate_ext_row_storage_private (er);
255 build_hash_table_private (er);
261 rcv_external_rows_private (er);
266 print_received_rows_private (er);
273 #define __FUNC__ "rcv_ext_storage_private"
278 int loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
279 int *rcv_row_counts = er->rcv_row_counts;
280 int *rcv_nz_counts = er->rcv_nz_counts;
281 int **lengths = er->rcv_row_lengths, **numbers = er->rcv_row_numbers;
284 if (logFile != NULL && er->debug)
288 for (i = 0; i < loCount; ++i)
290 int nabor = loNabors[i];
291 MPI_Irecv (rcv_row_counts + i, 1, MPI_INT, nabor, ROW_CT_TAG, comm_dh,
293 MPI_Irecv (rcv_nz_counts + i, 1, MPI_INT, nabor, NZ_CT_TAG, comm_dh,
296 MPI_Waitall (loCount, er->req1, er->status);
297 MPI_Waitall (loCount, er->req2, er->status);
302 "\nEXR rcv_ext_storage_private:: <nabor,rowCount,nzCount>\nEXR ");
303 for (i = 0; i < loCount; ++i)
305 fprintf (logFile,
"<%i,%i,%i> ", loNabors[i], rcv_row_counts[i],
311 for (i = 0; i < loCount; ++i)
313 int nz = rcv_nz_counts[i];
314 int nabor = loNabors[i];
315 lengths[i] = (
int *) MALLOC_DH (nz *
sizeof (
int));
317 numbers[i] = (
int *) MALLOC_DH (nz *
sizeof (
int));
319 MPI_Irecv (lengths[i], nz, MPI_INT, nabor, ROW_LENGTH_TAG, comm_dh,
321 MPI_Irecv (numbers[i], nz, MPI_INT, nabor, ROW_NUMBER_TAG, comm_dh,
324 MPI_Waitall (loCount, er->req1, er->status);
325 MPI_Waitall (loCount, er->req2, er->status);
330 for (i = 0; i < loCount; ++i)
333 "\nEXR rows <number,length> to be received from P_%i\nEXR ",
335 nz = rcv_row_counts[i];
336 for (j = 0; j < nz; ++j)
337 fprintf (logFile,
"<%i,%i> ", numbers[i][j], lengths[i][j]);
338 fprintf (logFile,
"\n");
346 #define __FUNC__ "allocate_ext_row_storage_private"
350 START_FUNC_DH
int i, nz = 0;
351 int loCount = er->sg->loCount;
352 int *rcv_nz_counts = er->rcv_nz_counts;
355 for (i = 0; i < loCount; ++i)
356 nz += rcv_nz_counts[i];
359 er->cvalExt = (
int *) MALLOC_DH (nz *
sizeof (
int));
361 er->fillExt = (
int *) MALLOC_DH (nz *
sizeof (
int));
363 er->avalExt = (REAL_DH *) MALLOC_DH (nz *
sizeof (REAL_DH));
368 #define __FUNC__ "build_hash_table_private"
372 START_FUNC_DH
int loCount = er->sg->loCount;
373 int i, j, offset, rowCt = 0;
376 int *extRowCval = er->cvalExt, *extRowFill = er->fillExt;
377 REAL_DH *extRowAval = er->avalExt;
378 int *rcv_row_counts = er->rcv_row_counts;
379 int **rcv_row_numbers = er->rcv_row_numbers;
380 int **rcv_row_lengths = er->rcv_row_lengths;
383 for (i = 0; i < loCount; ++i)
384 rowCt += rcv_row_counts[i];
387 Hash_dhCreate (&table, rowCt);
389 er->rowLookup = table;
393 for (i = 0; i < loCount; ++i)
397 int rowCount = rcv_row_counts[i];
400 for (j = 0; j < rowCount; ++j)
404 int row = rcv_row_numbers[i][j];
405 int rowLength = rcv_row_lengths[i][j];
406 record.iData = rowLength;
407 record.iDataPtr = extRowCval + offset;
408 record.iDataPtr2 = extRowFill + offset;
409 record.fDataPtr = extRowAval + offset;
410 Hash_dhInsert (table, row, &record);
418 #define __FUNC__ "rcv_external_rows_private"
422 START_FUNC_DH
int *rcv_nz_counts = er->rcv_nz_counts;
423 int i, loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
424 int nabor, nz = 0, offset = 0;
425 int *extRowCval = er->cvalExt, *extRowFill = er->fillExt;
426 double *extRowAval = er->avalExt;
430 for (i = 0; i < loCount; ++i)
433 nz = rcv_nz_counts[i];
434 MPI_Irecv (extRowCval + offset, nz, MPI_INT, nabor, CVAL_TAG, comm_dh,
436 MPI_Irecv (extRowFill + offset, nz, MPI_INT, nabor, FILL_TAG, comm_dh,
438 MPI_Irecv (extRowAval + offset, nz, MPI_DOUBLE, nabor, AVAL_TAG,
439 comm_dh, er->req3 + i);
444 MPI_Waitall (loCount, er->req1, er->status);
445 MPI_Waitall (loCount, er->req2, er->status);
446 MPI_Waitall (loCount, er->req3, er->status);
451 #define __FUNC__ "print_received_rows_private"
455 START_FUNC_DH
bool noValues = (Parser_dhHasSwitch (parser_dh,
"-noValues"));
456 int i, j, k, rwCt, idx = 0, nabor;
457 int loCount = er->sg->loCount, *loNabors = er->sg->loNabors;
461 "\nEXR ================= received rows, printed from buffers ==============\n");
464 for (i = 0; i < loCount; ++i)
466 rwCt = er->rcv_row_counts[i];
468 fprintf (logFile,
"\nEXR Rows received from P_%i:\n", nabor);
471 for (j = 0; j < rwCt; ++j)
473 int rowNum = er->rcv_row_numbers[i][j];
474 int rowLen = er->rcv_row_lengths[i][j];
475 fprintf (logFile,
"EXR %i :: ", 1 + rowNum);
476 for (k = 0; k < rowLen; ++k)
480 fprintf (logFile,
"%i,%i ; ", er->cvalExt[idx],
485 fprintf (logFile,
"%i,%i,%g ; ", er->cvalExt[idx],
486 er->fillExt[idx], er->avalExt[idx]);
490 fprintf (logFile,
"\n");
495 "\nEXR =============== received rows, printed from hash table =============\n");
496 for (i = 0; i < n; ++i)
498 int len, *cval, *fill;
500 ExternalRows_dhGetRow (er, i, &len, &cval, &fill, &aval);
504 fprintf (logFile,
"EXR %i :: ", i + 1);
505 for (j = 0; j < len; ++j)
509 fprintf (logFile,
"%i,%i ; ", cval[j], fill[j]);
513 fprintf (logFile,
"%i,%i,%g ; ", cval[j], fill[j], aval[j]);
516 fprintf (logFile,
"\n");
531 #define __FUNC__ "ExternalRows_dhSendRows"
535 START_FUNC_DH
if (er->sg->hiCount > 0)
540 send_ext_storage_private (er);
544 send_external_rows_private (er);
547 waitfor_sends_private (er);
554 #define __FUNC__ "send_ext_storage_private"
558 START_FUNC_DH
int nz, i, j;
559 int *nzCounts, *nzNumbers;
560 int hiCount = er->sg->hiCount, *hiNabors = er->sg->hiNabors;
561 int *rp = er->F->rp, *diag = er->F->diag;
563 int beg_row = er->F->beg_row;
564 int rowCount = er->F->bdry_count;
565 int first_bdry = er->F->first_bdry;
568 if (logFile != NULL && er->debug)
572 nzCounts = er->my_row_counts = (
int *) MALLOC_DH (rowCount *
sizeof (
int));
574 nzNumbers = er->my_row_numbers =
575 (
int *) MALLOC_DH (rowCount *
sizeof (
int));
580 for (i = first_bdry, j = 0; i < m; ++i, ++j)
582 int tmp = (rp[i + 1] - diag[i]);
590 fprintf (logFile,
"EXR send_ext_storage_private:: rowCount = %i\n",
592 fprintf (logFile,
"EXR send_ext_storage_private:: nz Count = %i\n", nz);
596 for (i = 0; i < hiCount; ++i)
598 int nabor = hiNabors[i];
599 MPI_Isend (&rowCount, 1, MPI_INT, nabor, ROW_CT_TAG, comm_dh,
601 MPI_Isend (&nz, 1, MPI_INT, nabor, NZ_CT_TAG, comm_dh, er->req2 + i);
605 for (i = 0, j = first_bdry; j < m; ++i, ++j)
607 nzNumbers[i] = j + beg_row;
615 for (i = 0; i < hiCount; ++i)
617 int nabor = hiNabors[i];
618 MPI_Isend (nzNumbers, rowCount, MPI_INT, nabor, ROW_NUMBER_TAG, comm_dh,
620 MPI_Isend (nzCounts, rowCount, MPI_INT, nabor, ROW_LENGTH_TAG, comm_dh,
628 #define __FUNC__ "send_external_rows_private"
633 int i, j, hiCount = er->sg->hiCount, *hiNabors = er->sg->hiNabors;
634 int offset, nz = er->nzSend;
635 int *cvalSend, *fillSend;
637 int *cval = er->F->cval, *fill = er->F->fill;
639 int *rp = er->F->rp, *diag = er->F->diag;
640 int first_bdry = er->F->first_bdry;
641 REAL_DH *aval = er->F->aval;
644 if (logFile != NULL && er->debug)
648 cvalSend = er->cvalSend = (
int *) MALLOC_DH (nz *
sizeof (
int));
650 fillSend = er->fillSend = (
int *) MALLOC_DH (nz *
sizeof (
int));
652 avalSend = er->avalSend = (
double *) MALLOC_DH (nz *
sizeof (
double));
657 for (i = first_bdry, j = 0; i < m; ++i, ++j)
659 int tmp = (rp[i + 1] - diag[i]);
661 memcpy (cvalSend + offset, cval + diag[i], tmp *
sizeof (
int));
662 memcpy (fillSend + offset, fill + diag[i], tmp *
sizeof (
int));
663 memcpy (avalSend + offset, aval + diag[i], tmp *
sizeof (
double));
669 int beg_row = er->F->beg_row;
671 bool noValues = (Parser_dhHasSwitch (parser_dh,
"-noValues"));
674 "\nEXR ======================= send buffers ======================\n");
676 for (i = first_bdry, j = 0; i < m; ++i, ++j)
678 int tmp = (rp[i + 1] - diag[i]);
679 fprintf (logFile,
"EXR %i :: ", i + beg_row);
681 for (j = 0; j < tmp; ++j)
685 fprintf (logFile,
"%i,%i ; ", cvalSend[idx], fillSend[idx]);
689 fprintf (logFile,
"%i,%i,%g ; ", cvalSend[idx],
690 fillSend[idx], avalSend[idx]);
694 fprintf (logFile,
"\n");
699 for (i = 0; i < hiCount; ++i)
701 int nabor = hiNabors[i];
702 MPI_Isend (cvalSend, nz, MPI_INT, nabor, CVAL_TAG, comm_dh,
704 MPI_Isend (fillSend, nz, MPI_INT, nabor, FILL_TAG, comm_dh,
706 MPI_Isend (avalSend, nz, MPI_DOUBLE, nabor, AVAL_TAG, comm_dh,
713 #define __FUNC__ "waitfor_sends_private"
717 START_FUNC_DH MPI_Status * status = er->status;
718 int hiCount = er->sg->hiCount;
722 MPI_Waitall (hiCount, er->req1, status);
723 MPI_Waitall (hiCount, er->req2, status);
724 MPI_Waitall (hiCount, er->req3, status);
725 MPI_Waitall (hiCount, er->req4, status);
726 MPI_Waitall (hiCount, er->cval_req, status);
727 MPI_Waitall (hiCount, er->fill_req, status);
728 MPI_Waitall (hiCount, er->aval_req, status);