43 #include "Euclid_dh.h"
47 #include "Factor_dh.h"
48 #include "getRow_dh.h"
50 #include "Parser_dh.h"
51 #include "SortedList_dh.h"
52 #include "SubdomainGraph_dh.h"
53 #include "ExternalRows_dh.h"
54 #include "krylov_dh.h"
56 static void get_runtime_params_private (
Euclid_dh ctx);
57 static void invert_diagonals_private (
Euclid_dh ctx);
58 static void compute_rho_private (
Euclid_dh ctx);
59 static void factor_private (
Euclid_dh ctx);
61 static void reduce_timings_private (
Euclid_dh ctx);
64 #define __FUNC__ "Euclid_dhCreate"
88 ctx->isScaled =
false;
94 strcpy (ctx->algo_par,
"pilu");
95 strcpy (ctx->algo_ilu,
"iluk");
97 ctx->droptol = DEFAULT_DROP_TOL;
98 ctx->sparseTolA = 0.0;
99 ctx->sparseTolF = 0.0;
101 ctx->pivotFix = PIVOT_FIX_DEFAULT;
107 strcpy (ctx->krylovMethod,
"bicgstab");
115 ctx->printStats = (Parser_dhHasSwitch (parser_dh,
"-printStats"));
119 for (i = 0; i < TIMING_BINS; ++i)
120 ctx->timing[i] = 0.0;
121 for (i = 0; i < STATS_BINS; ++i)
124 ctx->timingsWereReduced =
false;
131 #define __FUNC__ "Euclid_dhDestroy"
136 if (Parser_dhHasSwitch (parser_dh,
"-eu_stats") || ctx->logging)
139 Parser_dhInsert (parser_dh,
"-eu_mem",
"1");
141 Euclid_dhPrintHypreReport (ctx, stdout);
145 if (ctx->setupCount > 1 && ctx->printStats)
147 Euclid_dhPrintStatsShorter (ctx, stdout);
153 Factor_dhDestroy (ctx->F);
158 SubdomainGraph_dhDestroy (ctx->sg);
161 if (ctx->scale != NULL)
163 FREE_DH (ctx->scale);
166 if (ctx->work != NULL)
171 if (ctx->work2 != NULL)
173 FREE_DH (ctx->work2);
176 if (ctx->slist != NULL)
178 SortedList_dhDestroy (ctx->slist);
181 if (ctx->extRows != NULL)
183 ExternalRows_dhDestroy (ctx->extRows);
197 #define __FUNC__ "Euclid_dhSetup"
201 START_FUNC_DH
int m, n, beg_row;
203 bool isSetup = ctx->isSetup;
210 if (ctx->setupCount && ctx->printStats)
212 Euclid_dhPrintStatsShorter (ctx, stdout);
222 for (i = 0; i < STATS_BINS; ++i)
229 ctx->timing[SOLVE_START_T] = MPI_Wtime ();
231 ctx->timing[TOTAL_SOLVE_T] += ctx->timing[TOTAL_SOLVE_TEMP_T];
232 ctx->timing[TOTAL_SOLVE_TEMP_T] = 0.0;
236 Factor_dhDestroy (ctx->F);
243 SET_V_ERROR (
"must set ctx->A before calling init");
245 EuclidGetDimensions (ctx->A, &beg_row, &m, &n);
250 if (Parser_dhHasSwitch (parser_dh,
"-print_size"))
253 (
"setting up linear system; global rows: %i local rows: %i (on P_0)\n",
257 sprintf (msgBuf_dh,
"localRow= %i; globalRows= %i; beg_row= %i", m, n,
259 SET_INFO (msgBuf_dh);
261 bj = Parser_dhHasSwitch (parser_dh,
"-bj");
275 Parser_dhReadInt (parser_dh,
"-blocks", &blocks);
277 SubdomainGraph_dhCreate (&(ctx->sg));
279 SubdomainGraph_dhInit (ctx->sg, blocks, bj, ctx->A);
284 SubdomainGraph_dhCreate (&(ctx->sg));
286 SubdomainGraph_dhInit (ctx->sg, -1, bj, ctx->A);
289 ctx->timing[SUB_GRAPH_T] += (MPI_Wtime () - t1);
298 if (Parser_dhHasSwitch (parser_dh,
"-doNotFactor"))
300 goto END_OF_FUNCTION;
309 get_runtime_params_private (ctx);
312 if (!strcmp (ctx->algo_par,
"bj"))
319 if (ctx->scale == NULL)
321 ctx->scale = (REAL_DH *) MALLOC_DH (m *
sizeof (REAL_DH));
326 for (i = 0; i < m; ++i)
333 if (ctx->work == NULL)
335 ctx->work = (REAL_DH *) MALLOC_DH (m *
sizeof (REAL_DH));
338 if (ctx->work2 == NULL)
340 ctx->work2 = (REAL_DH *) MALLOC_DH (m *
sizeof (REAL_DH));
349 factor_private (ctx);
351 ctx->timing[FACTOR_T] += (MPI_Wtime () - t1);
356 if (strcmp (ctx->algo_par,
"none"))
358 invert_diagonals_private (ctx);
370 if (Parser_dhHasSwitch (parser_dh,
"-computeRho") || np_dh == 1)
372 if (strcmp (ctx->algo_par,
"none"))
375 compute_rho_private (ctx);
377 ctx->timing[COMPUTE_RHO_T] += (MPI_Wtime () - t1);
387 if (!strcmp (ctx->algo_par,
"pilu") && np_dh > 1)
390 Factor_dhSolveSetup (ctx->F, ctx->sg);
392 ctx->timing[SOLVE_SETUP_T] += (MPI_Wtime () - t1);
400 ctx->timing[SETUP_T] += (MPI_Wtime () - ctx->timing[SOLVE_START_T]);
401 ctx->setupCount += 1;
409 #define __FUNC__ "get_runtime_params_private"
411 get_runtime_params_private (
Euclid_dh ctx)
413 START_FUNC_DH
char *tmp;
416 Parser_dhReadInt (parser_dh,
"-maxIts", &(ctx->maxIts));
417 Parser_dhReadDouble (parser_dh,
"-rtol", &(ctx->rtol));
418 Parser_dhReadDouble (parser_dh,
"-atol", &(ctx->atol));
422 Parser_dhReadString (parser_dh,
"-par", &tmp);
425 strcpy (ctx->algo_par, tmp);
427 if (Parser_dhHasSwitch (parser_dh,
"-bj"))
429 strcpy (ctx->algo_par,
"bj");
434 Parser_dhReadDouble (parser_dh,
"-rho", &(ctx->rho_init));
436 Parser_dhReadInt (parser_dh,
"-level", &ctx->level);
437 Parser_dhReadInt (parser_dh,
"-pc_ilu_levels", &ctx->level);
439 if (Parser_dhHasSwitch (parser_dh,
"-ilut"))
441 Parser_dhReadDouble (parser_dh,
"-ilut", &ctx->droptol);
442 ctx->isScaled =
true;
443 strcpy (ctx->algo_ilu,
"ilut");
449 if (!strcmp (ctx->algo_par,
"none"))
451 strcpy (ctx->algo_ilu,
"none");
453 else if (!strcmp (ctx->algo_ilu,
"none"))
455 strcpy (ctx->algo_par,
"none");
459 Parser_dhReadDouble (parser_dh,
"-sparseA", &(ctx->sparseTolA));
461 Parser_dhReadDouble (parser_dh,
"-sparseF", &(ctx->sparseTolF));
463 Parser_dhReadDouble (parser_dh,
"-pivotMin", &(ctx->pivotMin));
465 Parser_dhReadDouble (parser_dh,
"-pivotFix", &(ctx->pivotFix));
469 if (ctx->sparseTolA || !strcmp (ctx->algo_ilu,
"ilut"))
471 ctx->isScaled =
true;
476 Parser_dhReadString (parser_dh,
"-ksp_type", &tmp);
479 strcpy (ctx->krylovMethod, tmp);
481 if (!strcmp (ctx->krylovMethod,
"bcgs"))
483 strcpy (ctx->krylovMethod,
"bicgstab");
489 #define __FUNC__ "invert_diagonals_private"
493 START_FUNC_DH REAL_DH * aval = ctx->F->aval;
494 int *diag = ctx->F->diag;
495 if (aval == NULL || diag == NULL)
497 SET_INFO (
"can't invert diags; either F->aval or F->diag is NULL");
501 int i, m = ctx->F->m;
502 for (i = 0; i < m; ++i)
504 aval[diag[i]] = 1.0 / aval[diag[i]];
512 #define __FUNC__ "compute_rho_private"
516 START_FUNC_DH
if (ctx->F != NULL)
518 double bufLocal[3], bufGlobal[3];
521 ctx->stats[NZF_STATS] = (double) ctx->F->rp[m];
522 bufLocal[0] = ctx->stats[NZA_STATS];
523 bufLocal[1] = ctx->stats[NZF_STATS];
524 bufLocal[2] = ctx->stats[NZA_USED_STATS];
528 bufGlobal[0] = bufLocal[0];
529 bufGlobal[1] = bufLocal[1];
530 bufGlobal[2] = bufLocal[2];
534 MPI_Reduce (bufLocal, bufGlobal, 3, MPI_DOUBLE, MPI_SUM, 0,
542 if (bufGlobal[0] && bufGlobal[1])
544 ctx->rho_final = bufGlobal[1] / bufGlobal[0];
552 if (bufGlobal[0] && bufGlobal[2])
554 ctx->stats[NZA_RATIO_STATS] =
555 100.0 * bufGlobal[2] / bufGlobal[0];
559 ctx->stats[NZA_RATIO_STATS] = 100.0;
566 #define __FUNC__ "factor_private"
574 if (!strcmp (ctx->algo_par,
"none"))
587 br = ctx->sg->beg_rowP[myid_dh];
588 id = ctx->sg->o2n_sub[myid_dh];
590 Factor_dhInit (ctx->A,
true,
true, ctx->rho_init,
id, br, &(ctx->F));
592 ctx->F->bdry_count = ctx->sg->bdry_count[myid_dh];
593 ctx->F->first_bdry = ctx->F->m - ctx->F->bdry_count;
594 if (!strcmp (ctx->algo_par,
"bj"))
595 ctx->F->blockJacobi =
true;
596 if (Parser_dhHasSwitch (parser_dh,
"-bj"))
597 ctx->F->blockJacobi =
true;
607 if (!strcmp (ctx->algo_ilu,
"iluk"))
613 if (Parser_dhHasSwitch (parser_dh,
"-mpi"))
615 if (ctx->sg != NULL && ctx->sg->blocks > 1)
618 (
"only use -mpi, which invokes ilu_mpi_pilu(), for np = 1 and -blocks 1");
627 iluk_seq_block (ctx);
634 else if (!strcmp (ctx->algo_ilu,
"ilut"))
645 sprintf (msgBuf_dh,
"factorization method: %s is not implemented",
647 SET_V_ERROR (msgBuf_dh);
657 if (!strcmp (ctx->algo_par,
"bj"))
666 else if (!strcmp (ctx->algo_ilu,
"iluk"))
668 bool bj = ctx->F->blockJacobi;
672 SortedList_dhCreate (&(ctx->slist));
674 SortedList_dhInit (ctx->slist, ctx->sg);
676 ExternalRows_dhCreate (&(ctx->extRows));
678 ExternalRows_dhInit (ctx->extRows, ctx);
683 ctx->to = ctx->F->first_bdry;
703 ExternalRows_dhRecvRows (ctx->extRows);
708 ctx->from = ctx->F->first_bdry;
718 ExternalRows_dhSendRows (ctx->extRows);
728 SortedList_dhDestroy (ctx->slist);
731 ExternalRows_dhDestroy (ctx->extRows);
739 sprintf (msgBuf_dh,
"factorization method: %s is not implemented",
741 SET_V_ERROR (msgBuf_dh);
752 #define __FUNC__ "discard_indices_private"
758 int *rp = ctx->F->rp, *cval = ctx->F->cval;
759 double *aval = ctx->F->aval;
760 int m = F->m, *nabors = ctx->nabors, nc = ctx->naborCount;
761 int i, j, k, idx, count = 0, start_of_row;
762 int beg_row = ctx->beg_row, end_row = beg_row + m;
763 int *diag = ctx->F->diag;
770 for (i = 0; i < m; ++i)
772 for (j = rp[i]; j < rp[i + 1]; ++j)
775 if (col < beg_row || col >= end_row)
778 int owner = find_owner_private_mpi (ctx, col);
781 for (k = 0; k < nc; ++k)
783 if (nabors[k] == owner)
800 "deleting %i indices that would alter the subdomain graph", count);
801 SET_INFO (msgBuf_dh);
806 for (i = 0; i < m; ++i)
808 for (j = start_of_row; j < rp[i + 1]; ++j)
811 double val = aval[j];
819 start_of_row = rp[i + 1];
824 for (i = 0; i < m; ++i)
826 for (j = rp[i]; j < rp[i + 1]; ++j)
828 if (cval[j] == i + beg_row)
840 #define __FUNC__ "Euclid_dhSolve"
844 START_FUNC_DH
int itsOUT;
847 if (!strcmp (ctx->krylovMethod,
"cg"))
849 cg_euclid (A, ctx, x->vals, b->vals, &itsOUT);
852 else if (!strcmp (ctx->krylovMethod,
"bicgstab"))
854 bicgstab_euclid (A, ctx, x->vals, b->vals, &itsOUT);
859 sprintf (msgBuf_dh,
"unknown krylov solver: %s", ctx->krylovMethod);
860 SET_V_ERROR (msgBuf_dh);
866 #define __FUNC__ "Euclid_dhPrintStats"
868 Euclid_dhPrintStats (
Euclid_dh ctx, FILE * fp)
870 START_FUNC_DH
double *timing;
873 nz = Factor_dhReadNz (ctx->F);
875 timing = ctx->timing;
878 ctx->timing[TOTAL_SOLVE_T] += ctx->timing[TOTAL_SOLVE_TEMP_T];
879 ctx->timing[TOTAL_SOLVE_TEMP_T] = 0.0;
881 reduce_timings_private (ctx);
885 "\n==================== Euclid report (start) ====================\n");
886 fprintf_dh (fp,
"\nruntime parameters\n");
887 fprintf_dh (fp,
"------------------\n");
888 fprintf_dh (fp,
" setups: %i\n", ctx->setupCount);
889 fprintf_dh (fp,
" tri solves: %i\n", ctx->itsTotal);
890 fprintf_dh (fp,
" parallelization method: %s\n", ctx->algo_par);
891 fprintf_dh (fp,
" factorization method: %s\n", ctx->algo_ilu);
892 fprintf_dh (fp,
" matrix was row scaled: %i\n", ctx->isScaled);
894 fprintf_dh (fp,
" matrix row count: %i\n", ctx->n);
895 fprintf_dh (fp,
" nzF: %i\n", nz);
896 fprintf_dh (fp,
" rho: %g\n", ctx->rho_final);
897 fprintf_dh (fp,
" level: %i\n", ctx->level);
898 fprintf_dh (fp,
" sparseA: %g\n", ctx->sparseTolA);
900 fprintf_dh (fp,
"\nEuclid timing report\n");
901 fprintf_dh (fp,
"--------------------\n");
902 fprintf_dh (fp,
" solves total: %0.2f (see docs)\n",
903 timing[TOTAL_SOLVE_T]);
904 fprintf_dh (fp,
" tri solves: %0.2f\n", timing[TRI_SOLVE_T]);
905 fprintf_dh (fp,
" setups: %0.2f\n", timing[SETUP_T]);
906 fprintf_dh (fp,
" subdomain graph setup: %0.2f\n",
907 timing[SUB_GRAPH_T]);
908 fprintf_dh (fp,
" factorization: %0.2f\n", timing[FACTOR_T]);
909 fprintf_dh (fp,
" solve setup: %0.2f\n",
910 timing[SOLVE_SETUP_T]);
911 fprintf_dh (fp,
" rho: %0.2f\n",
912 ctx->timing[COMPUTE_RHO_T]);
913 fprintf_dh (fp,
" misc (should be small): %0.2f\n",
914 timing[SETUP_T] - (timing[SUB_GRAPH_T] + timing[FACTOR_T] +
915 timing[SOLVE_SETUP_T] +
916 timing[COMPUTE_RHO_T]));
920 SubdomainGraph_dhPrintStats (ctx->sg, fp);
922 SubdomainGraph_dhPrintRatios (ctx->sg, fp);
927 fprintf_dh (fp,
"\nApplicable if Euclid's internal solvers were used:\n");
928 fprintf_dh (fp,
"---------------------------------------------------\n");
929 fprintf_dh (fp,
" solve method: %s\n", ctx->krylovMethod);
930 fprintf_dh (fp,
" maxIts: %i\n", ctx->maxIts);
931 fprintf_dh (fp,
" rtol: %g\n", ctx->rtol);
932 fprintf_dh (fp,
" atol: %g\n", ctx->atol);
934 "\n==================== Euclid report (end) ======================\n");
943 #define __FUNC__ "Euclid_dhPrintStatsShort"
945 Euclid_dhPrintStatsShort (
Euclid_dh ctx,
double setup,
double solve,
948 START_FUNC_DH
double *timing = ctx->timing;
959 blocks = ctx->sg->blocks;
961 reduce_timings_private (ctx);
966 apply_total = timing[TRI_SOLVE_T];
967 apply_per_it = apply_total / (double) ctx->its;
969 perIt = solve / (
double) ctx->its;
971 fprintf_dh (fp,
"\n");
972 fprintf_dh (fp,
"%6s %6s %6s %6s %6s %6s %6s %6s %6s %6s XX\n",
973 "method",
"subdms",
"level",
"its",
"setup",
"solve",
"total",
974 "perIt",
"perIt",
"rows");
976 "------ ----- ----- ----- ----- ----- ----- ----- ----- ----- XX\n");
977 fprintf_dh (fp,
"%6s %6i %6i %6i %6.2f %6.2f %6.2f %6.4f %6.5f %6g XXX\n", ctx->algo_par,
993 fprintf_dh (fp,
"\n");
994 fprintf_dh (fp,
"%6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s XX\n",
995 "",
"",
"",
"",
"",
"setup",
"setup",
"",
"",
"",
"",
"",
"");
997 fprintf_dh (fp,
"%6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s %6s XX\n",
998 "method",
"subdms",
"level",
"its",
"total",
"factor",
999 "other",
"apply",
"perIt",
"rho",
"A_tol",
"A_%",
"rows");
1001 "------ ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- ----- XX\n");
1004 fprintf_dh (fp,
"%6s %6i %6i %6i %6.2f %6.2f %6.2f %6.2f %6.4f %6.1f %6g %6.2f %6g XXX\n", ctx->algo_par,
1023 fprintf_dh (fp,
"\n%6s %6s %6s %6s %6s %6s WW\n",
"method",
"level",
1024 "subGph",
"factor",
"solveS",
"perIt");
1025 fprintf_dh (fp,
"------ ----- ----- ----- ----- ----- WW\n");
1026 fprintf_dh (fp,
"%6s %6i %6.2f %6.2f %6.2f %6.4f WWW\n",
1029 timing[SUB_GRAPH_T],
1030 timing[FACTOR_T], timing[SOLVE_SETUP_T], apply_per_it);
1037 #define __FUNC__ "Euclid_dhPrintStatsShorter"
1039 Euclid_dhPrintStatsShorter (
Euclid_dh ctx, FILE * fp)
1041 START_FUNC_DH
double *stats = ctx->stats;
1044 double rho = ctx->rho_final;
1045 double nzUsedRatio = stats[NZA_RATIO_STATS];
1048 fprintf_dh (fp,
"\nStats from last linear solve: YY\n");
1049 fprintf_dh (fp,
"%6s %6s %6s YY\n",
"its",
"rho",
"A_%");
1050 fprintf_dh (fp,
" ----- ----- ----- YY\n");
1051 fprintf_dh (fp,
"%6i %6.2f %6.2f YYY\n", its, rho, nzUsedRatio);
1056 #define __FUNC__ "Euclid_dhPrintScaling"
1058 Euclid_dhPrintScaling (
Euclid_dh ctx, FILE * fp)
1060 START_FUNC_DH
int i, m = ctx->m;
1065 if (ctx->scale == NULL)
1067 SET_V_ERROR (
"ctx->scale is NULL; was Euclid_dhSetup() called?");
1070 fprintf (fp,
"\n---------- 1st %i row scaling values:\n", m);
1071 for (i = 0; i < m; ++i)
1073 fprintf (fp,
" %i %g \n", i + 1, ctx->scale[i]);
1079 #define __FUNC__ "reduce_timings_private"
1083 START_FUNC_DH
if (np_dh > 1)
1085 double bufOUT[TIMING_BINS];
1087 memcpy (bufOUT, ctx->timing, TIMING_BINS * sizeof (
double));
1088 MPI_Reduce (bufOUT, ctx->timing, TIMING_BINS, MPI_DOUBLE, MPI_MAX, 0,
1092 ctx->timingsWereReduced =
true;
1096 #define __FUNC__ "Euclid_dhPrintHypreReport"
1098 Euclid_dhPrintHypreReport (
Euclid_dh ctx, FILE * fp)
1100 START_FUNC_DH
double *timing;
1103 nz = Factor_dhReadNz (ctx->F);
1105 timing = ctx->timing;
1108 ctx->timing[TOTAL_SOLVE_T] += ctx->timing[TOTAL_SOLVE_TEMP_T];
1109 ctx->timing[TOTAL_SOLVE_TEMP_T] = 0.0;
1111 reduce_timings_private (ctx);
1118 "@@@@@@@@@@@@@@@@@@@@@@ Euclid statistical report (start)\n");
1119 fprintf_dh (fp,
"\nruntime parameters\n");
1120 fprintf_dh (fp,
"------------------\n");
1121 fprintf_dh (fp,
" setups: %i\n", ctx->setupCount);
1122 fprintf_dh (fp,
" tri solves: %i\n", ctx->itsTotal);
1123 fprintf_dh (fp,
" parallelization method: %s\n", ctx->algo_par);
1124 fprintf_dh (fp,
" factorization method: %s\n", ctx->algo_ilu);
1125 if (!strcmp (ctx->algo_ilu,
"iluk"))
1127 fprintf_dh (fp,
" level: %i\n", ctx->level);
1132 fprintf_dh (fp,
" matrix was row scaled\n");
1135 fprintf_dh (fp,
" global matrix row count: %i\n", ctx->n);
1136 fprintf_dh (fp,
" nzF: %i\n", nz);
1137 fprintf_dh (fp,
" rho: %g\n", ctx->rho_final);
1138 fprintf_dh (fp,
" sparseA: %g\n", ctx->sparseTolA);
1140 fprintf_dh (fp,
"\nEuclid timing report\n");
1141 fprintf_dh (fp,
"--------------------\n");
1142 fprintf_dh (fp,
" solves total: %0.2f (see docs)\n",
1143 timing[TOTAL_SOLVE_T]);
1144 fprintf_dh (fp,
" tri solves: %0.2f\n", timing[TRI_SOLVE_T]);
1145 fprintf_dh (fp,
" setups: %0.2f\n", timing[SETUP_T]);
1146 fprintf_dh (fp,
" subdomain graph setup: %0.2f\n",
1147 timing[SUB_GRAPH_T]);
1148 fprintf_dh (fp,
" factorization: %0.2f\n",
1150 fprintf_dh (fp,
" solve setup: %0.2f\n",
1151 timing[SOLVE_SETUP_T]);
1152 fprintf_dh (fp,
" rho: %0.2f\n",
1153 ctx->timing[COMPUTE_RHO_T]);
1154 fprintf_dh (fp,
" misc (should be small): %0.2f\n",
1155 timing[SETUP_T] - (timing[SUB_GRAPH_T] + timing[FACTOR_T] +
1156 timing[SOLVE_SETUP_T] +
1157 timing[COMPUTE_RHO_T]));
1159 if (ctx->sg != NULL)
1161 SubdomainGraph_dhPrintStats (ctx->sg, fp);
1163 SubdomainGraph_dhPrintRatios (ctx->sg, fp);
1168 "@@@@@@@@@@@@@@@@@@@@@@ Euclid statistical report (end)\n");
1175 #define __FUNC__ "Euclid_dhPrintTestData"
1177 Euclid_dhPrintTestData (
Euclid_dh ctx, FILE * fp)
1186 fprintf (fp,
" setups: %i\n", ctx->setupCount);
1187 fprintf (fp,
" tri solves: %i\n", ctx->its);
1188 fprintf (fp,
" parallelization method: %s\n", ctx->algo_par);
1189 fprintf (fp,
" factorization method: %s\n", ctx->algo_ilu);
1190 fprintf (fp,
" level: %i\n", ctx->level);
1191 fprintf (fp,
" row scaling: %i\n", ctx->isScaled);
1193 SubdomainGraph_dhPrintRatios (ctx->sg, fp);