52 #include "Zoltan2_config.h"
57 #include <Teuchos_RCP.hpp>
58 #include <Teuchos_ArrayView.hpp>
59 #include <Teuchos_ParameterList.hpp>
60 #include <Teuchos_DefaultComm.hpp>
61 #include <Teuchos_Comm.hpp>
62 #include <Teuchos_CommHelpers.hpp>
65 #include <Tpetra_MultiVector.hpp>
66 #include <Tpetra_KokkosCompat_DefaultNode.hpp>
80 using Teuchos::ArrayView;
81 using Teuchos::CommandLineProcessor;
83 typedef Tpetra::MultiVector<zscalar_t, zlno_t, zgno_t, znode_t>
tMVector_t;
84 typedef Tpetra::Map<zlno_t, zgno_t, znode_t>
tMap_t;
98 const string& delimiters =
" \f\n\r\t\v" )
100 return s.substr( 0, s.find_last_not_of( delimiters ) + 1 );
105 const string& delimiters =
" \f\n\r\t\v" )
107 return s.substr( s.find_first_not_of( delimiters ) );
112 const string& delimiters =
" \f\n\r\t\v" )
117 void readGeoGenParams(
string paramFileName, Teuchos::ParameterList &geoparams,
const RCP<
const Teuchos::Comm<int> > & comm){
118 std::string input =
"";
120 for(
int i = 0; i < 25000; ++i){
125 if(comm->getRank() == 0){
127 std::fstream inParam(paramFileName.c_str());
134 std::string tmp =
"";
135 getline (inParam,tmp);
136 while (!inParam.eof()){
143 getline (inParam,tmp);
146 for (
size_t i = 0; i < input.size(); ++i){
154 int size = input.size();
158 comm->broadcast(0,
sizeof(
int), (
char*) &size);
160 throw "File " + paramFileName +
" cannot be opened.";
162 comm->broadcast(0, size, inp);
163 std::istringstream inParam(inp);
165 getline (inParam,str);
166 while (!inParam.eof()){
168 size_t pos = str.find(
'=');
169 if(pos == string::npos){
170 throw "Invalid Line:" + str +
" in parameter file";
172 string paramname =
trim_copy(str.substr(0,pos));
173 string paramvalue =
trim_copy(str.substr(pos + 1));
174 geoparams.set(paramname, paramvalue);
176 getline (inParam,str);
203 ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
204 int num_wgts,
float *obj_wgts,
int *ierr)
209 size_t localLen = dots->
coordinates->getLocalLength();
211 dots->
coordinates->getMap()->getLocalElementList().getRawPtr();
213 if (
sizeof(ZOLTAN_ID_TYPE) ==
sizeof(
zgno_t))
214 memcpy(gids, ids,
sizeof(ZOLTAN_ID_TYPE) * localLen);
216 for (
size_t i=0; i < localLen; i++)
217 gids[i] = static_cast<ZOLTAN_ID_TYPE>(ids[i]);
220 float *wgts = obj_wgts;
221 for (
size_t i=0; i < localLen; i++)
222 for (
int w=0; w < num_wgts; w++)
229 int numObj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids,
230 int dim,
double *coords,
int *ierr)
236 double *val = coords;
240 for (
int i=0; i < numObj; i++){
241 *val++ =
static_cast<double>(x[i]);
242 *val++ =
static_cast<double>(y[i]);
243 *val++ =
static_cast<double>(z[i]);
249 double *val = coords;
252 for (
int i=0; i < numObj; i++){
253 *val++ =
static_cast<double>(x[i]);
254 *val++ =
static_cast<double>(y[i]);
272 const RCP<
const Teuchos::Comm<int> > & comm,
273 vector<float> &wgts,
weightTypes how,
float scale,
int rank)
277 float val = scale + rank%2;
278 for (
zlno_t i=0; i < len; i++)
282 for (
int i=0; i < 10; i++){
283 float val = (i + 10)*scale;
284 for (
zlno_t j=i; j < len; j += 10)
289 float val = scale + rank;
290 for (
zlno_t i=0; i < len; i++)
301 const RCP<
const Teuchos::Comm<int> > & comm,
304 int rank = comm->getRank();
305 int nprocs = comm->getSize();
307 double k = log(numGlobalCoords) / 3;
308 double xdimf = exp(k) + 0.5;
309 ssize_t xdim =
static_cast<ssize_t
>(floor(xdimf));
311 ssize_t zdim = numGlobalCoords / (xdim*ydim);
312 ssize_t num=xdim*ydim*zdim;
313 ssize_t diff = numGlobalCoords - num;
317 if (zdim > xdim && zdim > ydim){
319 newdiff = diff - (xdim*ydim);
324 else if (ydim > xdim && ydim > zdim){
326 newdiff = diff - (xdim*zdim);
333 newdiff = diff - (ydim*zdim);
343 diff = numGlobalCoords - num;
345 diff /= -numGlobalCoords;
347 diff /= numGlobalCoords;
351 std::cout <<
"Warning: Difference " << diff*100 <<
" percent" << std::endl;
352 std::cout <<
"Mesh size: " << xdim <<
"x" << ydim <<
"x" <<
353 zdim <<
", " << num <<
" vertices." << std::endl;
358 ssize_t numLocalCoords = num / nprocs;
359 ssize_t leftOver = num % nprocs;
362 if (rank <= leftOver)
363 gid0 = rank * (numLocalCoords+1);
365 gid0 = (leftOver * (numLocalCoords+1)) +
366 ((rank - leftOver) * numLocalCoords);
371 ssize_t gid1 = gid0 + numLocalCoords;
376 ArrayView<zgno_t> idArray(ids, numLocalCoords);
379 for (ssize_t i=gid0; i < gid1; i++)
382 RCP<const tMap_t> idMap = rcp(
new tMap_t(num, idArray, 0, comm));
389 if (!x)
throw bad_alloc();
396 zgno_t xyPlane = xdim*ydim;
397 zgno_t zStart = gid0 / xyPlane;
398 zgno_t rem = gid0 % xyPlane;
405 for (
zscalar_t zval=zStart; next < numLocalCoords && zval < zdim; zval+=1.){
406 for (
zscalar_t yval=yStart; next < numLocalCoords && yval < ydim; yval+=1.){
407 for (
zscalar_t xval=xStart; next < numLocalCoords && xval < xdim;xval+=1.){
418 ArrayView<const zscalar_t> xArray(x, numLocalCoords*3);
426 int main(
int narg,
char *arg[])
430 Tpetra::ScopeGuard tscope(&narg, &arg);
431 Teuchos::RCP<const Teuchos::Comm<int> > comm = Tpetra::getDefaultComm();
432 int rank = comm->getRank();
433 int nprocs = comm->getSize();
436 MEMORY_CHECK(rank==0 || rank==nprocs-1,
"After initializing MPI");
439 std::cout <<
"Number of processes: " << nprocs << std::endl;
442 zgno_t numGlobalCoords = 1000;
445 string memoryOn(
"memoryOn");
446 string memoryOff(
"memoryOff");
448 int numGlobalParts = nprocs;
452 string balanceCount(
"balance_object_count");
453 string balanceWeight(
"balance_object_weight");
454 string mcnorm1(
"multicriteria_minimize_total_weight");
455 string mcnorm2(
"multicriteria_balance_total_maximum");
456 string mcnorm3(
"multicriteria_minimize_maximum_weight");
457 string objective(balanceWeight);
460 CommandLineProcessor commandLine(
false,
true);
463 int input_option = 0;
464 commandLine.setOption(
"input_option", &input_option,
465 "whether to use mesh creation, geometric generator, or file input");
466 string inputFile =
"";
468 commandLine.setOption(
"input_file", &inputFile,
469 "the input file for geometric generator or file input");
472 commandLine.setOption(
"size", &numGlobalCoords,
473 "Approximate number of global coordinates.");
474 commandLine.setOption(
"numParts", &numGlobalParts,
475 "Number of parts (default is one per proc).");
476 commandLine.setOption(
"nWeights", &nWeights,
477 "Number of weights per coordinate, zero implies uniform weights.");
478 commandLine.setOption(
"debug", &debugLevel,
"Zoltan1 debug level");
479 commandLine.setOption(
"remap",
"no-remap", &remap,
480 "Zoltan1 REMAP parameter; disabled by default for scalability testing");
481 commandLine.setOption(
"timers", &dummyTimer,
"ignored");
482 commandLine.setOption(memoryOn.c_str(), memoryOff.c_str(), &doMemory,
483 "do memory profiling");
485 string doc(balanceCount);
486 doc.append(
": ignore weights\n");
487 doc.append(balanceWeight);
488 doc.append(
": balance on first weight\n");
490 doc.append(
": given multiple weights, balance their total.\n");
492 doc.append(
": given multiple weights, "
493 "balance the maximum for each coordinate.\n");
495 doc.append(
": given multiple weights, balance the L2 norm of the weights.\n");
496 commandLine.setOption(
"objective", &objective, doc.c_str());
498 CommandLineProcessor::EParseCommandLineReturn rc =
499 commandLine.parse(narg, arg);
503 if (rc != Teuchos::CommandLineProcessor::PARSE_SUCCESSFUL) {
504 if (rc == Teuchos::CommandLineProcessor::PARSE_HELP_PRINTED) {
505 if (rank==0) std::cout <<
"PASS" << std::endl;
509 if (rank==0) std::cout <<
"FAIL" << std::endl;
517 size_t numLocalCoords = 0;
518 if (input_option == 0){
520 numLocalCoords = dots.
coordinates->getLocalLength();
524 for (
int p=0; p < nprocs; p++){
526 std::cout <<
"Rank " << rank <<
", " << numLocalCoords <<
"coords" << std::endl;
530 for (
zlno_t i=0; i < numLocalCoords; i++)
531 std::cout <<
" " << x[i] <<
" " << y[i] <<
" " << z[i] << std::endl;
544 for (
int i=0; i < nWeights; i++){
545 dots.
weights[i].resize(numLocalCoords);
555 else if(input_option == 1){
556 Teuchos::ParameterList geoparams(
"geo params");
565 for(
int i = 0; i < coord_dim; ++i){
566 coords[i] =
new zscalar_t[numLocalCoords];
572 for(
int i = 0; i < nWeights; ++i){
573 weight[i] =
new zscalar_t[numLocalCoords];
580 RCP<Tpetra::Map<zlno_t, zgno_t, znode_t> > mp = rcp(
581 new Tpetra::Map<zlno_t, zgno_t, znode_t> (numGlobalCoords, numLocalCoords, 0, comm));
583 Teuchos::Array<Teuchos::ArrayView<const zscalar_t> > coordView(coord_dim);
584 for (
int i=0; i < coord_dim; i++){
585 if(numLocalCoords > 0){
586 Teuchos::ArrayView<const zscalar_t> a(coords[i], numLocalCoords);
589 Teuchos::ArrayView<const zscalar_t> a;
600 for (
int i = 0; i < nWeights;++i){
602 dots.
weights[i].push_back(weight[i][j]);
607 for(
int i = 0; i < nWeights; ++i)
618 numLocalCoords = coords->getLocalLength();
619 numGlobalCoords = coords->getGlobalLength();
622 MEMORY_CHECK(doMemory && rank==0,
"After creating input");
627 int aok = Zoltan_Initialize(narg, arg, &ver);
630 printf(
"Zoltan_Initialize failed\n");
634 struct Zoltan_Struct *zz;
635 zz = Zoltan_Create(MPI_COMM_WORLD);
637 Zoltan_Set_Param(zz,
"LB_METHOD",
"RCB");
638 Zoltan_Set_Param(zz,
"LB_APPROACH",
"PARTITION");
639 Zoltan_Set_Param(zz,
"CHECK_GEOM",
"0");
640 Zoltan_Set_Param(zz,
"NUM_GID_ENTRIES",
"1");
641 Zoltan_Set_Param(zz,
"NUM_LID_ENTRIES",
"0");
642 Zoltan_Set_Param(zz,
"RETURN_LISTS",
"PART");
643 std::ostringstream oss;
644 oss << numGlobalParts;
645 Zoltan_Set_Param(zz,
"NUM_GLOBAL_PARTS", oss.str().c_str());
648 Zoltan_Set_Param(zz,
"DEBUG_LEVEL", oss.str().c_str());
651 Zoltan_Set_Param(zz,
"REMAP",
"1");
653 Zoltan_Set_Param(zz,
"REMAP",
"0");
655 if (objective != balanceCount){
658 Zoltan_Set_Param(zz,
"OBJ_WEIGHT_DIM", oss.str().c_str());
660 if (objective == mcnorm1)
661 Zoltan_Set_Param(zz,
"RCB_MULTICRITERIA_NORM",
"1");
662 else if (objective == mcnorm2)
663 Zoltan_Set_Param(zz,
"RCB_MULTICRITERIA_NORM",
"2");
664 else if (objective == mcnorm3)
665 Zoltan_Set_Param(zz,
"RCB_MULTICRITERIA_NORM",
"3");
668 Zoltan_Set_Param(zz,
"OBJ_WEIGHT_DIM",
"0");
671 Zoltan_Set_Num_Obj_Fn(zz,
getNumObj, &dots);
672 Zoltan_Set_Obj_List_Fn(zz,
getObjList, &dots);
673 Zoltan_Set_Num_Geom_Fn(zz,
getDim, &dots);
674 Zoltan_Set_Geom_Multi_Fn(zz,
getCoords, &dots);
676 int changes, numGidEntries, numLidEntries, numImport, numExport;
677 ZOLTAN_ID_PTR importGlobalGids, importLocalGids;
678 ZOLTAN_ID_PTR exportGlobalGids, exportLocalGids;
679 int *importProcs, *importToPart, *exportProcs, *exportToPart;
681 MEMORY_CHECK(doMemory && rank==0,
"Before Zoltan_LB_Partition");
683 if (rank == 0) std::cout <<
"Calling Zoltan_LB_Partition" << std::endl;
684 aok = Zoltan_LB_Partition(zz, &changes, &numGidEntries, &numLidEntries,
685 &numImport, &importGlobalGids, &importLocalGids,
686 &importProcs, &importToPart,
687 &numExport, &exportGlobalGids, &exportLocalGids,
688 &exportProcs, &exportToPart);
689 if (rank == 0) std::cout <<
"Returned from Zoltan_LB_Partition" << std::endl;
691 MEMORY_CHECK(doMemory && rank==0,
"After Zoltan_LB_Partition");
695 float *sumWgtPerPart =
new float[numGlobalParts];
696 float *gsumWgtPerPart =
new float[numGlobalParts];
697 for (
int i = 0; i < numGlobalParts; i++) sumWgtPerPart[i] = 0.;
699 for (
size_t i = 0; i < numLocalCoords; i++)
700 sumWgtPerPart[exportToPart[i]] += (nWeights ? dots.
weights[0][i]: 1.);
702 Teuchos::reduceAll<int, float>(*comm, Teuchos::REDUCE_SUM, numGlobalParts,
703 sumWgtPerPart, gsumWgtPerPart);
705 float maxSumWgtPerPart = 0.;
706 float minSumWgtPerPart = std::numeric_limits<float>::max();
708 int maxSumWgtPart=0, minSumWgtPart=0;
709 for (
int i = 0; i < numGlobalParts; i++) {
710 if (gsumWgtPerPart[i] > maxSumWgtPerPart) {
711 maxSumWgtPerPart = gsumWgtPerPart[i];
714 if (gsumWgtPerPart[i] < minSumWgtPerPart) {
715 minSumWgtPerPart = gsumWgtPerPart[i];
718 totWgt += gsumWgtPerPart[i];
722 std::cout << std::endl << std::endl
723 <<
"Part loads (per part for " << numGlobalParts <<
" parts):"
725 <<
" min = " << minSumWgtPerPart
726 <<
" in part " << minSumWgtPart << std::endl
727 <<
" max = " << maxSumWgtPerPart
728 <<
" in part " << maxSumWgtPart << std::endl
729 <<
" tot = " << totWgt << std::endl
730 <<
" avg = " << totWgt / numGlobalParts
731 << std::endl << std::endl << std::endl;
733 delete [] sumWgtPerPart;
734 delete [] gsumWgtPerPart;
737 MEMORY_CHECK(doMemory && rank==0,
"After Zoltan_Destroy");
740 for (
int i = 0; i < nWeights; i++)
744 MEMORY_CHECK(doMemory && rank==0,
"After destroying input");
748 std::cout <<
"FAIL" << std::endl;
750 std::cout <<
"PASS" << std::endl;
void getCoords(void *data, int numGid, int numLid, int numObj, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int dim, double *coords_, int *ierr)
void getLocalCoordinatesCopy(scalar_t **c)
string trim_right_copy(const string &s, const string &delimiters=" \f\n\r\t\v")
void readGeoGenParams(string paramFileName, Teuchos::ParameterList &geoparams, const RCP< const Teuchos::Comm< int > > &comm)
int getCoordinateDimension()
int main(int narg, char **arg)
common code used by tests
string trim_copy(const string &s, const string &delimiters=" \f\n\r\t\v")
int getNumObj(void *data, int *ierr)
int getDim(void *data, int *ierr)
vector< vector< float > > weights
void getObjList(void *data, int numGid, int numLid, ZOLTAN_ID_PTR gids, ZOLTAN_ID_PTR lids, int num_wgts, float *obj_wgts, int *ierr)
Tpetra::Map::local_ordinal_type zlno_t
static const std::string fail
string trim_left_copy(const string &s, const string &delimiters=" \f\n\r\t\v")
int getNumWeights()
##END Predistribution functions######################//
lno_t getNumLocalCoords()
gno_t getNumGlobalCoords()
A gathering of useful namespace methods.
Tpetra::Map::node_type znode_t
Tpetra::MultiVector< zscalar_t, zlno_t, zgno_t, znode_t > tMVector_t
Tpetra::Map::global_ordinal_type zgno_t
void getLocalWeightsCopy(scalar_t **w)
#define MEMORY_CHECK(iPrint, msg)
std::string testDataFilePath(".")