45 #ifndef __READMATRIXFROMBINARYFILE_HPP
46 #define __READMATRIXFROMBINARYFILE_HPP
55 #include "Tpetra_Details_makeColMap.hpp"
57 template <
typename global_ordinal_type,
typename local_ordinal_type,
typename scalar_type,
typename map_type>
59 distribute (Teuchos::ArrayRCP<size_t>& myNumEntriesPerRow,
60 Teuchos::ArrayRCP<size_t>& myRowPtr,
61 Teuchos::ArrayRCP<global_ordinal_type>& myColInd,
62 Teuchos::ArrayRCP<scalar_type>& myValues,
63 const Teuchos::RCP<const map_type>& pRowMap,
64 global_ordinal_type *rowPtr,
65 global_ordinal_type *colInd,
66 const bool debug=
false)
69 int maxNumEnt = INT_MAX/
sizeof(global_ordinal_type);
71 Teuchos::RCP<const Teuchos::Comm<int>> pComm = pRowMap->getComm ();
72 const int numProcs = pComm->getSize ();
73 const int myRank = pComm->getRank ();
74 const int rootRank = 0;
76 Teuchos::ArrayView<const global_ordinal_type> myRows = pRowMap->getLocalElementList();
77 const size_t myNumRows = myRows.size();
79 myNumEntriesPerRow = Teuchos::ArrayRCP<size_t> (myNumRows);
81 if (myRank != rootRank) {
83 Teuchos::send (*pComm, myNumRows, rootRank);
87 Teuchos::send (*pComm, static_cast<int> (myNumRows), myRows.getRawPtr(), rootRank);
88 Teuchos::receive (*pComm, rootRank, static_cast<int> (myNumRows), myNumEntriesPerRow.getRawPtr());
90 const global_ordinal_type myNumEntries =
91 std::accumulate (myNumEntriesPerRow.begin(),
92 myNumEntriesPerRow.end(), 0);
94 myColInd = Teuchos::ArrayRCP<global_ordinal_type> (myNumEntries);
95 myValues = Teuchos::ArrayRCP<scalar_type> (myNumEntries, 1.0);
96 if (myNumEntries > 0) {
98 if(myNumEntries < maxNumEnt)
99 Teuchos::receive (*pComm, rootRank, static_cast<int> (myNumEntries), &myColInd[0]);
101 int nchunks = myNumEntries/maxNumEnt;
102 if(myNumEntries % maxNumEnt != 0)
104 for(
int i = 0; i < nchunks-1; i++) {
105 Teuchos::receive (*pComm, rootRank, maxNumEnt, &myColInd[maxNumEnt*i]);
106 std::cout <<
"Chunk " << i <<
" received by myRank "<< myRank <<
", size: " << maxNumEnt <<
"\n";
108 int lastsize = (int)(myNumEntries - (nchunks-1)*maxNumEnt);
109 Teuchos::receive (*pComm, rootRank, lastsize, &myColInd[maxNumEnt*(nchunks-1)]);
110 std::cout <<
"Chunk " << nchunks-1 <<
" received by myRank " << myRank <<
", size: " << lastsize <<
"\n";
120 std::cout <<
"-- Proc 0: Copying my data from global arrays" << std::endl;
123 for (
size_t k = 0; k < myNumRows; ++k) {
124 const global_ordinal_type myCurRow = myRows[k];
125 const global_ordinal_type numEntriesInThisRow = rowPtr[myCurRow+1] - rowPtr[myCurRow];
126 myNumEntriesPerRow[k] = numEntriesInThisRow;
130 size_t myNumEntries = std::accumulate (myNumEntriesPerRow.begin(),
131 myNumEntriesPerRow.end(), 0);
133 std::cout <<
"-- Proc 0: I own " << myNumRows <<
" rows and "
134 << myNumEntries <<
" entries" << std::endl;
136 myColInd = Teuchos::ArrayRCP<global_ordinal_type> (myNumEntries);
137 myValues = Teuchos::ArrayRCP<scalar_type> (myNumEntries, 1.0);
139 global_ordinal_type myCurPos = 0;
140 for (
size_t k = 0; k < myNumRows; ++k) {
141 const global_ordinal_type curNumEntries = myNumEntriesPerRow[k];
142 const global_ordinal_type myRow = myRows[k];
143 global_ordinal_type curPos = rowPtr[myRow];
145 if (curNumEntries > 0) {
146 for(global_ordinal_type ii = 0; ii < curNumEntries; ++ii) {
147 myColInd[myCurPos++] = colInd[curPos++];
152 for (
int p = 1; p < numProcs; ++p) {
154 std::cout <<
"-- Proc 0: Processing proc " << p << std::endl;
157 size_t theirNumRows = 0;
158 Teuchos::receive (*pComm, p, &theirNumRows);
160 std::cout <<
"-- Proc 0: Proc " << p <<
" owns "
161 << theirNumRows <<
" rows" << std::endl;
164 if (theirNumRows != 0) {
165 Teuchos::ArrayRCP<global_ordinal_type> theirRows(theirNumRows);
166 Teuchos::receive (*pComm, p, Teuchos::as<int> (theirNumRows), theirRows.getRawPtr ());
168 Teuchos::ArrayRCP<size_t> theirNumEntriesPerRow = Teuchos::ArrayRCP<size_t> (theirNumRows);
169 for (
size_t k = 0; k < theirNumRows; ++k) {
170 theirNumEntriesPerRow[k] = rowPtr[theirRows[k]+1] - rowPtr[theirRows[k]];
173 Teuchos::send (*pComm, static_cast<int> (theirNumRows), theirNumEntriesPerRow.getRawPtr(), p);
175 const global_ordinal_type theirNumEntries =
176 std::accumulate (theirNumEntriesPerRow.begin(),
177 theirNumEntriesPerRow.end(), 0);
180 std::cout <<
"-- Proc 0: Proc " << p <<
" owns "
181 << theirNumEntries <<
" entries" << std::endl;
184 if (theirNumEntries == 0) {
188 Teuchos::ArrayRCP<global_ordinal_type> theirColInd (theirNumEntries);
190 global_ordinal_type theirCurPos = 0;
191 for (
size_t k = 0; k < theirNumRows; k++) {
192 const global_ordinal_type curNumEntries = theirNumEntriesPerRow[k];
193 const global_ordinal_type theirRow = theirRows[k];
194 global_ordinal_type curPos = rowPtr[theirRow];
196 if (curNumEntries > 0) {
198 for(global_ordinal_type ii = 0; ii < curNumEntries; ++ii) {
199 theirColInd[theirCurPos++] = colInd[curPos++];
205 if(theirNumEntries < maxNumEnt)
206 Teuchos::send (*pComm, static_cast<int> (theirNumEntries), &theirColInd[0], p);
208 int nchunks = theirNumEntries/maxNumEnt;
209 if(theirNumEntries % maxNumEnt != 0)
211 for(
int i = 0; i < nchunks-1; i++) {
212 Teuchos::send (*pComm, maxNumEnt, &theirColInd[maxNumEnt*i], p);
213 std::cout <<
"Chunk " << i <<
" sent to Rank "<< p <<
", size: " << maxNumEnt <<
"\n";
215 int lastsize = (int)(theirNumEntries - (nchunks-1)*maxNumEnt);
216 Teuchos::send (*pComm, lastsize, &theirColInd[maxNumEnt*(nchunks-1)], p);
217 std::cout <<
"Chunk " << nchunks-1 <<
" sent to Rank "<< p <<
", size: " << lastsize <<
"\n";
221 std::cout <<
"-- Proc 0: Finished with proc " << p << std::endl;
228 if (debug && myRank == 0) {
229 std::cout <<
"-- Proc 0: About to fill in myRowPtr" << std::endl;
232 myRowPtr = Teuchos::ArrayRCP<size_t> (myNumRows+1);
234 for (
size_t k = 1; k < myNumRows+1; ++k) {
235 myRowPtr[k] = myRowPtr[k-1] + myNumEntriesPerRow[k-1];
237 if (debug && myRank == 0) {
238 std::cout <<
"-- Proc 0: Done with distribute" << std::endl;
242 template <
typename crs_matrix_type>
243 Teuchos::RCP<crs_matrix_type>
244 readBinaryFile(std::string filename,
const Teuchos::RCP<
const Teuchos::Comm<int>> pComm,
bool callFillComplete=
true,
bool debug=
false)
246 typedef typename crs_matrix_type::global_ordinal_type global_ordinal_type;
247 typedef typename crs_matrix_type::local_ordinal_type local_ordinal_type;
248 typedef typename crs_matrix_type::scalar_type scalar_type;
249 typedef typename crs_matrix_type::node_type node_type;
251 typedef typename Tpetra::Map<local_ordinal_type, global_ordinal_type, node_type> map_type;
253 const int myRank = pComm->getRank();
254 const int rootRank = 0;
256 if (debug && myRank == rootRank) {
257 std::cout <<
"Binary CRS reader: readSparse:" << std::endl
258 <<
"-- Reading started" << std::endl;
261 Teuchos::RCP<std::ifstream> in;
262 global_ordinal_type globalNumRows;
263 global_ordinal_type globalNumNonzeros;
265 if (myRank == rootRank) {
268 in = Teuchos::RCP<std::ifstream>(
new std::ifstream(filename, std::ios::in | std::ios::binary));
271 in->read((
char *)&globalNumRows,
sizeof(global_ordinal_type));
272 in->read((
char *)&globalNumNonzeros,
sizeof(global_ordinal_type));
274 TEUCHOS_TEST_FOR_EXCEPTION(globalNumRows <= 0 || globalNumNonzeros <= 0, std::invalid_argument,
275 "Global number of rows or nonzeros have nonpositive value." << globalNumRows <<
" " << globalNumNonzeros <<
" " <<
sizeof(global_ordinal_type) );
278 broadcast (*pComm, rootRank, 1, &globalNumRows);
279 broadcast (*pComm, rootRank, 1, &globalNumNonzeros);
281 global_ordinal_type *rowPtr = 0;
282 global_ordinal_type *colInd = 0;
284 if (myRank == rootRank) {
286 rowPtr =
new global_ordinal_type[globalNumRows+1];
287 colInd =
new global_ordinal_type[globalNumNonzeros];
289 in->read((
char*)rowPtr,
sizeof(global_ordinal_type)*(globalNumRows+1));
290 in->read((
char*)colInd,
sizeof(global_ordinal_type)*(globalNumNonzeros));
293 Teuchos::RCP<const map_type> pRowMap = Teuchos::rcp (
new map_type (static_cast<Tpetra::global_size_t> (globalNumRows),
294 static_cast<global_ordinal_type> (0),
295 pComm, Tpetra::GloballyDistributed));
297 Teuchos::RCP<const map_type> pRangeMap = Teuchos::rcp (
new map_type (static_cast<Tpetra::global_size_t> (globalNumRows),
298 static_cast<global_ordinal_type> (0),
299 pComm, Tpetra::GloballyDistributed));
301 Teuchos::RCP<const map_type> pDomainMap = pRangeMap;
303 Teuchos::ArrayView<const global_ordinal_type> myRows = pRowMap->getLocalElementList ();
304 const size_t myNumRows = myRows.size ();
306 Teuchos::ArrayRCP<size_t> myNumEntriesPerRow(myNumRows);
307 Teuchos::ArrayRCP<size_t> myRowPtr;
308 Teuchos::ArrayRCP<global_ordinal_type> myColInd;
309 Teuchos::ArrayRCP<scalar_type> myValues;
311 distribute<global_ordinal_type, local_ordinal_type, scalar_type, map_type>(myNumEntriesPerRow, myRowPtr, myColInd, myValues, pRowMap, rowPtr, colInd, debug);
314 if (debug && myRank == rootRank) {
315 std::cout <<
"-- Inserting matrix entries on each processor";
316 if (callFillComplete) {
317 std::cout <<
" and calling fillComplete()";
319 std::cout << std::endl;
322 Teuchos::RCP<crs_matrix_type> pMatrix = Teuchos::rcp (
new crs_matrix_type (pRowMap, myNumEntriesPerRow()));
324 const global_ordinal_type indexBase = pRowMap->getIndexBase ();
325 for (
size_t i = 0; i < myNumRows; ++i) {
326 const size_t myCurPos = myRowPtr[i];
327 const local_ordinal_type curNumEntries = myNumEntriesPerRow[i];
328 Teuchos::ArrayView<global_ordinal_type> curColInd = myColInd.view (myCurPos, curNumEntries);
329 Teuchos::ArrayView<scalar_type> curValues = myValues.view (myCurPos, curNumEntries);
331 for (
size_t k = 0; k < curNumEntries; ++k) {
332 curColInd[k] += indexBase;
335 if (curNumEntries > 0) {
336 pMatrix->insertGlobalValues (myRows[i], curColInd, curValues);
340 if (debug && myRank == rootRank) {
341 std::cout <<
"-- Done with inserting." << std::endl;
344 myNumEntriesPerRow = Teuchos::null;
345 myRowPtr = Teuchos::null;
346 myColInd = Teuchos::null;
347 myValues = Teuchos::null;
349 if (callFillComplete) {
350 pMatrix->fillComplete (pDomainMap, pRangeMap);
353 if (debug && myRank == rootRank) {
354 std::cout <<
"-- Done with fill complete." << std::endl;
358 if(myRank == rootRank) {
366 template <
typename crs_matrix_type>
367 Teuchos::RCP<crs_matrix_type>
368 readBinaryFileFast(std::string filename,
const Teuchos::RCP<
const Teuchos::Comm<int>> pComm,
bool callFillComplete=
true,
bool debug=
false)
370 typedef typename crs_matrix_type::global_ordinal_type global_ordinal_type;
371 typedef typename crs_matrix_type::local_ordinal_type local_ordinal_type;
372 typedef typename crs_matrix_type::scalar_type scalar_type;
373 typedef typename crs_matrix_type::node_type node_type;
374 typedef typename crs_matrix_type::crs_graph_type crs_graph_type;
376 typedef typename Tpetra::Map<local_ordinal_type, global_ordinal_type, node_type> map_type;
378 const int myRank = pComm->getRank();
379 const int rootRank = 0;
381 if (debug && myRank == rootRank) {
382 std::cout <<
"Binary CRS reader: readSparse:" << std::endl
383 <<
"-- Reading started" << std::endl;
386 Teuchos::RCP<std::ifstream> in;
387 global_ordinal_type globalNumRows;
388 global_ordinal_type globalNumNonzeros;
390 if (myRank == rootRank) {
393 in = Teuchos::RCP<std::ifstream>(
new std::ifstream(filename, std::ios::in | std::ios::binary));
396 in->read((
char *)&globalNumRows,
sizeof(global_ordinal_type));
397 in->read((
char *)&globalNumNonzeros,
sizeof(global_ordinal_type));
399 TEUCHOS_TEST_FOR_EXCEPTION(globalNumRows <= 0 || globalNumNonzeros <= 0, std::invalid_argument,
400 "Global number of rows or nonzeros have nonpositive value." << globalNumRows <<
" " << globalNumNonzeros <<
" " <<
sizeof(global_ordinal_type) );
403 broadcast (*pComm, rootRank, 1, &globalNumRows);
404 broadcast (*pComm, rootRank, 1, &globalNumNonzeros);
406 global_ordinal_type *rowPtr = 0;
407 global_ordinal_type *colInd = 0;
409 if (myRank == rootRank) {
411 rowPtr =
new global_ordinal_type[globalNumRows+1];
412 colInd =
new global_ordinal_type[globalNumNonzeros];
414 in->read((
char*)rowPtr,
sizeof(global_ordinal_type)*(globalNumRows+1));
415 in->read((
char*)colInd,
sizeof(global_ordinal_type)*(globalNumNonzeros));
420 Teuchos::RCP<const map_type> pRowMap = Teuchos::rcp (
new map_type (static_cast<Tpetra::global_size_t> (globalNumRows),
421 static_cast<global_ordinal_type> (0),
422 pComm, Tpetra::GloballyDistributed));
424 Teuchos::RCP<const map_type> pRangeMap = Teuchos::rcp (
new map_type (static_cast<Tpetra::global_size_t> (globalNumRows),
425 static_cast<global_ordinal_type> (0),
426 pComm, Tpetra::GloballyDistributed));
428 Teuchos::RCP<const map_type> pDomainMap = pRangeMap;
430 Teuchos::ArrayView<const global_ordinal_type> myRows = pRowMap->getLocalElementList ();
431 const size_t myNumRows = myRows.size ();
433 Teuchos::ArrayRCP<size_t> myNumEntriesPerRow(myNumRows);
434 Teuchos::ArrayRCP<size_t> myRowPtr;
435 Teuchos::ArrayRCP<global_ordinal_type> myColInd;
436 Teuchos::ArrayRCP<scalar_type> myValues;
439 distribute<global_ordinal_type, local_ordinal_type, scalar_type, map_type>(myNumEntriesPerRow, myRowPtr, myColInd, myValues, pRowMap, rowPtr, colInd, debug);
442 if (debug && myRank == rootRank) {
443 std::cout <<
"-- Inserting matrix entries on each processor";
444 if (callFillComplete) {
445 std::cout <<
" and calling fillComplete()";
447 std::cout << std::endl;
451 std::vector<bool> mark(globalNumRows,
false);
452 size_t myNumEntries = myRowPtr[myNumRows];
453 for(
size_t i = 0; i < myNumEntries; i++)
454 mark[myColInd[i]] =
true;
456 local_ordinal_type myNumCols = 0;
457 for(global_ordinal_type i = 0; i < globalNumRows; i++)
461 Kokkos::View<global_ordinal_type*, typename node_type::memory_space> myColGIDs(
"myColGIDs", myNumCols);
462 auto myColGIDs_host = Kokkos::create_mirror_view(Kokkos::HostSpace(), myColGIDs);
465 for(global_ordinal_type i = 0; i < globalNumRows; i++)
466 if(mark[i] ==
true) {
467 myColGIDs_host(myNumCols)= i;
471 Kokkos::deep_copy(myColGIDs, myColGIDs_host);
473 Teuchos::RCP<const map_type> pColumnMap;
474 Tpetra::Details::makeColMap(pColumnMap, pDomainMap, myColGIDs);
476 std::vector<local_ordinal_type> map(globalNumRows);
477 for(global_ordinal_type i = 0; i < globalNumRows; i++) {
479 map[i] = pColumnMap->getLocalElement(i);
482 Teuchos::ArrayRCP<local_ordinal_type> myLclColInd(myNumEntries);
483 for(
size_t i = 0; i < myNumEntries; i++)
484 myLclColInd[i] = map[myColInd[i]];
486 local_ordinal_type *cur = myLclColInd.getRawPtr();
488 for(
size_t i = 0; i < myNumRows; i++) {
489 size_t start = myRowPtr[i];
490 size_t end = myRowPtr[i+1];
492 std::sort(&cur[start], &cur[end]);
495 Teuchos::RCP<crs_graph_type> graph(
new crs_graph_type(pRowMap, pColumnMap, myRowPtr, myLclColInd));
496 graph->fillComplete(pDomainMap, pRangeMap);
498 Kokkos::View<scalar_type*, typename node_type::memory_space> values(
"values", myNumEntries);
499 Kokkos::deep_copy(values, 1.0);
501 Teuchos::RCP<crs_matrix_type> pMatrix (
new crs_matrix_type(graph, values));
502 pMatrix->fillComplete(pDomainMap, pRangeMap);
505 if (debug && myRank == rootRank) {
506 std::cout <<
"-- Done with fill complete." << std::endl;
509 if(myRank == rootRank) {
517 template <
typename crs_matrix_type>
518 Teuchos::RCP<crs_matrix_type>
519 readMatrixFromBinaryFile(std::string filename,
const Teuchos::RCP<
const Teuchos::Comm<int>> pComm,
bool binary=
true,
bool debug=
false)
521 return readBinaryFileFast<crs_matrix_type>(filename, pComm,
true, debug);
void distribute(Teuchos::ArrayRCP< size_t > &myNumEntriesPerRow, Teuchos::ArrayRCP< size_t > &myRowPtr, Teuchos::ArrayRCP< global_ordinal_type > &myColInd, Teuchos::ArrayRCP< scalar_type > &myValues, const Teuchos::RCP< const map_type > &pRowMap, global_ordinal_type *rowPtr, global_ordinal_type *colInd, const bool debug=false)
Teuchos::RCP< crs_matrix_type > readBinaryFileFast(std::string filename, const Teuchos::RCP< const Teuchos::Comm< int >> pComm, bool callFillComplete=true, bool debug=false)
Teuchos::RCP< crs_matrix_type > readMatrixFromBinaryFile(std::string filename, const Teuchos::RCP< const Teuchos::Comm< int >> pComm, bool binary=true, bool debug=false)
Teuchos::RCP< crs_matrix_type > readBinaryFile(std::string filename, const Teuchos::RCP< const Teuchos::Comm< int >> pComm, bool callFillComplete=true, bool debug=false)