MueLu  Version of the Day
 All Classes Namespaces Files Functions Variables Typedefs Enumerations Enumerator Friends Macros Groups Pages
MueLu_AMGXOperator_decl.hpp
Go to the documentation of this file.
1 // @HEADER
2 //
3 // ***********************************************************************
4 //
5 // MueLu: A package for multigrid based preconditioning
6 // Copyright 2012 Sandia Corporation
7 //
8 // Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
9 // the U.S. Government retains certain rights in this software.
10 //
11 // Redistribution and use in source and binary forms, with or without
12 // modification, are permitted provided that the following conditions are
13 // met:
14 //
15 // 1. Redistributions of source code must retain the above copyright
16 // notice, this list of conditions and the following disclaimer.
17 //
18 // 2. Redistributions in binary form must reproduce the above copyright
19 // notice, this list of conditions and the following disclaimer in the
20 // documentation and/or other materials provided with the distribution.
21 //
22 // 3. Neither the name of the Corporation nor the names of the
23 // contributors may be used to endorse or promote products derived from
24 // this software without specific prior written permission.
25 //
26 // THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
27 // EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
28 // IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
29 // PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
30 // CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
31 // EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
32 // PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
33 // PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
34 // LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
35 // NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
36 // SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
37 //
38 // Questions? Contact
39 // Jonathan Hu (jhu@sandia.gov)
40 // Andrey Prokopenko (aprokop@sandia.gov)
41 // Ray Tuminaro (rstumin@sandia.gov)
42 //
43 // ***********************************************************************
44 //
45 // @HEADER
46 #ifndef MUELU_AMGXOPERATOR_DECL_HPP
47 #define MUELU_AMGXOPERATOR_DECL_HPP
48 
49 #if defined (HAVE_MUELU_AMGX)
51 
52 #include <Tpetra_Operator.hpp>
53 #include <Tpetra_CrsMatrix.hpp>
54 #include <Tpetra_MultiVector.hpp>
55 #include <Tpetra_Distributor.hpp>
56 #include <Tpetra_HashTable.hpp>
57 #include <Tpetra_Import.hpp>
58 #include <Tpetra_Import_Util.hpp>
59 
60 #include "MueLu_Exceptions.hpp"
61 #include "MueLu_TimeMonitor.hpp"
62 #include "MueLu_TpetraOperator.hpp"
63 #include "MueLu_VerboseObject.hpp"
64 
65 #include <cuda_runtime.h>
66 #include <amgx_c.h>
67 
68 namespace MueLu {
69 
70 
77  template <class Scalar,
78  class LocalOrdinal,
79  class GlobalOrdinal,
80  class Node>
81  class AMGXOperator : public TpetraOperator<Scalar, LocalOrdinal, GlobalOrdinal, Node>, public BaseClass {
82  private:
83  typedef Scalar SC;
84  typedef LocalOrdinal LO;
85  typedef GlobalOrdinal GO;
86  typedef Node NO;
87 
88  typedef Tpetra::Map<LO,GO,NO> Map;
89  typedef Tpetra::MultiVector<SC,LO,GO,NO> MultiVector;
90 
91  public:
92 
94 
95 
97  AMGXOperator(const Teuchos::RCP<Tpetra::CrsMatrix<SC,LO,GO,NO> > &InA, Teuchos::ParameterList &paramListIn) { }
98 
100  virtual ~AMGXOperator() {}
101 
103 
106  throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
107  }
108 
111  throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
112  }
113 
115 
121  throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
122  }
123 
125  bool hasTransposeApply() const{
126  throw Exceptions::RuntimeError("Cannot use AMGXOperator with scalar != double and/or global ordinal != int \n");
127  }
128 
130  throw Exceptions::RuntimeError("AMGXOperator does not hold a MueLu::Hierarchy object \n");
131  }
132 
133  private:
134  };
135 
142  template<class Node>
143  class AMGXOperator<double, int, int, Node> : public TpetraOperator<double, int, int, Node> {
144  private:
145  typedef double SC;
146  typedef int LO;
147  typedef int GO;
148  typedef Node NO;
149 
150  typedef Tpetra::Map<LO,GO,NO> Map;
151  typedef Tpetra::MultiVector<SC,LO,GO,NO> MultiVector;
152 
153 
154  void printMaps(Teuchos::RCP<const Teuchos::Comm<int> >& comm, const std::vector<std::vector<int> >& vec, const std::vector<int>& perm,
155  const int* nbrs, const Map& map, const std::string& label) {
156  for (int p = 0; p < comm->getSize(); p++) {
157  if (comm->getRank() == p) {
158  std::cout << "========\n" << label << ", lid (gid), PID " << p << "\n========" << std::endl;
159 
160  for (size_t i = 0; i < vec.size(); ++i) {
161  std::cout << " neighbor " << nbrs[i] << " :";
162  for (size_t j = 0; j < vec[i].size(); ++j)
163  std::cout << " " << vec[i][j] << " (" << map.getGlobalElement(perm[vec[i][j]]) << ")";
164  std::cout << std::endl;
165  }
166  std::cout << std::endl;
167  } else {
168  sleep(1);
169  }
170  comm->barrier();
171  }
172  }
173 
174  public:
175 
177 
178  AMGXOperator(const Teuchos::RCP<Tpetra::CrsMatrix<SC,LO,GO,NO> > &inA, Teuchos::ParameterList &paramListIn) {
179  RCP<const Teuchos::Comm<int> > comm = inA->getRowMap()->getComm();
180  int numProcs = comm->getSize();
181  int myRank = comm->getRank();
182 
183 
184  RCP<Teuchos::Time> amgxTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: initialize");
185  amgxTimer->start();
186  // Initialize
187  //AMGX_SAFE_CALL(AMGX_initialize());
188  //AMGX_SAFE_CALL(AMGX_initialize_plugins());
189 
190 
191  /*system*/
192  //AMGX_SAFE_CALL(AMGX_register_print_callback(&print_callback));
193  AMGX_SAFE_CALL(AMGX_install_signal_handler());
194  Teuchos::ParameterList configs = paramListIn.sublist("amgx:params", true);
195  if (configs.isParameter("json file")) {
196  AMGX_SAFE_CALL(AMGX_config_create_from_file(&Config_, (const char *) &configs.get<std::string>("json file")[0]));
197  } else {
198  std::ostringstream oss;
199  oss << "";
201  for (itr = configs.begin(); itr != configs.end(); ++itr) {
202  const std::string& name = configs.name(itr);
203  const ParameterEntry& entry = configs.entry(itr);
204  oss << name << "=" << filterValueToString(entry) << ", ";
205  }
206  oss << "\0";
207  std::string configString = oss.str();
208  if (configString == "") {
209  //print msg that using defaults
210  //GetOStream(Warnings0) << "Warning: No configuration parameters specified, using default AMGX configuration parameters. \n";
211  }
212  AMGX_SAFE_CALL(AMGX_config_create(&Config_, configString.c_str()));
213  }
214 
215  // TODO: we probably need to add "exception_handling=1" to the parameter list
216  // to switch on internal error handling (with no need for AMGX_SAFE_CALL)
217 
218  //AMGX_SAFE_CALL(AMGX_config_add_parameters(&Config_, "exception_handling=1"))
219 
220 #define NEW_COMM
221 #ifdef NEW_COMM
222  // NOTE: MPI communicator used in AMGX_resources_create must exist in the scope of AMGX_matrix_comm_from_maps_one_ring
223  // FIXME: fix for serial comm
224  RCP<const Teuchos::MpiComm<int> > tmpic = Teuchos::rcp_dynamic_cast<const Teuchos::MpiComm<int> >(comm->duplicate());
225  TEUCHOS_TEST_FOR_EXCEPTION(tmpic.is_null(), Exceptions::RuntimeError, "Communicator is not MpiComm");
226 
227  RCP<const Teuchos::OpaqueWrapper<MPI_Comm> > rawMpiComm = tmpic->getRawMpiComm();
228  MPI_Comm mpiComm = *rawMpiComm;
229 #endif
230 
231  // Construct AMGX resources
232  if (numProcs == 1) {
233  AMGX_resources_create_simple(&Resources_, Config_);
234 
235  } else {
236  int numGPUDevices;
237  cudaGetDeviceCount(&numGPUDevices);
238  int device[] = {(comm->getRank() % numGPUDevices)};
239 
240  AMGX_config_add_parameters(&Config_, "communicator=MPI");
241 #ifdef NEW_COMM
242  AMGX_resources_create(&Resources_, Config_, &mpiComm, 1/* number of GPU devices utilized by this rank */, device);
243 #else
244  AMGX_resources_create(&Resources_, Config_, MPI_COMM_WORLD, 1/* number of GPU devices utilized by this rank */, device);
245 #endif
246  }
247 
248  AMGX_Mode mode = AMGX_mode_dDDI;
249  AMGX_solver_create(&Solver_, Resources_, mode, Config_);
250  AMGX_matrix_create(&A_, Resources_, mode);
251  AMGX_vector_create(&X_, Resources_, mode);
252  AMGX_vector_create(&Y_, Resources_, mode);
253 
254  amgxTimer->stop();
255  amgxTimer->incrementNumCalls();
256 
257  std::vector<int> amgx2muelu;
258 
259  // Construct AMGX communication pattern
260  if (numProcs > 1) {
261  RCP<const Tpetra::Import<LO,GO,NO> > importer = inA->getCrsGraph()->getImporter();
262 
263  TEUCHOS_TEST_FOR_EXCEPTION(importer.is_null(), MueLu::Exceptions::RuntimeError, "The matrix A has no Import object.");
264 
265  Tpetra::Distributor distributor = importer->getDistributor();
266 
267  Array<int> sendRanks = distributor.getProcsTo();
268  Array<int> recvRanks = distributor.getProcsFrom();
269 
270  std::sort(sendRanks.begin(), sendRanks.end());
271  std::sort(recvRanks.begin(), recvRanks.end());
272 
273  bool match = true;
274  if (sendRanks.size() != recvRanks.size()) {
275  match = false;
276  } else {
277  for (int i = 0; i < sendRanks.size(); i++) {
278  if (recvRanks[i] != sendRanks[i])
279  match = false;
280  break;
281  }
282  }
283  TEUCHOS_TEST_FOR_EXCEPTION(!match, MueLu::Exceptions::RuntimeError, "AMGX requires that the processors that we send to and receive from are the same. "
284  "This is not the case: we send to {" << sendRanks << "} and receive from {" << recvRanks << "}");
285 
286  int num_neighbors = sendRanks.size(); // does not include the calling process
287  const int* neighbors = &sendRanks[0];
288 
289  // Later on, we'll have to organize the send and recv data by PIDs,
290  // i.e, a vector V of vectors, where V[i] is PID i's vector of data.
291  // Hence we need to be able to quickly look up an array index
292  // associated with each PID.
293  Tpetra::Details::HashTable<int,int> hashTable(3*num_neighbors);
294  for (int i = 0; i < num_neighbors; i++)
295  hashTable.add(neighbors[i], i);
296 
297  // Get some information out
298  ArrayView<const int> exportLIDs = importer->getExportLIDs();
299  ArrayView<const int> exportPIDs = importer->getExportPIDs();
300  Array<int> importPIDs;
301  Tpetra::Import_Util::getPids(*importer, importPIDs, true/* make local -1 */);
302 
303  // Construct the reordering for AMGX as in AMGX_matrix_upload_all documentation
304  RCP<const Map> rowMap = inA->getRowMap();
305  RCP<const Map> colMap = inA->getColMap();
306 
307  int N = rowMap->getNodeNumElements(), Nc = colMap->getNodeNumElements();
308  muelu2amgx_.resize(Nc, -1);
309 
310  int numUniqExports = 0;
311  for (int i = 0; i < exportLIDs.size(); i++)
312  if (muelu2amgx_[exportLIDs[i]] == -1) {
313  numUniqExports++;
314  muelu2amgx_[exportLIDs[i]] = -2;
315  }
316 
317  int localOffset = 0, exportOffset = N - numUniqExports;
318  // Go through exported LIDs and put them at the end of LIDs
319  for (int i = 0; i < exportLIDs.size(); i++)
320  if (muelu2amgx_[exportLIDs[i]] < 0) // exportLIDs are not unique
321  muelu2amgx_[exportLIDs[i]] = exportOffset++;
322  // Go through all non-export LIDs, and put them at the beginning of LIDs
323  for (int i = 0; i < N; i++)
324  if (muelu2amgx_[i] == -1)
325  muelu2amgx_[i] = localOffset++;
326  // Go through the tail (imported LIDs), and order those by neighbors
327  int importOffset = N;
328  for (int k = 0; k < num_neighbors; k++)
329  for (int i = 0; i < importPIDs.size(); i++)
330  if (importPIDs[i] != -1 && hashTable.get(importPIDs[i]) == k)
331  muelu2amgx_[i] = importOffset++;
332 
333  amgx2muelu.resize(muelu2amgx_.size());
334  for (int i = 0; i < (int)muelu2amgx_.size(); i++)
335  amgx2muelu[muelu2amgx_[i]] = i;
336 
337  // Construct send arrays
338  std::vector<std::vector<int> > sendDatas (num_neighbors);
339  std::vector<int> send_sizes(num_neighbors, 0);
340  for (int i = 0; i < exportPIDs.size(); i++) {
341  int index = hashTable.get(exportPIDs[i]);
342  sendDatas [index].push_back(muelu2amgx_[exportLIDs[i]]);
343  send_sizes[index]++;
344  }
345  // FIXME: sendDatas must be sorted (based on GIDs)
346 
347  std::vector<const int*> send_maps(num_neighbors);
348  for (int i = 0; i < num_neighbors; i++)
349  send_maps[i] = &(sendDatas[i][0]);
350 
351  // Debugging
352  // printMaps(comm, sendDatas, amgx2muelu, neighbors, *importer->getTargetMap(), "send_map_vector");
353 
354  // Construct recv arrays
355  std::vector<std::vector<int> > recvDatas (num_neighbors);
356  std::vector<int> recv_sizes(num_neighbors, 0);
357  for (int i = 0; i < importPIDs.size(); i++)
358  if (importPIDs[i] != -1) {
359  int index = hashTable.get(importPIDs[i]);
360  recvDatas [index].push_back(muelu2amgx_[i]);
361  recv_sizes[index]++;
362  }
363  // FIXME: recvDatas must be sorted (based on GIDs)
364 
365  std::vector<const int*> recv_maps(num_neighbors);
366  for (int i = 0; i < num_neighbors; i++)
367  recv_maps[i] = &(recvDatas[i][0]);
368 
369  // Debugging
370  // printMaps(comm, recvDatas, amgx2muelu, neighbors, *importer->getTargetMap(), "recv_map_vector");
371 
372  AMGX_SAFE_CALL(AMGX_matrix_comm_from_maps_one_ring(A_, 1, num_neighbors, neighbors, &send_sizes[0], &send_maps[0], &recv_sizes[0], &recv_maps[0]));
373 
374  AMGX_vector_bind(X_, A_);
375  AMGX_vector_bind(Y_, A_);
376  }
377 
378  RCP<Teuchos::Time> matrixTransformTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transform matrix");
379  matrixTransformTimer->start();
380 
384  inA->getAllValues(ia_s, ja, a);
385 
386  ArrayRCP<int> ia(ia_s.size());
387  for (int i = 0; i < ia.size(); i++)
388  ia[i] = Teuchos::as<int>(ia_s[i]);
389 
390  N_ = inA->getNodeNumRows();
391  int nnz = inA->getNodeNumEntries();
392 
393  matrixTransformTimer->stop();
394  matrixTransformTimer->incrementNumCalls();
395 
396 
397  // Upload matrix
398  // TODO Do we need to pin memory here through AMGX_pin_memory?
399  RCP<Teuchos::Time> matrixTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer matrix CPU->GPU");
400  matrixTimer->start();
401  if (numProcs == 1) {
402  AMGX_matrix_upload_all(A_, N_, nnz, 1, 1, &ia[0], &ja[0], &a[0], NULL);
403 
404  } else {
405  // Transform the matrix
406  std::vector<int> ia_new(ia.size());
407  std::vector<int> ja_new(ja.size());
408  std::vector<double> a_new (a.size());
409 
410  ia_new[0] = 0;
411  for (int i = 0; i < N_; i++) {
412  int oldRow = amgx2muelu[i];
413 
414  ia_new[i+1] = ia_new[i] + (ia[oldRow+1] - ia[oldRow]);
415 
416  for (int j = ia[oldRow]; j < ia[oldRow+1]; j++) {
417  int offset = j - ia[oldRow];
418  ja_new[ia_new[i] + offset] = muelu2amgx_[ja[j]];
419  a_new [ia_new[i] + offset] = a[j];
420  }
421  // Do bubble sort on two arrays
422  // NOTE: There are multiple possible optimizations here (even of bubble sort)
423  bool swapped;
424  do {
425  swapped = false;
426 
427  for (int j = ia_new[i]; j < ia_new[i+1]-1; j++)
428  if (ja_new[j] > ja_new[j+1]) {
429  std::swap(ja_new[j], ja_new[j+1]);
430  std::swap(a_new [j], a_new [j+1]);
431  swapped = true;
432  }
433  } while (swapped == true);
434  }
435 
436  AMGX_matrix_upload_all(A_, N_, nnz, 1, 1, &ia_new[0], &ja_new[0], &a_new[0], NULL);
437  }
438  matrixTimer->stop();
439  matrixTimer->incrementNumCalls();
440 
441  domainMap_ = inA->getDomainMap();
442  rangeMap_ = inA->getRangeMap();
443 
444  RCP<Teuchos::Time> realSetupTimer = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: setup (total)");
445  realSetupTimer->start();
446  AMGX_solver_setup(Solver_, A_);
447  realSetupTimer->stop();
448  realSetupTimer->incrementNumCalls();
449 
450  vectorTimer1_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer vectors CPU->GPU");
451  vectorTimer2_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: transfer vector GPU->CPU");
452  solverTimer_ = Teuchos::TimeMonitor::getNewTimer("MueLu: AMGX: Solve (total)");
453  }
454 
456  virtual ~AMGXOperator()
457  {
458  // Comment this out if you need rebuild to work. This causes AMGX_solver_destroy memory issues.
459  AMGX_SAFE_CALL(AMGX_solver_destroy(Solver_));
460  AMGX_SAFE_CALL(AMGX_vector_destroy(X_));
461  AMGX_SAFE_CALL(AMGX_vector_destroy(Y_));
462  AMGX_SAFE_CALL(AMGX_matrix_destroy(A_));
463  AMGX_SAFE_CALL(AMGX_resources_destroy(Resources_));
464  AMGX_SAFE_CALL(AMGX_config_destroy(Config_));
465  }
466 
467 
470 
473 
475 
481 
483  bool hasTransposeApply() const;
484 
486  throw Exceptions::RuntimeError("AMGXOperator does not hold a MueLu::Hierarchy object \n");
487  }
488 
489  std::string filterValueToString(const Teuchos::ParameterEntry& entry ) {
490  return ( entry.isList() ? std::string("...") : toString(entry.getAny()) );
491  }
492 
493  int sizeA() {
494  int sizeX, sizeY, n;
495  AMGX_matrix_get_size(A_, &n, &sizeX, &sizeY);
496  return n;
497  }
498 
499  int iters() {
500  int it;
501  AMGX_solver_get_iterations_number(Solver_, &it);
502  return it;
503  }
504 
505  AMGX_SOLVE_STATUS getStatus() {
506  AMGX_SOLVE_STATUS status;
507  AMGX_solver_get_status(Solver_, &status);
508  return status;
509  }
510 
511 
512  private:
513  AMGX_solver_handle Solver_;
514  AMGX_resources_handle Resources_;
515  AMGX_config_handle Config_;
516  AMGX_matrix_handle A_;
517  AMGX_vector_handle X_;
518  AMGX_vector_handle Y_;
519  int N_;
520 
523 
524  std::vector<int> muelu2amgx_;
525 
529  };
530 
531 } // namespace
532 
533 #endif //HAVE_MUELU_AMGX
534 #endif // MUELU_AMGXOPERATOR_DECL_HPP
RCP< MueLu::Hierarchy< SC, LO, GO, NO > > GetHierarchy() const
const std::string & name() const
ConstIterator end() const
MueLu::DefaultLocalOrdinal LocalOrdinal
std::string toString(const T &what)
Little helper function to convert non-string types to strings.
AMGXOperator(const Teuchos::RCP< Tpetra::CrsMatrix< SC, LO, GO, NO > > &inA, Teuchos::ParameterList &paramListIn)
T & get(const std::string &name, T def_value)
void printMaps(Teuchos::RCP< const Teuchos::Comm< int > > &comm, const std::vector< std::vector< int > > &vec, const std::vector< int > &perm, const int *nbrs, const Map &map, const std::string &label)
#define TEUCHOS_TEST_FOR_EXCEPTION(throw_exception_test, Exception, msg)
Teuchos::RCP< const Map > getRangeMap() const
Returns the Tpetra::Map object associated with the range of this operator.
size_type size() const
size_type size() const
virtual ~AMGXOperator()
Destructor.
MueLu::DefaultNode Node
Tpetra::Map< LO, GO, NO > Map
static RCP< Time > getNewTimer(const std::string &name)
bool isParameter(const std::string &name) const
void start(bool reset=false)
MueLu::DefaultScalar Scalar
MueLu::DefaultGlobalOrdinal GlobalOrdinal
double stop()
void apply(const MultiVector &X, MultiVector &Y, Teuchos::ETransp mode=Teuchos::NO_TRANS, Scalar alpha=Teuchos::ScalarTraits< Scalar >::one(), Scalar beta=Teuchos::ScalarTraits< Scalar >::zero()) const
Returns a solution for the linear system AX=Y in the Tpetra::MultiVector X.
params_t::ConstIterator ConstIterator
ConstIterator begin() const
iterator end()
std::string filterValueToString(const Teuchos::ParameterEntry &entry)
const ParameterEntry & entry(ConstIterator i) const
any & getAny(bool activeQry=true)
Teuchos::RCP< const Map > getDomainMap() const
Returns the Tpetra::Map object associated with the domain of this operator.
AMGXOperator(const Teuchos::RCP< Tpetra::CrsMatrix< SC, LO, GO, NO > > &InA, Teuchos::ParameterList &paramListIn)
Constructor.
size_type size() const
Wraps an existing MueLu::Hierarchy as a Tpetra::Operator.
Tpetra::MultiVector< SC, LO, GO, NO > MultiVector
ParameterList & sublist(const std::string &name, bool mustAlreadyExist=false, const std::string &docString="")
Exception throws to report errors in the internal logical of the program.
Tpetra::MultiVector< SC, LO, GO, NO > MultiVector
RCP< MueLu::Hierarchy< SC, LO, GO, NO > > GetHierarchy() const
void incrementNumCalls()
iterator begin()
bool hasTransposeApply() const
Indicates whether this operator supports applying the adjoint operator.
Adapter for AmgX library from Nvidia.
bool is_null() const