1 #ifndef _ZOLTAN2_DISTANCE2_HPP_
2 #define _ZOLTAN2_DISTANCE2_HPP_
5 #include <unordered_map>
22 #include "Tpetra_Core.hpp"
23 #include "Teuchos_RCP.hpp"
24 #include "Tpetra_Import.hpp"
25 #include "Tpetra_FEMultiVector.hpp"
27 #include "Kokkos_Core.hpp"
28 #include "KokkosSparse_CrsMatrix.hpp"
29 #include "KokkosKernels_Handle.hpp"
30 #include "KokkosKernels_IOUtils.hpp"
31 #include "KokkosGraph_Distance2Color.hpp"
32 #include "KokkosGraph_Distance2ColorHandle.hpp"
41 template <
typename Adapter>
51 using map_t = Tpetra::Map<lno_t,gno_t>;
53 using femv_t = Tpetra::FEMultiVector<femv_scalar_t, lno_t, gno_t>;
57 using host_exec =
typename femv_t::host_view_type::device_type::execution_space;
58 using host_mem =
typename femv_t::host_view_type::device_type::memory_space;
63 template <
class ExecutionSpace,
typename MemorySpace>
64 void localColoring(
const size_t nVtx,
65 Kokkos::View<
lno_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> adjs_view,
66 Kokkos::View<
offset_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> offset_view,
67 Teuchos::RCP<femv_t> femv,
68 Kokkos::View<
lno_t*, Kokkos::Device<ExecutionSpace, MemorySpace> > vertex_list,
69 size_t vertex_list_size = 0,
70 bool use_vertex_based_coloring =
false){
71 using KernelHandle = KokkosKernels::Experimental::KokkosKernelsHandle
78 kh.create_distance2_graph_coloring_handle(KokkosGraph::COLORING_D2_NB_BIT);
82 if(vertex_list_size != 0){
83 kh.get_distance2_graph_coloring_handle()->set_vertex_list(vertex_list, vertex_list_size);
90 auto femvColors = femv->template getLocalView<Kokkos::Device<ExecutionSpace,MemorySpace> >(Tpetra::Access::ReadWrite);
91 auto sv = subview(femvColors, Kokkos::ALL, 0);
92 kh.get_distance2_graph_coloring_handle()->set_vertex_colors(sv);
95 KokkosGraph::Experimental::graph_color_distance2(&kh, nVtx, offset_view, adjs_view);
100 std::cout<<
"\nKokkosKernels Coloring: "
101 <<kh.get_distance2_graph_coloring_handle()->get_overall_coloring_time()
107 virtual void colorInterior(
const size_t nVtx,
108 Kokkos::View<lno_t*,device_type> adjs_view,
109 Kokkos::View<offset_t*, device_type> offset_view,
110 Teuchos::RCP<femv_t> femv,
111 Kokkos::View<lno_t*, device_type> vertex_list,
112 size_t vertex_list_size = 0,
115 this->localColoring<execution_space, memory_space>(nVtx,
125 virtual void colorInterior_serial(
const size_t nVtx,
126 typename Kokkos::View<lno_t*, device_type>::HostMirror adjs_view,
127 typename Kokkos::View<offset_t*, device_type>::HostMirror offset_view,
128 Teuchos::RCP<femv_t> femv,
129 typename Kokkos::View<lno_t*, device_type>::HostMirror vertex_list,
130 size_t vertex_list_size = 0,
132 this->localColoring<host_exec, host_mem>(nVtx,
143 template<
class ExecutionSpace,
typename MemorySpace>
145 Kokkos::View<
offset_t*, Kokkos::Device<ExecutionSpace,MemorySpace>> dist_offsets,
146 Kokkos::View<
lno_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> dist_adjs,
147 Kokkos::View<
int*, Kokkos::Device<ExecutionSpace, MemorySpace>> femv_colors,
148 Kokkos::View<
lno_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> boundary_verts_view,
150 Kokkos::Device<ExecutionSpace, MemorySpace> > verts_to_recolor_view,
152 Kokkos::Device<ExecutionSpace, MemorySpace>,
153 Kokkos::MemoryTraits<Kokkos::Atomic> > verts_to_recolor_size_atomic,
155 Kokkos::Device<ExecutionSpace, MemorySpace> > verts_to_send_view,
156 Kokkos::View<
size_t*,
157 Kokkos::Device<ExecutionSpace, MemorySpace>,
158 Kokkos::MemoryTraits<Kokkos::Atomic> > verts_to_send_size_atomic,
159 Kokkos::View<
size_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> recoloringSize,
160 Kokkos::View<
int*, Kokkos::Device<ExecutionSpace, MemorySpace>> rand,
161 Kokkos::View<
gno_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> gid,
162 Kokkos::View<
gno_t*, Kokkos::Device<ExecutionSpace, MemorySpace>> ghost_degrees,
163 bool recolor_degrees){
164 Kokkos::RangePolicy<ExecutionSpace> policy(0, boundary_verts_view.extent(0));
165 size_t local_recoloring_size;
166 Kokkos::parallel_reduce(
"D2 conflict detection",policy, KOKKOS_LAMBDA(
const uint64_t& i,
size_t& recoloring_size){
168 const size_t curr_lid = boundary_verts_view(i);
169 const int curr_color = femv_colors(curr_lid);
170 const size_t vid_d1_adj_begin = dist_offsets(curr_lid);
171 const size_t vid_d1_adj_end = dist_offsets(curr_lid+1);
172 const size_t curr_degree = vid_d1_adj_end - vid_d1_adj_begin;
173 for(
size_t vid_d1_adj = vid_d1_adj_begin; vid_d1_adj < vid_d1_adj_end; vid_d1_adj++){
175 size_t vid_d1 = dist_adjs(vid_d1_adj);
176 size_t vid_d1_degree = 0;
178 if(vid_d1 < n_local){
179 vid_d1_degree = dist_offsets(vid_d1+1) - dist_offsets(vid_d1);
181 vid_d1_degree = ghost_degrees(vid_d1-n_local);
183 if( vid_d1 != curr_lid && femv_colors(vid_d1) == curr_color){
184 if(curr_degree < vid_d1_degree && recolor_degrees){
185 femv_colors(curr_lid) = 0;
188 }
else if (vid_d1_degree < curr_degree && recolor_degrees){
189 femv_colors(vid_d1) = 0;
191 }
else if(rand(curr_lid) < rand(vid_d1)){
192 femv_colors(curr_lid) = 0;
195 }
else if(rand(vid_d1) < rand(curr_lid)){
196 femv_colors(vid_d1) = 0;
199 if(gid(curr_lid) >= gid(vid_d1)){
200 femv_colors(curr_lid) = 0;
204 femv_colors(vid_d1) = 0;
209 size_t d2_adj_begin = 0;
210 size_t d2_adj_end = 0;
211 d2_adj_begin = dist_offsets(vid_d1);
212 d2_adj_end = dist_offsets(vid_d1+1);
218 for(
size_t vid_d2_adj = d2_adj_begin; vid_d2_adj < d2_adj_end; vid_d2_adj++){
220 const size_t vid_d2 = dist_adjs(vid_d2_adj);
221 size_t vid_d2_degree = 0;
223 if(vid_d2 < n_local){
224 vid_d2_degree = dist_offsets(vid_d2+1) - dist_offsets(vid_d2);
226 vid_d2_degree = ghost_degrees(vid_d2-n_local);
228 if(curr_lid != vid_d2 && femv_colors(vid_d2) == curr_color){
229 if(curr_degree < vid_d2_degree && recolor_degrees){
231 femv_colors(curr_lid) = 0;
234 }
else if(vid_d2_degree < curr_degree && recolor_degrees){
235 femv_colors(vid_d2) = 0;
237 }
else if(rand(curr_lid) < rand(vid_d2)){
239 femv_colors(curr_lid) = 0;
242 }
else if(rand(vid_d2) < rand(curr_lid)){
243 femv_colors(vid_d2) = 0;
246 if(gid(curr_lid) >= gid(vid_d2)){
248 femv_colors(curr_lid) = 0;
252 femv_colors(vid_d2) = 0;
260 },local_recoloring_size);
261 Kokkos::deep_copy(recoloringSize,local_recoloring_size);
265 Kokkos::parallel_for(
"rebuild verts_to_send and verts_to_recolor",
266 Kokkos::RangePolicy<ExecutionSpace>(0,femv_colors.size()),
267 KOKKOS_LAMBDA(
const uint64_t& i){
268 if(femv_colors(i) == 0){
271 verts_to_send_view(verts_to_send_size_atomic(0)++) = i;
274 verts_to_recolor_view(verts_to_recolor_size_atomic(0)++) = i;
283 Kokkos::View<offset_t*, device_type > dist_offsets_dev,
284 Kokkos::View<lno_t*, device_type > dist_adjs_dev,
285 Kokkos::View<int*,device_type > femv_colors,
286 Kokkos::View<lno_t*, device_type > boundary_verts_view,
291 Kokkos::MemoryTraits<Kokkos::Atomic>> verts_to_recolor_size_atomic,
294 Kokkos::View<
size_t*,
296 Kokkos::MemoryTraits<Kokkos::Atomic>> verts_to_send_size_atomic,
297 Kokkos::View<size_t*, device_type> recoloringSize,
304 bool recolor_degrees){
306 this->detectD2Conflicts<execution_space, memory_space>(n_local,
311 verts_to_recolor_view,
312 verts_to_recolor_size_atomic,
314 verts_to_send_size_atomic,
323 typename Kokkos::View<offset_t*, device_type >::HostMirror dist_offsets_host,
324 typename Kokkos::View<lno_t*, device_type >::HostMirror dist_adjs_host,
325 typename Kokkos::View<int*,device_type >::HostMirror femv_colors,
326 typename Kokkos::View<lno_t*, device_type >::HostMirror boundary_verts_view,
327 typename Kokkos::View<lno_t*,device_type>::HostMirror verts_to_recolor,
328 typename Kokkos::View<int*,device_type>::HostMirror verts_to_recolor_size,
329 typename Kokkos::View<lno_t*,device_type>::HostMirror verts_to_send,
330 typename Kokkos::View<size_t*,device_type>::HostMirror verts_to_send_size,
331 typename Kokkos::View<size_t*, device_type>::HostMirror recoloringSize,
332 typename Kokkos::View<int*, device_type>::HostMirror rand,
333 typename Kokkos::View<gno_t*,device_type>::HostMirror gid,
334 typename Kokkos::View<gno_t*,device_type>::HostMirror ghost_degrees,
335 bool recolor_degrees) {
336 this->detectD2Conflicts<host_exec, host_mem>(n_local,
342 verts_to_recolor_size,
354 Kokkos::View<offset_t*, device_type> dist_offsets_dev,
355 Kokkos::View<lno_t*, device_type> dist_adjs_dev,
356 typename Kokkos::View<offset_t*, device_type>::HostMirror dist_offsets_host,
357 typename Kokkos::View<lno_t*, device_type>::HostMirror dist_adjs_host,
358 Kokkos::View<lno_t*, device_type>& boundary_verts,
361 Kokkos::View<
size_t*,
363 Kokkos::MemoryTraits<Kokkos::Atomic>> verts_to_send_size_atomic){
366 gno_t boundary_size_temp = 0;
367 for(
size_t i = 0; i < n_local; i++){
368 for(
offset_t j = dist_offsets_host(i); j < dist_offsets_host(i+1); j++){
369 if((
size_t)dist_adjs_host(j) >= n_local){
370 boundary_size_temp++;
374 for(
offset_t k = dist_offsets_host(dist_adjs_host(j)); k < dist_offsets_host(dist_adjs_host(j)+1); k++){
375 if((
size_t)dist_adjs_host(k) >= n_local){
376 boundary_size_temp++;
386 boundary_verts = Kokkos::View<lno_t*, device_type>(
"boundary verts",boundary_size_temp);
387 typename Kokkos::View<lno_t*, device_type>::HostMirror boundary_verts_host = Kokkos::create_mirror_view(boundary_verts);
390 boundary_size_temp = 0;
393 for(
size_t i = 0; i < n_local; i++){
394 for(
offset_t j = dist_offsets_host(i); j < dist_offsets_host(i+1); j++){
395 if((
size_t)dist_adjs_host(j) >= n_local){
396 boundary_verts_host(boundary_size_temp++) = i;
400 for(
offset_t k = dist_offsets_host(dist_adjs_host(j)); k < dist_offsets_host(dist_adjs_host(j)+1); k++){
401 if((
size_t)dist_adjs_host(k) >= n_local){
402 boundary_verts_host(boundary_size_temp++) = i;
411 Kokkos::deep_copy(boundary_verts, boundary_verts_host);
414 Kokkos::parallel_for(
"init verts to send",
415 Kokkos::RangePolicy<execution_space, int>(0,n_local),
416 KOKKOS_LAMBDA(
const int& i){
417 for(
offset_t j = dist_offsets_dev(i); j < dist_offsets_dev(i+1); j++){
418 if((
size_t)dist_adjs_dev(j) >= n_local){
419 verts_to_send_view(verts_to_send_size_atomic(0)++) = i;
423 for(
offset_t k = dist_offsets_dev(dist_adjs_dev(j)); k < dist_offsets_dev(dist_adjs_dev(j)+1); k++){
424 if((
size_t)dist_adjs_dev(k) >= n_local){
425 verts_to_send_view(verts_to_send_size_atomic(0)++) = i;
438 const RCP<const base_adapter_t> &adapter_,
439 const RCP<Teuchos::ParameterList> &pl_,
440 const RCP<Environment> &env_,
441 const RCP<
const Teuchos::Comm<int> > &comm_)
Zoltan2::BaseAdapter< userTypes_t > base_adapter_t
Tpetra::FEMultiVector< femv_scalar_t, lno_t, gno_t > femv_t
virtual void constructBoundary(const size_t n_local, Kokkos::View< offset_t *, device_type > dist_offsets_dev, Kokkos::View< lno_t *, device_type > dist_adjs_dev, typename Kokkos::View< offset_t *, device_type >::HostMirror dist_offsets_host, typename Kokkos::View< lno_t *, device_type >::HostMirror dist_adjs_host, Kokkos::View< lno_t *, device_type > &boundary_verts, Kokkos::View< lno_t *, device_type > verts_to_send_view, Kokkos::View< size_t *, device_type, Kokkos::MemoryTraits< Kokkos::Atomic >> verts_to_send_size_atomic)
typename Adapter::gno_t gno_t
virtual void detectConflicts_serial(const size_t n_local, typename Kokkos::View< offset_t *, device_type >::HostMirror dist_offsets_host, typename Kokkos::View< lno_t *, device_type >::HostMirror dist_adjs_host, typename Kokkos::View< int *, device_type >::HostMirror femv_colors, typename Kokkos::View< lno_t *, device_type >::HostMirror boundary_verts_view, typename Kokkos::View< lno_t *, device_type >::HostMirror verts_to_recolor, typename Kokkos::View< int *, device_type >::HostMirror verts_to_recolor_size, typename Kokkos::View< lno_t *, device_type >::HostMirror verts_to_send, typename Kokkos::View< size_t *, device_type >::HostMirror verts_to_send_size, typename Kokkos::View< size_t *, device_type >::HostMirror recoloringSize, typename Kokkos::View< int *, device_type >::HostMirror rand, typename Kokkos::View< gno_t *, device_type >::HostMirror gid, typename Kokkos::View< gno_t *, device_type >::HostMirror ghost_degrees, bool recolor_degrees)
typename Adapter::lno_t lno_t
map_t::global_ordinal_type gno_t
typename Adapter::offset_t offset_t
virtual void detectConflicts(const size_t n_local, Kokkos::View< offset_t *, device_type > dist_offsets_dev, Kokkos::View< lno_t *, device_type > dist_adjs_dev, Kokkos::View< int *, device_type > femv_colors, Kokkos::View< lno_t *, device_type > boundary_verts_view, Kokkos::View< lno_t *, device_type > verts_to_recolor_view, Kokkos::View< int *, device_type, Kokkos::MemoryTraits< Kokkos::Atomic >> verts_to_recolor_size_atomic, Kokkos::View< lno_t *, device_type > verts_to_send_view, Kokkos::View< size_t *, device_type, Kokkos::MemoryTraits< Kokkos::Atomic >> verts_to_send_size_atomic, Kokkos::View< size_t *, device_type > recoloringSize, Kokkos::View< int *, device_type > rand, Kokkos::View< gno_t *, device_type > gid, Kokkos::View< gno_t *, device_type > ghost_degrees, bool recolor_degrees)
typename femv_t::host_view_type::device_type::memory_space host_mem
map_t::local_ordinal_type lno_t
typename femv_t::device_type device_type
Traits class to handle conversions between gno_t/lno_t and TPL data types (e.g., ParMETIS's idx_t...
typename femv_t::host_view_type::device_type::execution_space host_exec
AlgDistance2(const RCP< const base_adapter_t > &adapter_, const RCP< Teuchos::ParameterList > &pl_, const RCP< Environment > &env_, const RCP< const Teuchos::Comm< int > > &comm_)
Defines the ColoringSolution class.
Defines the GraphModel interface.
void detectD2Conflicts(const size_t n_local, Kokkos::View< offset_t *, Kokkos::Device< ExecutionSpace, MemorySpace >> dist_offsets, Kokkos::View< lno_t *, Kokkos::Device< ExecutionSpace, MemorySpace >> dist_adjs, Kokkos::View< int *, Kokkos::Device< ExecutionSpace, MemorySpace >> femv_colors, Kokkos::View< lno_t *, Kokkos::Device< ExecutionSpace, MemorySpace >> boundary_verts_view, Kokkos::View< lno_t *, Kokkos::Device< ExecutionSpace, MemorySpace > > verts_to_recolor_view, Kokkos::View< int *, Kokkos::Device< ExecutionSpace, MemorySpace >, Kokkos::MemoryTraits< Kokkos::Atomic > > verts_to_recolor_size_atomic, Kokkos::View< lno_t *, Kokkos::Device< ExecutionSpace, MemorySpace > > verts_to_send_view, Kokkos::View< size_t *, Kokkos::Device< ExecutionSpace, MemorySpace >, Kokkos::MemoryTraits< Kokkos::Atomic > > verts_to_send_size_atomic, Kokkos::View< size_t *, Kokkos::Device< ExecutionSpace, MemorySpace >> recoloringSize, Kokkos::View< int *, Kokkos::Device< ExecutionSpace, MemorySpace >> rand, Kokkos::View< gno_t *, Kokkos::Device< ExecutionSpace, MemorySpace >> gid, Kokkos::View< gno_t *, Kokkos::Device< ExecutionSpace, MemorySpace >> ghost_degrees, bool recolor_degrees)
A gathering of useful namespace methods.
typename Adapter::base_adapter_t base_adapter_t
typename Adapter::lno_t lno_t
typename device_type::execution_space execution_space
Tpetra::Map< lno_t, gno_t > map_t
typename Adapter::scalar_t scalar_t
typename device_type::memory_space memory_space
AlltoAll communication methods.
typename Adapter::offset_t offset_t