Tpetra_withLocalAccess.hpp

Go to the documentation of this file.

/*
// @HEADER
// ***********************************************************************
//
//          Tpetra: Templated Linear Algebra Services Package
//                 Copyright (2008) Sandia Corporation
//
// Under the terms of Contract DE-AC04-94AL85000 with Sandia Corporation,
// the U.S. Government retains certain rights in this software.
//
// Redistribution and use in source and binary forms, with or without
// modification, are permitted provided that the following conditions are
// met:
//
// 1. Redistributions of source code must retain the above copyright
// notice, this list of conditions and the following disclaimer.
//
// 2. Redistributions in binary form must reproduce the above copyright
// notice, this list of conditions and the following disclaimer in the
// documentation and/or other materials provided with the distribution.
//
// 3. Neither the name of the Corporation nor the names of the
// contributors may be used to endorse or promote products derived from
// this software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY SANDIA CORPORATION "AS IS" AND ANY
// EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
// IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL SANDIA CORPORATION OR THE
// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF
// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING
// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
//
// Questions? Contact Michael A. Heroux (maherou@sandia.gov)
//
// ************************************************************************
// @HEADER
*/

#ifndef TPETRA_WITHLOCALACCESS_HPP
#define TPETRA_WITHLOCALACCESS_HPP

#include "TpetraCore_config.h"
#include <functional>
#include <type_traits>


namespace Tpetra {

  // Generic classes needed to implement withLocalAccess

  namespace Details {
    enum class AccessMode {
      ReadOnly,
      WriteOnly,
      ReadWrite
    };

    template<class GlobalObjectType>
    struct DefaultMemorySpace {
      using type = typename GlobalObjectType::device_type::memory_space;

      // Given a global object, get its (default) memory space instance.
      static type space (const GlobalObjectType& /* G */) {
        // This stub just assumes that 'type' is default constructible.
        // In Kokkos, default-constructing a memory space instance just
        // gives the default memory space.
        return type ();
      }
    };

#ifndef DOXYGEN_SHOULD_SKIP_THIS
    template<class GlobalObjectType,
             class MemorySpace,
             const AccessMode am>
    class LocalAccess; // forward declaration
#endif // DOXYGEN_SHOULD_SKIP_THIS

    template<class LocalAccessType>
    struct GetMasterLocalObject {};

    template<class LocalAccessType>
    typename GetMasterLocalObject<LocalAccessType>::master_local_object_type
    getMasterLocalObject (LocalAccessType LA) {
      return GetMasterLocalObject<LocalAccessType>::get (LA);
    }

    template<class LocalAccessType>
    struct GetNonowningLocalObject {};

    template<class LocalAccessType>
    typename GetNonowningLocalObject<LocalAccessType>::
    nonowning_local_object_type
    getNonowningLocalObject (LocalAccessType LA,
      const typename GetMasterLocalObject<LocalAccessType>::
        master_local_object_type& master)
    {
      return GetNonowningLocalObject<LocalAccessType>::get (LA, master);
    }
  } // namespace Details

  template<class LocalAccessType>
  using with_local_access_function_argument_type =
    typename Details::GetNonowningLocalObject<LocalAccessType>::
      nonowning_local_object_type;

  // Users call readOnly, writeOnly, and readWrite, in order to declare
  // how they intend to access a global object's local data.

  template<class GlobalObjectType>
  Details::LocalAccess<
    GlobalObjectType,
    typename Details::DefaultMemorySpace<GlobalObjectType>::type,
    Details::AccessMode::ReadOnly>
  readOnly (GlobalObjectType&);

  template<class GlobalObjectType>
  Details::LocalAccess<
    GlobalObjectType,
    typename Details::DefaultMemorySpace<GlobalObjectType>::type,
    Details::AccessMode::ReadOnly>
  readOnly (const GlobalObjectType&);

  template<class GlobalObjectType>
  Details::LocalAccess<
    GlobalObjectType,
    typename Details::DefaultMemorySpace<GlobalObjectType>::type,
    Details::AccessMode::WriteOnly>
  writeOnly (GlobalObjectType&);

  template<class GlobalObjectType>
  Details::LocalAccess<
    GlobalObjectType,
    typename Details::DefaultMemorySpace<GlobalObjectType>::type,
    Details::AccessMode::ReadWrite>
  readWrite (GlobalObjectType&);

  // LocalAccess struct

  namespace Details {
    template<class GlobalObjectType,
             class MemorySpace,
             const AccessMode am>
    class LocalAccess {
    public:
      using global_object_type = GlobalObjectType;
      using memory_space = typename MemorySpace::memory_space;
      static constexpr AccessMode access_mode = am;

    private:
      using canonical_this_type = LocalAccess<global_object_type,
                                              memory_space,
                                              access_mode>;
    public:
      LocalAccess (global_object_type& G,
                   memory_space space = memory_space (),
                   const bool isValid = true) :
        G_ (G),
        space_ (space),
        valid_ (isValid)
      {}

      using function_argument_type =
        with_local_access_function_argument_type<canonical_this_type>;

    public:
      LocalAccess<GlobalObjectType, MemorySpace, am>
      valid (const bool isValid) const {
        return {this->G_, this->space_, isValid};
      }

      template<class NewMemorySpace>
      LocalAccess<GlobalObjectType, NewMemorySpace, am>
      on (NewMemorySpace space) const {
        return {this->G_, space, this->valid_};
      }

      bool isValid () const { return this->valid_; }

      memory_space getSpace () const { return space_; }

    public:
      global_object_type& G_;

      memory_space space_;
      bool valid_;
    };
  } // namespace Details

  // Implementations of readOnly, writeOnly, and readWrite

  template<class GOT>
  Details::LocalAccess<
    GOT,
    typename Details::DefaultMemorySpace<GOT>::type,
    Details::AccessMode::ReadOnly>
  readOnly (GOT& G)
  {
    return {G, Details::DefaultMemorySpace<GOT>::space (G), true};
  }

  template<class GOT>
  Details::LocalAccess<
    GOT,
    typename Details::DefaultMemorySpace<GOT>::type,
    Details::AccessMode::ReadOnly>
  readOnly (const GOT& G)
  {
    GOT& G_nc = const_cast<GOT&> (G);
    return {G_nc, Details::DefaultMemorySpace<GOT>::space (G_nc), true};
  }

  template<class GOT>
  Details::LocalAccess<
    GOT,
    typename Details::DefaultMemorySpace<GOT>::type,
    Details::AccessMode::WriteOnly>
  writeOnly (GOT& G)
  {
    return {G, Details::DefaultMemorySpace<GOT>::space (G), true};
  }

  template<class GOT>
  Details::LocalAccess<
    GOT,
    typename Details::DefaultMemorySpace<GOT>::type,
    Details::AccessMode::ReadWrite>
  readWrite (GOT& G)
  {
    return {G, Details::DefaultMemorySpace<GOT>::space (G), true};
  }

  // Implementation of withLocalAccess

  namespace Details {

    // Use std::tuple as a compile-time list (Greenspun's 10th Rule)

    // cons<T, std::tuple<Args...>>::type is std::tuple<T, Args...>.
    // This is the usual Lisp CONS function, but for std::tuple.  I
    // got the idea from
    // https://stackoverflow.com/questions/18701798/building-and-accessing-a-list-of-types-at-compile-time
    // but without "struct Void {};", and with head of new list
    // ordered first instead of last (hence "cons").
    template<class ...> struct cons;

    // (CONS SomeType NIL)
    template<class T, template <class ...> class List>
    struct cons<T, List<>> {
      using type = List<T>;
    };

    // (CONS SomeType ListOfTypes)
    template <class T, template <class ...> class List, class ...Types>
    struct cons<T, List<Types...>>
    {
      typedef List<T, Types...> type;
    };

    // Map from std::tuple<Ts...> to std::function<void (Ts...)>.

    // I got inspiration from
    // https://stackoverflow.com/questions/15418841/how-do-i-strip-a-tuple-back-into-a-variadic-template-list-of-types
    //
    // The only significant change was from "using type = T<Ts...>;",
    // to "using type = std::function<void (Ts...)>;".
    template<class T>
    struct tuple_to_function_type { };

    template<typename... Ts>
    struct tuple_to_function_type<std::tuple<Ts...> >
    {
      using type = std::function<void (Ts...)>;
    };

    // Map from a list of zero or more LocalAccess types, to the
    // corresponding list of arguments for the user function to give to
    // withLocalAccess.
    template<class ... Args>
    struct ArgsToFunction {};

    template<>
    struct ArgsToFunction<> {
      using arg_list_type = std::tuple<>;

      // Implementers should only use this.
      using type = std::function<void ()>;
    };

    template<class FirstLocalAccessType, class ... Rest>
    struct ArgsToFunction<FirstLocalAccessType, Rest...> {
      using head_arg_type =
        with_local_access_function_argument_type<FirstLocalAccessType>;
      using tail_arg_list_type =
        typename ArgsToFunction<Rest...>::arg_list_type;
      using arg_list_type =
        typename cons<head_arg_type, tail_arg_list_type>::type;

      // Implementers should only use this.
      using type = typename tuple_to_function_type<arg_list_type>::type;
    };

    template<class ... LocalAccessTypes>
    struct WithLocalAccess {
      using current_user_function_type =
        typename Details::ArgsToFunction<LocalAccessTypes...>::type;

      static void
      withLocalAccess (LocalAccessTypes...,
                       typename Details::ArgsToFunction<LocalAccessTypes...>::type);
    };

    template<>
    struct WithLocalAccess<> {
      using current_user_function_type =
        typename Details::ArgsToFunction<>::type;

      static void
      withLocalAccess (current_user_function_type userFunction)
      {
        userFunction ();
      }
    };

    template<class FirstLocalAccessType, class ... Rest>
    struct WithLocalAccess<FirstLocalAccessType, Rest...> {
      using current_user_function_type =
        typename Details::ArgsToFunction<FirstLocalAccessType, Rest...>::type;

      static void
      withLocalAccess (current_user_function_type userFunction,
                       FirstLocalAccessType first,
                       Rest... rest)
      {
        // The "master" local object is the scope guard for local
        // data.  Its constructor may allocate temporary storage, copy
        // data to the desired memory space, etc.  Its destructor will
        // put everything back.  "Put everything back" could be a
        // no-op, or it could copy data back so where they need to go
        // and/or free temporary storage.
        //
        // Users define this function and the type it returns by
        // specializing GetMasterLocalObject for LocalAccess
        // specializations.
        auto first_lcl_master = getMasterLocalObject (first);

        // The "nonowning" local object is a nonowning view of the
        // "master" local object.  This is the only local object that
        // users see, and they see it as input to their function.
        // Subsequent slices / subviews view this nonowning local
        // object.  All such nonowning views must have lifetime
        // contained within the lifetime of the master local object.
        //
        // Users define this function and the type it returns by
        // specializing GetNonowningLocalObject (see above).
        //
        // Constraining the nonowning views' lifetime to this scope
        // means that master local object types may use low-cost
        // ownership models, like that of std::unique_ptr.  There
        // should be no need for reference counting (in the manner of
        // std::shared_ptr) or Herb Sutter's deferred_heap.
        auto first_lcl_view = getNonowningLocalObject (first, first_lcl_master);

        // Curry the user's function by fixing the first argument.

        // The commented-out implementation requires C++14, because it
        // uses a generic lambda (the special case where parameters
        // are "auto").  We do have the types of the arguments,
        // though, from ArgsToFunction, so we don't need this feature.

        // WithLocalAccess<Rest...>::withLocalAccess
        //   (rest...,
        //    [=] (auto ... args) {
        //      userFunction (first_lcl_view, args...);
        //    });

        WithLocalAccess<Rest...>::withLocalAccess
          ([=] (typename Rest::function_argument_type... args) {
             userFunction (first_lcl_view, args...);
           },
           rest...);
      }
    };
  } // namespace Details

  // withLocalAccess function declaration and definition

  template<class ... LocalAccessTypes>
  void
  withLocalAccess
    (typename Details::ArgsToFunction<LocalAccessTypes...>::type userFunction,
     LocalAccessTypes... localAccesses)
  {
    using impl_type = Details::WithLocalAccess<LocalAccessTypes...>;
    impl_type::withLocalAccess (userFunction, localAccesses...);
  }

} // namespace Tpetra

#endif // TPETRA_WITHLOCALACCESS_HPP