FM / native

/usr/local/lib64/python3.6/site-packages/torch/include/ATen/native/

/usr/local/lib64/python3.6/site-packages/torch/include/ATen/native

Name	Size	Mode	Actions
cpu/	-	0755	rm
cuda/	-	0755	rm
quantized/	-	0755	rm
Activation.h	3069	0644	edit dl rm
AdaptivePooling.h	1165	0644	edit dl rm
BatchLinearAlgebra.h	8246	0644	edit dl rm
batch_norm.h	1285	0644	edit dl rm
BinaryOps.h	4916	0644	edit dl rm
BucketizationUtils.h	4248	0644	edit dl rm
ComplexHelper.h	3797	0644	edit dl rm
CompositeRandomAccessor.h	888	0644	edit dl rm
CompositeRandomAccessorCommon.h	6713	0644	edit dl rm
ConvUtils.h	5350	0644	edit dl rm
Copy.h	356	0644	edit dl rm
CPUBlas.h	4199	0644	edit dl rm
CPUFallback.h	2404	0644	edit dl rm
Cross.h	262	0644	edit dl rm
DilatedConvolutionUtils.h	6416	0644	edit dl rm
DispatchStub.h	7672	0644	edit dl rm
Distance.h	732	0644	edit dl rm
Distributions.h	21654	0644	edit dl rm
DistributionTemplates.h	18623	0644	edit dl rm
EmbeddingBag.h	1320	0644	edit dl rm
Fill.h	384	0644	edit dl rm
ForeachUtils.h	5962	0644	edit dl rm
FunctionOfAMatrixUtils.h	436	0644	edit dl rm
GridSampler.h	10525	0644	edit dl rm
group_norm.h	896	0644	edit dl rm
Histogram.h	492	0644	edit dl rm
im2col.h	2838	0644	edit dl rm
im2col_shape_check.h	6181	0644	edit dl rm
IndexingUtils.h	5373	0644	edit dl rm
layer_norm.h	2892	0644	edit dl rm
Lerp.h	553	0644	edit dl rm
LinearAlgebra.h	603	0644	edit dl rm
LinearAlgebraUtils.h	25236	0644	edit dl rm
LossMulti.h	2197	0644	edit dl rm
Math.h	91356	0644	edit dl rm
MathBitFallThroughLists.h	4086	0644	edit dl rm
MathBitsFallback.h	7326	0644	edit dl rm
MaxPooling.h	1234	0644	edit dl rm
Normalization.h	302	0644	edit dl rm
PointwiseOps.h	749	0644	edit dl rm
Pool.h	10922	0644	edit dl rm
Pow.h	1694	0644	edit dl rm
ReduceAllOps.h	378	0644	edit dl rm
ReduceOps.h	1745	0644	edit dl rm
ReduceOpsUtils.h	12245	0644	edit dl rm
Repeat.h	1286	0644	edit dl rm
Resize.h	6501	0644	edit dl rm
ResizeCommon.h	1321	0644	edit dl rm
RNN.h	2467	0644	edit dl rm
ScatterGatherChecks.h	3641	0644	edit dl rm
SegmentReduce.h	685	0644	edit dl rm
SharedReduceOps.h	15785	0644	edit dl rm
SobolEngineOpsUtils.h	1723	0644	edit dl rm
Sorting.h	536	0644	edit dl rm
SortingUtils.h	5722	0644	edit dl rm
SpectralOpsUtils.h	3146	0644	edit dl rm
StridedRandomAccessor.h	6847	0644	edit dl rm
TensorAdvancedIndexing.h	3072	0644	edit dl rm
TensorCompare.h	1333	0644	edit dl rm
TensorDimApply.h	1832	0644	edit dl rm
TensorFactories.h	3382	0644	edit dl rm
TensorIterator.h	46	0644	edit dl rm
TensorIteratorDynamicCasting.h	2025	0644	edit dl rm
TensorShape.h	1049	0644	edit dl rm
TensorTransformations.h	938	0644	edit dl rm
TriangularOpsUtils.h	2000	0644	edit dl rm
TypeProperties.h	496	0644	edit dl rm
UnaryOps.h	4464	0644	edit dl rm
Unfold2d.h	551	0644	edit dl rm
Unfold3d.h	852	0644	edit dl rm
UnfoldBackward.h	5398	0644	edit dl rm
UpSample.h	13599	0644	edit dl rm
vol2col.h	3642	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/ATen/native/DispatchStub.h (7672B)

#pragma once

#include <c10/core/Backend.h>
#include <c10/core/ScalarType.h>
#include <c10/util/Exception.h>

#include <type_traits>
#include <atomic>

// Implements instruction set specific function dispatch.
//
// Kernels that may make use of specialized instruction sets (e.g. AVX2) are
// compiled multiple times with different compiler flags (e.g. -mavx2). A
// DispatchStub contains a table of function pointers for a kernel. At runtime,
// the fastest available kernel is chosen based on the features reported by
// cpuinfo.
//
// Example:
//
// In native/MyKernel.h:
//   using fn_type = void(*)(const Tensor& x);
//   DECLARE_DISPATCH(fn_type, stub);
//
// In native/MyKernel.cpp
//   DEFINE_DISPATCH(stub);
//
// In native/cpu/MyKernel.cpp:
//   namespace {
//     // use anonymous namespace so that different cpu versions won't conflict
//     void kernel(const Tensor& x) { ... }
//   }
//   REGISTER_DISPATCH(stub, &kernel);
//
// To call:
//   stub(kCPU, tensor);
//
// TODO: CPU instruction set selection should be folded into whatever
// the main dispatch mechanism is.

// ignore warnings about DispatchStub::DEFAULT, AVX, AVX2 defined elsewhere
#if defined(__clang__)
#pragma clang diagnostic push
#pragma clang diagnostic ignored "-Wundefined-var-template"
#endif

namespace at { namespace native {

enum class CPUCapability {
  DEFAULT = 0,
#ifdef HAVE_VSX_CPU_DEFINITION
  VSX = 1,
#else
  AVX2 = 1,
  AVX512 = 2,
#endif
  NUM_OPTIONS
};

CPUCapability get_cpu_capability();

template <typename FnPtr, typename T>
struct DispatchStub;

/**
 * The sole purpose of this class is to outline methods that don't need to be
 * specialized or otherwise inlined and duplicated (by the compiler due to
 * template expansion), since it causes size bloat if there are a significant
 * number of specialization of the DispatchStub<> class.
 */
struct TORCH_API DispatchStubImpl {
  void* get_call_ptr(
    DeviceType device_type
    , void *DEFAULT
#ifdef HAVE_AVX512_CPU_DEFINITION
      , void *AVX512
#endif
#ifdef HAVE_AVX2_CPU_DEFINITION
      , void *AVX2
#endif
#ifdef HAVE_VSX_CPU_DEFINITION
      , void *VSX
#endif
  );

/**
   * The CPU Dispatch actual method is chosen in decreasing order of preference by
   * DispatchStubImpl::choose_cpu_impl() in case none is found by
   * DispatchStubImpl::get_call_ptr() in cpu_dispatch_ptr.
   */
  void* choose_cpu_impl(
    void *DEFAULT
#ifdef HAVE_AVX512_CPU_DEFINITION
    , void *AVX512
#endif
#ifdef HAVE_AVX2_CPU_DEFINITION
    , void *AVX2
#endif
#ifdef HAVE_VSX_CPU_DEFINITION
    , void *VSX
#endif
  );

// Fixing dispatch error in Windows debug builds.
  // See https://github.com/pytorch/pytorch/issues/22681 for more details.
  #if defined(_MSC_VER) && defined(_DEBUG)
    std::atomic<void*> cpu_dispatch_ptr;
    void* cuda_dispatch_ptr;
    void* hip_dispatch_ptr;
  #else
    std::atomic<void*> cpu_dispatch_ptr{nullptr};
    void* cuda_dispatch_ptr = nullptr;
    void* hip_dispatch_ptr = nullptr;
  #endif
};

template <typename rT, typename T, typename... Args>
struct DispatchStub<rT (*)(Args...), T> {
  using FnPtr = rT (*) (Args...);

DispatchStub() = default;
  DispatchStub(const DispatchStub&) = delete;
  DispatchStub& operator=(const DispatchStub&) = delete;

private:
  FnPtr get_call_ptr(DeviceType device_type) {
    return reinterpret_cast<FnPtr>(
      impl.get_call_ptr(device_type
      , reinterpret_cast<void*>(DEFAULT)
#ifdef HAVE_AVX512_CPU_DEFINITION
      , reinterpret_cast<void*>(AVX512)
#endif
#ifdef HAVE_AVX2_CPU_DEFINITION
      , reinterpret_cast<void*>(AVX2)
#endif
#ifdef HAVE_VSX_CPU_DEFINITION
      , reinterpret_cast<void*>(VSX)
#endif
      )
    );
  }

public:
  template <typename... ArgTypes>
  rT operator()(DeviceType device_type, ArgTypes&&... args) {
    FnPtr call_ptr = get_call_ptr(device_type);
    return (*call_ptr)(std::forward<ArgTypes>(args)...);
  }

void set_cuda_dispatch_ptr(FnPtr fn_ptr) {
    impl.cuda_dispatch_ptr = reinterpret_cast<void*>(fn_ptr);
  }

void set_hip_dispatch_ptr(FnPtr fn_ptr) {
    impl.hip_dispatch_ptr = reinterpret_cast<void*>(fn_ptr);
  }

static FnPtr DEFAULT;
#ifdef HAVE_AVX512_CPU_DEFINITION
  static FnPtr AVX512;
#endif
#ifdef HAVE_AVX2_CPU_DEFINITION
  static FnPtr AVX2;
#endif
#ifdef HAVE_VSX_CPU_DEFINITION
  static FnPtr VSX;
#endif
private:
  DispatchStubImpl impl;
};

namespace {
template <typename FnPtr, typename T>
struct RegisterCUDADispatch {
  RegisterCUDADispatch(DispatchStub<FnPtr, T>& stub, FnPtr value) {
    stub.set_cuda_dispatch_ptr(value);
  }
};

template <typename FnPtr, typename T>
struct RegisterHIPDispatch {
  RegisterHIPDispatch(DispatchStub<FnPtr, T>& stub, FnPtr value) {
    // TODO: make this point at hip_dispatch_ptr
    stub.set_cuda_dispatch_ptr(value);
  }
};
} // anonymous namespace

// Compiler will complain if you put things like std::tuple<Tensor, Tensor> in
// the `fn` argument of DECLARE_DISPATCH. Some possible workarounds, e.g.,
// adding parentheses and using helper struct to get rid of the parentheses, do
// not work with MSVC. So do a `using`-declaration if you need to pass in such
// `fn`, e.g., grid_sampler_2d_backward_cpu_kernel in GridSampleKernel.h.
#define DECLARE_DISPATCH(fn, name)         \
  struct name : DispatchStub<fn, name> {   \
    name() = default;                      \
    name(const name&) = delete;            \
    name& operator=(const name&) = delete; \
  };                                       \
  extern TORCH_API struct name name

#define DEFINE_DISPATCH(name) struct name name

#define REGISTER_ARCH_DISPATCH(name, arch, fn) \
  template <> decltype(fn) DispatchStub<decltype(fn), struct name>::arch = fn;

#ifdef HAVE_AVX512_CPU_DEFINITION
#define REGISTER_AVX512_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, AVX512, fn)
#else
#define REGISTER_AVX512_DISPATCH(name, fn)
#endif

#ifdef HAVE_AVX2_CPU_DEFINITION
#define REGISTER_AVX2_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, AVX2, fn)
#else
#define REGISTER_AVX2_DISPATCH(name, fn)
#endif

#ifdef HAVE_VSX_CPU_DEFINITION
#define REGISTER_VSX_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, VSX, fn)
#else
#define REGISTER_VSX_DISPATCH(name, fn)
#endif

#define REGISTER_NO_CPU_DISPATCH(name, fn_type)                                \
  REGISTER_ARCH_DISPATCH(name, DEFAULT, static_cast<fn_type>(nullptr))         \
  REGISTER_AVX512_DISPATCH(name, static_cast<fn_type>(nullptr))                \
  REGISTER_AVX2_DISPATCH(name, static_cast<fn_type>(nullptr))                  \
  REGISTER_VSX_DISPATCH(name, static_cast<fn_type>(nullptr))

#define REGISTER_CUDA_DISPATCH(name, fn) \
  static RegisterCUDADispatch<decltype(fn), struct name> name ## __register(name, fn);

#define REGISTER_HIP_DISPATCH(name, fn) \
  static RegisterHIPDispatch<decltype(fn), struct name> name ## __register(name, fn);

// NB: This macro must be used in an actual 'cu' file; if you try using
// it from a 'cpp' file it will not work!
#if defined(__CUDACC__)
#define REGISTER_DISPATCH(name, fn) REGISTER_CUDA_DISPATCH(name, fn)
#elif defined(__HIPCC__)
// TODO: cut this over to HIP dispatch once we stop pretending that CUDA
// is HIP in the PyTorch HIPify build.
#define REGISTER_DISPATCH(name, fn) REGISTER_CUDA_DISPATCH(name, fn)
// #define REGISTER_DISPATCH(name, fn) REGISTER_HIP_DISPATCH(name, fn)
#elif defined(CPU_CAPABILITY)
#define REGISTER_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, CPU_CAPABILITY, fn)
#define REGISTER_NO_AVX512_DISPATCH(name, fn_type)                             \
  REGISTER_AVX512_DISPATCH(name, static_cast<fn_type>(nullptr))
#endif

}} // namespace at::native

#if defined(__clang__)
#pragma clang diagnostic pop
#endif

#pragma once #include #include #include #include #include // Implements instruction set specific function dispatch. // // Kernels that may make use of specialized instruction sets (e.g. AVX2) are // compiled multiple times with different compiler flags (e.g. -mavx2). A // DispatchStub contains a table of function pointers for a kernel. At runtime, // the fastest available kernel is chosen based on the features reported by // cpuinfo. // // Example: // // In native/MyKernel.h: // using fn_type = void(*)(const Tensor& x); // DECLARE_DISPATCH(fn_type, stub); // // In native/MyKernel.cpp // DEFINE_DISPATCH(stub); // // In native/cpu/MyKernel.cpp: // namespace { // // use anonymous namespace so that different cpu versions won't conflict // void kernel(const Tensor& x) { ... } // } // REGISTER_DISPATCH(stub, &kernel); // // To call: // stub(kCPU, tensor); // // TODO: CPU instruction set selection should be folded into whatever // the main dispatch mechanism is. // ignore warnings about DispatchStub::DEFAULT, AVX, AVX2 defined elsewhere #if defined(__clang__) #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wundefined-var-template" #endif namespace at { namespace native { enum class CPUCapability { DEFAULT = 0, #ifdef HAVE_VSX_CPU_DEFINITION VSX = 1, #else AVX2 = 1, AVX512 = 2, #endif NUM_OPTIONS }; CPUCapability get_cpu_capability(); template struct DispatchStub; /** * The sole purpose of this class is to outline methods that don't need to be * specialized or otherwise inlined and duplicated (by the compiler due to * template expansion), since it causes size bloat if there are a significant * number of specialization of the DispatchStub<> class. */ struct TORCH_API DispatchStubImpl { void* get_call_ptr( DeviceType device_type , void *DEFAULT #ifdef HAVE_AVX512_CPU_DEFINITION , void *AVX512 #endif #ifdef HAVE_AVX2_CPU_DEFINITION , void *AVX2 #endif #ifdef HAVE_VSX_CPU_DEFINITION , void *VSX #endif ); /** * The CPU Dispatch actual method is chosen in decreasing order of preference by * DispatchStubImpl::choose_cpu_impl() in case none is found by * DispatchStubImpl::get_call_ptr() in cpu_dispatch_ptr. */ void* choose_cpu_impl( void *DEFAULT #ifdef HAVE_AVX512_CPU_DEFINITION , void *AVX512 #endif #ifdef HAVE_AVX2_CPU_DEFINITION , void *AVX2 #endif #ifdef HAVE_VSX_CPU_DEFINITION , void *VSX #endif ); // Fixing dispatch error in Windows debug builds. // See https://github.com/pytorch/pytorch/issues/22681 for more details. #if defined(_MSC_VER) && defined(_DEBUG) std::atomic cpu_dispatch_ptr; void* cuda_dispatch_ptr; void* hip_dispatch_ptr; #else std::atomic cpu_dispatch_ptr{nullptr}; void* cuda_dispatch_ptr = nullptr; void* hip_dispatch_ptr = nullptr; #endif }; template struct DispatchStub { using FnPtr = rT (*) (Args...); DispatchStub() = default; DispatchStub(const DispatchStub&) = delete; DispatchStub& operator=(const DispatchStub&) = delete; private: FnPtr get_call_ptr(DeviceType device_type) { return reinterpret_cast( impl.get_call_ptr(device_type , reinterpret_cast(DEFAULT) #ifdef HAVE_AVX512_CPU_DEFINITION , reinterpret_cast(AVX512) #endif #ifdef HAVE_AVX2_CPU_DEFINITION , reinterpret_cast(AVX2) #endif #ifdef HAVE_VSX_CPU_DEFINITION , reinterpret_cast(VSX) #endif ) ); } public: template rT operator()(DeviceType device_type, ArgTypes&&... args) { FnPtr call_ptr = get_call_ptr(device_type); return (*call_ptr)(std::forward(args)...); } void set_cuda_dispatch_ptr(FnPtr fn_ptr) { impl.cuda_dispatch_ptr = reinterpret_cast(fn_ptr); } void set_hip_dispatch_ptr(FnPtr fn_ptr) { impl.hip_dispatch_ptr = reinterpret_cast(fn_ptr); } static FnPtr DEFAULT; #ifdef HAVE_AVX512_CPU_DEFINITION static FnPtr AVX512; #endif #ifdef HAVE_AVX2_CPU_DEFINITION static FnPtr AVX2; #endif #ifdef HAVE_VSX_CPU_DEFINITION static FnPtr VSX; #endif private: DispatchStubImpl impl; }; namespace { template struct RegisterCUDADispatch { RegisterCUDADispatch(DispatchStub& stub, FnPtr value) { stub.set_cuda_dispatch_ptr(value); } }; template struct RegisterHIPDispatch { RegisterHIPDispatch(DispatchStub& stub, FnPtr value) { // TODO: make this point at hip_dispatch_ptr stub.set_cuda_dispatch_ptr(value); } }; } // anonymous namespace // Compiler will complain if you put things like std::tuple in // the `fn` argument of DECLARE_DISPATCH. Some possible workarounds, e.g., // adding parentheses and using helper struct to get rid of the parentheses, do // not work with MSVC. So do a `using`-declaration if you need to pass in such // `fn`, e.g., grid_sampler_2d_backward_cpu_kernel in GridSampleKernel.h. #define DECLARE_DISPATCH(fn, name) \ struct name : DispatchStub { \ name() = default; \ name(const name&) = delete; \ name& operator=(const name&) = delete; \ }; \ extern TORCH_API struct name name #define DEFINE_DISPATCH(name) struct name name #define REGISTER_ARCH_DISPATCH(name, arch, fn) \ template <> decltype(fn) DispatchStub::arch = fn; #ifdef HAVE_AVX512_CPU_DEFINITION #define REGISTER_AVX512_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, AVX512, fn) #else #define REGISTER_AVX512_DISPATCH(name, fn) #endif #ifdef HAVE_AVX2_CPU_DEFINITION #define REGISTER_AVX2_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, AVX2, fn) #else #define REGISTER_AVX2_DISPATCH(name, fn) #endif #ifdef HAVE_VSX_CPU_DEFINITION #define REGISTER_VSX_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, VSX, fn) #else #define REGISTER_VSX_DISPATCH(name, fn) #endif #define REGISTER_NO_CPU_DISPATCH(name, fn_type) \ REGISTER_ARCH_DISPATCH(name, DEFAULT, static_cast(nullptr)) \ REGISTER_AVX512_DISPATCH(name, static_cast(nullptr)) \ REGISTER_AVX2_DISPATCH(name, static_cast(nullptr)) \ REGISTER_VSX_DISPATCH(name, static_cast(nullptr)) #define REGISTER_CUDA_DISPATCH(name, fn) \ static RegisterCUDADispatch name ## __register(name, fn); #define REGISTER_HIP_DISPATCH(name, fn) \ static RegisterHIPDispatch name ## __register(name, fn); // NB: This macro must be used in an actual 'cu' file; if you try using // it from a 'cpp' file it will not work! #if defined(__CUDACC__) #define REGISTER_DISPATCH(name, fn) REGISTER_CUDA_DISPATCH(name, fn) #elif defined(__HIPCC__) // TODO: cut this over to HIP dispatch once we stop pretending that CUDA // is HIP in the PyTorch HIPify build. #define REGISTER_DISPATCH(name, fn) REGISTER_CUDA_DISPATCH(name, fn) // #define REGISTER_DISPATCH(name, fn) REGISTER_HIP_DISPATCH(name, fn) #elif defined(CPU_CAPABILITY) #define REGISTER_DISPATCH(name, fn) REGISTER_ARCH_DISPATCH(name, CPU_CAPABILITY, fn) #define REGISTER_NO_AVX512_DISPATCH(name, fn_type) \ REGISTER_AVX512_DISPATCH(name, static_cast(nullptr)) #endif }} // namespace at::native #if defined(__clang__) #pragma clang diagnostic pop #endif