FM / operators

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/operators/

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/operators

Name	Size	Mode	Actions
abs_op.h	705	0644	edit dl rm
accumulate_op.h	1073	0644	edit dl rm
accuracy_op.h	652	0644	edit dl rm
acos_op.h	711	0644	edit dl rm
activation_ops_cudnn.h	4122	0644	edit dl rm
affine_channel_op.h	3450	0644	edit dl rm
alias_with_name.h	1234	0644	edit dl rm
apmeter_op.h	1027	0644	edit dl rm
arg_ops.h	2319	0644	edit dl rm
asin_op.h	711	0644	edit dl rm
assert_op.h	1335	0644	edit dl rm
async_net_barrier_op.h	904	0644	edit dl rm
atan_op.h	711	0644	edit dl rm
batch_box_cox_op.h	2287	0644	edit dl rm
batch_bucketize_op.h	720	0644	edit dl rm
batch_gather_ops.h	5264	0644	edit dl rm
batch_matmul_op.h	9602	0644	edit dl rm
batch_moments_op.h	3364	0644	edit dl rm
batch_permutation_op.h	954	0644	edit dl rm
batch_sparse_to_dense_op.h	6147	0644	edit dl rm
bbox_transform_op.h	2668	0644	edit dl rm
bisect_percentile_op.h	4921	0644	edit dl rm
boolean_mask_ops.h	2665	0644	edit dl rm
boolean_unmask_ops.h	378	0644	edit dl rm
box_with_nms_limit_op.h	4960	0644	edit dl rm
bucketize_op.h	1361	0644	edit dl rm
byte_weight_dequant_op.h	1722	0644	edit dl rm
cast_op.h	1393	0644	edit dl rm
cbrt_op.h	723	0644	edit dl rm
cc_bmm_bg_op.h	3894	0644	edit dl rm
ceil_op.h	782	0644	edit dl rm
channel_backprop_stats_op.h	737	0644	edit dl rm
channel_shuffle_op.h	1902	0644	edit dl rm
channel_stats_op.h	1807	0644	edit dl rm
clip_op.h	1639	0644	edit dl rm
collect_and_distribute_fpn_rpn_proposals_op.h	6875	0644	edit dl rm
concat_split_op.h	11850	0644	edit dl rm
conditional_op.h	487	0644	edit dl rm
conv_op.h	3125	0644	edit dl rm
conv_op_cache_cudnn.h	1935	0644	edit dl rm
conv_op_impl.h	28729	0644	edit dl rm
conv_op_shared.h	672	0644	edit dl rm
conv_pool_op_base.h	32109	0644	edit dl rm
conv_transpose_op.h	1727	0644	edit dl rm
conv_transpose_op_impl.h	18264	0644	edit dl rm
conv_transpose_op_mobile.h	1470	0644	edit dl rm
conv_transpose_op_mobile_impl.h	19587	0644	edit dl rm
conv_transpose_unpool_op_base.h	10303	0644	edit dl rm
copy_op.h	1296	0644	edit dl rm
copy_rows_to_tensor_op.h	2599	0644	edit dl rm
cosh_op.h	711	0644	edit dl rm
cosine_embedding_criterion_op.h	1127	0644	edit dl rm
cos_op.h	705	0644	edit dl rm
counter_ops.h	4596	0644	edit dl rm
create_scope_op.h	5232	0644	edit dl rm
cross_entropy_op.h	4420	0644	edit dl rm
ctc_beam_search_decoder_op.h	1102	0644	edit dl rm
ctc_greedy_decoder_op.h	817	0644	edit dl rm
cube_op.h	723	0644	edit dl rm
dataset_ops.h	5501	0644	edit dl rm
data_couple.h	464	0644	edit dl rm
deform_conv_op.h	3543	0644	edit dl rm
deform_conv_op_impl.h	13171	0644	edit dl rm
dense_vector_to_id_list_op.h	1797	0644	edit dl rm
distance_op.h	8419	0644	edit dl rm
do_op.h	6981	0644	edit dl rm
dropout_op.h	1516	0644	edit dl rm
elementwise_add_op.h	2024	0644	edit dl rm
elementwise_div_op.h	1224	0644	edit dl rm
elementwise_linear_op.h	1170	0644	edit dl rm
elementwise_logical_ops.h	5083	0644	edit dl rm
elementwise_mul_op.h	1224	0644	edit dl rm
elementwise_ops.h	19115	0644	edit dl rm
elementwise_ops_utils.h	1008	0644	edit dl rm
elementwise_op_test.h	9237	0644	edit dl rm
elementwise_sub_op.h	2025	0644	edit dl rm
elu_op.h	875	0644	edit dl rm
enforce_finite_op.h	2303	0644	edit dl rm
ensure_clipped_op.h	1608	0644	edit dl rm
ensure_cpu_output_op.h	1465	0644	edit dl rm
erf_op.h	751	0644	edit dl rm
expand_op.h	3877	0644	edit dl rm
expand_squeeze_dims_op.h	3451	0644	edit dl rm
exp_op.h	425	0644	edit dl rm
fc_inference.h	775	0644	edit dl rm
feature_maps_ops.h	32437	0644	edit dl rm
feed_blob_op.h	802	0644	edit dl rm
filler_op.h	18431	0644	edit dl rm
find_duplicate_elements_op.h	1563	0644	edit dl rm
find_op.h	2055	0644	edit dl rm
flatten_op.h	1525	0644	edit dl rm
flexible_top_k.h	936	0644	edit dl rm
floor_op.h	788	0644	edit dl rm
free_op.h	777	0644	edit dl rm
fully_connected_op.h	9351	0644	edit dl rm
fused_rowwise_8bit_conversion_ops.h	6601	0644	edit dl rm
fused_rowwise_nbitfake_conversion_ops.h	4375	0644	edit dl rm
fused_rowwise_nbit_conversion_ops.h	8723	0644	edit dl rm
fused_rowwise_random_quantization_ops.h	2607	0644	edit dl rm
gather_fused_8bit_rowwise_op.h	2179	0644	edit dl rm
gather_op.h	7505	0644	edit dl rm
gather_ranges_to_dense_op.h	8188	0644	edit dl rm
gelu_op.h	1452	0644	edit dl rm
generate_proposals_op.h	6256	0644	edit dl rm
generate_proposals_op_util_boxes.h	14309	0644	edit dl rm
generate_proposals_op_util_nms.h	26214	0644	edit dl rm
generate_proposals_op_util_nms_gpu.h	2128	0644	edit dl rm
given_tensor_byte_string_to_uint8_fill_op.h	2150	0644	edit dl rm
given_tensor_fill_op.h	3002	0644	edit dl rm
glu_op.h	1458	0644	edit dl rm
group_norm_op.h	8967	0644	edit dl rm
gru_unit_op.h	6626	0644	edit dl rm
half_float_ops.h	2732	0644	edit dl rm
hard_sigmoid_op.h	994	0644	edit dl rm
heatmap_max_keypoint_op.h	939	0644	edit dl rm
histogram_op.h	2421	0644	edit dl rm
h_softmax_op.h	4954	0644	edit dl rm
if_op.h	1764	0644	edit dl rm
im2col_op.h	8943	0644	edit dl rm
index_hash_ops.h	2232	0644	edit dl rm
index_ops.h	3155	0644	edit dl rm
inference_lstm_op.h	9881	0644	edit dl rm
instance_norm_op.h	7441	0644	edit dl rm
integral_image_op.h	923	0644	edit dl rm
is_empty_op.h	558	0644	edit dl rm
jsd_op.h	721	0644	edit dl rm
key_split_ops.h	1400	0644	edit dl rm
layer_norm_op.h	8098	0644	edit dl rm
leaky_relu_op.h	1111	0644	edit dl rm
lengths_pad_op.h	2574	0644	edit dl rm
lengths_reducer_fused_8bit_rowwise_ops.h	5532	0644	edit dl rm
lengths_reducer_fused_nbit_rowwise_ops.h	23465	0644	edit dl rm
lengths_reducer_ops.h	23315	0644	edit dl rm
lengths_reducer_rowwise_8bit_ops.h	6180	0644	edit dl rm
lengths_tile_op.h	582	0644	edit dl rm
lengths_top_k_op.h	1358	0644	edit dl rm
length_split_op.h	2259	0644	edit dl rm
listwise_l2r_op.h	1677	0644	edit dl rm
load_save_op.h	14091	0644	edit dl rm
load_save_op_util.h	1642	0644	edit dl rm
locally_connected_op.h	3872	0644	edit dl rm
locally_connected_op_impl.h	26495	0644	edit dl rm
locally_connected_op_util.h	1332	0644	edit dl rm
local_response_normalization_op.h	2804	0644	edit dl rm
log1p_op.h	717	0644	edit dl rm
logit_op.h	1129	0644	edit dl rm
log_op.h	431	0644	edit dl rm
loss_op.h	1058	0644	edit dl rm
lpnorm_op.h	1279	0644	edit dl rm
lstm_unit_op.h	6733	0644	edit dl rm
lstm_utils.h	9424	0644	edit dl rm
map_ops.h	8011	0644	edit dl rm
margin_ranking_criterion_op.h	1113	0644	edit dl rm
matmul_op.h	2843	0644	edit dl rm
max_pool_with_index_gpu.h	1155	0644	edit dl rm
mean_op.h	3252	0644	edit dl rm
merge_id_lists_op.h	2570	0644	edit dl rm
minmax_ops.h	3829	0644	edit dl rm
mish_op.h	794	0644	edit dl rm
mod_op.h	984	0644	edit dl rm
moments_op.h	4051	0644	edit dl rm
multi_class_accuracy_op.h	539	0644	edit dl rm
negate_gradient_op.h	566	0644	edit dl rm
negative_op.h	451	0644	edit dl rm
ngram_ops.h	2644	0644	edit dl rm
normalize_l1_op.h	1075	0644	edit dl rm
normalize_op.h	3013	0644	edit dl rm
no_default_engine_op.h	1063	0644	edit dl rm
numpy_tile_op.h	3643	0644	edit dl rm
one_hot_ops.h	2562	0644	edit dl rm
onnx_while_op.h	10655	0644	edit dl rm
operator_fallback_gpu.h	4155	0644	edit dl rm
op_utils_cudnn.h	2112	0644	edit dl rm
order_switch_ops.h	2149	0644	edit dl rm
pack_rnn_sequence_op.h	3074	0644	edit dl rm
pack_segments.h	2729	0644	edit dl rm
pad_op.h	2902	0644	edit dl rm
partition_ops.h	9958	0644	edit dl rm
percentile_op.h	1009	0644	edit dl rm
perplexity_op.h	447	0644	edit dl rm
piecewise_linear_transform_op.h	8281	0644	edit dl rm
pool_op.h	8525	0644	edit dl rm
pool_op_util.h	1105	0644	edit dl rm
pow_op.h	4677	0644	edit dl rm
prefetch_op.h	4661	0644	edit dl rm
prelu_op.h	1067	0644	edit dl rm
prepend_dim_op.h	2760	0644	edit dl rm
quantile_op.h	4120	0644	edit dl rm
quant_decode_op.h	5370	0644	edit dl rm
rank_loss_op.h	820	0644	edit dl rm
reciprocal_op.h	721	0644	edit dl rm
reducer_functors.h	24556	0644	edit dl rm
reduce_front_back_max_ops.h	4399	0644	edit dl rm
reduce_front_back_sum_mean_ops.h	5337	0644	edit dl rm
reduce_ops.h	9962	0644	edit dl rm
reduction_ops.h	5944	0644	edit dl rm
relu_n_op.h	990	0644	edit dl rm
relu_op.h	624	0644	edit dl rm
remove_data_blocks_op.h	2651	0644	edit dl rm
replace_nan_op.h	1170	0644	edit dl rm
reshape_op.h	5723	0644	edit dl rm
resize_3d_op.h	2677	0644	edit dl rm
resize_op.h	2307	0644	edit dl rm
reverse_packed_segs_op.h	2772	0644	edit dl rm
rmac_regions_op.h	708	0644	edit dl rm
rms_norm_op.h	2968	0644	edit dl rm
roi_align_gradient_op.h	1486	0644	edit dl rm
roi_align_op.h	2857	0644	edit dl rm
roi_align_rotated_gradient_op.h	1369	0644	edit dl rm
roi_align_rotated_op.h	1636	0644	edit dl rm
roi_pool_op.h	2503	0644	edit dl rm
rowmul_op.h	1947	0644	edit dl rm
rsqrt_op.h	729	0644	edit dl rm
scale_blobs_op.h	1458	0644	edit dl rm
scale_op.h	1019	0644	edit dl rm
segment_reduction_op.h	71022	0644	edit dl rm
self_binning_histogram_op.h	6258	0644	edit dl rm
selu_op.h	1545	0644	edit dl rm
sequence_ops.h	8264	0644	edit dl rm
shape_op.h	1638	0644	edit dl rm
sigmoid_op.h	639	0644	edit dl rm
sinh_op.h	711	0644	edit dl rm
sinusoid_position_encoding_op.h	2834	0644	edit dl rm
sin_op.h	705	0644	edit dl rm
slice_op.h	10071	0644	edit dl rm
softmax_op.h	1174	0644	edit dl rm
softmax_utils.h	447	0644	edit dl rm
softmax_with_loss_op.h	2883	0644	edit dl rm
softplus_op.h	781	0644	edit dl rm
softsign_op.h	675	0644	edit dl rm
space_batch_op.h	6848	0644	edit dl rm
sparse_dropout_with_replacement_op.h	1122	0644	edit dl rm
sparse_itemwise_dropout_with_replacement_op.h	1163	0644	edit dl rm
sparse_lp_regularizer_op.h	1130	0644	edit dl rm
sparse_normalize_op.h	834	0644	edit dl rm
sparse_to_dense_mask_op.h	10051	0644	edit dl rm
sparse_to_dense_op.h	3977	0644	edit dl rm
spatial_batch_norm_op.h	15175	0644	edit dl rm
spatial_softmax_with_loss_op.h	2182	0644	edit dl rm
sqrt_op.h	448	0644	edit dl rm
sqr_op.h	431	0644	edit dl rm
square_root_divide_op.h	1857	0644	edit dl rm
stats_put_ops.h	2813	0644	edit dl rm
stop_gradient.h	548	0644	edit dl rm
string_ops.h	2067	0644	edit dl rm
stump_func_op.h	2112	0644	edit dl rm
summarize_op.h	1875	0644	edit dl rm
swish_op.h	772	0644	edit dl rm
tanh_op.h	723	0644	edit dl rm
tan_op.h	705	0644	edit dl rm
tensor_protos_db_input.h	3633	0644	edit dl rm
text_file_reader_utils.h	2900	0644	edit dl rm
thresholded_relu_op.h	1137	0644	edit dl rm
tile_op.h	8741	0644	edit dl rm
top_k.h	1061	0644	edit dl rm
transpose_op.h	2082	0644	edit dl rm
tt_linear_op.h	6501	0644	edit dl rm
unique_ops.h	1666	0644	edit dl rm
unsafe_coalesce.h	2481	0644	edit dl rm
upsample_op.h	2246	0644	edit dl rm
utility_ops.h	49994	0644	edit dl rm
variable_length_sequence_padding.h	1378	0644	edit dl rm
weighted_multi_sampling_op.h	602	0644	edit dl rm
weighted_sample_op.h	739	0644	edit dl rm
while_op.h	1961	0644	edit dl rm
zero_gradient_op.h	347	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/caffe2/operators/reducer_functors.h (24556B)

#ifndef CAFFE2_OPERATORS_RECUDER_FUNCTORS_H_
#define CAFFE2_OPERATORS_RECUDER_FUNCTORS_H_

#include <array>

#include "caffe2/core/context.h"
#include "caffe2/core/tensor.h"
#include "caffe2/utils/eigen_utils.h"
#include "caffe2/utils/math.h"
#include "caffe2/utils/proto_utils.h"

namespace caffe2 {

////////////////////////////////////////////////////////////////////////////////
// Range reducers: can leverage that input segment is continuous and provide
// special implementation
////////////////////////////////////////////////////////////////////////////////

// Put forward and backward in the same template?
template <typename T, class Context>
class SumRangeReducer;
template <typename T, class Context>
class SumRangeReducerGradient;

template <typename T>
class SumRangeReducer<T, CPUContext> {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* in,
      T* out,
      CPUContext* /*context*/) {
    // do we need to go through wrapper in math.h?
    EigenVectorMap<T> out_vec(out, block_size);
    out_vec = ConstEigenMatrixMap<T>(in, block_size, blocks).rowwise().sum();
  }
};

template <typename T, class Context>
class SumRangeReducerGradient {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* segment_grad,
      T* data_grad,
      const T* /*data_in*/, // unused
      const T* /*data_out*/, // unused
      Context* context) {
    // do we have some op that does it smartly with minimum number of memcpy?
    for (int64_t i = 0; i < blocks; ++i) {
      context->template CopySameDevice<T>(
          block_size, segment_grad, data_grad + block_size * i);
    }
  }
};

struct SumRangeReducerDef {
  template <typename T, class Context>
  using Reducer = SumRangeReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = SumRangeReducerGradient<T, Context>;
  static constexpr const char* name = "Sum";
  static constexpr const char* doc =
      "Summation is done element-wise across slices of the input tensor and "
      "doesn't change the shape of the individual blocks.";
};

// Put forward and backward in the same template?
template <typename T, class Context>
class LogSumExpRangeReducer;
template <typename T, class Context>
class LogSumExpRangeReducerGradient;

template <typename T>
class LogSumExpRangeReducer<T, CPUContext> {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* in,
      T* out,
      CPUContext* /*context*/) {
    for (int j = 0; j < block_size; ++j) {
      T max_value = std::numeric_limits<T>::lowest();
      for (int i = 0; i < blocks; ++i) {
        max_value = std::max(max_value, in[i * block_size + j]);
      }
      T scaled_exp_sum = 0;
      for (int i = 0; i < blocks; ++i) {
        scaled_exp_sum += std::exp(in[i * block_size + j] - max_value);
      }
      *(out++) = std::log(scaled_exp_sum) + max_value;
    }
  }
  T r{1};
};

template <typename T, class Context>
class LogSumExpRangeReducerGradient {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* segment_grad, // GO
      T* data_grad, // GI
      const T* data_in, // I
      const T* data_out, // O
      Context* /*context*/) {
    for (int j = 0; j < block_size; ++j) {
      const T out_grad = *(segment_grad++);
      const T offset = *(data_out++);
      for (int i = 0; i < blocks; ++i) {
        auto idx = i * block_size + j;
        data_grad[idx] = out_grad * std::exp(data_in[idx] - offset);
      }
    }
  }
};

struct LogSumExpRangeReducerDef {
  template <typename T, class Context>
  using Reducer = LogSumExpRangeReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = LogSumExpRangeReducerGradient<T, Context>;
  static constexpr const char* name = "LogSumExp";
  static constexpr const char* doc =
      "LogSumExp computes the element-wise log of the sum of exponentials of "
      "input slices. Operation doesn't change the shape of individual blocks.";
};

template <typename T, class Context>
class LogMeanExpRangeReducer;
template <typename T, class Context>
class LogMeanExpRangeReducerGradient;

template <typename T>
class LogMeanExpRangeReducer<T, CPUContext> {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* in,
      T* out,
      CPUContext* /*context*/) {
    for (int j = 0; j < block_size; ++j) {
      T max_value = std::numeric_limits<T>::lowest();
      for (int i = 0; i < blocks; ++i) {
        max_value = std::max(max_value, in[i * block_size + j]);
      }
      T scaled_exp_sum = 0;
      for (int i = 0; i < blocks; ++i) {
        scaled_exp_sum += std::exp(in[i * block_size + j] - max_value);
      }
      scaled_exp_sum /= blocks;
      *(out++) = std::log(scaled_exp_sum) + max_value;
    }
  }
};

template <typename T, class Context>
class LogMeanExpRangeReducerGradient {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* segment_grad, // GO
      T* data_grad, // GI
      const T* data_in, // I
      const T* data_out, // O
      Context* /*context*/) {
    for (int j = 0; j < block_size; ++j) {
      const T out_grad = *(segment_grad++);
      const T offset = *(data_out++);
      for (int i = 0; i < blocks; ++i) {
        auto idx = i * block_size + j;
        data_grad[idx] = out_grad * std::exp(data_in[idx] - offset) / blocks;
      }
    }
  }
};

struct LogMeanExpRangeReducerDef {
  template <typename T, class Context>
  using Reducer = LogMeanExpRangeReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = LogMeanExpRangeReducerGradient<T, Context>;
  static constexpr const char* name = "LogMeanExp";
  static constexpr const char* doc =
      "LogMeanExp computes the element-wise log of the mean of exponentials of "
      "input slices. Operation doesn't change the shape of individual blocks.";
};

template <typename T, class Context>
class MeanRangeReducer;
template <typename T, class Context>
class MeanRangeReducerGradient;

template <typename T>
class MeanRangeReducer<T, CPUContext> {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* in,
      T* out,
      CPUContext* /*context*/) {
    for (int j = 0; j < block_size; ++j) {
      T avg_value = 0;
      for (int i = 0; i < blocks; ++i) {
        avg_value += in[i * block_size + j] / blocks;
      }
      *(out++) = avg_value;
    }
  }
};

template <typename T, class Context>
class MeanRangeReducerGradient {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* segment_grad, // GO
      T* data_grad, // GI
      const T* /*data_in*/, // I
      const T* /*data_out*/, // O
      Context* /*context*/) {
    const auto in_grad = 1.0 / blocks;
    for (int j = 0; j < block_size; ++j) {
      const T out_grad = *(segment_grad++);
      for (int i = 0; i < blocks; ++i) {
        auto idx = i * block_size + j;
        data_grad[idx] = out_grad * in_grad;
      }
    }
  }
};

struct MeanRangeReducerDef {
  template <typename T, class Context>
  using Reducer = MeanRangeReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = MeanRangeReducerGradient<T, Context>;
  static constexpr const char* name = "Mean";
  static constexpr const char* doc =
      "Mean computation is done element-wise, so that each element of the "
      "output slice corresponds to the average value of the respective "
      "elements in the input slices. Operation doesn't change the shape of "
      "individual blocks.";
};

template <typename T, class Context>
class MaxRangeReducer;
template <typename T, class Context>
class MaxRangeReducerGradient;

template <typename T>
class MaxRangeReducer<T, CPUContext> {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* in,
      T* out,
      CPUContext* /*context*/) {
    for (int j = 0; j < block_size; ++j) {
      T max_value = std::numeric_limits<T>::lowest();
      for (int i = 0; i < blocks; ++i) {
        max_value = std::max(max_value, in[i * block_size + j]);
      }
      *(out++) = max_value;
    }
  }
};

template <typename T, class Context>
class MaxRangeReducerGradient {
 public:
  void operator()(
      const int64_t block_size,
      const int64_t blocks,
      const T* segment_grad, // GO
      T* data_grad, // GI
      const T* data_in, // I
      const T* data_out, // O
      Context* /*context*/) {
    std::memset(
        static_cast<void*>(data_grad), 0, blocks * block_size * sizeof(T));
    for (int j = 0; j < block_size; ++j) {
      const T out_grad = *(segment_grad++);
      const T out = data_out[j];
      for (int i = 0; i < blocks; ++i) {
        auto idx = i * block_size + j;
        if (out == data_in[idx]) {
          data_grad[idx] = out_grad;
        }
      }
    }
  }
};

struct MaxRangeReducerDef {
  template <typename T, class Context>
  using Reducer = MaxRangeReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = MaxRangeReducerGradient<T, Context>;
  static constexpr const char* name = "Max";
  static constexpr const char* doc =
      "Max computation is done element-wise, so that each element of the "
      "output slice corresponds to the max value of the respective "
      "elements in the input slices. Operation doesn't change the shape of "
      "individual blocks. This implementation imitates torch nn.Max operator. "
      "If the maximum value occurs more than once, the operator will return "
      "the first occurrence of value. When computing the gradient using the "
      "backward propagation, the gradient input corresponding to the first "
      "occurrence of the maximum value will be used.";
};

////////////////////////////////////////////////////////////////////////////////
// Incremental reducers: consume elements one by one
////////////////////////////////////////////////////////////////////////////////

// Base implementation, everything can be overwritten
class BaseReducer {
 public:
  static constexpr int kInputCount = 1;

struct Meta {
    int64_t block_size;
    vector<int64_t> block_shape;
    bool first_dim;

explicit Meta(bool first = true) : first_dim(first) {}

void computeMeta(at::IntArrayRef dims, size_t skip_dims) {
      first_dim ? block_shape.assign(dims.begin() + skip_dims, dims.end())
                : block_shape.assign(dims.begin(), dims.end() - skip_dims);
      block_size = first_dim ? size_from_dim_(skip_dims, dims)
                             : size_from_dim_(dims.size() - skip_dims, dims);
    }

void observeInput(int input, const Tensor& value, int skip_dims) {
      DCHECK_EQ(0, input);
      auto dims = value.sizes();
      computeMeta(dims, skip_dims);
    }

void appendOutputShape(vector<int64_t>* output_shape) {
      output_shape->insert(
          output_shape->end(), block_shape.begin(), block_shape.end());
    }

vector<int64_t> getOutputShape(const TensorShape& in, int skip_dims) {
      vector<int64_t> dims(in.dims().begin(), in.dims().end());
      computeMeta(dims, skip_dims);
      return block_shape;
    }
  };

template <int FixedSize>
  void finish(const Meta& /*meta*/, CPUContext* /*context*/) {}
};

class BaseReducerGradient {
 public:
  // which of the original inputs are required for gradient computation
  static constexpr std::array<int, 0> originalInputs() {
    return std::array<int, 0>();
  }

static constexpr bool computeLength() {
    return false;
  }

static int numAuxInputsWithGrads(const OperatorDef& /*def*/) {
    return 0;
  }

static bool requiresDataInput(const OperatorDef& /*def*/) {
    return false;
  }

// True if the backward op requires the output of the forward op.
  static bool requiresForwardOutput() {
    return false;
  }

struct Meta {
    int64_t block_size;
    vector<int64_t> block_shape;
    bool first_dim;

Meta(const Tensor& out_grad, int skip_dims, bool first_dim = true)
        : first_dim(first_dim) {
      auto dims = out_grad.sizes();
      first_dim ? block_shape.assign(dims.begin() + skip_dims, dims.end())
                : block_shape.assign(dims.begin(), dims.end() - skip_dims);
      block_size = first_dim
          ? out_grad.size_from_dim(skip_dims)
          : out_grad.size_from_dim(out_grad.dim() - skip_dims);
    }

void observeOriginalInput(
        int /*original_input*/,
        const Tensor& /*value*/,
        Tensor* /*input_grad*/, // optional grad to populate
        int /*skip_dims*/) {}

void appendGradShape(vector<int64_t>* output_shape) {
      output_shape->insert(
          output_shape->end(), block_shape.begin(), block_shape.end());
    }
  };
};

// Put forward and backward in the same template?
template <typename T, class Context>
class SumReducer;
template <typename T, class Context>
class SumReducerGradient;

template <typename T>
class SumReducer<T, CPUContext> : public BaseReducer {
 public:
  using FixedDispatch = FixedValues<1>;

SumReducer(const Meta& meta, T* out, CPUContext* /*context*/)
      : current_size_(0), out_(out) {
    // add a wrapper in Context for it
    if (meta.first_dim) {
      memset(out, 0, sizeof(T) * meta.block_size);
    }
  }
  template <int FixedSize>
  void process(
      const Meta& meta,
      const T* in,
      int64_t /*offset*/,
      CPUContext* context) {
    if (meta.first_dim) {
      math::AxpyFixedSize<T, CPUContext, FixedSize>(
          meta.block_size, 1, in, out_, context);
    } else {
      math::Sum<T, CPUContext>(
          meta.block_size, in, out_ + current_size_++, context);
    }
  }

private:
  int current_size_;
  T* out_;
};

template <typename T, class Context>
class SumReducerGradient : public BaseReducerGradient {
 public:
  using FixedDispatch = FixedValues<1>;

SumReducerGradient(
      const Meta& /*meta*/,
      const T* s_grad,
      CPUContext* /*context*/)
      : s_grad_(s_grad) {}

template <int FixedSize>
  void fillGrad(
      const Meta& meta,
      T* data_grad,
      int64_t offset,
      Context* context,
      const int length) {
    if (FixedSize == 1) { // static if
      *data_grad = *s_grad_;
    } else if (meta.first_dim) {
      context->template CopySameDevice<T>(meta.block_size, s_grad_, data_grad);
    } else {
      math::Set<T, Context>(length, s_grad_[offset], data_grad, context);
    }
  }

private:
  const T* s_grad_;
};

struct SumReducerDef {
  template <typename T, class Context>
  using Reducer = SumReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = SumReducerGradient<T, Context>;
  static constexpr const char* name = "Sum";
  static constexpr const char* doc =
      "Summation is done element-wise across slices of the input tensor and "
      "doesn't change the shape of the individual blocks.";
  static void PopulateSchema(OpSchema& /*schema*/) {}
};

// Put forward and backward in the same template?
template <typename T, class Context>
class WeightedSumReducer;
template <typename T, class Context>
class WeightedSumReducerGradient;

template <typename T>
class WeightedSumReducer<T, CPUContext> : public BaseReducer {
 public:
  static constexpr int kInputCount = 2;

using FixedDispatch = FixedValues<1>;

struct Meta : BaseReducer::Meta {
    const T* scalars;

bool first_dim;

explicit Meta(bool first = true) : first_dim(first) {}

void observeInput(int input, const Tensor& value, int skip_dims) {
      if (input == 1) {
        CAFFE_ENFORCE_EQ(
            skip_dims, value.dim(), "SCALARS mustn't have extra dimensions");
        scalars = value.data<T>();
        return;
      }
      BaseReducer::Meta::observeInput(input, value, skip_dims);
    }
  };

WeightedSumReducer(const Meta& meta, T* out, CPUContext* /*context*/)
      : out_(out) {
    // do we have a wrapper for it?
    memset(out, 0, sizeof(T) * meta.block_size);
  }
  template <int FixedSize>
  void
  process(const Meta& meta, const T* in, int64_t offset, CPUContext* context) {
    CAFFE_ENFORCE(
        meta.first_dim,
        "WeightedSumReducer implemented only for "
        "front dimensions reduction");
    math::AxpyFixedSize<T, CPUContext, FixedSize>(
        meta.block_size, meta.scalars[offset], in, out_, context);
  }

private:
  T* out_;
};

template <typename T, class Context>
class WeightedSumReducerGradient : public BaseReducerGradient {
 public:
  // which of the original inputs are required for gradient computation
  static constexpr std::array<int, 1> originalInputs() {
    return {{1}};
  }

static int numAuxInputsWithGrads(const OperatorDef& def) {
    return GetFlagArgument(def, "grad_on_weights");
  }

static bool requiresDataInput(const OperatorDef& def) {
    return numAuxInputsWithGrads(def) > 0;
  }

using FixedDispatch = FixedValues<1>;

struct Meta : public BaseReducerGradient::Meta {
    const T* scalars;
    T* scalars_grad;

using BaseReducerGradient::Meta::Meta;

void observeOriginalInput(
        int original_input,
        const Tensor& value,
        Tensor* input_grad, // optional grad to populate
        int /*skip_dims*/) {
      CAFFE_ENFORCE_EQ(1, original_input);
      scalars = value.data<T>();
      if (input_grad) {
        input_grad->ResizeLike(value);
        scalars_grad = input_grad->template mutable_data<T>();
      }
    }
  };

WeightedSumReducerGradient(
      const Meta& /*meta*/,
      const T* s_grad,
      CPUContext* /*context*/)
      : s_grad_(s_grad) {}

template <int FixedSize>
  void fillGrad(
      const Meta& meta,
      T* data_grad,
      int64_t offset,
      Context* context,
      const int /*length*/) {
    math::ScaleFixedSize<T, CPUContext, FixedSize>(
        meta.block_size, meta.scalars[offset], s_grad_, data_grad, context);
  }

// Special version which is called with the main input too, used only if
  // additional input grad is requested
  template <int FixedSize>
  void fillGradWithMainInput(
      const Meta& meta,
      const T* data,
      T* data_grad,
      int64_t offset,
      Context* context,
      const int /*length*/) {
    math::ScaleFixedSize<T, CPUContext, FixedSize>(
        meta.block_size, meta.scalars[offset], s_grad_, data_grad, context);
    math::Dot(
        meta.block_size, s_grad_, data, meta.scalars_grad + offset, context);
  }

private:
  const T* s_grad_;
};

struct WeightedSumReducerDef {
  template <typename T, class Context>
  using Reducer = WeightedSumReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = WeightedSumReducerGradient<T, Context>;
  static constexpr const char* name = "WeightedSum";
  static constexpr const char* doc =
      "Input slices are first scaled by SCALARS and then summed element-wise. "
      "It doesn't change the shape of the individual blocks.";
  static void PopulateSchema(OpSchema& schema) {
    schema.Input(0, "DATA", "Input tensor for the summation");
    schema.Input(
        1,
        "SCALARS",
        "Scalar multipliers for the input slices. Must be a vector with the "
        "length matching the number of slices");
    schema.Arg(
        "grad_on_weights",
        "Produce also gradient for `weights`. For now it's only supported in "
        "`Lengths`-based operators");
  }
};

template <typename T, class Context>
class MeanReducer;
template <typename T, class Context>
class MeanReducerGradient;

template <typename T>
class MeanReducer<T, CPUContext> : public BaseReducer {
 public:
  using FixedDispatch = FixedValues<1>;

MeanReducer(const Meta& meta, T* out, CPUContext* /*context*/)
      : out_(out), current_size_(0) {
    if (meta.first_dim) {
      memset(out, 0, sizeof(T) * meta.block_size);
    }
  }

template <int FixedSize>
  void process(
      const Meta& meta,
      const T* in,
      int64_t /*offset*/,
      CPUContext* context) {
    if (meta.first_dim) {
      math::AxpyFixedSize<T, CPUContext, FixedSize>(
          meta.block_size, 1, in, out_, context);
    } else {
      math::Sum<T, CPUContext>(
          meta.block_size, in, out_ + current_size_, context);
    }
    current_size_++;
  }

template <int FixedSize>
  void finish(const Meta& meta, CPUContext* context) {
    if (meta.first_dim) {
      if (current_size_ > 0) {
        math::ScaleFixedSize<T, CPUContext, FixedSize>(
            meta.block_size, 1.0 / current_size_, out_, out_, context);
      }
    } else {
      math::ScaleFixedSize<T, CPUContext, FixedSize>(
          current_size_, 1.0 / meta.block_size, out_, out_, context);
    }
  }

private:
  T* out_;
  int current_size_;
};

template <typename T, class Context>
class MeanReducerGradient : public BaseReducerGradient {
 public:
  static constexpr bool computeLength() {
    return true;
  }

using FixedDispatch = FixedValues<1>;

MeanReducerGradient(
      const Meta& /*meta*/,
      const T* s_grad,
      CPUContext* /*context*/)
      : s_grad_(s_grad) {}

template <int FixedSize>
  void fillGrad(
      const Meta& meta,
      T* data_grad,
      int64_t offset,
      Context* context,
      const int length) {
    CAFFE_ENFORCE_GT(length, 0, "Segment length must be > 0");
    if (meta.first_dim) {
      math::ScaleFixedSize<T, CPUContext, FixedSize>(
          meta.block_size, 1.0 / length, s_grad_, data_grad, context);
    } else {
      math::Set<T, CPUContext>(
          length, s_grad_[offset] * 1.0f / length, data_grad, context);
    }
  }

private:
  const T* s_grad_;
};

struct MeanReducerDef {
  template <typename T, class Context>
  using Reducer = MeanReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = MeanReducerGradient<T, Context>;
  static constexpr const char* name = "Mean";
  static constexpr const char* doc =
      "Mean computes the element-wise mean of the input slices. "
      "Operation doesn't change the shape of the individual blocks.";
  static void PopulateSchema(OpSchema& /*schema*/) {}
};

template <typename T, class Context>
class MaxReducer;
template <typename T, class Context>
class MaxReducerGradient;

template <typename T>
class MaxReducer<T, CPUContext> : public BaseReducer {
 public:
  using FixedDispatch = FixedValues<1>;

MaxReducer(const Meta& meta, T* out, CPUContext* /*context*/)
      : out_(out), current_size_(0) {
    // add a wrapper in Context for it
    memset(out, 0, sizeof(T) * meta.block_size);
  }

template <int FixedSize>
  void process(
      const Meta& meta,
      const T* in,
      int64_t /*offset*/,
      CPUContext* context) {
    CAFFE_ENFORCE(
        meta.first_dim,
        "MaxReducer implemented only for front dimensions reduction");
    if (current_size_ > 0) {
      EigenVectorMap<T> output_vec(out_, meta.block_size);
      output_vec =
          output_vec.cwiseMax(ConstEigenVectorMap<T>(in, meta.block_size));
    } else {
      memcpy(out_, in, sizeof(T) * meta.block_size);
    }
    ++current_size_;
  }

private:
  T* out_;
  int current_size_;
};

template <typename T, class Context>
class MaxReducerGradient : public BaseReducerGradient {
 public:
  static bool requiresDataInput(const OperatorDef& /*def*/) {
    return true;
  }

static bool requiresForwardOutput() {
    return true;
  }

using FixedDispatch = FixedValues<1>;

MaxReducerGradient(
      const Meta& /*meta*/,
      const T* s_grad,
      CPUContext* /*context*/)
      : s_grad_(s_grad) {}

template <int FixedSize>
  void fillGradWithMainInputAndForwardOutput(
      const Meta& meta,
      const T* data,
      T* data_grad,
      const T* forward_output,
      int64_t /*offset*/,
      Context* /*context*/,
      const int /*length*/) {
    for (int64_t i = 0; i < meta.block_size; ++i) {
      data_grad[i] = data[i] == forward_output[i] ? s_grad_[i] : 0;
    }
  }

private:
  const T* s_grad_;
};

struct MaxReducerDef {
  template <typename T, class Context>
  using Reducer = MaxReducer<T, Context>;
  template <typename T, class Context>
  using ReducerGradient = MaxReducerGradient<T, Context>;
  static constexpr const char* name = "Max";
  static constexpr const char* doc =
      "Max computes the element-wise max of the input slices. "
      "Operation doesn't change the shape of the individual blocks.";
  static void PopulateSchema(OpSchema& /*schema*/) {}
};

} // namespace caffe2

#endif // CAFFE2_OPERATORS_RECUDER_FUNCTORS_H_

cmd: