FM / sgd

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd/

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd

Name	Size	Mode	Actions
adadelta_op.h	5852	0644	edit dl rm
adagrad_fused.h	16632	0644	edit dl rm
adagrad_op.h	17993	0644	edit dl rm
adam_op.h	26165	0644	edit dl rm
clip_tensor_op.h	1853	0644	edit dl rm
decay_adagrad_op.h	3215	0644	edit dl rm
fp16_momentum_sgd_op.h	2292	0644	edit dl rm
fp32_momentum_sgd_op.h	2015	0644	edit dl rm
ftrl_op.h	2219	0644	edit dl rm
gftrl_op.h	1028	0644	edit dl rm
iter_op.h	3379	0644	edit dl rm
lars_op.h	2479	0644	edit dl rm
learning_rate_adaption_op.h	1974	0644	edit dl rm
learning_rate_functors.h	15135	0644	edit dl rm
learning_rate_op.h	12696	0644	edit dl rm
math_lp.h	655	0644	edit dl rm
momentum_sgd_op.h	6035	0644	edit dl rm
rmsprop_op.h	1980	0644	edit dl rm
rowwise_adagrad_fused.h	29595	0644	edit dl rm
rowwise_counter.h	2083	0644	edit dl rm
storm_op.h	5915	0644	edit dl rm
weight_scale_op.h	2552	0644	edit dl rm
wngrad_op.h	7272	0644	edit dl rm
yellowfin_op.h	10284	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd/rowwise_counter.h (2083B)

#pragma once

#include "caffe2/core/operator.h"

namespace caffe2 {

class RowWiseCounterOp final : public Operator<CPUContext> {
 public:
  RowWiseCounterOp(const OperatorDef& operator_def, Workspace* ws)
      : Operator<CPUContext>(operator_def, ws),
        counter_halflife_(
            this->template GetSingleArgument<int64_t>("counter_halflife", -1)),
        counter_neg_log_rho_(0.0) {
    if (counter_halflife_ > 0) {
      counter_neg_log_rho_ = std::log(2.0) / counter_halflife_;
    }
  }

bool RunOnDevice() override {
    CAFFE_ENFORCE_EQ(Input(PREV_ITER).numel(), Input(COUNTER).numel());
    CAFFE_ENFORCE_EQ(Input(ITER).numel(), 1);
    return DispatchHelper<TensorTypes<int32_t, int64_t>>::call(
        this, Input(INDICES));
  }

template <typename SIndex>
  bool DoRunWithType() {
    auto* prev_iter =
        Output(OUTPUT_PREV_ITER)->template mutable_data<int64_t>();
    auto* counter = Output(OUTPUT_COUNTER)->template mutable_data<double>();

const int64_t curr_iter = Input(ITER).template data<int64_t>()[0];
    const auto* indices = Input(INDICES).template data<SIndex>();

auto n = Input(INDICES).numel();
    if (n == 0) {
      return true;
    }
    if (counter_halflife_ <= 0) {
      return true;
    }

for (auto i = 0; i < n; ++i) {
      const std::size_t idx = indices[i];
      CAFFE_ENFORCE_GE(
          Input(COUNTER).numel(),
          idx,
          this->debug_def().input(COUNTER),
          ", out of bound,  idx:",
          idx,
          " for input i:",
          i,
          " max size:",
          Input(COUNTER).numel());
      const int64_t iter_delta =
          std::max<int64_t>(0, curr_iter - prev_iter[idx]);

counter[idx] =
          1.0 + std::exp(-iter_delta * counter_neg_log_rho_) * counter[idx];
      prev_iter[idx] = std::max<int64_t>(curr_iter, prev_iter[idx]);
    }
    return true;
  }

protected:
  int64_t counter_halflife_;
  double counter_neg_log_rho_;
  INPUT_TAGS(PREV_ITER, COUNTER, INDICES, ITER);
  OUTPUT_TAGS(OUTPUT_PREV_ITER, OUTPUT_COUNTER);
};

} // namespace caffe2

#pragma once #include "caffe2/core/operator.h" namespace caffe2 { class RowWiseCounterOp final : public Operator { public: RowWiseCounterOp(const OperatorDef& operator_def, Workspace* ws) : Operator(operator_def, ws), counter_halflife_( this->template GetSingleArgument("counter_halflife", -1)), counter_neg_log_rho_(0.0) { if (counter_halflife_ > 0) { counter_neg_log_rho_ = std::log(2.0) / counter_halflife_; } } bool RunOnDevice() override { CAFFE_ENFORCE_EQ(Input(PREV_ITER).numel(), Input(COUNTER).numel()); CAFFE_ENFORCE_EQ(Input(ITER).numel(), 1); return DispatchHelper>::call( this, Input(INDICES)); } template bool DoRunWithType() { auto* prev_iter = Output(OUTPUT_PREV_ITER)->template mutable_data(); auto* counter = Output(OUTPUT_COUNTER)->template mutable_data(); const int64_t curr_iter = Input(ITER).template data()[0]; const auto* indices = Input(INDICES).template data(); auto n = Input(INDICES).numel(); if (n == 0) { return true; } if (counter_halflife_ <= 0) { return true; } for (auto i = 0; i < n; ++i) { const std::size_t idx = indices[i]; CAFFE_ENFORCE_GE( Input(COUNTER).numel(), idx, this->debug_def().input(COUNTER), ", out of bound, idx:", idx, " for input i:", i, " max size:", Input(COUNTER).numel()); const int64_t iter_delta = std::max(0, curr_iter - prev_iter[idx]); counter[idx] = 1.0 + std::exp(-iter_delta * counter_neg_log_rho_) * counter[idx]; prev_iter[idx] = std::max(curr_iter, prev_iter[idx]); } return true; } protected: int64_t counter_halflife_; double counter_neg_log_rho_; INPUT_TAGS(PREV_ITER, COUNTER, INDICES, ITER); OUTPUT_TAGS(OUTPUT_PREV_ITER, OUTPUT_COUNTER); }; } // namespace caffe2