FM / sgd

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd/

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd

Name	Size	Mode	Actions
adadelta_op.h	5852	0644	edit dl rm
adagrad_fused.h	16632	0644	edit dl rm
adagrad_op.h	17993	0644	edit dl rm
adam_op.h	26165	0644	edit dl rm
clip_tensor_op.h	1853	0644	edit dl rm
decay_adagrad_op.h	3215	0644	edit dl rm
fp16_momentum_sgd_op.h	2292	0644	edit dl rm
fp32_momentum_sgd_op.h	2015	0644	edit dl rm
ftrl_op.h	2219	0644	edit dl rm
gftrl_op.h	1028	0644	edit dl rm
iter_op.h	3379	0644	edit dl rm
lars_op.h	2479	0644	edit dl rm
learning_rate_adaption_op.h	1974	0644	edit dl rm
learning_rate_functors.h	15135	0644	edit dl rm
learning_rate_op.h	12696	0644	edit dl rm
math_lp.h	655	0644	edit dl rm
momentum_sgd_op.h	6035	0644	edit dl rm
rmsprop_op.h	1980	0644	edit dl rm
rowwise_adagrad_fused.h	29595	0644	edit dl rm
rowwise_counter.h	2083	0644	edit dl rm
storm_op.h	5915	0644	edit dl rm
weight_scale_op.h	2552	0644	edit dl rm
wngrad_op.h	7272	0644	edit dl rm
yellowfin_op.h	10284	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd/decay_adagrad_op.h (3215B)

#pragma once

#include "caffe2/core/operator.h"
#include "caffe2/utils/eigen_utils.h"

namespace caffe2 {

template <typename Context>
void decay_adagrad_compute(
    int N,
    const float* w,
    const float* g,
    const float* m,
    const float* v,
    float* nw,
    float* nm,
    float* nv,
    float beta1,
    float beta2,
    float eps_hat,
    float weight_decay,
    float c,
    const float* lr,
    Context* /*context*/) {
    ConstEigenVectorArrayMap<float> w_arr(w, N);
    ConstEigenVectorArrayMap<float> g_arr(g, N);
    ConstEigenVectorArrayMap<float> m_arr(m, N);
    ConstEigenVectorArrayMap<float> v_arr(v, N);
    EigenVectorArrayMap<float> nw_arr(nw, N);
    EigenVectorArrayMap<float> nm_arr(nm, N);
    EigenVectorArrayMap<float> nv_arr(nv, N);
    nm_arr = m_arr * beta1 + g_arr * (1.0f - beta1);
    nv_arr = v_arr + g_arr.square();
    nw_arr = w_arr + *lr * (nm_arr / c / (nv_arr.sqrt() + eps_hat) + weight_decay * w_arr);
}

template <typename T, class Context>
class DecayAdagradOp final : public Operator<Context> {
 public:
  USE_OPERATOR_CONTEXT_FUNCTIONS;
  DecayAdagradOp(const OperatorDef& operator_def, Workspace* ws)
      : Operator<Context>(operator_def, ws),
        beta1_(this->template GetSingleArgument<float>("beta1", 0.9f)),
        beta2_(this->template GetSingleArgument<float>("beta2", 0.999f)),
        epsilon_(this->template GetSingleArgument<float>("epsilon", 1e-5f)),
        weight_decay_(this->template GetSingleArgument<float>("weight_decay", 0.0f)),
        bias_correction_first_(this->template GetSingleArgument<bool>("bias_correction_first", true)) {}

bool RunOnDevice() override {
    // Iter live on the CPU
    CAFFE_ENFORCE(OperatorBase::InputIsTensorType(ITER, CPU));
    CAFFE_ENFORCE(Input(LR).numel() == 1);
    CAFFE_ENFORCE(Input(GRAD).numel() == Input(PARAM).numel());
    CAFFE_ENFORCE(Input(GRAD).numel() == Input(MOMENT_1).numel());
    CAFFE_ENFORCE(Input(GRAD).numel() == Input(MOMENT_2).numel());
    Output(OUTPUT_PARAM)->ResizeLike(Input(PARAM));
    Output(OUTPUT_MOMENT_1)->ResizeLike(Input(MOMENT_1));
    Output(OUTPUT_MOMENT_2)->ResizeLike(Input(MOMENT_2));

const auto iter =
        OperatorBase::Input<Tensor>(ITER, CPU).template data<int64_t>()[0];

const auto t = iter + 1;
    const auto c = (bias_correction_first_)? (T(1.) - std::pow(beta1_, t)) : 1.0;
    decay_adagrad_compute<Context>(
        Input(GRAD).numel(),
        Input(PARAM).template data<T>(),
        Input(GRAD).template data<T>(),
        Input(MOMENT_1).template data<T>(),
        Input(MOMENT_2).template data<T>(),
        Output(OUTPUT_PARAM)->template mutable_data<T>(),
        Output(OUTPUT_MOMENT_1)->template mutable_data<T>(),
        Output(OUTPUT_MOMENT_2)->template mutable_data<T>(),
        beta1_,
        beta2_,
        epsilon_,
        weight_decay_,
        c,
        Input(LR).template data<T>(),
        &context_);

return true;
  }

protected:
  T beta1_{0.9};
  T beta2_{0.999};
  T epsilon_{1e-8};
  T weight_decay_{0.0};
  bool bias_correction_first_{true};
  INPUT_TAGS(PARAM, MOMENT_1, MOMENT_2, GRAD, LR, ITER);
  OUTPUT_TAGS(OUTPUT_PARAM, OUTPUT_MOMENT_1, OUTPUT_MOMENT_2);
};

} // namespace caffe2

#pragma once #include "caffe2/core/operator.h" #include "caffe2/utils/eigen_utils.h" namespace caffe2 { template void decay_adagrad_compute( int N, const float* w, const float* g, const float* m, const float* v, float* nw, float* nm, float* nv, float beta1, float beta2, float eps_hat, float weight_decay, float c, const float* lr, Context* /*context*/) { ConstEigenVectorArrayMap w_arr(w, N); ConstEigenVectorArrayMap g_arr(g, N); ConstEigenVectorArrayMap m_arr(m, N); ConstEigenVectorArrayMap v_arr(v, N); EigenVectorArrayMap nw_arr(nw, N); EigenVectorArrayMap nm_arr(nm, N); EigenVectorArrayMap nv_arr(nv, N); nm_arr = m_arr * beta1 + g_arr * (1.0f - beta1); nv_arr = v_arr + g_arr.square(); nw_arr = w_arr + *lr * (nm_arr / c / (nv_arr.sqrt() + eps_hat) + weight_decay * w_arr); } template class DecayAdagradOp final : public Operator { public: USE_OPERATOR_CONTEXT_FUNCTIONS; DecayAdagradOp(const OperatorDef& operator_def, Workspace* ws) : Operator(operator_def, ws), beta1_(this->template GetSingleArgument("beta1", 0.9f)), beta2_(this->template GetSingleArgument("beta2", 0.999f)), epsilon_(this->template GetSingleArgument("epsilon", 1e-5f)), weight_decay_(this->template GetSingleArgument("weight_decay", 0.0f)), bias_correction_first_(this->template GetSingleArgument("bias_correction_first", true)) {} bool RunOnDevice() override { // Iter live on the CPU CAFFE_ENFORCE(OperatorBase::InputIsTensorType(ITER, CPU)); CAFFE_ENFORCE(Input(LR).numel() == 1); CAFFE_ENFORCE(Input(GRAD).numel() == Input(PARAM).numel()); CAFFE_ENFORCE(Input(GRAD).numel() == Input(MOMENT_1).numel()); CAFFE_ENFORCE(Input(GRAD).numel() == Input(MOMENT_2).numel()); Output(OUTPUT_PARAM)->ResizeLike(Input(PARAM)); Output(OUTPUT_MOMENT_1)->ResizeLike(Input(MOMENT_1)); Output(OUTPUT_MOMENT_2)->ResizeLike(Input(MOMENT_2)); const auto iter = OperatorBase::Input(ITER, CPU).template data()[0]; const auto t = iter + 1; const auto c = (bias_correction_first_)? (T(1.) - std::pow(beta1_, t)) : 1.0; decay_adagrad_compute( Input(GRAD).numel(), Input(PARAM).template data(), Input(GRAD).template data(), Input(MOMENT_1).template data(), Input(MOMENT_2).template data(), Output(OUTPUT_PARAM)->template mutable_data(), Output(OUTPUT_MOMENT_1)->template mutable_data(), Output(OUTPUT_MOMENT_2)->template mutable_data(), beta1_, beta2_, epsilon_, weight_decay_, c, Input(LR).template data(), &context_); return true; } protected: T beta1_{0.9}; T beta2_{0.999}; T epsilon_{1e-8}; T weight_decay_{0.0}; bool bias_correction_first_{true}; INPUT_TAGS(PARAM, MOMENT_1, MOMENT_2, GRAD, LR, ITER); OUTPUT_TAGS(OUTPUT_PARAM, OUTPUT_MOMENT_1, OUTPUT_MOMENT_2); }; } // namespace caffe2