FM / sgd

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd/

/usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd

Name	Size	Mode	Actions
adadelta_op.h	5852	0644	edit dl rm
adagrad_fused.h	16632	0644	edit dl rm
adagrad_op.h	17993	0644	edit dl rm
adam_op.h	26165	0644	edit dl rm
clip_tensor_op.h	1853	0644	edit dl rm
decay_adagrad_op.h	3215	0644	edit dl rm
fp16_momentum_sgd_op.h	2292	0644	edit dl rm
fp32_momentum_sgd_op.h	2015	0644	edit dl rm
ftrl_op.h	2219	0644	edit dl rm
gftrl_op.h	1028	0644	edit dl rm
iter_op.h	3379	0644	edit dl rm
lars_op.h	2479	0644	edit dl rm
learning_rate_adaption_op.h	1974	0644	edit dl rm
learning_rate_functors.h	15135	0644	edit dl rm
learning_rate_op.h	12696	0644	edit dl rm
math_lp.h	655	0644	edit dl rm
momentum_sgd_op.h	6035	0644	edit dl rm
rmsprop_op.h	1980	0644	edit dl rm
rowwise_adagrad_fused.h	29595	0644	edit dl rm
rowwise_counter.h	2083	0644	edit dl rm
storm_op.h	5915	0644	edit dl rm
weight_scale_op.h	2552	0644	edit dl rm
wngrad_op.h	7272	0644	edit dl rm
yellowfin_op.h	10284	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/torch/include/caffe2/sgd/fp16_momentum_sgd_op.h (2292B)

#pragma once

#include "caffe2/core/operator.h"
#include "caffe2/core/timer.h"

namespace caffe2 {

template <class Context>
void fp16_momentum_sgd_update(
    int N,
    const at::Half* g,
    const at::Half* m,
    at::Half* ng,
    at::Half* nm,
    const float* lr,
    float momentum,
    bool nesterov,
    float weight_decay,
    bool fp32_update,
    at::Half* param,
    Context* /*context*/) {}

template <typename T, class Context>
class FP16MomentumSGDUpdateOp final : public Operator<Context> {
 public:
  USE_OPERATOR_CONTEXT_FUNCTIONS;
  FP16MomentumSGDUpdateOp(const OperatorDef& operator_def, Workspace* ws)
      : Operator<Context>(operator_def, ws),
        momentum_(this->template GetSingleArgument<float>("momentum", 0.0)),
        weight_decay_(
            this->template GetSingleArgument<float>("weight_decay", 0.0)),
        nesterov_(this->template GetSingleArgument<int>("nesterov", 0)),
        // when set, fp32_update will read in the fp16 data but
        // perform all the compute in fp32 precision.
        fp32_update_(this->template GetSingleArgument<int>("fp32_update", 0)) {}

bool RunOnDevice() override {
    auto device_type = Context::GetDeviceType();
    // Iter live on the CPU
    CAFFE_ENFORCE(OperatorBase::InputIsTensorType(GRAD, device_type));
    CAFFE_ENFORCE(OperatorBase::InputIsTensorType(MOMENTUM, device_type));
    CAFFE_ENFORCE(Input(LR).size() == 1);
    CAFFE_ENFORCE(Input(GRAD).size() == Input(MOMENTUM).size());
    Output(OUTPUT_GRAD)->ResizeLike(Input(GRAD));
    Output(OUTPUT_MOMENTUM)->ResizeLike(Input(MOMENTUM));

fp16_momentum_sgd_update<Context>(
        Input(GRAD).size(),
        Input(GRAD).template data<T>(),
        Input(MOMENTUM).template data<T>(),
        Output(OUTPUT_GRAD)->template mutable_data<T>(),
        Output(OUTPUT_MOMENTUM)->template mutable_data<T>(),
        Input(LR).template data<float>(),
        momentum_,
        nesterov_,
        weight_decay_,
        fp32_update_,
        Output(OUTPUT_PARAM)->template mutable_data<T>(),
        &context_);

return true;
  }

protected:
  float momentum_{0.9};
  float weight_decay_{0.0};
  bool nesterov_;
  bool fp32_update_;
  INPUT_TAGS(GRAD, MOMENTUM, LR, PARAM);
  OUTPUT_TAGS(OUTPUT_GRAD, OUTPUT_MOMENTUM, OUTPUT_PARAM);
};
}

#pragma once #include "caffe2/core/operator.h" #include "caffe2/core/timer.h" namespace caffe2 { template void fp16_momentum_sgd_update( int N, const at::Half* g, const at::Half* m, at::Half* ng, at::Half* nm, const float* lr, float momentum, bool nesterov, float weight_decay, bool fp32_update, at::Half* param, Context* /*context*/) {} template class FP16MomentumSGDUpdateOp final : public Operator { public: USE_OPERATOR_CONTEXT_FUNCTIONS; FP16MomentumSGDUpdateOp(const OperatorDef& operator_def, Workspace* ws) : Operator(operator_def, ws), momentum_(this->template GetSingleArgument("momentum", 0.0)), weight_decay_( this->template GetSingleArgument("weight_decay", 0.0)), nesterov_(this->template GetSingleArgument("nesterov", 0)), // when set, fp32_update will read in the fp16 data but // perform all the compute in fp32 precision. fp32_update_(this->template GetSingleArgument("fp32_update", 0)) {} bool RunOnDevice() override { auto device_type = Context::GetDeviceType(); // Iter live on the CPU CAFFE_ENFORCE(OperatorBase::InputIsTensorType(GRAD, device_type)); CAFFE_ENFORCE(OperatorBase::InputIsTensorType(MOMENTUM, device_type)); CAFFE_ENFORCE(Input(LR).size() == 1); CAFFE_ENFORCE(Input(GRAD).size() == Input(MOMENTUM).size()); Output(OUTPUT_GRAD)->ResizeLike(Input(GRAD)); Output(OUTPUT_MOMENTUM)->ResizeLike(Input(MOMENTUM)); fp16_momentum_sgd_update( Input(GRAD).size(), Input(GRAD).template data(), Input(MOMENTUM).template data(), Output(OUTPUT_GRAD)->template mutable_data(), Output(OUTPUT_MOMENTUM)->template mutable_data(), Input(LR).template data(), momentum_, nesterov_, weight_decay_, fp32_update_, Output(OUTPUT_PARAM)->template mutable_data(), &context_); return true; } protected: float momentum_{0.9}; float weight_decay_{0.0}; bool nesterov_; bool fp32_update_; INPUT_TAGS(GRAD, MOMENTUM, LR, PARAM); OUTPUT_TAGS(OUTPUT_GRAD, OUTPUT_MOMENTUM, OUTPUT_PARAM); }; }