FM / __pycache_

/usr/local/lib64/python3.6/site-packages/caffe2/python/__pycache__/

/usr/local/lib64/python3.6/site-packages/caffe2/python/__pycache__

Name	Size	Mode	Actions
allcompare_test.cpython-36.pyc	2710	0644	edit dl rm
attention.cpython-36.pyc	4938	0644	edit dl rm
benchmark_generator.cpython-36.pyc	3910	0644	edit dl rm
binarysize.cpython-36.pyc	4853	0644	edit dl rm
brew.cpython-36.pyc	3849	0644	edit dl rm
brew_test.cpython-36.pyc	10452	0644	edit dl rm
build.cpython-36.pyc	340	0644	edit dl rm
cached_reader.cpython-36.pyc	2954	0644	edit dl rm
caffe_translator.cpython-36.pyc	24165	0644	edit dl rm
caffe_translator_test.cpython-36.pyc	2929	0644	edit dl rm
checkpoint.cpython-36.pyc	27973	0644	edit dl rm
checkpoint_test.cpython-36.pyc	9462	0644	edit dl rm
cnn.cpython-36.pyc	8435	0644	edit dl rm
context.cpython-36.pyc	4008	0644	edit dl rm
context_test.cpython-36.pyc	2665	0644	edit dl rm
control.cpython-36.pyc	14466	0644	edit dl rm
control_ops_grad.cpython-36.pyc	16365	0644	edit dl rm
control_ops_grad_test.cpython-36.pyc	1502	0644	edit dl rm
control_ops_util.cpython-36.pyc	8332	0644	edit dl rm
control_test.cpython-36.pyc	11870	0644	edit dl rm
convert.cpython-36.pyc	148	0644	edit dl rm
convert_test.cpython-36.pyc	568	0644	edit dl rm
convnet_benchmarks.cpython-36.pyc	12945	0644	edit dl rm
convnet_benchmarks_test.cpython-36.pyc	1090	0644	edit dl rm
core.cpython-36.pyc	94428	0644	edit dl rm
core_gradients_test.cpython-36.pyc	24825	0644	edit dl rm
core_test.cpython-36.pyc	33942	0644	edit dl rm
crf.cpython-36.pyc	7622	0644	edit dl rm
crf_predict.cpython-36.pyc	1044	0644	edit dl rm
crf_viterbi_test.cpython-36.pyc	1677	0644	edit dl rm
dataio.cpython-36.pyc	23978	0644	edit dl rm
dataio_test.cpython-36.pyc	13267	0644	edit dl rm
dataset.cpython-36.pyc	12879	0644	edit dl rm
data_parallel_model.cpython-36.pyc	51416	0644	edit dl rm
data_parallel_model_test.cpython-36.pyc	38923	0644	edit dl rm
data_workers.cpython-36.pyc	13142	0644	edit dl rm
data_workers_test.cpython-36.pyc	4522	0644	edit dl rm
db_file_reader.cpython-36.pyc	5361	0644	edit dl rm
db_test.cpython-36.pyc	1328	0644	edit dl rm
device_checker.cpython-36.pyc	4014	0644	edit dl rm
dyndep.cpython-36.pyc	1568	0644	edit dl rm
embedding_generation_benchmark.cpython-36.pyc	4203	0644	edit dl rm
experiment_util.cpython-36.pyc	3233	0644	edit dl rm
extension_loader.cpython-36.pyc	543	0644	edit dl rm
fakefp16_transform_lib.cpython-36.pyc	546	0644	edit dl rm
filler_test.cpython-36.pyc	925	0644	edit dl rm
functional.cpython-36.pyc	3531	0644	edit dl rm
functional_test.cpython-36.pyc	4132	0644	edit dl rm
fused_8bit_rowwise_conversion_ops_test.cpython-36.pyc	3631	0644	edit dl rm
gradient_checker.cpython-36.pyc	10622	0644	edit dl rm
gradient_check_test.cpython-36.pyc	16788	0644	edit dl rm
gru_cell.cpython-36.pyc	2596	0644	edit dl rm
hip_test_util.cpython-36.pyc	669	0644	edit dl rm
hsm_util.cpython-36.pyc	1823	0644	edit dl rm
hypothesis_test.cpython-36.pyc	84312	0644	edit dl rm
hypothesis_test_util.cpython-36.pyc	20039	0644	edit dl rm
ideep_test_util.cpython-36.pyc	1081	0644	edit dl rm
layers_test.cpython-36.pyc	57372	0644	edit dl rm
layer_model_helper.cpython-36.pyc	21641	0644	edit dl rm
layer_model_instantiator.cpython-36.pyc	3665	0644	edit dl rm
layer_parameter_sharing_test.cpython-36.pyc	5626	0644	edit dl rm
layer_test_util.cpython-36.pyc	5273	0644	edit dl rm
lazy.cpython-36.pyc	424	0644	edit dl rm
lazy_dyndep.cpython-36.pyc	2475	0644	edit dl rm
lazy_dyndep_test.cpython-36.pyc	4807	0644	edit dl rm
lengths_reducer_fused_8bit_rowwise_ops_test.cpython-36.pyc	4392	0644	edit dl rm
lengths_reducer_rowwise_8bit_ops_test.cpython-36.pyc	3684	0644	edit dl rm
lstm_benchmark.cpython-36.pyc	7282	0644	edit dl rm
memonger.cpython-36.pyc	32170	0644	edit dl rm
memonger_test.cpython-36.pyc	23100	0644	edit dl rm
mkl_test_util.cpython-36.pyc	1191	0644	edit dl rm
model_device_test.cpython-36.pyc	3376	0644	edit dl rm
model_helper.cpython-36.pyc	18618	0644	edit dl rm
model_helper_test.cpython-36.pyc	1991	0644	edit dl rm
modifier_context.cpython-36.pyc	2650	0644	edit dl rm
muji.cpython-36.pyc	5566	0644	edit dl rm
muji_test.cpython-36.pyc	3567	0644	edit dl rm
net_builder.cpython-36.pyc	26786	0644	edit dl rm
net_builder_test.cpython-36.pyc	9255	0644	edit dl rm
net_drawer.cpython-36.pyc	10056	0644	edit dl rm
net_printer.cpython-36.pyc	14367	0644	edit dl rm
net_printer_test.cpython-36.pyc	3323	0644	edit dl rm
nomnigraph.cpython-36.pyc	5125	0644	edit dl rm
nomnigraph_test.cpython-36.pyc	14591	0644	edit dl rm
nomnigraph_transformations.cpython-36.pyc	2431	0644	edit dl rm
nomnigraph_transformations_test.cpython-36.pyc	4073	0644	edit dl rm
normalizer.cpython-36.pyc	1941	0644	edit dl rm
normalizer_context.cpython-36.pyc	1553	0644	edit dl rm
normalizer_test.cpython-36.pyc	870	0644	edit dl rm
numa_benchmark.cpython-36.pyc	1886	0644	edit dl rm
numa_test.cpython-36.pyc	1626	0644	edit dl rm
observer_test.cpython-36.pyc	4181	0644	edit dl rm
operator_fp_exceptions_test.cpython-36.pyc	1305	0644	edit dl rm
optimizer.cpython-36.pyc	45606	0644	edit dl rm
optimizer_context.cpython-36.pyc	2008	0644	edit dl rm
optimizer_test.cpython-36.pyc	24747	0644	edit dl rm
optimizer_test_util.cpython-36.pyc	6808	0644	edit dl rm
parallelize_bmuf_distributed_test.cpython-36.pyc	6990	0644	edit dl rm
parallel_workers.cpython-36.pyc	9148	0644	edit dl rm
parallel_workers_test.cpython-36.pyc	3824	0644	edit dl rm
pipeline.cpython-36.pyc	12940	0644	edit dl rm
pipeline_test.cpython-36.pyc	2715	0644	edit dl rm
predictor_constants.cpython-36.pyc	304	0644	edit dl rm
python_op_test.cpython-36.pyc	10502	0644	edit dl rm
queue_util.cpython-36.pyc	5055	0644	edit dl rm
record_queue.cpython-36.pyc	4225	0644	edit dl rm
recurrent.cpython-36.pyc	9893	0644	edit dl rm
regularizer.cpython-36.pyc	18604	0644	edit dl rm
regularizer_context.cpython-36.pyc	1562	0644	edit dl rm
regularizer_test.cpython-36.pyc	8825	0644	edit dl rm
rnn_cell.cpython-36.pyc	44785	0644	edit dl rm
schema.cpython-36.pyc	41763	0644	edit dl rm
schema_test.cpython-36.pyc	13787	0644	edit dl rm
scope.cpython-36.pyc	2600	0644	edit dl rm
scope_test.cpython-36.pyc	4068	0644	edit dl rm
session.cpython-36.pyc	7346	0644	edit dl rm
session_test.cpython-36.pyc	2353	0644	edit dl rm
sparse_to_dense_mask_test.cpython-36.pyc	5298	0644	edit dl rm
sparse_to_dense_test.cpython-36.pyc	3003	0644	edit dl rm
task.cpython-36.pyc	22207	0644	edit dl rm
task_test.cpython-36.pyc	1178	0644	edit dl rm
test_util.cpython-36.pyc	3729	0644	edit dl rm
text_file_reader.cpython-36.pyc	2239	0644	edit dl rm
timeout_guard.cpython-36.pyc	3106	0644	edit dl rm
toy_regression_test.cpython-36.pyc	2360	0644	edit dl rm
transformations.cpython-36.pyc	1833	0644	edit dl rm
transformations_test.cpython-36.pyc	9608	0644	edit dl rm
tt_core.cpython-36.pyc	6468	0644	edit dl rm
tt_core_test.cpython-36.pyc	1824	0644	edit dl rm
utils.cpython-36.pyc	12245	0644	edit dl rm
utils_test.cpython-36.pyc	1416	0644	edit dl rm
visualize.cpython-36.pyc	6027	0644	edit dl rm
workspace.cpython-36.pyc	22690	0644	edit dl rm
workspace_test.cpython-36.pyc	29055	0644	edit dl rm
_import_c_extension.cpython-36.pyc	1524	0644	edit dl rm
__init__.cpython-36.pyc	2607	0644	edit dl rm

Edit: /usr/local/lib64/python3.6/site-packages/caffe2/python/__pycache__/optimizer.cpython-36.pyc (45606B)

3 Eg�3�@sJddlZddlZddlmZmZddlZddlmZddl m Z mZmZm Z ddlmZddlmZdZedd d g�Zee�Zddd gZeje�Zdd�ZGdd�de�ZGdd�de�ZGdd�de�ZGdd�de�ZGdd�de�Z Gdd�de�Z!Gdd�de�Z"Gdd�de�Z#Gd d!�d!e�Z$Gd"d#�d#e�Z%Gd$d%�d%e�Z&Gd&d'�d'e�Z'Gd(d)�d)e�Z(Gd*d+�d+e�Z)Gd,d-�d-e�Z*d.d/�Z+d[d0d1�Z,d2d3�Z-d4d5�Z.d6d7�Z/d\d:d;�Z0dd?�Z2d^d@dA�Z3dBdC�Z4d_dEdF�Z5d`dHdI�Z6dadJdK�Z7dbdLdM�Z8dcdNdO�Z9dddPdQ�Z:dedRdS�Z;dfdTdU�ZdS)i�N)�defaultdict� namedtuple)� caffe2_pb2)�core�scope�utils� workspace)�parameter_info)� basestring�lr_injection�AuxOptimizerParams�local�sharedZSIMD_Q_FP16ZSIMD_Q_STOC_FP16ZSIMD_Q_STOC_MKL_FP16cCstj�dS)z� This function clears the _optimizer_instance_count. And keeps it empty. This functionality is needed in some situations where optimizer instance count might not reset even though the workplace is reset. N)�_optimizer_instance_count�clear�rr�C/usr/local/lib64/python3.6/site-packages/caffe2/python/optimizer.py�reset_optimizer_instance_countsrc@s�eZdZdd�Zd$dd�Zdd�Zd%d d �Zdd�Zed d��Z dd�Z d&dd�Zdd�Zd'dd�Z dd�Zedd��Zdd�Zd d!�Zd"d#�ZdS)(� OptimizercCsFtggd�|_t|jj|_t|jjd7<d|_d|_d|_dS)N)r r�F) r�_aux_paramsr� __class__�__name__� _instance_num�_lr_multiplier�_local_lr_multiplier�_local_lr_multiplier_on_gpu)�selfrrr�__init__#szOptimizer.__init__NcCsh|dkr2t|tj�s"tdj|��|jdk sVt�n$t|t�rFtj|�}tjd||d�}|j |||�dS)Nz6Expected parameter to be of type ParameterInfo, got {})Zparam_id�param�grad) � isinstancer Z ParameterInfo�AssertionError�formatr r rZ BlobReference�_run)r�net�param_init_netrr rrr�__call__4s zOptimizer.__call__cCstd��dS)NzNot Implemented)� Exception)rr%r&� param_inforrrr$AszOptimizer._run�cCs|jj}d||j||fS)Nz%s_%d_%s%s_cpu)rrr)r�base_str� node_name� classnamerrr�get_cpu_blob_nameDszOptimizer.get_cpu_blob_namecCs|jj}d||j|||fS)Nz%s_%d_%s%s_gpu%d)rrr)rr+Zgpu_idr,r-rrr�get_gpu_blob_nameHszOptimizer.get_gpu_blob_namecCstj|j�}|d=|S)Nr)�copy�deepcopy�__dict__)r�attrrrr� attributesRszOptimizer.attributescCsJtj�}|dkr|j|�Stj|j�r8|j||j|j�S|j||j�SdS)zo Returns a blob name that will be unique to the current device and optimizer instance. N) r�CurrentDeviceScoper.r�IsGPUDeviceType�device_typer/� device_idr,)rr+� current_scoperrr�make_unique_blob_nameYs zOptimizer.make_unique_blob_name�fixedrc Ks�|dkr|jd�}tj|||d�}|j|�sL|j|g|f||d�|��} n |j|�} |jdk r�|j|j|jd��} |j| | g|jd�dd�} |j dk r�t j�}|dk r�tj |j�r�|jr�|j|j |jd��}n|j }|j| |g|jd �dd�} | |fS) N�lr)�iter_val)Zbase_lr�policy� lr_multiplierZ scaled_lrr)� broadcast�local_lr_multiplierZlocal_scaled_lr)r:r�BuildUniqueMutexIter� BlobIsDefinedZLearningRateZ GetBlobRefrZCopyFromCPUInput�Mulrrr5rr6r7r) rr%r&�base_learning_rateZlearning_rate_blobr>r=�kwargs� iterationr<r?r9rArrr�build_lris@ zOptimizer.build_lrcCs ||_dS)z� Set the global learning rate multiplier. If a multiplier already existed, this will overwrite the existing multiplier. The multiplier is used for all future calls to _run(), unless it is overwritten. N)r)rr?rrr�add_lr_multiplier�szOptimizer.add_lr_multiplierFcCs||_||_dS)a� Set the local learning rate multiplier. This local multiplier is multiplied with the global learning rate multiplier if it exists. As with the global learning rate multiplier, this multiplier will be used for all future calls to _run(), so please call _clear_local_lr_multiplier() at the beginning of the optimizer's _run() before optionally calling this function. N)rr)rrA�is_gpu_blobrrr�_add_local_lr_multiplier�s z"Optimizer._add_local_lr_multipliercCsd|_d|_dS)NF)rr)rrrr�_clear_local_lr_multiplier�sz$Optimizer._clear_local_lr_multipliercCs4t|tj�stdj|��|r,|j||d�S|SdS)Nz,Dedup only works for sparse gradient, got {})Z aggregator)r!r� GradientSlicer"r#ZDeduplicateGradientSlices)r%�sparse_dedup_aggregatorr rrr�dedup�s zOptimizer.dedupcCs|jS)axReturns a list of auxiliary parameters. Returns: aux_params: A namedtuple, AuxParams. aux_params.local stores a list of blobs. Each blob is a local auxiliary parameter. A local auxiliary parameter is a parameter in parallel to a learning rate parameter. Take adagrad as an example, the local auxiliary parameter is the squared sum parameter, because every learning rate has a squared sum associated with it. aux_params.shared also stores a list of blobs. Each blob is a shared auxiliary parameter. A shared auxiliary parameter is a parameter that is shared across all the learning rate parameters. Take adam as an example, the iteration parameter is a shared parameter, because all the learning rates share the same iteration parameter. )r)rrrr�get_auxiliary_parameters�sz"Optimizer.get_auxiliary_parameterscOstd��dS)Nz9Optimizer Need to Implement `scale_learning_rate` method.)�NotImplementedError)r�argsrFrrr�scale_learning_rate�szOptimizer.scale_learning_ratecCsF|jgddg|d�}|jgddg|d�}|jgddg|d�}|||fS)N�weight_decayr)�shape�value�trust�lr_max)�ConstantFill)rr&rTrWrX�wdrrr�create_lars_inputs�s zOptimizer.create_lars_inputs)N)r*)Nr;r)F)r� __module__�__qualname__rr'r$r.r/�propertyr4r:rHrIrKrL�staticmethodrOrPrSr[rrrrr"s" 6 rcs.eZdZd�fdd� Zdd �Zd d�Z�ZS) �SgdOptimizer�{�G�z�?r;�TNcs<tt|�j�||_||_||_||_||_||_||_ dS)N) �superr`rrEr>�momentum�nesterovrN�lars�init_kwargs)rrEr>rdrerNrfrF)rrrr�s zSgdOptimizer.__init__cCsZ|j}|j}|jdkrdS|jdks4tdj|j��|j�|jdk r�t|tj �r�|jdksntdj|j��|j |ddtjtj �j�\}}}|j|||||g|jt|�d�|jdd�} tj�} |j| | dk o�tj| j�d�|jr�dnd }|j||f|j||jd �|j��\}} tj�}|dk�r4tjtj�}|jgdj|j|j|j �d gdd�}|j!j"j#|�|jdk�r�|j|t|�d dd�}|j!j$j#|�t|tj ��r|j%||j&|�}|jdk�r�|j'|j(||||j)g|j(||g|j|j*d�n|j+|||j)|j(|g|�nJ|jdk�r>|j,||||g|||g|j|j*d�n|}|j-||||g|�dS)Nrz*Expect positive base learning rate, got {}z'Lars offset must be nonnegative, got {}gg�?�_lars)�offset�lr_min)rJr)rEr>zONE_{}_{}{})rUrV� _momentum)rV)rdre��).�blobr rEr"r#rLrfr!rrMr[�np�finfo�float32�max�Larsr:�strrr5rKr6r7rdrHr>rg�DeviceOptionr�CPUrYr8r,rr�appendr rOrNZSparseMomentumSGDUpdate�values�indicesreZScatterWeightedSum�MomentumSGDUpdate�WeightedSum)rr%r&r)rr rZrWrX�lr_lars_multiplierr9Zlr_signr<�_�dev�ONE� momentum_dataZcoeffrrrr$sx zSgdOptimizer._runcCs|j|9_dS)N)rE)r�scalerrrrSdsz SgdOptimizer.scale_learning_rate)rar;rbTNN)rr\r]rr$rS� __classcell__rr)rrr`�s\r`cs&eZdZd �fdd� Zdd �Z�ZS)�MultiPrecisionSgdOptimizer皙��?�r;TNcs&tt|�jf|||||d�|��dS)N)rEr>rdrerN)rcr�r)rrErdr>rerNrF)rrrrjs z#MultiPrecisionSgdOptimizer.__init__cCs|j}|jdk r|jtjjnd}|dkr:tj||||�S|j}|jdkrNdS|jdksht dj |j��|j||f|j|jd�|j ��\}}|j|t|�ddd�} |jjj| �t|tj�s�t d��|j||d�} |j| | ||g| | |g|j|jd �|j||�dS) Nrz*Expect positive base learning rate, got {})rEr>rkg)rVz3MultiPrecisionSgd does not support sparse gradientsZ_fp32)rdre)rm� blob_copyr�DataType�FLOATr`r$r rEr"r#rHr>rgrYrsrr rvr!rMZHalfToFloatryrdre�FloatToHalf)rr%r&r)r� param_fp32r r<r|rZ grad_fp32rrrr$|s: zMultiPrecisionSgdOptimizer._run)r�r�r;TN)rr\r]rr$r�rr)rrr�isr�cs(eZdZd�fdd� Zd d d�Z�ZS)�FP16SgdOptimizer皙��?�r;T�-C��6?Ncs,tt|�jf|||||d�|��||_dS)N)rEr>rdrerN)rcr�rrT)rrErdr>rerTrNrF)rrrr�s zFP16SgdOptimizer.__init__FcCs�d}t|j�}|jd�dkr d}|r6d}|j}|j}nt|jdkrRd}|j}|j}nXtjj|jkrv|j}|jtjj}n4tjj|jkr�|jtjj}|j}ntdj |j ��|j} |jdkr�dS|jdks�tdj |j��|j ||f|j|jd�|j��\} }|j|t|�dd d �}|j|t|�d�} |jjj| �t| tj��sPtd��|dk�r�|j| | | |g| | |g|j|j|jd �n(|j| || |g| ||g|j|j|jd �dS)NrZspatbnrTzLUnrecognized parameter format to be updated by FP16 Optimizer. Parameter: {}z*Expect positive base learning rate, got {})rEr>Z_momentum_fp32g)rVrkz)FP16Sgd does not support sparse gradients)rdrerTrl)rsrm�findr�rr�r�ZFLOAT16r"r#�namer rErHr>rgrYr�rr rvr!rMZFP16MomentumSGDUpdaterdrerTZFP32MomentumSGDUpdate)rr%r&r)Zfp32_updateZfp32_update_flag� param_namerr�r r<r|Zmomentum_data_fp32rrrrr$�sj zFP16SgdOptimizer._run)r�r�r;Tr�N)F)rr\r]rr$r�rr)rrr��s r�c@seZdZdd�Zdd�ZdS)�WeightDecayBuildercCs ||_dS)N)rT)rrTrrrrszWeightDecayBuilder.__init__cCs�tj�}|dkrtjtj�}|jgdj|j|j �dgdd�}|jgdj|j|j �dg|j d�}t|jtj �rvtd��n|j|j||j|g|j�dS)Nz ONE_{}_{}rg�?)rUrVzwd_{}_{}z2Weight decay does not yet support sparse gradients)rr5rrtrrurYr#r7r8rTr!r rM� ValueErrorrzrm)rr%r&r)r}r~ZWDrrrr$s zWeightDecayBuilder._runN)rr\r]rr$rrrrr�sr�csFeZdZd�fd d � Zdd�Zd d�Zdd�Zdd�Zdd�Z�Z S)�AdagradOptimizer�{�G�z�?�-C��6?r�r;NFr*cs�x(t�j�D]\}}tjdj||��qWtt|�j�||_||_ ||_ t|�|_||_ ||_||_||_| |_| |_||_||_||_||_|j|�|j| �|j|�dS)Nz)AdagradOptimizer: input arguments: {}: {})�locals�items�logger�infor#rcr�r�alpha�epsilon�decay�floatrTr>rN�rowWise�enginerf�output_effective_lr�output_effective_lr_and_update�counter_halfliferg�weight_scale�_process_pruning_options�_process_swa_options�_process_ema_options)rr�r�r�rTr>rNr�r�rfr�r��pruning_options�swa_options�ema_optionsr�r�rF�k�v)rrrr7s( zAdagradOptimizer.__init__cCs^|rdnd|_|jrZ|jdd�|_|jdd�|_|jdd�|_|jdd�|_|jdd�|_dS)NTF�swa_avg_start_it�swa_avg_end_it�swa_feedback_start_it�swa_feedback_step�swa_feedback_end_it)�swa_enabled�getr�r�r�r�r�)rr�rrrr�bsz%AdagradOptimizer._process_swa_optionscCsP|rdnd|_|jrL|jdd�|_|jdd�|_|jdd�|_|jdd�|_dS)NTF� ema_start�ema_end�ema_step� ema_alpha)�ema_enabledr�r�r�r�r�)rr�rrrr�ksz%AdagradOptimizer._process_ema_optionscCs�d|_|dkri}nt|t�s,tdj|��|jdd�|_|jdd�|_|jdd�|_|jdd�|_ |jdg�|_ |jdg�|_|jd d �|_|jdk r�t |j�tjks�td��|jdks�td��|jdks�td��|j dks�td��d |_|jdk p�|jdk �rD|jdk �std��|jdk �s*td��|jdk�s>td��d |_|j �r�|jdk �rnt|j �t|j�k�svtd��|jdk�s�td��|jdk�s�td��d |_dS)NFzCpruning_options can only be provided as a dictionary, currently: {}�mask_tensor�mask_db_path�mask_db_type�mask_blob_name�prune_delays�prune_ratios�prune_block_sizerz"mask_tensor must be a numpy array!zHmask can be provided through either a numpy array or a db path, not bothTzPwhen mask is provided through db, db path, db type, and blob name are all neededz:Prune Delays and prune ratios should be of the same lengthz,Mask Tensor should be None with prune ratiosz-Mask DB Path should be None with prune ratios)�use_maskr!�dictr"r#r�r�r�r�r�r�r�r��typernZndarray�len)rr�rrrr�ssP z)AdagradOptimizer._process_pruning_optionsc Csh |j}|j}|jdkrdS|j�|jdk r�t|tj�r�|jdksLt d��|jdksft dj |j��|j|ddtj tj�j�\}}}|j|||||g|jt|�d�|jdd�} tj�} |j| | dk o�tj| j�d�|j||f|j|jd �|j��\}}|} |jdk�r|jjj| �|j�r�t j!d j |j"|��t#j$|g�\}}t|�|k�r�|j%|t|�d�}|j&|gt|�ddgd gd�}|j'|t|�dd dd�}n(|j'gt|�d|t|�dgdd�}n�t j!dj |j"|��|j"t(k�rX|jdk�s t dj |j"��t#j$|g�\}}t|�|k�s0t |��|t|�}|j)gt|�dd|d�}n|j'|gt|�ddd�}|j*dk�r(|jdk�s�t d��|j+dk �r:t|tj��s�|j,gt|�dg|j+|j+j-d�}nh|j+j.tj/�|_+|j0gt|�dg|j+|j+j-d�}|j1|tj2j3d�}|j'gt|�dgdtj2j4d gd�}n�|j5dk �sR|j6dk �r�|j7�pdt|�d|_7|j8g|j7|j5|j6dd�}t|tj��r(|j'gt|�dgdtj2j4d gd�}nr|j9�r |j'gt|�d gdDtj2j:d gd�}t|tj��r�t d!�n$|j,gt|�d"ggtj2j;dgd#�}ntd&�}|j'|t|�d'd dtj2j:d&�}n\|j'gt|�d%|t|�dgdtj2j>d(�}|j'gt|�d'|t|�dgdtj2j:d(�}|jj=j|�|jj=j|�|j�rlt|tj��slt d)��t#j$|g�\}}|t|�}d}t|tj��r�t?|�d k�r�d}t j@d*j t|�|��n|j}n0t?|�d k�r�d}t j@d+j t|�|��n|j}t j!d,j t|�||��t|tj��r�|jAdk�s6t d-��|jB||jC|�}|||jD|jE|g}||g}|j�r�|j*dk�r�d.}|dk�s�t d/j |��|||g7}n|jdk�r�||g7}d0}n:|j*dk�r�d1}|dk�s�t d/j |��|||g7}nd2}t j!d3j |t|��|j9�r*|||g7}|||g7}|dk�r^|jdEk�r^|jF|�|||jG||j"d4�nR|dk�r�|jdFk�r�|jF|�|||jG||j"|jd5�n|jF|�|||jG|j"d6�|jdk� r�|jH|||jD| g||g|jd7��n�||||g}||g}|jI�r6|j*dk�st d8��|jt|�d9�|jt|�d:�n.|jJ�rd|j*dk�sRt d;��|jt|�d9�|j*dk�rz||g7}|j9�r�|||g7}|||g7}|j*�r�|dk�s�t d��|jK|||jGtL|jA�|jM|j9|jN|j"d<�n�|dk� r|jO|||jGtL|jA�||j"d=�n|jO|||jGtL|jA�|j"d>�|jP� r�t|�d?}|jQ|�� sj|j'|g|dd�|jj=j|�|jR|||g||g|jS|jT|jU|jV|jWd@�|jX� r�t|�dA}|jQ|�� s�|j'|g|dd�|jj=j|�|jY|||g||g|jZ|j[|j\|j]dB�|j^� rd|j_||g|g|j^j`|j^jatL|j^jb�dC�|j^jc� rd|j_||g|g|j^j`|j^jatL|j^jb�dC�dS)GNrz,weight decay is not implemented for LARS yetz'Lars offset must be nonnegative, got {}gg�?rh)rirj)rJ)rEr>z5Using engine {} for rowWise Adagrad to train param {}Z_shapeZ_numrowsr)ZstartsZendsZ_avg_squared_sum)�input_as_shaperV)rUrVz5Using engine {} for regular Adagrad to train param {}z)weight decay is not tested for engine: {}�_squared_sum)rVrU)rVTz0weight decay is not implemented for use_mask yetZ_mask)rwrU)ZtoZ_mask_changed_blobF)rV�dtyperU)�dbZdb_typeZ absolute_pathZ_last_mask_updated_iterzMPrune Delays and Prune Ratios are currently not supportedfor sparse operatorsZ_empty_mask)rwr�rUzXIf mask is used, it needs a numpy array or a db file ora delay iter needs to be providedZ_update_counter)r�rVr�Z_prev_update_iter)rUrVr�z�If SparseAdagrad with rowWise=True, gradient must be a gradientslice. PLease ensure that rowWise is not enabled for the dense Adagrad optimizer, as it is not supported.z8SKIPPING weight decay on 1d sparse param: {}.shape is {}z7SKIPPING weight decay on 1d dense param: {}.shape is {}z"weight_decay for {} (shape:{}): {}z?Decay is not implemented for SparseAdagrad and must be set to 1ZMaskedRowWiseSparseAdagradz*weight decay is not implemented for {} yetZRowWiseSparseAdagradZMaskedSparseAdagrad� SparseAdagradzusing {} for {})r�rTr�)r�rTr�r�)r�r�)r�z@MaskedAdagrad doesn't support outputting effective_lr_and_update� _effective_lr�_updatez5MaskedAdagrad doesn't support outputting effective_lr)r�r�� block_sizeZdelaysr�r�)r�r�rTr�)r�r�r�Z_swa)Z avg_startZavg_endZfeedback_startZ feedback_stepZfeedback_end�_ema)r�r�r�r�)�stepsize�upper_bound_iterr�rlrlrl)drmr r�rLrfr!rrMrTr"r#r[rnrorprqrrr:rsrr5rKr6r7rHr>rgr�rrrvr�r�r�r�r�InferShapesAndTypesZShape�SlicerY�FP16_ENGINESZFloat16ConstantFillr�r�ZGivenTensorFillrUZastypeZuint8ZGivenTensorBoolFillZCastr�ZUINT8ZBOOLr�r�r��Loadr��INT64r�rQr ZDOUBLEr��warnr�rOrNrxrw�__getattr__r�ZRowWiseCounterr�r�Z MaskedAdagradr�r�r�ZAdagradr�rCZSWAr�r�r�r�r�r��EMAr�r�r�r�r�ZWeightScaler�r�r�Zto_aux) rr%r&r)rr rZrWrXr{r9r<Zlr_iterationrG�shapes�typesrUZnum_rows�param_squared_sumZ mask_blobZmask_changed_blobZlast_mask_updated_iterZupdate_counterZprev_update_iterr|Zparam_shaperTZ input_args�output_args�opZ param_swa� param_emarrrr$�s\ zAdagradOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrSVsz$AdagradOptimizer.scale_learning_raterl)r�r�rr�r;NFr*NFFNNNNrl) rr\r]rr�r�r�r$rSr�rr)rrr�6s0 @&r�c s.eZdZd�fdd � Zd d�Zdd �Z�ZS)�WngradOptimizer��?��&�.>r;Nr*�Y@Fc sNtt|�j�||_||_||_||_||_||_||_ ||_ | |_| |_dS)N) rcr�rr�r�r>rNr��moment_initrfr�r�rg)rr�r�r>rNr�r�rfr�r�rF)rrrr\s zWngradOptimizer.__init__cCs�|j}|j}|jdkrdS|j�|jdk r�t|tj�r�|jdksTtdj |j��|j |ddtjtj �j�\}}}|j|||||g|jt|�d�|jdd�} tj�} |j| | dk o�tj| j�d�|j||f|j|jd�|j��\}}|jgt|�d d g|jd�} |jjj| �t|tj��r^|j||j |�}|j!|| |j"|j#|g|| g|j$|j%d�nn|| g}|j&�r�|jt|�d �|jt|�d�n|j'�r�|jt|�d �|j(|| ||g||j$|j%d�dS)Nrz'Lars offset must be nonnegative, got {}gg�?rh)rirj)rJ)rEr>�_momentr)rUrV)r�r�r�r�))rmr r�rLrfr!rrMr"r#r[rnrorprqrrr:rsrr5rKr6r7rHr>rgrYr�rr rvrOrNZSparseWngradrxrwr�r�r�r�ZWngrad)rr%r&r)rr rZrWrXr{r9r<r|�momentr�rrrr$us^ zWngradOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS�sz#WngradOptimizer.scale_learning_rate) r�r�r;Nr*r�NFF)rr\r]rr$rSr�rr)rrr�[sDr�cs.eZdZd�fdd� Zdd �Zd d�Z�ZS) �StormOptimizer皙��?�$@�{�G�z�?r;Nc sBtt|�j�||_||_||_||_||_||_||_ ||_ dS)a�Constructor function to add STORM Optimizer Args: lr: learning rate scaling (called k in the original paper) momentum: momentum scaling (called c in the original paper) beta: initial value of denominator in adaptive learning rate ( called c in the original paper) grad_sq_init: initial value of gradient squared accumulator. policy: specifies how learning rate should be applied, options are 'fixed', 'step', 'exp', etc. sparse_dedup_aggregator: specifies deduplication strategy for gradient slices. Works while using sparse gradients. Options include 'mean' and 'sum'. lars: lars offset. N)rcr�rr<rd�beta�grad_sq_initr>rNrfrg) rr<rdr�r�r>rNrfrF)rrrr�szStormOptimizer.__init__cCs�|j}|j}|jdkrdS|j�|jdk r�t|tj�r�|jdksTtdj |j��|j |ddtjtj �j�\}}}|j|||||g|jt|�d�|jdd�} tj�} |j| | dk o�tj| j�d�|j||f|j|jd�|j��\}}|j|t|�d dd �} |jjj| �|jgt|�ddg|jd �}|jjj|�t|tj��r�|j||j |�}|j!|| ||j"|j#|g|| |g|j$|j%d�n&|j&|| |||g|| |g|j$|j%d�dS)Nrz'Lars offset must be nonnegative, got {}gg�?rh)rirj)rJ)rEr>r�)rVZ_grad_sq_sumr)rUrV)rdr�)'rmr r<rLrfr!rrMr"r#r[rnrorprqrrr:rsrr5rKr6r7rHr>rgrYrr rvr�rOrNZSparseStormrwrxrdr�ZStorm)rr%r&r)rr rZrWrXr{r9r<r|r�Zgrad_sq_sumrrrr$�sV zStormOptimizer._runcCs|j|9_dS)N)r<)rr�rrrrS"sz"StormOptimizer.scale_learning_rate)r�r�r�r�r;NN)rr\r]rr$rSr�rr)rrr��s?r�cs.eZdZd �fdd� Zd d �Zdd�Z�ZS)�AdadeltaOptimizer�{�G�z�?�-C��6?�ffffff�?r;Nr*cs<tt|�j�||_||_||_||_||_||_||_ dS)auConstructor function to add Adadelta Optimizer Args: alpha: learning rate epsilon: attribute of Adadelta to avoid numerical issues decay: attribute of Adadelta to decay the squared gradient sum policy: specifies how learning rate should be applied, options are "fixed", "step", "exp", etc. sparse_dedup_aggregator: specifies deduplication strategy for gradient slices. Works while using sparse gradients. Options include "mean" and "sum". engine: the engine used, options include "", "CUDNN", etc. N) rcr�rr�r�r�r>rNr�rg)rr�r�r�r>rNr�rF)rrrr'szAdadeltaOptimizer.__init__c Cs |j}|j}|jdkrdS|j||f|j|jd�|j��\}}|j|gt|�ddd�}|j|gt|�ddd�} |jj j |�|jj j | �t|tj �r�|j||j|�}|j||| |j|j|g||| g|j|j|jd�n*|j||| ||g||| g|j|j|jd�dS)Nr)rEr>Z_squared_momentg)rVZ_squared_moment_update)r�r�r�)rmr r�rHr>rgrYrsrr rvr!rrMrOrNZSparseAdadeltarxrwr�r�r�ZAdadelta) rr%r&r)rr r<r|r�Z moment_updaterrrr$Gs< zAdadeltaOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrSssz%AdadeltaOptimizer.scale_learning_rate)r�r�r�r;Nr*)rr\r]rr$rSr�rr)rrr�&s,r�cs.eZdZd�fdd� Zdd �Zd d�Z�ZS) � FtrlOptimizer�{�G�z�?�-C��6?rNr*cs6tt|�j�||_||_||_||_||_||_dS)N) rcr�rr�r��lambda1�lambda2rNr�)rr�r�r�r�rNr�)rrrrys zFtrlOptimizer.__init__c Cs�|j}|j}|jdkrdS|j|gt|�ddgdd�}|jjj|�t|t j �r�|j||j|�}|j |||j|jg||g|j|j|j|j|jd�n,|j|||g||g|j|j|j|j|jd�dS)NrZ_ftrl_nz�g)�extra_shaperV)r�r�r�r�r�)rmr r�rYrsrr rvr!rrMrOrNZ SparseFtrlrxrwr�r�r�r�ZFtrl)rr%r&r)rr �nzrrrr$�s2 zFtrlOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS�sz!FtrlOptimizer.scale_learning_rate)r�r�rrNr*)rr\r]rr$rSr�rr)rrr�xs !r�cs2eZdZdZd �fdd� Zd d �Zdd�Z�ZS)�GFtrlOptimizerzGroup Lasso FTRL Optimizer.�{�G�z�?�-C��6?rNr*cs6tt|�j�||_||_||_||_||_||_dS)N) rcr�rr�r�r�r�rNr�)rr�r�r�r�rNr�)rrrr�s zGFtrlOptimizer.__init__c Csv|j}|j}|jdkrdS|j|gt|�ddgdd�}|jjj|�|j|||g||g|j |j|j |j|jd�dS)NrZ _gftrl_nzr�g)r�rV)r�r�r�r�r�) rmr r�rYrsrr rvZGFtrlr�r�r�r�)rr%r&r)rr r�rrrr$�s zGFtrlOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS�sz"GFtrlOptimizer.scale_learning_rate)r�r�rrNr*)rr\r]�__doc__rr$rSr�rr)rrr��s r�c s.eZdZd�fdd� Zd d�Zdd�Z�ZS)� AdamOptimizer��MbP?��?�+��?�:�0�yE>r;F�{�G�z�?TNr*cs�tt|�j�||_||_||_||_||_||_||_ ||_ | |_| |_||_ ||_| r~| rftd��|rrtd��|r~td��| |_||_dS)NzZSmart decay is not implemented for rowWise Adam. Set rowWise or use_smart_decay to False.zWSmart decay is not implemented for RAdam. Set enableRAdam or use_smart_decay to False.zbSmart decay is not implemented with lr_adaption. Set use_lr_adaption or use_smart_decay to False.)rcr�rr��beta1�beta2r�r>�use_lr_adaption�lr_alpha�normalized_lr_adaptionrNr�r��enableRAdamrQ�use_smart_decayrg)rr�rrr�r>rrrrNr�r�rrrF)rrrr�s,zAdamOptimizer.__init__c Cs�|j}|j}|jdkrdS|j||f|j|jd�|j��\}}|j|g|ddd�}|jr�tj |g�\} } |jg|d| |dgdd�}n|j|g|ddd�}|j r�tj |g�\} }|jg|d | |dgdtjj d �} |jjj| �|jjj|�|jjj|�|jjj|�|j�r4t|tj��s4td��|||g}|j �rP|j| �|j�rnt|�d}|j|�t|tj��r�|j||j|�}|j�r�d }n|j �r�d}nd}|dk�r�|j|�||||j|j||g||j|j|j|jd�n�|dk�r*|j|�|||| |j|j||g||j|j|jd�nB|j�sZ _first_momentg)rVZ_avg_second_moment)rUrV�_second_momentZ _last_seen)rUrVr�z�If SparseAdam with rowWise=True, gradient must be a gradientslice. PLease ensure that rowWise is not enabled for the dense Adam optimizer, as it is not supported.Z_effective_gradZRowWiseSparseAdamZSmartDecaySparseAdamZ SparseAdam)rrr�r)rrr�z7Currently, RowWiseSparseAdam is not supported by RAdam!)rr)$rmr r�rHr>rgrYr�rr�rrr�r�rr rvrr!rMr"rrsrOrNr�rxrwrrr�rZLearningRateAdaptionrrZAdam)rr%r&r)rr r<rG�m1r�r��m2r|Z last_seen�output_blobsZeffective_gradr�rrrr$s� " zAdamOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS�sz!AdamOptimizer.scale_learning_rate) r�r�r�r�r;Fr�TNFr*FF)rr\r]rr$rSr�rr)rrr��svr�c s6eZdZd�fd d � Zdd�Zd d�Zdd�Z�ZS)�DecayAdagradOptimizer�{�G�z�?��+��?皙��?NTr;r*c sRtt|�j�||_||_||_||_||_||_||_ | |_ | |_|j|�dS)N) rcrrr�rrr�rT�bias_correction_firstr>r�rgr�)rr�rrr�rTr�rr>r�rF)rrrr�s zDecayAdagradOptimizer.__init__cCsP|rdnd|_|jrL|jdd�|_|jdd�|_|jdd�|_|jdd�|_dS)NTFr�r�r�r�)r�r�r�r�r�r�)rr�rrrr��sz*DecayAdagradOptimizer._process_ema_optionsc Cs�|j}|j}|jdkrdS|j||f|j|jd�|j��\}}t|tj�r�|j |g|ddd�}|j jj|�||g} |j |||j|j|g| |jd�n�|j |g|ddd�} |j |g|ddd�}|j jj|�|j jj| �|j jj|�|| |g} |j|| ||||g| |j|j|j|j|jd �|j�r�t|�d }|j|��sf|j |g|dd�|j jj|�|j|||g||g|j|j|j|jd�dS)Nr)rEr>r�g)rV)r�Z_first_mo1mentr)rrr�rTrr�)r�r�r�r�)rmr r�rHr>rgr!rrMrYrr rvr�rxrwr�rZDecayAdagradrrrTrr�rsrCr�r�r�r�r�) rr%r&r)rr r<rGr�r rr r�rrrr$�sX zDecayAdagradOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS�sz)DecayAdagradOptimizer.scale_learning_rate) rr rrr NTr;r*)rr\r]rr�r$rSr�rr)rrr�s9rcs2eZdZdZd�fd d� Zdd �Zdd�Z�ZS)�YellowFinOptimizerz�YellowFin: An automatic tuner for momentum SGD See https://arxiv.org/abs/1706.03471 for more details. This implementation has separate learning rate and momentum per each parameter.皙��?��+��?�T�r;Nc sHtt|�j�||_||_||_||_||_||_||_ ||_ | |_dS)N)rcrrr��mur��curv_win_width�zero_debiasr�r>rNrg) rr�rr�rrr�r>rNrF)rrrr�szYellowFinOptimizer.__init__cCs�d}|j}|j}|j|g|ddd�}|jg|d|jgdd�}|j|g|ddd�} |j|g|ddd�} |jg|d d g|jd�}|jg|dd g|jd�}|jg|d|gdd�} |jd ks�t�t|tj �s�td��t j||d d�}|jj j|�|jjj|�|jjj|�|jjj|�|jjj|�|jjj| �|jjj| �|jjj| �|||||| | | g}|j|||g||j|j|j|jd�dS)N�r�g)rVZ _curv_win)rUrVZ_g_avgZ_g2_avgZ_lr_avgrZ_mu_avgZ_scalars_memoryrz+YellowFin does not support sparse gradients)r=)r�r�rr)rmr rYrr�rr"r!rrMrrBrrrvr Z YellowFinr�r�r)rr%r&r)ZSCALARS_MEMORY_SIZErr r�Zcurv_winZg_avgZg2_avgZlr_avgZmu_avgZscalars_memoryrGZyf_in_out_argsrrrr$sT zYellowFinOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS>sz&YellowFinOptimizer.scale_learning_rate��ư>)rrrrTrr;N)rr\r]r�rr$rSr�rr)rrr�s=rcs.eZdZd �fdd� Zd d �Zdd�Z�ZS)�RmsPropOptimizer�{�G�z�?��?��h㈵��>r;r*cs<tt|�j�||_||_||_||_||_||_||_ dS)N) rcrrr�r�rdr�r>r�rg)rr�r�rdr�r>r�rF)rrrrDs zRmsPropOptimizer.__init__c CsF|j}|j}|jdkst�t|tj�s0td��tj�}|dkrLtj t j�}|jgdj |j|j�dgdd�}|j||f|j|jd�|j��\}} |j|gt|�dd d �} |j|gt|�dd d �}|j|gt|�dd d �}|jjj|�|jjj|�|j||||g| ||g|j|j|j|jd �|j| |||g| ||g�dS)Nrz1RmsPropOptimizer doesn't support sparse gradientsz ONE_{}_{}rg�?)rUrV)rEr>Z_grad_og)rwZ _mean_squaresrk)r�rdr�r�)rmr r�r"r!rrMrr5rtrrurYr#r7r8rHr>rgrsrr rvZRmsPropr�rdr�r�ry) rr%r&r)rr r}r~r<r|Zgrad_o�msZmomrrrr$Ws@ zRmsPropOptimizer._runcCs|j|9_dS)N)r�)rr�rrrrS�sz$RmsPropOptimizer.scale_learning_rate)rrrr r;r*)rr\r]rr$rSr�rr)rrrCs1rcCs"tj|j�}|jtj|j��|S)N)rZInferBlobDevicesr%�updater&)�model�param_to_devicerrr�_get_param_to_device�sr%cCs�|}|p i}||kr||}ndt|tj�rj|}t|j�|krL|t|j�}q�t|j�|kr�|t|j�}nt|�}||kr�||}|dk s�tdj|��|S)Nz,Cannot infer device for {}: no op creates it)r!rrMrsrwrxr"r#)r�r r$Zdefault_device�deviceZ grad_namerrr�get_param_device�s r'cCs tjt�S)z� Gets current value for lr_injection, a multiplier for all base learning rates. Must set allow_lr_injection=True when building optimizer, as it relies on synchronization over CPU. )rZ FetchBlob�_LEARNING_RATE_INJECTIONrrrr�get_lr_injection�sr)cCs"tjttjt|�gtjd��dS)z� Sets lr_injection, a multiplier for all base learning rates. Must set allow_lr_injection=True when building optimizer, as it relies on synchronization over CPU. )r�N)rZFeedBlobr(rn�arrayr�rp)Zlr_injection_valuerrr�set_lr_injection�sr+cCs$tj|��g}x�t|�D]|\}}tt|j�|j|�}tj|��Pt|jtj �sX|jn|jj } dj|�} |jj | | �}|jj|�}|j|�WdQRXqWtjtjtj��b|jj|d�} |jj| ddd�}|jjgdgt|�d�}|jj||gd�}|jj||gd �}|SQRXWdQRXdS) Nzgrad_{}_squared_sum�grad_squared_full_sum�global_normg�?)�exponent� clip_norm)rUrV�max_norm� norm_ratio)rZ NameScope� enumerater'rsrmr �DeviceScoper!rMrwr#r%ZSumSqrElementsZEnsureCPUOutputrvrtrruZSum�Powr&rYr�ZMax�Div)r#�paramsZ name_scoper$�max_gradient_normZgrad_squared_sums�irr&r Zgrad_squared_sum_nameZgrad_squared_sumZgrad_squared_sum_cpur,r-r/r0r1rrr�_calc_norm_ratio�s* r9FTc Cs(t|�}|j�g}x,|j�D] }|r4|j|jkr4q|j|�qWd} |dk r^t||d||�} |r�|jjt �s�|j jgt dgdd�} nt } | dkr�| } n|jj| | gddd�} |j | �xj|D]b}t|j�}t||j|�}tj|��6|jo�|�r|j|j|j |�n||j|j |�WdQRXq�W|S)NZnorm_clipped_grad_updaterg�?)rUrVr?)r@)r%ZValidateZGetOptimizationParamInform�weightsrvr9r%rCr(r&rYrDrIrsr'r rr3� optimizer) r#r;�weights_only�use_param_info_optimr7�allow_lr_injectionr$r6r)r?rr�r&rrr�_build�sB r?cCst|t|d�ddd�dS)z�Adds a decay to weights in the model. This is a form of L2 regularization. Args: weight_decay: strength of the regularization )rTTF)r<r=N)r?r�)r#rTrrr�add_weight_decays r@cKst|f|�}t||||d�S)N)r7r>)r`r?)r#rEr7r>rFZ sgd_optimizerrrr� build_sgd.srAcKst|f|�}t||||d�S)N)r7r>)r�r?)r#rEr7r>rFZmulti_prec_sgd_optimizerrrr�build_multi_precision_sgd>srBcKst|f|�}t||�S)N)r�r?)r#rErFZfp16_sgd_optimizerrrr�build_fp16_sgdNsrC�SIMDcKs@|dkr$tjd�st�tjd�s$t�tfd|i|��}t||�S)NrDZFtrl_ENGINE_SIMDZSparseFtrl_ENGINE_SIMDr�)r� IsOperatorr"r�r?)r#r�rFZftrl_optimizerrrr� build_ftrlSs rFr*cKs2|dkrtjd�st�tfd|i|��}t||�S)NrDZGFtrl_ENGINE_SIMDr�)rrEr"r�r?)r#r�rFZgftrl_optimizerrrr�build_gftrl[srGcKs"tfd|i|��}t||||d�S)Nr�)r7r>)r�r?)r#rE� parametersr7r>rFZadagrad_optimizerrrr� build_adagradbsrIcKs"tfd|i|��}t||||d�S)Nr�)r7r>)r�r?)r#rErHr7r>rFZwngrad_optimizerrrr�build_wngradssrJcKs"tfd|i|��}t||||d�S)Nr<)r7r>)r�r?)r#rErHr7r>rFZstorm_optimizerrrr�build_storm�srKcKs"tfd|i|��}t||||d�S)Nr�)r7r>)r�r?)r#rErHr7r>rFZadadelta_optimizerrrr�build_adadelta�srLcKs"tfd|i|��}t||||d�S)Nr�)r7r>)r�r?)r#rEr7r>rFZadam_optimizerrrr� build_adam�srMcKs"tfd|i|��}t||||d�S)Nr�)r7r>)rr?)r#rEr7r>rFZdecay_adagrad_optimizerrrr�build_decay_adagrad�srN皙��?cKstfd|i|��}t||�S)Nr�)rr?)r#rErFZyellowfin_optimizerrrr�build_yellowfin�srPcKs"tfd|i|��}t||||d�S)Nr�)r7r>)rr?)r#rEr7r>rFZrms_prop_optimizerrrr�build_rms_prop�srQ)NN)FTNF)NF)NF)rD)r*)NNF)NNF)NNF)NNF)NF)NF)rO)NF)?r0�logging�collectionsrrZnumpyrnZcaffe2.protorZ caffe2.pythonrrrrZcaffe2.python.modelingr Z past.builtinsr r(r�intrr�� getLoggerrr�r�objectrr`r�r�r�r�r�r�r�r�r�r�rrrr%r'r)r+r9r?r@rArBrCrFrGrIrJrKrLrMrNrPrQrrrr�s� SuFl)chR8.(__J $ 2