/usr/local/lib64/python3.6/site-packages/torch/optim/__pycache__
Edit: /usr/local/lib64/python3.6/site-packages/torch/optim/__pycache__/lbfgs.cpython-36.pyc (8802B)
3
EgXC @ sH d dl Z d dlmZ ddlmZ dddZdd d ZG d
d deZdS ) N)reduce ) Optimizerc
C s |d k r|\}}n | |kr"| |fn|| f\}}|| d|| | | } | d || }
|
dkr|
j } | |kr||| || | || d| } n(| | | || | || d| } tt| ||S || d S d S )N r g @)sqrtminmax)
x1f1Zg1Zx2f2Zg2boundsZ
xmin_boundZ
xmax_boundZd1Z d2_squareZd2Zmin_pos r =/usr/local/lib64/python3.6/site-packages/torch/optim/lbfgs.py_cubic_interpolate s
*(r -C 6??& .> c ! C s& |j j } |jtjd}| |||\} }
d}|
j|}d|||f\}}}}d}d}x8||
k r| ||| | ks|dkr| |kr||g}|| g}||
jtjdg}||g}P t || | kr|g}| g}|
g}d}P |dkr||g}|| g}||
jtjdg}||g}P |d|| }|d }|} t||||| |||fd}| }| }|
jtjd}|}| |||\} }
|d7 }|
j|}|d7 }qXW ||
krd|g}|| g}||
g}d} |d |d
krd nd \} } x*| r ||
k r t |d |d | | k r
P t|d |d |d |d |d |d }d t|t| } tt|| |t| | k r| s|t|ks|t|krt |t| t |t| k rt|| }n t|| }d} nd} nd} | |||\} }
|d7 }|
j|}|d7 }| ||| | ks$| || krn||| < | || < |
jtjd|| < ||| < |d |d krdd
nd\} } nt || | krd}nJ||| || dkr|| || < || || < || || < || || < ||| < | || < |
jtjd|| < ||| < qW || }|| } || }
| |
||fS )N)
memory_formatr r FTg{Gz?
)r
g?)r r )r r )r r )r r )absr clonetorchcontiguous_formatdotr r )!obj_funcxtdfggtdc1c2tolerance_changeZmax_lsZd_normZf_newZg_new
ls_func_evalsZgtd_newZt_prevZf_prevZg_prevZgtd_prevdoneZls_iterZbracketZ bracket_fZ bracket_gZ bracket_gtdZmin_stepZmax_steptmpZinsuf_progressZlow_posZhigh_posZepsr r r
_strong_wolfe" s
$
""
$ r* c sb e Zd ZdZd fdd Zd
d Zd d
Zdd Zdd Zdd Z dd Z
e j dd Z
ZS )LBFGSa Implements L-BFGS algorithm, heavily inspired by `minFunc
`_.
.. warning::
This optimizer doesn't support per-parameter options and parameter
groups (there can be only one).
.. warning::
Right now all parameters have to be on a single device. This will be
improved in the future.
.. note::
This is a very memory intensive optimizer (it requires additional
``param_bytes * (history_size + 1)`` bytes). If it doesn't fit in memory
try reducing the history size, or use a different algorithm.
Args:
lr (float): learning rate (default: 1)
max_iter (int): maximal number of iterations per optimization step
(default: 20)
max_eval (int): maximal number of function evaluations per optimization
step (default: max_iter * 1.25).
tolerance_grad (float): termination tolerance on first order optimality
(default: 1e-5).
tolerance_change (float): termination tolerance on function
value/parameter changes (default: 1e-9).
history_size (int): update history size (default: 100).
line_search_fn (str): either 'strong_wolfe' or None (default: None).
r NHz>& .>d c
sl |d kr|d d }t |||||||d} tt| j|| t| jdkrRtd| jd d | _d | _d S )N )lrmax_itermax_evaltolerance_gradr& history_sizeline_search_fnr z>LBFGS doesn't support per-parameter options (parameter groups)r params) dictsuperr+ __init__len param_groups
ValueError_params _numel_cache)
selfr8 r2 r3 r4 r5 r& r6 r7 defaults) __class__r r r; s zLBFGS.__init__c C s$ | j d kr tdd | jd| _ | j S )Nc S s | |j S )N)numel)totalpr r r s z LBFGS._numel..r )r@ r r? )rA r r r _numel s
z LBFGS._numelc C sn g }x\| j D ]R}|jd kr.|j|j j }n&|jjrH|jj jd}n |jjd}|j| q W t j
|dS )Nr r r r ) r? ZgradnewrD Zzero_Z is_sparseZto_denseviewappendr cat)rA ZviewsrF rJ r r r _gather_flat_grad s
zLBFGS._gather_flat_gradc C sX d}x>| j D ]4}|j }|j|||| j||d ||7 }q W || j ksTtd S )Nr )alpha)r? rD add_Zview_asrH AssertionError)rA Z step_sizeupdateoffsetrF rD r r r _add_grad s zLBFGS._add_gradc C s dd | j D S )Nc S s g | ]}|j tjd qS ))r )r r r ).0rF r r r
s z&LBFGS._clone_param..)r? )rA r r r _clone_param s zLBFGS._clone_paramc C s( x"t | j|D ]\}}|j| qW d S )N)zipr? copy_)rA Z params_datarF Zpdatar r r
_set_param s zLBFGS._set_paramc C s0 | j || t| }| j }| j| ||fS )N)rS floatrM rY )rA closurer r r loss flat_gradr r r _directional_evaluate s
zLBFGS._directional_evaluatec & s t jdksttj jd }|d }|d }|d }|d }|d }|d }|d } jjd }
|
jd
d |
jd d } t| } d}
|
d
d7 < j }|j
j |k}|r| S |
j d }|
j d
}|
j d}|
j d}|
j d}|
j d}|
j d}|
j d}d}x>||k r^|d7 }|
d d7 < |
d dkrn|j
}g }g }g }d}nV|j|}|j|}|j|}|dkrt || kr|jd |jd |jd |j| |j| |jd| ||j| }t |} d|
krdg| |
d< |
d } |j
} xLt| d d d D ]8} || j| || | | < | j|| | | d q4W tj| | }} xBt| D ]6} || j| || }!| j|| | | |! d qW |dkr|jtjd}n
|j| | }|
d dkrtdd|j
j | }n|}|j|}"|"| kr2P d}#|dk r|dkrTtdn2j }$ fd d }%t |%|$||| ||"\} }}}#j || |j
j |k}nPj || ||krtj t } W dQ R X j }|j
j |k}d}#|
|#7 }
|
d
|#7 < ||krP |
|kr$P |r,P |j|j
j |krFP t
| | |k r"P q"W ||
d < ||
d
< ||
d< ||
d< ||
d< ||
d< ||
d< ||
d< | S ) zPerforms a single optimization step.
Args:
closure (callable): A closure that reevaluates the model
and returns the loss.
r r r2 r3 r4 r5 r& r7 r6 Z
func_evalsn_iterr r old_dirsold_stpsroH_diagprev_flat_grad prev_lossg|=g ?alN)rN )r Z strong_wolfez only 'strong_wolfe' is supportedc s j | ||S )N)r^ )r r r )r[ rA r r r s z LBFGS.step..obj_funcr r ) r< r= rP r Z enable_gradstater?
setdefaultrZ rM r r getnegsubmulr poprK rangerO r r rX r sum RuntimeErrorrV r* rS )&rA r[ groupr2 r3 r4 r5 r& r7 r6 rg Z orig_lossr\ Z
current_evalsr] Zopt_condr r r` ra rb rc rd re r_ ysZysZnum_oldrf qirZbe_ir# r' Zx_initr r )r[ rA r step s
z
LBFGS.step)r r, Nr- r. r/ N)__name__
__module__ __qualname____doc__r; rH rM rS rV rY r^ r Zno_gradrw
__classcell__r r )rC r r+ s r+ )N)r r r r )r functoolsr Z optimizerr r r* r+ r r r r s
#