Name	Size	Mode	Actions
anomaly_mode.cpython-36.pyc	5149	0644	editdlrm
forward_ad.cpython-36.pyc	4259	0644	editdlrm
function.cpython-36.pyc	22587	0644	editdlrm
functional.cpython-36.pyc	31543	0644	editdlrm
gradcheck.cpython-36.pyc	50452	0644	editdlrm
grad_mode.cpython-36.pyc	9602	0644	editdlrm
graph.cpython-36.pyc	6316	0644	editdlrm
profiler.cpython-36.pyc	25753	0644	editdlrm
profiler_legacy.cpython-36.pyc	7322	0644	editdlrm
profiler_util.cpython-36.pyc	26941	0644	editdlrm
variable.cpython-36.pyc	815	0644	editdlrm
__init__.cpython-36.pyc	11001	0644	editdlrm

3 Eg�X�@s~
dd
lZdd
l
jZddl
mZ
dd
ljjZddlmZ
dd
l Z dd
l Z ddlmZ
ddl mZmZmZ
Gdd�de�ZeZGdd �d e�ZGd d�dejee�ZGdd �d e�ZGdd�deeejee��Zdd�Zdd�ZGdd�de�Zd0dd�
Zdd�Zd1dd�
Zdd�Z edd �d!d"�Z!ed#d �d$ed%�Z"ed&d �d'd(d)�Z#ed*d �d+d"�Z$ed,d �d-d �d$d"�Z%Gd.d/�d/e�Z&d
S)2�N)
� _functions)
�with_metaclass)
�OrderedDict)�Any�List�Optionalc@sReZ
dZejd
�
dd�Zejd�
dd�Zdd�Zejd�
d d �Ze d�
dd �Z dS)�FunctionCtx)
�tensorsc
Gs |
|_d
S)a�Saves given tensors for a future call to :func:`~Function.backward`. **This should be called at most once, and only from inside the** :func:`forward` **method. This should only be called with input or output tensors** In :func:`backward`, saved tensors can be accessed through the :attr:`saved_tensors` attribute. Before returning them to the user, a check is made to ensure they weren't used in any in-place operation that modified their content. Arguments can also be ``None``. This is a no-op. See :ref:`extending-autograd` for more details on how to use this method. Example:: >>> class Func(Function): >>> @staticmethod >>> def forward(ctx, x: torch.Tensor, y: torch.Tensor, z: int): >>> w = x * y * z >>> out = x * y + y * z + w >>> ctx.save_for_backward(x, y, out) >>> ctx.z = z # z is not a tensor >>> ctx.w = w # w is neither input nor output >>> return out >>> >>> @staticmethod >>> def backward(ctx, grad_out): >>> x, y, out = ctx.saved_tensors >>> z = ctx.z >>> gx = grad_out * (y + y * z) >>> gy = grad_out * (x + z + x * z) >>> gz = None >>> return gx, gy, gz >>> >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double) >>> b = torch.tensor(2., requires_grad=True, dtype=torch.double) >>> c = 4 >>> d = Func.apply(a, b, c) N)
�to_save)�selfr �r�C/usr/local/lib64/python3.6/site-packages/torch/autograd/function.py�save_for_backwards)zFunctionCtx.save_for_backward)
�argsc
Gs |
|_d
S)a�Marks given tensors as modified in an in-place operation. **This should be called at most once, only from inside the** :func:`forward` **method, and all arguments should be inputs.** Every tensor that's been modified in-place in a call to :func:`forward` should be given to this function, to ensure correctness of our checks. It doesn't matter whether the function is called before or after modification. Examples:: >>> class Inplace(Function): >>> @staticmethod >>> def forward(ctx, x): >>> x_npy = x.numpy() # x_npy shares storage with x >>> x_npy += 1 >>> ctx.mark_dirty(x) >>> return x >>> >>> @staticmethod >>> @once_differentiable >>> def backward(ctx, grad_output): >>> return grad_output >>> >>> a = torch.tensor(1., requires_grad=True, dtype=torch.double).clone() >>> b = a * a >>> Inplace.apply(a) # This would lead to wrong gradients! >>> # but the engine would not know unless we mark_dirty >>> b.backward() # RuntimeError: one of the variables needed for gradient >>> # computation has been modified by an inplace operation N)
� dirty_tensors)rrrrr � mark_dirty9s!zFunctionCtx.mark_dirtyc
Gstj
d
�

dS)Nz�mark_shared_storage is deprecated. Tensors with shared storages are automatically tracked. Note that calls to `set_()` are not tracked)�warnings�warn)r�pairsrrr �mark_shared_storage\s

zFunctionCtx.mark_shared_storagec
Gs |
|_d
S)aMarks outputs as non-differentiable. **This should be called at most once, only from inside the** :func:`forward` **method, and all arguments should be tensor outputs.** This will mark outputs as not requiring gradients, increasing the efficiency of backward computation. You still need to accept a gradient for each output in :meth:`~Function.backward`, but it's always going to be a zero tensor with the same shape as the shape of a corresponding output. This is used e.g. for indices returned from a sort. See example:: >>> class Func(Function): >>> @staticmethod >>> def forward(ctx, x): >>> sorted, idx = x.sort() >>> ctx.mark_non_differentiable(idx) >>> ctx.save_for_backward(x, idx) >>> return sorted, idx >>> >>> @staticmethod >>> @once_differentiable >>> def backward(ctx, g1, g2): # still need to accept g2 >>> x, idx = ctx.saved_tensors >>> grad_input = torch.zeros_like(x) >>> grad_input.index_add_(0, idx, g1) >>> return grad_input N)
�non_differentiable)rrrrr �mark_non_differentiablebsz#FunctionCtx.mark_non_differentiable)
�valuecCs |
|_d
S)a�Sets whether to materialize output grad tensors. Default is ``True``. **This should be called only from inside the** :func:`forward` **method** If ``True``, undefined output grad tensors will be expanded to tensors full of zeros prior to calling the :func:`backward` method. Example:: >>> class SimpleFunc(Function): >>> @staticmethod >>> def forward(ctx, x): >>> return x.clone(), x.clone() >>> >>> @staticmethod >>> @once_differentiable >>> def backward(ctx, g1, g2): >>> return g1 + g2 # No check for None necessary >>> >>> # We modify SimpleFunc to handle non-materialized grad outputs >>> class Func(Function): >>> @staticmethod >>> def forward(ctx, x): >>> ctx.set_materialize_grads(False) >>> ctx.save_for_backward(x) >>> return x.clone(), x.clone() >>> >>> @staticmethod >>> @once_differentiable >>> def backward(ctx, g1, g2): >>> x, = ctx.saved_tensors >>> grad_input = torch.zeros_like(x) >>> if g1 is not None: # We must check for None now >>> grad_input += g1 >>> if g2 is not None: >>> grad_input += g2 >>> return grad_input >>> >>> a = torch.tensor(1., requires_grad=True) >>> b, _ = Func.apply(a) # induces g2 to be undefined N)
Zmaterialize_grads)rrrrr �set_materialize_grads�s*z!FunctionCtx.set_materialize_gradsN)�__name__� __module__�__qualname__�torch�Tensorrrrr�boolrrrrr rs +# rc@seZ
dZed
d��
ZdS)� _HookMixincCs*|dkrt�}t
j|�
}|
||j<||fS)
N)r�hooksZRemovableHandle�id)Zbackward_hooks�hook�handlerrr �_register_hook�s 



z_HookMixin._register_hookN)rrr�staticmethodr%rrrr r �sr c@seZ
dZd
d�Zdd�ZdS)�BackwardCFunctionc
GsL|jj
}|jj}|tj
k r,|tjk r,td
�
�
|tjk r:|n|}||f
|
��S)NzsImplementing both 'backward' and 'vjp' for a custom Function is not allowed. You should only implement one of them.)�_forward_cls�backward�vjp�Function�RuntimeError)rr�backward_fnZvjp_fnZuser_fnrrr �apply�s



zBackwardCFunction.applyc
Gs|jj
|f
|
��S)
N)r(�jvp)rrrrr � apply_jvp�szBackwardCFunction.apply_jvpN)rrrr.r0rrrr r'�s
r'cs eZ
dZd
Z�f
dd�Z�ZS)�FunctionMetaa
Function metaclass. This metaclass sets up the following properties: _backward_cls: The Function class corresponding to the differentiated version of this function (which is generated on the fly by this metaclass). cs4t|
d
t
f
d|i
�}||_tt|�j|
||�
dS)NZBackwardr()�typer'Z _backward_cls�superr1�__init__)�cls�name�bases�attrsr-)
� __class__rr r4�s

zFunctionMeta.__init__)rrr�__doc__r4� __classcell__rr)
r9r r1�s
r1c@sleZ
dZd
Zdd�Zdd�ZdZeeeeed�dd ��
Z eeeed �dd��
Z e Zeeeed �dd��
ZdS)r+aYBase class to create custom `autograd.Function` To create a custom `autograd.Function`, subclass this class and implement the :meth:`forward` and :meth`backward` static methods. Then, to use your custom op in the forward pass, call the class method ``apply``. Do not call :meth:`forward` directly. To ensure correctness and best performance, make sure you are calling the correct methods on ``ctx`` and validating your backward function using :func:`torch.autograd.gradcheck`. See :ref:`extending-autograd` for more details on how to use this class. Examples:: >>> class Exp(Function): >>> @staticmethod >>> def forward(ctx, i): >>> result = i.exp() >>> ctx.save_for_backward(result) >>> return result >>> >>> @staticmethod >>> def backward(ctx, grad_output): >>> result, = ctx.saved_tensors >>> return grad_output * result >>> >>> # Use it by calling the apply method: >>> output = Exp.apply(input) c
Os|j}t
j|�d
�t�
dS)Nz� should not be instantiated. Methods on autograd functionsare all static, so you should invoke them on the class itself. Instantiating an autograd function will raise an error in a future version of PyTorch.)r9rr�DeprecationWarning)rr�kwargsr5rrr r4�s

zFunction.__init__c
Ostd
�
�
dS)Nz�Legacy autograd function with non-static forward method is deprecated. Please use new-style autograd function with static forward method. (Example: https://pytorch.org/docs/stable/autograd.html#torch.autograd.Function))
r,)rrr=rrr �__call__
s

zFunction.__call__F)�ctxrr=�returnc
Ostd
�
�
dS)aW
Performs the operation. This function is to be overridden by all subclasses. It must accept a context ctx as the first argument, followed by any number of arguments (tensors or other types). The context can be used to store arbitrary data that can be then retrieved during the backward pass. zEYou must implement the forward function for custom autograd.Function.N)
�NotImplementedError)r?rr=rrr �forward
szFunction.forward)r?�grad_outputsr@c
Gstd
�
�
dS)a�Defines a formula for differentiating the operation with backward mode automatic differentiation. This function is to be overridden by all subclasses. It must accept a context :attr:`ctx` as the first argument, followed by as many outputs as the :func:`forward` returned (None will be passed in for non tensor outputs of the forward function), and it should return as many tensors, as there were inputs to :func:`forward`. Each argument is the gradient w.r.t the given output, and each returned value should be the gradient w.r.t. the corresponding input. If an input is not a Tensor or is a Tensor not requiring grads, you can just pass None as a gradient for that input. The context can be used to retrieve tensors saved during the forward pass. It also has an attribute :attr:`ctx.needs_input_grad` as a tuple of booleans representing whether each input needs gradient. E.g., :func:`backward` will have ``ctx.needs_input_grad[0] = True`` if the first input to :func:`forward` needs gradient computated w.r.t. the output. zwYou must implement either the backward or vjp method for your custom autograd.Function to use it with backward mode AD.N)
rA)r?rCrrr r)
szFunction.backward)r?�grad_inputsr@c
Gstd
�
�
dS)a�Defines a formula for differentiating the operation with forward mode automatic differentiation. This function is to be overridden by all subclasses. It must accept a context :attr:`ctx` as the first argument, followed by as many inputs as the :func:`forward` got (None will be passed in for non tensor inputs of the forward function), and it should return as many tensors as there were outputs to :func:`forward`. Each argument is the gradient w.r.t the given input, and each returned value should be the gradient w.r.t. the corresponding output. If an output is not a Tensor or the function is not differentiable with respect to that output, you can just pass None as a gradient for that input. You can use the :attr:`ctx` object to pass any value from the forward to this functions. z`You must implement the jvp function for custom autograd.Function to use it with forward mode AD.N)
rA)r?rDrrr r/:
szFunction.jvpN) rrrr:r4r>�is_traceabler&rrBr)r*r/rrrr r+�s



r+c
stj
��
�f
d
d��
}
|
S)Nc
s�tj
��
�
|f
|
��}WdQRXtj�s.|Std
d�|
D�
�
}|sH|St|t�sX|f
}tjdt|�
�}dd��|�f
dd�|D�
�S)Nc
ss |]}
t|
t
j�o|
jV
qdS)
N)� isinstancerr� requires_grad)�.0�argrrr � b
s
z7once_differentiable..wrapper..sRtrying to differentiate twice a function that was marked with @once_differentiablec

Ss|dk r|j�}d
|_
|S)NT)�detachrG)
�varrrr �fake_requires_gradq
s



z@once_differentiable..wrapper..fake_requires_gradc
sg|]}
�|
�
�qSrr)rH�
v)
rMrr � w
sz8once_differentiable..wrapper..) rZno_gradZis_grad_enabled�anyrF�tuplerZDelayedError�len)r?r�outputsrGZerr_fn)
�fn)
rMr �wrapperQ
s







z$once_differentiable..wrapper)� functools�wraps)rTrUr)
rTr �once_differentiableO
s'rXc

Cs d
|_|S)a�
Marks Function as traceable for the JIT. Traceable functions have additional restrictions - they can't pass any data-dependent values to backward (e.g. Prod passes the output, which makes it non-traceable), and their backward should be implemented entirely in terms of operations on autograd Tensors in all cases. DON'T USE THIS DECORATOR. IT IS FOR INTERNAL USE ONLY AND SHOULD BE HANDLED WITH CARE (or can give incorrect results otherwise). T)
rE)
Zfn_clsrrr � traceable{
s
rYcseZ
dZd�f
dd� Z�ZS)�InplaceFunctionFcstt
|�j�
|
|_dS)
N)r3rZr4�inplace)rr[)
r9rr r4�
s

zInplaceFunction.__init__)
F)rrrr4r;rr)
r9r rZ�
srZcs��
��fd
d���S)Nc
s����
r���
S�dkrdSt�t
tf�r^�
f
d
d��D�
}
t�d�rRt��
|
�St��
|
�
St�t�r|�
�fdd��D�
Stdtj��
�r�d�dnd �
�
dS) Nc
3s|]}
�|
�
V
qdS)
Nr)rH�
x)
�_maprr rJ�
sz,_nested_map.._map..�_fieldsc
si|]}
��
|
�
|
�qSrr)rHr\)r]�objrr � �
sz-_nested_map.._map..zAAuto nesting doesn't know how to process an input object of type z. Accepted types: z, or lists/tuples of them�) rF�listrQ�hasattrr2�dict� ValueErrorr�typename)r_Zmapped)r]� condition� condition_msgrT)
r_r r]�
s










z_nested_map.._mapr)rgrTrhr)r]rgrhrTr �_nested_map�
s
ric

Cst|d
�r|j
�S|S)N�_jit_unwrap)rcrj)
r_rrr �_jit_unwrap_structured�
s


rkFcs��
���fd
d���S)Nc
3s��dk r�|�
}�|�
r |V
n�|dkr,dSt|t
tf�rbx�|D]}
x�|
�
D] }|V
qNWq@Wnht|t�r�x\|j�D]}
x�|
�
D] }|V
q�WqvWn2�
r�|V
n&td
tj|�
�r�d�dnd�
�
dS)NzAAuto nesting doesn't know how to process an input object of type z. Accepted types: z, or lists/tuples of themra)rFrbrQrd�valuesrerrf)r_�
orL)�_iter� allow_unknownrgrh� conversionrr rn�
s&















z_iter_filter.._iterr)rgrorhrpr)rnrorgrhrpr �_iter_filter�
srqcs�f
d
d���||
�dS)Ncs�g}t|
d
�r|
j
|�
St|
ttf�s:|d|dd�fSx8|
D]0}|dkrX|j|�

q@�||�\}}|j|�

q@Wt|
�
|�
|fS)N� _jit_wrapr
�
)rcrrrFrbrQ�appendr2)�input�proto�res�
eZres_e)
�unflatten_helperrr ry�
s









z$_unflatten..unflatten_helperr
r)rurvr)
ryr � _unflatten�
srzc

Cs|dkpt|t
jj�S)
N)rFr�_C�Value)
rmrrr ��
sr}zjit's Values or None)
rhc

Cst|t
j�S)
N)rFrr)
r\rrr r}�
sZTensors)rhrpc

Cst|t
j�S)
N)rFrr)
r\rrr r}�
sTzTensors (permissive))rorhc

Cs|dkpt|t
j�S)
N)rFrr)
rmrrr r}�
szTensors or Nonec

Cst|t
j�S)
N)rFrr)
r\rrr r}�
sc


Cs|jS)
N)
�data)
rmrrr r}�
scs�eZ
dZ�f
d
d�Z�f
dd�Zeed�dd�ZeZeed�d d �Zedd�dd �Z e �f
dd��
Zeedd�dd�Zeedd�dd�Z edd�dd�Zedd�dd�Z�ZS)�NestedIOFunctionc
s8|
|_t
t|
�
�
}tt|�j|�}|j}t||j�}|S)
N)� _nested_inputrQ� _iter_tensorsr3r�_do_forward�_nested_outputrz)rruZ flat_inputZflat_outputZ nested_output�nested_tensors)
r9rr r��
s





zNestedIOFunction._do_forwardcs(||_t
t|�j|
|�}|s$|`|`|S)
N)�retain_variablesr3r�_do_backwardr��_to_save_nested)r� gradientsr��result)
r9rr r��
s





zNestedIOFunction._do_backward)r�r@c
Gs"t|
|j
�}|j|�}tt|�
�
S)
N)rzr��backward_extendedrQ�_iter_None_tensors)rr�Znested_gradientsr�rrr r)s


zNestedIOFunction.backward)rr@c
Gs*t|j
�
}|j|�}|`
||_tt|�
�
S)
N)�_map_tensor_datar��forward_extendedr�rQr�)rrr�r�rrr rB s




zNestedIOFunction.forwardNc
Gstt
|
�
�
|_|
|_dS)
N)rQr�r r�)rrrrr rs

z"NestedIOFunction.save_for_backwardc
stt
|�j}
t|
|j�S)
N)r3r� saved_tensorsrzr�)rZflat_tensors)
r9rr r�s
zNestedIOFunction.saved_tensors)rr=r@c
Ostt
|
|f�
�
|_dS)
N)rQr�r)rrr=rrr rs
zNestedIOFunction.mark_dirtyc
Ostt
|
|f�
�
|_dS)
N)rQr�r)rrr=rrr rs
z(NestedIOFunction.mark_non_differentiable)rur@c

Gst�
dS)
N)
rA)rrurrr r�s
z!NestedIOFunction.forward_extended)�grad_outputr@c

Gst�
dS)
N)
rA)rr�rrr r�"s
z"NestedIOFunction.backward_extended)rrrr�r�rr)r>rBr�propertyr�rrr�r�r;rr)
r9r r�
sr)
N)FNN)'rZtorch._Cr{rZtorch.utils.hooks�utilsr!Z torch._sixrrVr�collectionsr�typingrrr�objectrZ_ContextMethodMixinr Z _FunctionBaser'r2r1r+rXrYrZrirkrqrzZ_iter_jit_valuesr�Z_iter_tensors_permissiver�r�rrrrr �
sF







$q,