L iQ8&dZddlZddlZddlZddlmZmZmZmZddl Z ddl m Z m Z m Z ddlmZmZddlmZmZmZmZmZddlmZmZmZddlmZmZdd lm Z m!Z!m"Z"m#Z#dd l$m%Z%m&Z&m'Z'm(Z(erdd l)m*Z+ne,Z+ ddl-Z.ee j`d jbdDieed zZ0ee jddjbdDieedzZ2ee jfdjbdDieedzZ3ee jhdjbdDieedzZ4ee jjdjbdDieedzZ5ee jldjbdDieedzZ6ee jndZ7ee jpdZ8ee jrjtjvdZ;ee jrjtjxdZ< dEde dee,deee,d eee=d!e>d"ee d#e?e e ffd$Z@ dEde dee,deee,d eee=d!e>d"ee d#e fd%ZAed!d&de@eAeBd'(ZC dEde dee,deee,d eee=d!e>d"ee d#e?e e ffd)ZD dEde dee,deee,d eee=d!e>d"ee d#e fd*ZEed!d&deDeEeBd+(ZF dFde dee,d-eee,d.ee,d/ee,d0e>d!e>d#e?e e ffd1ZG dFde dee,d-eee,d.ee,d/ee,d0e>d!e>d#e fd2ZHed!d3deGeHeBd4(ZI dFde dee,d-eee,d.ee,d/ee,d0e>d!e>d#e?e e ffd5ZJ dFde dee,d-eee,d.ee,d/ee,d0e>d!e>d#e fd6ZKed!d3deJeKeBd7(ZL dFde dee,d-eee,d.ee,d/ee,d0e>d!e>d#e?e e ffd8ZM dFde dee,d-eee,d.ee,d/ee,d0e>d!e>d#e fd9ZNed!d3deMeNeBd:(ZOde dePe,d-ePe,d.ePe,deePe,d#ePe,f d;ZQ dGde dZS dGde dd#e f dAZU dHde d@ee,e=fdee,d-eee,d0e>d#e f dBZV dHde d@ee,e=fde,d-eee,d0e>d#e f dCZW dIde dee,d!e>d#e?e e ffdDZX dIde dee,d!e>d#e fdEZYed!dFdeXeYeBdG(ZZ dIde dee,d!e>d#e?e e ffdHZ[ dIde dee,d!e>d#e fdIZ\ed!dFde[e\eBdJ(Z] dIde dee,d!e>d#e?e e ffdKZ^ dIde dee,d!e>d#e fdLZ_ed!dFde^e_eBdM(Z`ee jdNZade dee,d#e fdOZbde dee,d#e fdPZc dJde dQe=dRe>dSe>d#e f dTZd dKde dQe=dRe>dSe>d#e f dUZe dJde dQe=dRe>dSe>d#e f dVZf dJde dQe=dRe>dSe>d#e f dWZg dJde dQe=dRe>dSe>d#e f dXZh dKde dQe=dRe>dSe>d#e f dYZi dIde dZe=d[e=dSe>d#e f d\ZjejZkee jd]ZldIde dSe>d#e fd^Zmee jd_ZndLde d`e,d#e fdaZo dMde dce=dde=dSe>d#e f deZpee jrjtjdfZqdIde dSe>d#e fdgZrdNde dhe=dSe>d#e fdiZsee jrjtjdjZtdIde dSe>d#e fdkZuee jdlZv dNde dhe=dSe>d#e fdmZwee jdnZx dOde doe=dSe>d#e fdpZyee jrjtjdqZzee jdrZ{ dPde dse=dte=dRe>dSe>d#e f duZ|ee jdvZ}ee jrjtjdwZee jrjtjdxZee jdyZdzZd{Zee jrjtjd|Zd}ed~e,de,d#e,fdZ dQde d`ee,de,dee+d#e f dZ dQde d`ee,de,dee+d#e f dZ dRde de=de>de=d`e,d#e f dZ dQde d`ee,de,dee+d#e f dZee jrjtjdZdZdZdIde dSe>d#e fdZee jrjtjdjbdDieZee j dZdIde dSe>d#e fdZdIde dSe>d#e fdZdIde dSe>d#e fdZde de de=d@e=d#e?e e ff dZ dSde de dee,dee=d@e=de>de>d#e fdZ dTde de dee dee=d@e=de>dede>dee de>dee,d#e fdZejr"ejjbdDiee_dePe,d#dfdZ dUde dee dee dee dee dRe>de=de=d#e fdZdePe,d#dfdZ dVde dee dee dee dee de>de=de=d#e fdZ dWde dePe,dee dee de=d#e f dZ dXde dePe,dee dee=d#e f dZ dWde de,dee dee de=d#e f dZ dYde de,dhe=de=de=d#e f dZ dZde de de de de,dede>d#e fdZejr"ejjbdDiee_ d[de de dee dee>de,dee>ded#e fdZ d\de de de>de>dee>de=dee>ded#e fd„Z d]de de dee e=fde>de=ded#e fdĄZ d^de de dee>dee>dede>d#e fdƄZ d_de de dee dee>de,dee>dede=d#e fdȄZ d`de de dee dee>dee>ded#e fdɄZ dade de dee dee>dee>dedee d#e fd˄Z dbde de dee>dee>dede=d#e fd̄Z dcde de dede=dee d#e f d΄Z ddde de dee>dee>dedee d#e fdτZ ddde de dee>dee>dedee d#e fdЄZ dede de de de=dee>dee>ded#e fdԄZ dfde de de=dee>dee>ded#e fdՄZ dgde de dee>dee>ded#e f dքZ dgde de dee>dee>ded#e f dׄZ d`de de dee dee>dee>ded#e fd؄Z dede de de de=dee>dee>ded#e fdلZ dhde de dQe,de=dee dee>dee>ded#e fdڄZee jdd۫Zee jfdܫZee jhdݫZee jjdޫZe dide dee,dee=dedee>d#e f dZe dide deePe,dee=dedee>d#e f dZ didZejr"ejjbdDiee_d#e>fdZe djde dee,deePe=dedee>dee>de>d#e fdZe djde deePe,deePe=dedee>dee>de>d#e fdZe djde dee,dee=dedee>dee>de>d#e fdZe djde deePe,dee=dedee>dee>de>d#e fdZ djde dee,deePe=dedee>dee>de>d#e fdZejr"ejjbdDiee_e dXde dee,dee=d#e fdZe dXde deePe,dee=d#e fdZdXdZejr"ejjbdDiee_e dXde dee,dee=d#e fdZe dXde deePe,dee=d#e fdZe dXde dee,deePe=d#e fdZe dXde deePe,deePe=d#e fdZdXdZejr"ejjbdDiee_dd,dFdZdd,dFdZ dkde de dededee>d#e f dZ dlde dePe,dee>d#e fdZ dmde dePe,ded[ee=d#e f dZde_ee jdZee jdZee jdZee jrjtjdZ dnde de de de=dQe=de=de>dee>dee>ded#e fdZddbdddde de de deee e ge fde=de>ded#e fd Z dode dQe=d`e,de=d ee d#e f d Zǐd ePe,d eded#dfdZ dpde dee,d/ee,d.ee,d-ee,d#e f dZ dpde dee,dee,d/ee,d.ee,d-ee,d#e fdZ dlde de de de dee d#ePe f dZ dqde de de de de de dee dee dee d#e?e e e ffdZee jrjtjdjbdDiedzZ͐d e d!e d[e d"ee d#ee d$e,f d%Z drd&ee d'ed(ee+d)ed*e+d+e>d#ee fd,Zdee d#ee+fd-ZАd"e jd.e,d/e,d#dfd0Z dsd e d!e d[e d1e,d$e,d2ee d3ee d4ee d5ee d6e>d7e=d8e d9ee dRe>d"ee d:e>d#ee d;e>dee d?ee d@ee dAe>dBe>d#e?e ee ff4dCZy#e/$rdZ.YwxYw(tzFunctional interface.N)CallableOptional TYPE_CHECKINGUnion)_VFsym_intTensor) _add_docstr _infer_size) _overloadboolean_dispatchBroadcastingList1BroadcastingList2BroadcastingList3)reproducibility_notessparse_support_notes tf32_notes) _reductiongrad)_list_with_default_pair_single_triple)handle_torch_functionhas_torch_functionhas_torch_function_unaryhas_torch_function_variadic)_dtypeaz conv1d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor Applies a 1D convolution over an input signal composed of several input planes. {tf32_note} See :class:`~torch.nn.Conv1d` for details and output shape. Note: {cudnn_reproducibility_note} Note: This operator supports complex data types i.e. ``complex32, complex64, complex128``. ai Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kW)` bias: optional bias of shape :math:`(\text{out\_channels})`. Default: ``None`` stride: the stride of the convolving kernel. Can be a single number or a one-element tuple `(sW,)`. Default: 1 padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, single number or a one-element tuple `(padW,)`. Default: 0 ``padding='valid'`` is the same as no padding. ``padding='same'`` pads the input so the output has the same shape as the input. However, this mode doesn't support any stride values other than 1. .. warning:: For ``padding='same'``, if the ``weight`` is even-length and ``dilation`` is odd in any dimension, a full :func:`pad` operation may be needed internally. Lowering performance. dilation: the spacing between kernel elements. Can be a single number or a one-element tuple `(dW,)`. Default: 1 groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the number of groups. Default: 1 Examples:: >>> inputs = torch.randn(33, 16, 30) >>> filters = torch.randn(20, 16, 5) >>> F.conv1d(inputs, filters) ay conv2d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor Applies a 2D convolution over an input image composed of several input planes. {tf32_note} See :class:`~torch.nn.Conv2d` for details and output shape. Note: {cudnn_reproducibility_note} Note: This operator supports complex data types i.e. ``complex32, complex64, complex128``. a Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kH , kW)` bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: ``None`` stride: the stride of the convolving kernel. Can be a single number or a tuple `(sH, sW)`. Default: 1 padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, single number or a tuple `(padH, padW)`. Default: 0 ``padding='valid'`` is the same as no padding. ``padding='same'`` pads the input so the output has the same shape as the input. However, this mode doesn't support any stride values other than 1. .. warning:: For ``padding='same'``, if the ``weight`` is even-length and ``dilation`` is odd in any dimension, a full :func:`pad` operation may be needed internally. Lowering performance. dilation: the spacing between kernel elements. Can be a single number or a tuple `(dH, dW)`. Default: 1 groups: split input into groups, both :math:`\text{in\_channels}` and :math:`\text{out\_channels}` should be divisible by the number of groups. Default: 1 Examples:: >>> # With square kernels and equal stride >>> filters = torch.randn(8, 4, 3, 3) >>> inputs = torch.randn(1, 4, 5, 5) >>> F.conv2d(inputs, filters, padding=1) ay conv3d(input, weight, bias=None, stride=1, padding=0, dilation=1, groups=1) -> Tensor Applies a 3D convolution over an input image composed of several input planes. {tf32_note} See :class:`~torch.nn.Conv3d` for details and output shape. Note: {cudnn_reproducibility_note} Note: This operator supports complex data types i.e. ``complex32, complex64, complex128``. a Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)` weight: filters of shape :math:`(\text{out\_channels} , \frac{\text{in\_channels}}{\text{groups}} , kT , kH , kW)` bias: optional bias tensor of shape :math:`(\text{out\_channels})`. Default: None stride: the stride of the convolving kernel. Can be a single number or a tuple `(sT, sH, sW)`. Default: 1 padding: implicit paddings on both sides of the input. Can be a string {'valid', 'same'}, single number or a tuple `(padT, padH, padW)`. Default: 0 ``padding='valid'`` is the same as no padding. ``padding='same'`` pads the input so the output has the same shape as the input. However, this mode doesn't support any stride values other than 1. .. warning:: For ``padding='same'``, if the ``weight`` is even-length and ``dilation`` is odd in any dimension, a full :func:`pad` operation may be needed internally. Lowering performance. dilation: the spacing between kernel elements. Can be a single number or a tuple `(dT, dH, dW)`. Default: 1 groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the number of groups. Default: 1 Examples:: >>> filters = torch.randn(33, 16, 3, 3, 3) >>> inputs = torch.randn(20, 16, 50, 10, 20) >>> F.conv3d(inputs, filters) az conv_transpose1d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor Applies a 1D transposed convolution operator over an input signal composed of several input planes, sometimes also called "deconvolution". {tf32_note} See :class:`~torch.nn.ConvTranspose1d` for details and output shape. Note: {cudnn_reproducibility_note} al Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kW)` bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None stride: the stride of the convolving kernel. Can be a single number or a tuple ``(sW,)``. Default: 1 padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both sides of each dimension in the input. Can be a single number or a tuple ``(padW,)``. Default: 0 output_padding: additional size added to one side of each dimension in the output shape. Can be a single number or a tuple ``(out_padW)``. Default: 0 groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the number of groups. Default: 1 dilation: the spacing between kernel elements. Can be a single number or a tuple ``(dW,)``. Default: 1 Examples:: >>> inputs = torch.randn(20, 16, 50) >>> weights = torch.randn(16, 33, 5) >>> F.conv_transpose1d(inputs, weights) ay conv_transpose2d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor Applies a 2D transposed convolution operator over an input image composed of several input planes, sometimes also called "deconvolution". {tf32_note} See :class:`~torch.nn.ConvTranspose2d` for details and output shape. Note: {cudnn_reproducibility_note} a Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kH , kW)` bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None stride: the stride of the convolving kernel. Can be a single number or a tuple ``(sH, sW)``. Default: 1 padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both sides of each dimension in the input. Can be a single number or a tuple ``(padH, padW)``. Default: 0 output_padding: additional size added to one side of each dimension in the output shape. Can be a single number or a tuple ``(out_padH, out_padW)``. Default: 0 groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the number of groups. Default: 1 dilation: the spacing between kernel elements. Can be a single number or a tuple ``(dH, dW)``. Default: 1 Examples:: >>> # With square kernels and equal stride >>> inputs = torch.randn(1, 4, 5, 5) >>> weights = torch.randn(4, 8, 3, 3) >>> F.conv_transpose2d(inputs, weights, padding=1) ax conv_transpose3d(input, weight, bias=None, stride=1, padding=0, output_padding=0, groups=1, dilation=1) -> Tensor Applies a 3D transposed convolution operator over an input image composed of several input planes, sometimes also called "deconvolution" {tf32_note} See :class:`~torch.nn.ConvTranspose3d` for details and output shape. Note: {cudnn_reproducibility_note} a Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iT , iH , iW)` weight: filters of shape :math:`(\text{in\_channels} , \frac{\text{out\_channels}}{\text{groups}} , kT , kH , kW)` bias: optional bias of shape :math:`(\text{out\_channels})`. Default: None stride: the stride of the convolving kernel. Can be a single number or a tuple ``(sT, sH, sW)``. Default: 1 padding: ``dilation * (kernel_size - 1) - padding`` zero-padding will be added to both sides of each dimension in the input. Can be a single number or a tuple ``(padT, padH, padW)``. Default: 0 output_padding: additional size added to one side of each dimension in the output shape. Can be a single number or a tuple ``(out_padT, out_padH, out_padW)``. Default: 0 groups: split input into groups, :math:`\text{in\_channels}` should be divisible by the number of groups. Default: 1 dilation: the spacing between kernel elements. Can be a single number or a tuple `(dT, dH, dW)`. Default: 1 Examples:: >>> inputs = torch.randn(20, 16, 50, 10, 20) >>> weights = torch.randn(16, 33, 3, 3, 3) >>> F.conv_transpose3d(inputs, weights) a Applies a 1-dimensional sequence convolution over an input sequence. Input and output dimensions are (Time, Batch, Channels) - hence TBC. Args: input: input tensor of shape :math:`(\text{sequence length} \times batch \times \text{in\_channels})` weight: filter of shape (:math:`\text{kernel width} \times \text{in\_channels} \times \text{out\_channels}`) bias: bias of shape (:math:`\text{out\_channels}`) pad: number of timesteps to pad. Default: 0 a avg_pool1d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True) -> Tensor Applies a 1D average pooling over an input signal composed of several input planes. See :class:`~torch.nn.AvgPool1d` for details and output shape. Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)` kernel_size: the size of the window. Can be a single number or a tuple `(kW,)` stride: the stride of the window. Can be a single number or a tuple `(sW,)`. Default: :attr:`kernel_size` padding: implicit zero paddings on both sides of the input. Can be a single number or a tuple `(padW,)`. Should be at most half of effective kernel size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 ceil_mode: when True, will use `ceil` instead of `floor` to compute the output shape. Default: ``False`` count_include_pad: when True, will include the zero-padding in the averaging calculation. Default: ``True`` Examples:: >>> # pool of square window of size=3, stride=2 >>> input = torch.tensor([[[1, 2, 3, 4, 5, 6, 7]]], dtype=torch.float32) >>> F.avg_pool1d(input, kernel_size=3, stride=2) tensor([[[ 2., 4., 6.]]]) at avg_pool2d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor Applies 2D average-pooling operation in :math:`kH \times kW` regions by step size :math:`sH \times sW` steps. The number of output features is equal to the number of input planes. See :class:`~torch.nn.AvgPool2d` for details and output shape. Args: input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)` kernel_size: size of the pooling region. Can be a single number, a single-element tuple or a tuple `(kH, kW)` stride: stride of the pooling operation. Can be a single number, a single-element tuple or a tuple `(sH, sW)`. Default: :attr:`kernel_size` padding: implicit zero paddings on both sides of the input. Can be a single number, a single-element tuple or a tuple `(padH, padW)`. Should be at most half of effective kernel size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 ceil_mode: when True, will use `ceil` instead of `floor` in the formula to compute the output shape. Default: ``False`` count_include_pad: when True, will include the zero-padding in the averaging calculation. Default: ``True`` divisor_override: if specified, it will be used as divisor, otherwise size of the pooling region will be used. Default: None aJ avg_pool3d(input, kernel_size, stride=None, padding=0, ceil_mode=False, count_include_pad=True, divisor_override=None) -> Tensor Applies 3D average-pooling operation in :math:`kT \times kH \times kW` regions by step size :math:`sT \times sH \times sW` steps. The number of output features is equal to :math:`\lfloor\frac{\text{input planes}}{sT}\rfloor`. See :class:`~torch.nn.AvgPool3d` for details and output shape. Args: input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iT \times iH , iW)` kernel_size: size of the pooling region. Can be a single number or a tuple `(kT, kH, kW)` stride: stride of the pooling operation. Can be a single number or a tuple `(sT, sH, sW)`. Default: :attr:`kernel_size` padding: implicit zero paddings on both sides of the input. Can be a single number or a tuple `(padT, padH, padW)`. Should be at most half of effective kernel size, that is :math:`((kernelSize - 1) * dilation + 1) / 2`. Default: 0 ceil_mode: when True, will use `ceil` instead of `floor` in the formula to compute the output shape count_include_pad: when True, will include the zero-padding in the averaging calculation divisor_override: if specified, it will be used as divisor, otherwise size of the pooling region will be used. Default: None Finput kernel_size output_size output_ratioreturn_indices_random_samplesreturnc `t||rtt||f||||||S| | td|h|Jt |dkDr tdt |}t |jd|dzt |jd|dzg}|c|jd k(rdn|jd}tj||jd d|j|j }tjjj||||S) a fractional_max_pool2d(input, kernel_size, output_size=None, output_ratio=None, return_indices=False, _random_samples=None) Applies 2D fractional max pooling over an input signal composed of several input planes. Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham The max-pooling operation is applied in :math:`kH \times kW` regions by a stochastic step size determined by the target output size. The number of output features is equal to the number of input planes. Args: kernel_size: the size of the window to take a max over. Can be a single number :math:`k` (for a square kernel of :math:`k \times k`) or a tuple `(kH, kW)` output_size: the target output size of the image of the form :math:`oH \times oW`. Can be a tuple `(oH, oW)` or a single number :math:`oH` for a square image :math:`oH \times oH` output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given. This has to be a number or tuple in the range (0, 1) return_indices: if ``True``, will return the indices along with the outputs. Useful to pass to :func:`~torch.nn.functional.max_unpool2d`. Examples:: >>> input = torch.randn(20, 16, 50, 32) >>> # pool of square window of size=3, and target output size 13x12 >>> F.fractional_max_pool2d(input, 3, output_size=(13, 12)) >>> # pool of square window and target output size being half of input image size >>> F.fractional_max_pool2d(input, 3, output_ratio=(0.5, 0.5)) .. _Fractional MaxPooling: http://arxiv.org/abs/1412.6071 r!r"r#r$zRfractional_max_pool2d requires specifying either an output_size or an output_ratiozWfractional_max_pool2d requires output_ratio to either be a single Int or tuple of Ints.rdtypedevice)rr"fractional_max_pool2d_with_indices ValueErrorlenrintsizedimtorchrandr/r0_C_nnfractional_max_pool2drr r!r"r#r$ _output_ration_batchs Y/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/torch/nn/functional.pyr1r1sAP#5/:$ . O $  #%)+  |3 `  ''' | q i l+  2q!11 2  2q!11 2 yy{a'!UZZ]** UZZ^Qekk%,,  88<< - - {K c rt||rtt||f||||||St||||||dSNr'r)rrr;r1rr r!r"r#r$s r?_fractional_max_pool2drD\#5/:$ ! O $  #%)+   . {K~   r@r;)arg_name arg_indexdefaultif_trueif_false module_name func_namec lt||rtt||f||||||S| | td|n|Jt |}t |j d|dzt |j d|dzt |j d|dzg}|c|jd k(rdn|j d}tj||j d d |j|j }tjjj||||S) a fractional_max_pool3d(input, kernel_size, output_size=None, output_ratio=None, return_indices=False, _random_samples=None) Applies 3D fractional max pooling over an input signal composed of several input planes. Fractional MaxPooling is described in detail in the paper `Fractional MaxPooling`_ by Ben Graham The max-pooling operation is applied in :math:`kT \times kH \times kW` regions by a stochastic step size determined by the target output size. The number of output features is equal to the number of input planes. Args: kernel_size: the size of the window to take a max over. Can be a single number :math:`k` (for a square kernel of :math:`k \times k \times k`) or a tuple `(kT, kH, kW)` output_size: the target output size of the form :math:`oT \times oH \times oW`. Can be a tuple `(oT, oH, oW)` or a single number :math:`oH` for a cubic output :math:`oH \times oH \times oH` output_ratio: If one wants to have an output size as a ratio of the input size, this option can be given. This has to be a number or tuple in the range (0, 1) return_indices: if ``True``, will return the indices along with the outputs. Useful to pass to :func:`~torch.nn.functional.max_unpool3d`. Shape: - Input: :math:`(N, C, T_{in}, H_{in}, W_{in})` or :math:`(C, T_{in}, H_{in}, W_{in})`. - Output: :math:`(N, C, T_{out}, H_{out}, W_{out})` or :math:`(C, T_{out}, H_{out}, W_{out})`, where :math:`(T_{out}, H_{out}, W_{out})=\text{output\_size}` or :math:`(T_{out}, H_{out}, W_{out})=\text{output\_ratio} \times (T_{in}, H_{in}, W_{in})` Examples:: >>> input = torch.randn(20, 16, 50, 32, 16) >>> # pool of cubic window of size=3, and target output size 13x12x11 >>> F.fractional_max_pool3d(input, 3, output_size=(13, 12, 11)) >>> # pool of cubic window and target output size being half of input size >>> F.fractional_max_pool3d(input, 3, output_ratio=(0.5, 0.5, 0.5)) .. _Fractional MaxPooling: http://arxiv.org/abs/1412.6071 r'zRfractional_max_pool3d requires specifying either an output_size or an output_ratior-rr)r+r*r(rFr,r.)rr"fractional_max_pool3d_with_indicesr2rr4r5r6r7r8r/r0r9r:fractional_max_pool3dr<s r?rPrPs<^#5/:$ . O $  #%)+  |3 `  ''' -  2q!11 2  2q!11 2  2q!11 2 yy{a'!UZZ]** UZZ^Qekk%,,  88<< - - {K r@c rt||rtt||f||||||St||||||dSrB)rrrQrPrCs r?_fractional_max_pool3drSnrEr@rQr+stridepaddingdilation ceil_modec t|rtt|f||||||| S|+tjj t tg}tj||||||S)aS max_pool1d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) Applies a 1D max pooling over an input signal composed of several input planes. .. note:: The order of :attr:`ceil_mode` and :attr:`return_indices` is different from what seen in :class:`~torch.nn.MaxPool1d`, and will change in a future release. See :class:`~torch.nn.MaxPool1d` for details. Args: input: input tensor of shape :math:`(\text{minibatch} , \text{in\_channels} , iW)`, minibatch dim optional. kernel_size: the size of the window. Can be a single number or a tuple `(kW,)` stride: the stride of the window. Can be a single number or a tuple `(sW,)`. Default: :attr:`kernel_size` padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. dilation: The stride between elements within a sliding window, must be > 0. ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This ensures that every element in the input tensor is covered by a sliding window. return_indices: If ``True``, will return the argmax along with the max values. Useful for :class:`torch.nn.functional.max_unpool1d` later rTrUrVrWr#)rrmax_pool1d_with_indicesr7jitannotatelistr4rr rTrUrVrWr#s r?rZrZsxD &$ # H  )  ~##DIr2  ( ( {FGXy r@c t|rtt|f||||||| S|+tjj t tg}tj||||||SNrY)rr max_pool1dr7r[r\r]r4r^s r? _max_pool1drbr &$  H  )  ~##DIr2   E;9 UUr@rac t|rtt|f||||||| S|+tjj t tg}tjjj||||||S)a` max_pool2d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) Applies a 2D max pooling over an input signal composed of several input planes. .. note:: The order of :attr:`ceil_mode` and :attr:`return_indices` is different from what seen in :class:`~torch.nn.MaxPool2d`, and will change in a future release. See :class:`~torch.nn.MaxPool2d` for details. Args: input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iH , iW)`, minibatch dim optional. kernel_size: size of the pooling region. Can be a single number or a tuple `(kH, kW)` stride: stride of the pooling operation. Can be a single number or a tuple `(sH, sW)`. Default: :attr:`kernel_size` padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. dilation: The stride between elements within a sliding window, must be > 0. ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This ensures that every element in the input tensor is covered by a sliding window. return_indices: If ``True``, will return the argmax along with the max values. Useful for :class:`torch.nn.functional.max_unpool2d` later rY) rrmax_pool2d_with_indicesr7r[r\r]r4r9r:r^s r?rfrfD &$ # H  )  ~##DIr2 88<< / / {FGXy r@c t|rtt|f||||||| S|+tjj t tg}tj||||||Sr`)rr max_pool2dr7r[r\r]r4r^s r? _max_pool2drj rcr@ric t|rtt|f||||||| S|+tjj t tg}tjjj||||||S)ay max_pool3d(input, kernel_size, stride=None, padding=0, dilation=1, ceil_mode=False, return_indices=False) Applies a 3D max pooling over an input signal composed of several input planes. .. note:: The order of :attr:`ceil_mode` and :attr:`return_indices` is different from what seen in :class:`~torch.nn.MaxPool3d`, and will change in a future release. See :class:`~torch.nn.MaxPool3d` for details. Args: input: input tensor :math:`(\text{minibatch} , \text{in\_channels} , iD, iH , iW)`, minibatch dim optional. kernel_size: size of the pooling region. Can be a single number or a tuple `(kT, kH, kW)` stride: stride of the pooling operation. Can be a single number or a tuple `(sT, sH, sW)`. Default: :attr:`kernel_size` padding: Implicit negative infinity padding to be added on both sides, must be >= 0 and <= kernel_size / 2. dilation: The stride between elements within a sliding window, must be > 0. ceil_mode: If ``True``, will use `ceil` instead of `floor` to compute the output shape. This ensures that every element in the input tensor is covered by a sliding window. return_indices: If ``True``, will return the argmax along with the max values. Useful for :class:`torch.nn.functional.max_unpool3d` later rY) rrmax_pool3d_with_indicesr7r[r\r]r4r9r:r^s r?rlrlErgr@c t|rtt|f||||||| S|+tjj t tg}tj||||||Sr`)rr max_pool3dr7r[r\r]r4r^s r? _max_pool3drozrcr@rnc |j}tjjtt g}t t|D];}|j|t| |zdz ||z||zd||zz =||}|St|t|dzk(r|dd}t|t|k7r3tdt|dt|dzdt|dt t|D]@}||||z } ||||z} | ||cxkr| kr(ntd|d|d | d | d |}|S) Nr+r(z,output_size should be a sequence containing z or z# elements, but it has a length of ''zinvalid output_size "z" (dim z must be between  and )) r5r7r[r\r]r4ranger3appendr2) rr rTrUr! input_size default_sizedretmin_sizemax_sizes r?_unpool_output_sizer|sJ99%%d3i4L 3{# $  [))A- . 2fQi ?!n '!*n   $ J! { s;/!3 3%ab/K { s;/ />{#$D[)9A)=(>>abefqbrasstv s;'( A#A2H#A2H{1~88 +K=sBST\S]]bckbllmn   Jr@indicesc t|rtt|f||||||St|}| t|}n|}t|}t |||||}t |t r|dgz}n|dz}tjjj|jd|jd|jdS)zjCompute a partial inverse of :class:`MaxPool1d`. See :class:`~torch.nn.MaxUnpool1d` for details. rTrUr!r+)r+r*) rr max_unpool1drr| isinstancer]r7r9r: max_unpool2d unsqueezesqueezerr}r rTrUr!_strides r?rrs &$  H   #  +&K &/gG%e['7KXK+t$!QC' !D( 88<< $ $ W..r2K  gbkr@c t|rtt|f||||||St|}| t|}n|}t|}t |||||}t j jj|||S)zjCompute a partial inverse of :class:`MaxPool2d`. See :class:`~torch.nn.MaxUnpool2d` for details. r)rrrrr|r7r9r:rs r?rrs &$  H   #   $K -GnG%e['7KXK 88<< $ $UG[ AAr@c  t|rtt|f||||||St|}| t|}n|}t|}t |||||}t j jj|||||S)zjCompute a partial inverse of :class:`MaxPool3d`. See :class:`~torch.nn.MaxUnpool3d` for details. r)rr max_unpool3drr|r7r9r:rs r?rr s &$  H   #  +&K &/gG%e['7KXK 88<< $ $UG['7 SSr@ norm_typec t|rtt|f|||||St|\}}}|t |j |||d|}nt |j ||d|}t j|tt j|zj||z|zj d|z S)z Apply a 3D power-average pooling over an input signal composed of several input planes. If the sum of all inputs to the power of `p` is zero, the gradient is set to zero as well. See :class:`~torch.nn.LPPool3d` for details. rTrWrrUrW?) rr lp_pool3dr avg_pool3dpowr7signreluabsmul) rrr rTrWkdkwkhouts r?rr-s &$  H     %JBB 9-{FAyQ IIi +qI  C4 #/ /44R"Wr\BFFsYWr@c t|rtt|f|||||St|\}}|t |j |||d|}nt |j ||d|}t j|tt j|zj||zj d|z S)z Apply a 2D power-average pooling over an input signal composed of several input planes. If the sum of all inputs to the power of `p` is zero, the gradient is set to zero as well. See :class:`~torch.nn.LPPool2d` for details. rrrr) rr lp_pool2dr avg_pool2drr7rrrr)rrr rTrWrrrs r?rrSs &$  H     ; FB 9-{FAyQ IIi +qI  JJsOd599S>2 2 7 7R @ D DS9_ UUr@c nt|rtt|f|||||S|t|j |||d|}nt|j ||d|}t j |tt j|zj|j d|z S)zApply a 1D power-average pooling over an input signal composed of several input planes. If the sum of all inputs to the power of `p` is zero, the gradient is set to zero as well. See :class:`~torch.nn.LPPool1d` for details. rrrr) rr lp_pool1d avg_pool1drr7rrrr)rrr rTrWrs r?rrws &$  H     9-{FAyQ IIi +qI  C4 #/ /44[AEEcIoVr@cnt|rtt|f|||Stj||S)a adaptive_max_pool1d(input, output_size, return_indices=False) Applies a 1D adaptive max pooling over an input signal composed of several input planes. See :class:`~torch.nn.AdaptiveMaxPool1d` for details and output shape. Args: output_size: the target output size (single integer) return_indices: whether to return pooling indices. Default: ``False`` r#)rr adaptive_max_pool1d_with_indicesr7adaptive_max_pool1drr!r#s r?rrs?" &$ , H  )     $ $UK 88r@c`t|rtt|f|||St||dSNrr)rrrrrs r?_adaptive_max_pool1dr>  &$  H  )    ,E; ? BBr@r(rct|rtt|f|||St||j }t j jj||S)aadaptive_max_pool2d(input, output_size, return_indices=False) Applies a 2D adaptive max pooling over an input signal composed of several input planes. See :class:`~torch.nn.AdaptiveMaxPool2d` for details and output shape. Args: output_size: the target output size (single integer or double-integer tuple) return_indices: whether to return pooling indices. Default: ``False`` r) rr adaptive_max_pool2d_with_indicesrr5r7r9r:adaptive_max_pool2drs r?rrsY" &$ , H  )   %[%**,?K 88<< + +E; ??r@c`t|rtt|f|||St||dSr)rrrrrs r?_adaptive_max_pool2drrr@rct|rtt|f|||St||j }t j jj||S)a adaptive_max_pool3d(input, output_size, return_indices=False) Applies a 3D adaptive max pooling over an input signal composed of several input planes. See :class:`~torch.nn.AdaptiveMaxPool3d` for details and output shape. Args: output_size: the target output size (single integer or triple-integer tuple) return_indices: whether to return pooling indices. Default: ``False`` r) rr adaptive_max_pool3d_with_indicesrr5r7r9r:adaptive_max_pool3drs r?rr sY$ &$ , H  )   %[%**,?K 88<< + +E; ??r@c`t|rtt|f|||St||dSr)rrrrrs r?_adaptive_max_pool3dr(rr@ra adaptive_avg_pool1d(input, output_size) -> Tensor Applies a 1D adaptive average pooling over an input signal composed of several input planes. See :class:`~torch.nn.AdaptiveAvgPool1d` for details and output shape. Args: output_size: the target output size (single integer) ct|rtt|f||St||j }t j jj||S)aApply a 2D adaptive average pooling over an input signal composed of several input planes. See :class:`~torch.nn.AdaptiveAvgPool2d` for details and output shape. Args: output_size: the target output size (single integer or double-integer tuple) )rradaptive_avg_pool2drr5r7r9r:rr! _output_sizes r?rrSM &$%85(E;WW%k5::<@L 88<< + +E< @@r@ct|rtt|f||St||j }t j jj||S)aApply a 3D adaptive average pooling over an input signal composed of several input planes. See :class:`~torch.nn.AdaptiveAvgPool3d` for details and output shape. Args: output_size: the target output size (single integer or triple-integer tuple) )rradaptive_avg_pool3drr5r7r9r:rs r?rrbrr@ptraininginplacect|rtt|f||||S|dks|dkDrtd||rt j |||St j|||S)aDuring training, randomly zeroes some elements of the input tensor with probability :attr:`p`. Uses samples from a Bernoulli distribution. See :class:`~torch.nn.Dropout` for details. Args: p: probability of an element to be zeroed. Default: 0.5 training: apply dropout if is ``True``. Default: ``True`` inplace: If set to ``True``, will do this operation in-place. Default: ``False`` rrrr7dropout probability has to be between 0 and 1, but got )rrdropoutr2rdropout_rrrrs r?rrrsx" &$ eXuHg   3w!c'RSTRUVWW,3 UAx(9<UAx9Xr@ct|rtt|f||||S|dks|dkDrtd||rt j |||St j|||S)z\Apply alpha dropout to the input. See :class:`~torch.nn.AlphaDropout` for details. rrrr)rr alpha_dropoutr2ralpha_dropout_rs r?rrs &$ E8Ua(G   3w!c'RSTRUVWW  5!X.  ua 2r@ct|rtt|f||||S|dks|dkDrtd||j }|dvrt d|d|dk(}|s$|r|j d n|jd }|rtj|||ntj|||}|s$|r|jd n|jd }|S) aRandomly zero out entire channels (a channel is a 1D feature map). For example, the :math:`j`-th channel of the :math:`i`-th sample in the batched input is a 1D tensor :math:`\text{input}[i, j]` of the input tensor. Each channel will be zeroed out independently on every forward call with probability :attr:`p` using samples from a Bernoulli distribution. See :class:`~torch.nn.Dropout1d` for details. Args: p: probability of a channel to be zeroed. Default: 0.5 training: apply dropout if is ``True``. Default: ``True`` inplace: If set to ``True``, will do this operation in-place. Default: ``False`` rrrrr(r,z3dropout1d: Expected 2D or 3D input, but received a zD input. Note that dropout1d exists to provide channel-wise dropout on inputs with 1 spatial dimension, a channel dimension, and an optional batch dimension (i.e. 2D or 3D inputs).r,r) rr dropout1dr2r6 RuntimeError unsqueeze_rrfeature_dropout_feature_dropoutsqueeze_r)rrrrinp_dim is_batchedresults r?rrs( &$ x!h   3w!c'RSTRUVWWiikGfA'K& &  AJ '.  #EOOA4F  UAx0  8 4 '.#FNN14E Mr@cnt|rtt|f||||S|dks|dkDrtd||j }|dvrd|d}t j ||dk(rt j d |rtj|||}|Stj|||}|S) aRandomly zero out entire channels (a channel is a 2D feature map). For example, the :math:`j`-th channel of the :math:`i`-th sample in the batched input is a 2D tensor :math:`\text{input}[i, j]` of the input tensor. Each channel will be zeroed out independently on every forward call with probability :attr:`p` using samples from a Bernoulli distribution. See :class:`~torch.nn.Dropout2d` for details. Args: p: probability of a channel to be zeroed. Default: 0.5 training: apply dropout if is ``True``. Default: ``True`` inplace: If set to ``True``, will do this operation in-place. Default: ``False`` rrrr)r,rFzdropout2d: Received a aU-D input to dropout2d, which is deprecated and will result in an error in a future release. To retain the behavior and silence this warning, please use dropout instead. Note that dropout2d exists to provide channel-wise dropout on inputs with 2 spatial dimensions, a channel dimension, and an optional batch dimension (i.e. 3D or 4D inputs).r,adropout2d: Received a 3D input to dropout2d and assuming that channel-wise 1D dropout behavior is desired - input is interpreted as shape (N, C, L), where C is the channel dim. This behavior will change in a future release to interpret the input as one without a batch dimension, i.e. shape (C, H, W). To maintain the 1D channel-wise dropout behavior, please switch to using dropout1d instead.) rr dropout2dr2r6warningswarnrrr)rrrrrwarn_msgrs r?rrs( &$ x!h   3w!c'RSTRUVWWiikGf$WI.[ [   h !|  W   UAx0 M 8 4 Mr@ct|rtt|f||||S|dks|dkDrtd||j }|dvrd|d}t j ||dk(}|s$|r|jd n|jd }|rtj|||ntj|||}|s$|r|jd n|jd }|S) aRandomly zero out entire channels (a channel is a 3D feature map). For example, the :math:`j`-th channel of the :math:`i`-th sample in the batched input is a 3D tensor :math:`\text{input}[i, j]` of the input tensor. Each channel will be zeroed out independently on every forward call with probability :attr:`p` using samples from a Bernoulli distribution. See :class:`~torch.nn.Dropout3d` for details. Args: p: probability of a channel to be zeroed. Default: 0.5 training: apply dropout if is ``True``. Default: ``True`` inplace: If set to ``True``, will do this operation in-place. Default: ``False`` rrrr)rFzdropout3d: Received a aU-D input to dropout3d, which is deprecated and will result in an error in a future release. To retain the behavior and silence this warning, please use dropout instead. Note that dropout3d exists to provide channel-wise dropout on inputs with 3 spatial dimensions, a channel dimension, and an optional batch dimension (i.e. 4D or 5D inputs).rr)rr dropout3dr2r6rrrrrrrrr)rrrrrrrrs r?rrs( &$ x!h   3w!c'RSTRUVWWiikGf$WI.[ [   hAJ '.  #EOOA4F  UAx0  8 4 '.#FNN14E Mr@ct|rtt|f||||S|dks|dkDrtd||rt j |||St j|||S)aRandomly masks out entire channels (a channel is a feature map). For example, the :math:`j`-th channel of the :math:`i`-th sample in the batch input is a tensor :math:`\text{input}[i, j]` of the input tensor. Instead of setting activations to zero, as in regular Dropout, the activations are set to the negative saturation value of the SELU activation function. Each element will be masked independently on every forward call with probability :attr:`p` using samples from a Bernoulli distribution. The elements to be masked are randomized on every forward call, and scaled and shifted to maintain zero mean and unit variance. See :class:`~torch.nn.FeatureAlphaDropout` for details. Args: p: dropout probability of a channel to be zeroed. Default: 0.5 training: apply dropout if is ``True``. Default: ``True`` inplace: If set to ``True``, will do this operation in-place. Default: ``False`` rrrr)rrfeature_alpha_dropoutr2rfeature_alpha_dropout_rs r?rrGs2 &$ ! H     3w!c'RSTRUVWW  ""5!X6 & &ua :r@ thresholdvaluect|rtt|f||||S|rtj|||}|Stj |||}|S)zsApply a threshold to each element of the input Tensor. See :class:`~torch.nn.Threshold` for more details. r)rr _thresholdr threshold_r)rrrrrs r?rrrs_ &$ %E7  y%8 Mui7 Mr@zX threshold_(input, threshold, value) -> Tensor In-place version of :func:`~threshold`. ct|rtt|f||S|rtj|}|Stj|}|S)zrelu(input, inplace=False) -> Tensor Applies the rectified linear unit function element-wise. See :class:`~torch.nn.ReLU` for more details. r)rrrr7relu_rrrs r?rrsL  &$TE8UGLLU# ME" Mr@z< relu_(input) -> Tensor In-place version of :func:`~relu`. r6ct|rtt|f||S|jdk(r t dt j jj||S)a glu(input, dim=-1) -> Tensor The gated linear unit. Computes: .. math :: \text{GLU}(a, b) = a \otimes \sigma(b) where `input` is split in half along `dim` to form `a` and `b`, :math:`\sigma` is the sigmoid function and :math:`\otimes` is the element-wise product between matrices. See `Language Modeling with Gated Convolutional Networks `_. Args: input (Tensor): input tensor dim (int): dimension on which to split the input. Default: -1 r6rz>glu does not support scalars because halving size must be even)rrglur6rr7r9r:)rr6s r?rrsZ$ &$S5(EsCC yy{a L   88<<  E3 ''r@rmin_valmax_valct|rtt|f||||S||kDr td|r-tj j j|||}|Stj j j|||}|S)z hardtanh(input, min_val=-1., max_val=1., inplace=False) -> Tensor Applies the HardTanh function element-wise. See :class:`~torch.nn.Hardtanh` for more details. )rrrz&min_val cannot be greater than max_val)rrhardtanhr2r7r9r: hardtanh_)rrrrrs r?rrs &$ uhwQX  ABB''w@ M&&ugw? Mr@z] hardtanh_(input, min_val=-1., max_val=1.) -> Tensor In-place version of :func:`~hardtanh`. ct|rtt|f||S|r+tjj j |}|Stjj j|}|S)zrelu6(input, inplace=False) -> Tensor Applies the element-wise function :math:`\text{ReLU6}(x) = \min(\max(0,x), 6)`. See :class:`~torch.nn.ReLU6` for more details. r)rrrelu6r7r9r:relu6_rs r?rrs` &$UUHeWMM$$U+ M##E* Mr@alphact|rtt|f|||S|r,tjj j ||}|Stjj j||}|S)zuApply the Exponential Linear Unit (ELU) function element-wise. See :class:`~torch.nn.ELU` for more details. rr)rrelur7r9r:elu_rrrrs r?rrsg  &$S5(EPWXX""5%0 M!!%/ Mr@zD elu_(input, alpha=1.) -> Tensor In-place version of :func:`~elu`. ct|rtt|f||S|rtj|}|Stj|}|S)a>selu(input, inplace=False) -> Tensor Applies element-wise, :math:`\text{SELU}(x) = scale * (\max(0,x) + \min(0, \alpha * (\exp(x) - 1)))`, with :math:`\alpha=1.6732632423543772848170429916717` and :math:`scale=1.0507009873554804934193349852946`. See :class:`~torch.nn.SELU` for more details. r)rrselur7selu_rs r?rrsL &$TE8UGLLU# ME" Mr@z< selu_(input) -> Tensor In-place version of :func:`~selu`. ct|rtt|f|||S|rtj||}|Stj||}|S)zcelu(input, alpha=1., inplace=False) -> Tensor Applies element-wise, :math:`\text{CELU}(x) = \max(0,x) + \min(0, \alpha * (\exp(x/\alpha) - 1))`. See :class:`~torch.nn.CELU` for more details. r)rrcelur7celu_rs r?rr2sY &$ 5(E  UE* ME5) Mr@zF celu_(input, alpha=1.) -> Tensor In-place version of :func:`~celu`. negative_slopect|rtt|f|||S|r,tjj j ||}|Stjj j||}|S)z leaky_relu(input, negative_slope=0.01, inplace=False) -> Tensor Applies element-wise, :math:`\text{LeakyReLU}(x) = \max(0, x) + \text{negative\_slope} * \min(0, x)` See :class:`~torch.nn.LeakyReLU` for more details. )rr)rr leaky_relur7r9r: leaky_relu_)rrrrs r?rrSsn &$ %PW  ))%@ M((? Mr@z] leaky_relu_(input, negative_slope=0.01) -> Tensor In-place version of :func:`~leaky_relu`. aprelu(input, weight) -> Tensor Applies element-wise the function :math:`\text{PReLU}(x) = \max(0,x) + \text{weight} * \min(0,x)` where weight is a learnable parameter. .. note:: `weight` is expected to be a scalar or 1-D tensor. If `weight` is 1-D, its size must match the number of input channels, determined by `input.size(1)` when `input.dim() >= 2`, otherwise 1. In the 1-D case, note that when `input` has dim > 2, `weight` can be expanded to the shape of `input` in a way that is not possible using normal :ref:`broadcasting semantics`. See :class:`~torch.nn.PReLU` for more details. lowerupperc t|rtt|f|||||S|rtj||||}|Stj||||}|S)zrrelu(input, lower=1./8, upper=1./3, training=False, inplace=False) -> Tensor Randomized leaky ReLU. See :class:`~torch.nn.RReLU` for more details. )rrrr)rrrrelur7rrelu_)rrrrrrs r?rrsk &$  H   eUE8< MUE5(; Mr@zf rrelu_(input, lower=1./8, upper=1./3, training=False) -> Tensor In-place version of :func:`~rrelu`. z logsigmoid(input) -> Tensor Applies element-wise :math:`\text{LogSigmoid}(x_i) = \log \left(\frac{1}{1 + \exp(-x_i)}\right)` See :class:`~torch.nn.LogSigmoid` for more details. a gelu(input, approximate = 'none') -> Tensor When the approximate argument is 'none', it applies element-wise the function :math:`\text{GELU}(x) = x * \Phi(x)` where :math:`\Phi(x)` is the Cumulative Distribution Function for Gaussian Distribution. When the approximate argument is 'tanh', Gelu is estimated with .. math:: \text{GELU}(x) = 0.5 * x * (1 + \text{Tanh}(\sqrt{2 / \pi} * (x + 0.044715 * x^3))) See `Gaussian Error Linear Units (GELUs) `_. z hardshrink(input, lambd=0.5) -> Tensor Applies the hard shrinkage function element-wise See :class:`~torch.nn.Hardshrink` for more details. cbt|rtt|f|S||jz S)ztanhshrink(input) -> Tensor Applies element-wise, :math:`\text{Tanhshrink}(x) = x - \text{Tanh}(x)` See :class:`~torch.nn.Tanhshrink` for more details. )rr tanhshrinktanhrs r?rrs. &$Z%5AA 5::< r@cht|rtt|f|S||jdzz S)zsoftsign(input) -> Tensor Applies element-wise, the function :math:`\text{SoftSign}(x) = \frac{x}{1 + |x|}` See :class:`~torch.nn.Softsign` for more details. r+)rrsoftsignrr s r?r r s2 &$Xx?? EIIK!O $$r@aJ softplus(input, beta=1, threshold=20) -> Tensor Applies element-wise, the function :math:`\text{Softplus}(x) = \frac{1}{\beta} * \log(1 + \exp(\beta * x))`. For numerical stability the implementation reverts to the linear function when :math:`input \times \beta > threshold`. See :class:`~torch.nn.Softplus` for more details. namendim stacklevelcftjd|d||dk(s |dk(s|dk(rd}|Sd}|S)NzImplicit dimension choice for zF has been deprecated. Change the call to include dim=X as an argument.rrr+r,)rr)r r rrys r?_get_softmax_dimrsO MM (/; ;  qyDAI J Jr@ _stacklevelr/ct|rtt|f||||S|td|j |}|| j |}|S| j ||}|S)atApply a softmin function. Note that :math:`\text{Softmin}(x) = \text{Softmax}(-x)`. See softmax definition for mathematical formula. See :class:`~torch.nn.Softmin` for more details. Args: input (Tensor): input dim (int): A dimension along which softmin will be computed (so every slice along dim will sum to 1). dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. If specified, the input tensor is casted to :attr:`dtype` before the operation is performed. This is useful for preventing data type overflows. Default: None. r6rr/softminr/)rrrrr6softmaxrr6rr/rys r?rrs( &$ eXu#;e   {y%))+{C }vs# Jvs%0 Jr@ct|rtt|f||||S|td|j |}||j|}|S|j||}|S)aApply a softmax function. Softmax is defined as: :math:`\text{Softmax}(x_{i}) = \frac{\exp(x_i)}{\sum_j \exp(x_j)}` It is applied to all slices along dim, and will re-scale them so that the elements lie in the range `[0, 1]` and sum to 1. See :class:`~torch.nn.Softmax` for more details. Args: input (Tensor): input dim (int): A dimension along which softmax will be computed. dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. If specified, the input tensor is casted to :attr:`dtype` before the operation is performed. This is useful for preventing data type overflows. Default: None. .. note:: This function doesn't work directly with NLLLoss, which expects the Log to be computed between the Softmax and itself. Use log_softmax instead (it's faster and has better numerical properties). rrr)rrrrr6rs r?rr0sv< &$ eXu#;e   {y%))+{C }mmC  JmmCum- Jr@logitstauhardepsc t|rtt|f|||||S|dk7rtjdt j |t jjj }||z|z }|j|}|rd|j|dd}t j|t jj||d}||jz |z} | S|} | S) a Sample from the Gumbel-Softmax distribution (`Link 1`_ `Link 2`_) and optionally discretize. Args: logits: `[..., num_features]` unnormalized log probabilities tau: non-negative scalar temperature hard: if ``True``, the returned samples will be discretized as one-hot vectors, but will be differentiated as if it is the soft sample in autograd dim (int): A dimension along which softmax will be computed. Default: -1. Returns: Sampled tensor of same shape as `logits` from the Gumbel-Softmax distribution. If ``hard=True``, the returned samples will be one-hot, otherwise they will be probability distributions that sum to 1 across `dim`. .. note:: This function is here for legacy reasons, may be removed from nn.Functional in the future. .. note:: The main trick for `hard` is to do `y_hard - y_soft.detach() + y_soft` It achieves two things: - makes the output value exactly one-hot (since we add then subtract y_soft value) - makes the gradient equal to y_soft gradient (since we strip all other gradients) Examples:: >>> logits = torch.randn(20, 32) >>> # Sample soft categorical using reparametrization trick: >>> F.gumbel_softmax(logits, tau=1, hard=False) >>> # Sample hard categorical using "Straight-through" trick: >>> F.gumbel_softmax(logits, tau=1, hard=True) .. _Link 1: https://arxiv.org/abs/1611.00712 .. _Link 2: https://arxiv.org/abs/1611.01144 )rrrr6绽|=z0`eps` parameter is deprecated and has no effect.) memory_formatTkeepdimr+r)rrgumbel_softmaxrrr7 empty_likelegacy_contiguous_format exponential_logrmax zeros_likescatter_detach) rrrrr6gumbelsy_softindexy_hardrys r?r#r#[s\ '$ VIv3TsPS   e| HI   &0N0N O    3&G __S !F  3 -a0!! %"@"@ (3s # v}}&/ J Jr@ct|rtt|f||||S|td|j |}||j|}|S|j||}|S)aApply a softmax followed by a logarithm. While mathematically equivalent to log(softmax(x)), doing these two operations separately is slower and numerically unstable. This function uses an alternative formulation to compute the output and gradient correctly. See :class:`~torch.nn.LogSoftmax` for more details. Args: input (Tensor): input dim (int): A dimension along which log_softmax will be computed. dtype (:class:`torch.dtype`, optional): the desired data type of returned tensor. If specified, the input tensor is cast to :attr:`dtype` before the operation is performed. This is useful for preventing data type overflows. Default: None. r log_softmaxr)rrr1rr6rs r?r1r1s|* &$ %5c{RW   {}eiik;G }$ J51 Jr@z softshrink(input, lambd=0.5) -> Tensor Applies the soft shrinkage function elementwise See :class:`~torch.nn.Softshrink` for more details. c"|jS)ztanh(input) -> Tensor Applies element-wise, :math:`\text{Tanh}(x) = \tanh(x) = \frac{\exp(x) - \exp(-x)}{\exp(x) + \exp(-x)}` See :class:`~torch.nn.Tanh` for more details. )rr s r?rrs ::<r@c"|jS)zsigmoid(input) -> Tensor Applies the element-wise function :math:`\text{Sigmoid}(x) = \frac{1}{1 + \exp(-x)}` See :class:`~torch.nn.Sigmoid` for more details. )sigmoidr s r?r4r4s ==?r@ct|rtt|f||S|r)tjj j |Stjj j|S)aApply the Hardsigmoid function element-wise. .. math:: \text{Hardsigmoid}(x) = \begin{cases} 0 & \text{if~} x \le -3, \\ 1 & \text{if~} x \ge +3, \\ x / 6 + 1 / 2 & \text{otherwise} \end{cases} Args: inplace: If set to ``True``, will do this operation in-place. Default: ``False`` See :class:`~torch.nn.Hardsigmoid` for more details. r)rr hardsigmoidr7r9r: hardsigmoid_rrs r?r6r6sT &$[5(E7SSxx||((// 88<< # #E **r@au linear(input, weight, bias=None) -> Tensor Applies a linear transformation to the incoming data: :math:`y = xA^T + b`. This operation supports 2-D :attr:`weight` with :ref:`sparse layout` {sparse_beta_warning} This operator supports :ref:`TensorFloat32`. Shape: - Input: :math:`(*, in\_features)` where `*` means any number of additional dimensions, including none - Weight: :math:`(out\_features, in\_features)` or :math:`(in\_features)` - Bias: :math:`(out\_features)` or :math:`()` - Output: :math:`(*, out\_features)` or :math:`(*)`, based on the shape of the weight a bilinear(input1, input2, weight, bias=None) -> Tensor Applies a bilinear transformation to the incoming data: :math:`y = x_1^T A x_2 + b` Shape: - input1: :math:`(N, *, H_{in1})` where :math:`H_{in1}=\text{in1\_features}` and :math:`*` means any number of additional dimensions. All but the last dimension of the inputs should be the same. - input2: :math:`(N, *, H_{in2})` where :math:`H_{in2}=\text{in2\_features}` - weight: :math:`(\text{out\_features}, \text{in1\_features}, \text{in2\_features})` - bias: :math:`(\text{out\_features})` - output: :math:`(N, *, H_{out})` where :math:`H_{out}=\text{out\_features}` and all but the last dimension are the same shape as the input. ct|rtt|f||S|r)tjj j |Stjj j|S)aApply the Sigmoid Linear Unit (SiLU) function, element-wise. The SiLU function is also known as the swish function. .. math:: \text{silu}(x) = x * \sigma(x), \text{where } \sigma(x) \text{ is the logistic sigmoid.} .. note:: See `Gaussian Error Linear Units (GELUs) `_ where the SiLU (Sigmoid Linear Unit) was originally coined, and see `Sigmoid-Weighted Linear Units for Neural Network Function Approximation in Reinforcement Learning `_ and `Swish: a Self-Gated Activation Function `_ where the SiLU was experimented with later. See :class:`~torch.nn.SiLU` for more details. r)rrsilur7r9r:silu_r8s r?r:r:- sT$ &$TE8UGLLxx||!!%(( 88<<  U ##r@ct|rtt|f||S|r)tjj j |Stjj j|S)auApply the Mish function, element-wise. Mish: A Self Regularized Non-Monotonic Neural Activation Function. .. math:: \text{Mish}(x) = x * \text{Tanh}(\text{Softplus}(x)) .. note:: See `Mish: A Self Regularized Non-Monotonic Neural Activation Function `_ See :class:`~torch.nn.Mish` for more details. r)rrmishr7r9r:mish_r8s r?r=r=F sT &$TE8UGLLxx||!!%(( 88<<  U ##r@ct|rtt|f||S|r)tjj j |Stjj j|S)aApply hardswish function, element-wise. Follows implementation as described in the paper: `Searching for MobileNetV3`_. .. math:: \text{Hardswish}(x) = \begin{cases} 0 & \text{if~} x \le -3, \\ x & \text{if~} x \ge +3, \\ x \cdot (x + 3) /6 & \text{otherwise} \end{cases} See :class:`~torch.nn.Hardswish` for more details. .. _`Searching for MobileNetV3`: https://arxiv.org/abs/1905.02244 r)rr hardswishr7r9r: hardswish_r8s r?r@r@Z sT$ &$Y%QQxx||&&u-- 88<< ! !% ((r@weightmax_normcPtj|j|||yN)r7embedding_renorm_r+)rBrrCrs r?_no_grad_embedding_renorm_rGs s  FMMOUHiHr@ padding_idxscale_grad_by_freqsparsec tt||rtt||f||||||| S|V|dkDr||jdks?Jd|dkr3||jd k\sJd|jd|z}nd}||j }t ||||t j|||||S)awGenerate a simple lookup table that looks up embeddings in a fixed dictionary and size. This module is often used to retrieve word embeddings using indices. The input to the module is a list of indices, and the embedding matrix, and the output is the corresponding word embeddings. See :class:`torch.nn.Embedding` for more details. .. note:: Note that the analytical gradients of this function with respect to entries in :attr:`weight` at the row specified by :attr:`padding_idx` are expected to differ from the numerical ones. .. note:: Note that `:class:`torch.nn.Embedding` differs from this function in that it initializes the row of :attr:`weight` specified by :attr:`padding_idx` to all zeros on construction. Args: input (LongTensor): Tensor containing indices into the embedding matrix weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1, and number of columns equal to the embedding size padding_idx (int, optional): If specified, the entries at :attr:`padding_idx` do not contribute to the gradient; therefore, the embedding vector at :attr:`padding_idx` is not updated during training, i.e. it remains as a fixed "pad". max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm` is renormalized to have norm :attr:`max_norm`. Note: this will modify :attr:`weight` in-place. norm_type (float, optional): The p of the p-norm to compute for the :attr:`max_norm` option. Default ``2``. scale_grad_by_freq (bool, optional): If given, this will scale gradients by the inverse of frequency of the words in the mini-batch. Default ``False``. sparse (bool, optional): If ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under :class:`torch.nn.Embedding` for more details regarding sparse gradients. Shape: - Input: LongTensor of arbitrary shape containing the indices to extract - Weight: Embedding matrix of floating point type with shape `(V, embedding_dim)`, where V = maximum index + 1 and embedding_dim = the embedding size - Output: `(*, embedding_dim)`, where `*` is the input shape Examples:: >>> # a batch of 2 samples of 4 indices each >>> input = torch.tensor([[1, 2, 4, 5], [4, 3, 2, 9]]) >>> # an embedding matrix containing 10 tensors of size 3 >>> embedding_matrix = torch.rand(10, 3) >>> # xdoctest: +IGNORE_WANT("non-deterministic") >>> F.embedding(input, embedding_matrix) tensor([[[ 0.8490, 0.9625, 0.6753], [ 0.9666, 0.7761, 0.6108], [ 0.6246, 0.9751, 0.3618], [ 0.4161, 0.2419, 0.7383]], [[ 0.6246, 0.9751, 0.3618], [ 0.0237, 0.7794, 0.0528], [ 0.9666, 0.7761, 0.6108], [ 0.3385, 0.8612, 0.1867]]]) >>> # example with padding_idx >>> weights = torch.rand(10, 3) >>> weights[0, :].zero_() >>> embedding_matrix = weights >>> input = torch.tensor([[0, 2, 0, 5]]) >>> F.embedding(input, embedding_matrix, padding_idx=0) tensor([[[ 0.0000, 0.0000, 0.0000], [ 0.5609, 0.5384, 0.8720], [ 0.0000, 0.0000, 0.0000], [ 0.6262, 0.2438, 0.7471]]]) )rHrCrrIrJrz)Padding_idx must be within num_embeddingsr*)rr embeddingr5 contiguousrGr7)rrBrHrCrrIrJs r?rLrL| s\#5&1$  FO  #1   ?Q/ ; /1_6;;q>/1 ; 1!++a.;6K    " #65(IF ??65+7I6 RRr@meanoffsetsmodeper_sample_weightsinclude_last_offsetc zt||||r tt||||f|||||||||| |  S|jtj k(r)|j rtjd||}}|G|j|jk7r&td|jd|jd|jdk(std|jtjjs|jdk(rx|jrld} |j!}|j#j%d }|Y|js td |j#j%d }n!|jdk(r|Bd } tjjst't)|} td | t j*d |j-|jd|j|j.}|j%d }|l|j%d }nZ|jdk(r+| td|jdk7r'tdtd|j|dk(rd } n4|dk(rd} n,|dk(rd} |r td|rtdtd|t1||||||dk7rt3d|dt j||||| ||| | \} }}}| S)aWCompute sums, means or maxes of `bags` of embeddings. Calculation is done without instantiating the intermediate embeddings. See :class:`torch.nn.EmbeddingBag` for more details. Note: {backward_reproducibility_note} Args: input (LongTensor): Tensor containing bags of indices into the embedding matrix weight (Tensor): The embedding matrix with number of rows equal to the maximum possible index + 1, and number of columns equal to the embedding size offsets (LongTensor, optional): Only used when :attr:`input` is 1D. :attr:`offsets` determines the starting index position of each bag (sequence) in :attr:`input`. max_norm (float, optional): If given, each embedding vector with norm larger than :attr:`max_norm` is renormalized to have norm :attr:`max_norm`. Note: this will modify :attr:`weight` in-place. norm_type (float, optional): The ``p`` in the ``p``-norm to compute for the :attr:`max_norm` option. Default ``2``. scale_grad_by_freq (bool, optional): if given, this will scale gradients by the inverse of frequency of the words in the mini-batch. Default ``False``. Note: this option is not supported when ``mode="max"``. mode (str, optional): ``"sum"``, ``"mean"`` or ``"max"``. Specifies the way to reduce the bag. Default: ``"mean"`` sparse (bool, optional): if ``True``, gradient w.r.t. :attr:`weight` will be a sparse tensor. See Notes under :class:`torch.nn.Embedding` for more details regarding sparse gradients. Note: this option is not supported when ``mode="max"``. per_sample_weights (Tensor, optional): a tensor of float / double weights, or None to indicate all weights should be taken to be 1. If specified, :attr:`per_sample_weights` must have exactly the same shape as input and is treated as having the same :attr:`offsets`, if those are not None. include_last_offset (bool, optional): if ``True``, the size of offsets is equal to the number of bags + 1. The last element is the size of the input, or the ending index position of the last bag (sequence). padding_idx (int, optional): If specified, the entries at :attr:`padding_idx` do not contribute to the gradient; therefore, the embedding vector at :attr:`padding_idx` is not updated during training, i.e. it remains as a fixed "pad". Note that the embedding vector at :attr:`padding_idx` is excluded from the reduction. Shape: - :attr:`input` (LongTensor) and :attr:`offsets` (LongTensor, optional) - If :attr:`input` is 2D of shape `(B, N)`, it will be treated as ``B`` bags (sequences) each of fixed length ``N``, and this will return ``B`` values aggregated in a way depending on the :attr:`mode`. :attr:`offsets` is ignored and required to be ``None`` in this case. - If :attr:`input` is 1D of shape `(N)`, it will be treated as a concatenation of multiple bags (sequences). :attr:`offsets` is required to be a 1D tensor containing the starting index positions of each bag in :attr:`input`. Therefore, for :attr:`offsets` of shape `(B)`, :attr:`input` will be viewed as having ``B`` bags. Empty bags (i.e., having 0-length) will have returned vectors filled by zeros. - :attr:`weight` (Tensor): the learnable weights of the module of shape `(num_embeddings, embedding_dim)` - :attr:`per_sample_weights` (Tensor, optional). Has the same shape as :attr:`input`. - :attr:`output`: aggregated embedding values of shape `(B, embedding_dim)` Examples:: >>> # an Embedding module containing 10 tensors of size 3 >>> embedding_matrix = torch.rand(10, 3) >>> # a batch of 2 samples of 4 indices each >>> input = torch.tensor([1, 2, 4, 5, 4, 3, 2, 9]) >>> offsets = torch.tensor([0, 4]) >>> # xdoctest: +IGNORE_WANT("non-deterministic") >>> F.embedding_bag(input, embedding_matrix, offsets) tensor([[ 0.3397, 0.3552, 0.5545], [ 0.5893, 0.4386, 0.5882]]) >>> # example with padding_idx >>> embedding_matrix = torch.rand(10, 3) >>> input = torch.tensor([2, 2, 2, 2, 4, 3, 2, 9]) >>> offsets = torch.tensor([0, 4]) >>> F.embedding_bag(input, embedding_matrix, offsets, padding_idx=2, mode='sum') tensor([[ 0.0000, 0.0000, 0.0000], [-0.7082, 3.2145, -2.6251]]) ) rOrCrrIrPrJrQrRrHzArgument order of nn.functional.embedding_bag was changed. Usage `embedding_bag(weight, input, ...)` is deprecated, and should now be `embedding_bag(input, weight, ...)`.z&embedding_bag: If per_sample_weights (z>) is not None, then it must have the same shape as the input (rsr(z:weight has to be a 2D Tensor, but got Tensor of dimension Tr*zGIf input is nested, then per_sample_weights must be nested if specifiedz zif input is 2D, then offsets has to be None, as input is treated is a mini-batch of fixed length sequences. However, found offsets of type rr+r.z*offsets has to be a 1D Tensor but got Nonezoffsets has to be a 1D Tensorz=input has to be 1D or 2D Tensor, but got Tensor of dimension sumrNr(z?max mode does not support scaling the gradient by the frequencyz(max mode does not support sparse weightsz&mode has to be one of sum, mean or maxzoembedding_bag: per_sample_weights was not None. per_sample_weights is only supported for mode='sum' (got mode='z,'). Please open a feature request on GitHub.)rr embedding_bagr/r7longis_floating_pointrrr5r2shaper6r[ is_scripting is_nestedrOvaluesreshapestrtypearangenumelr0rGNotImplementedError)rrBrOrCrrIrPrJrQrRrHtype_str mode_enumry_s r?rUrU six#5&';MN$  FG%7 8  11 3#  $||uzz!e&=&=&?  E v%%**,:L:Q:Q:S*S45G5M5M4NO>>Ckk]! M  ::<1 H W   99 ! ! # q(8U__"--/ &&r*  )%// ]"4!:!:!vSV   >>%!163 ??r@ num_groupsc t|||rtt|||f|||||S|jdkrt d|jt |j d|j dz|z|gt|j ddztj|||||tjjjS)zxApply Group Normalization for last certain number of dimensions. See :class:`~torch.nn.GroupNorm` for details. r{r(z=Expected at least 2 dimensions for input tensor but received rr+N) rr group_normr6rrjr5r]r7rqrrrs)rrrBrmrs r?rrk s#5&$7$       yy{QKEIIK= Y   AA &* 4jA uzz|AB  !    z64enn.B.B.J.J r@betakc t|rtt|f|||||S|j}|dkrt d|d|j dk(r|S|j |}|dk(rJ|jd}t|dd|dz|dz dzf}t||dfdjd}nz|j}|j|dd|d|dd }t|dddd|dz|dz dzf}t||ddfdjd}|j|}|j |j|j|}||z S) a Apply local response normalization over an input signal. The input signal is composed of several input planes, where channels occupy the second dimension. Normalization is applied across channels. See :class:`~torch.nn.LocalResponseNorm` for details. )rrrr,zIExpected 3D or higher dimensionality input (got z dimensions)rr+r()rTr*)rrlocal_response_normr6r2r`rrpadrrr5viewraddr)rr5rrrr6divsizess r?rr s| &$ %5$e$RS   ))+C QwWX[W\\h i   {{} ))E C axmmA#1dai$(q9:tQi2::1= hhuQxE!HeAh;#1aDAIqQ?@tQl15==a@hhuo ''%.  Q  # #D )C 3;r@ log_probstargets input_lengthstarget_lengthsblank reduction zero_infinityc t||||rtt||||f||||||| Stj|||||t j ||S)a(Compute the Connectionist Temporal Classification loss. See :class:`~torch.nn.CTCLoss` for details. Note: {cudnn_reproducibility_note} Note: {backward_reproducibility_note} Args: log_probs: :math:`(T, N, C)` or :math:`(T, C)` where `C = number of characters in alphabet including blank`, `T = input length`, and `N = batch size`. The logarithmized probabilities of the outputs (e.g. obtained with :func:`torch.nn.functional.log_softmax`). targets: :math:`(N, S)` or `(sum(target_lengths))`. May be an empty tensor if all entries in `target_lengths` are zero. In the second form, the targets are assumed to be concatenated. input_lengths: :math:`(N)` or :math:`()`. Lengths of the inputs (must each be :math:`\leq T`) target_lengths: :math:`(N)` or :math:`()`. Lengths of the targets blank (int, optional): Blank label. Default :math:`0`. reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the output losses will be divided by the target lengths and then the mean over the batch is taken, ``'sum'``: the output will be summed. Default: ``'mean'`` zero_infinity (bool, optional): Whether to zero infinite losses and the associated gradients. Default: ``False`` Infinite losses mainly occur when the inputs are too short to be aligned to the targets. Example:: >>> log_probs = torch.randn(50, 16, 20).log_softmax(2).detach().requires_grad_() >>> targets = torch.randint(1, 20, (16, 30), dtype=torch.long) >>> input_lengths = torch.full((16,), 50, dtype=torch.long) >>> target_lengths = torch.randint(10, 30, (16,), dtype=torch.long) >>> loss = F.ctc_loss(log_probs, targets, input_lengths, target_lengths) >>> loss.backward() )rrr)rrctc_lossr7 _Reductionget_enum)rrrrrrrs r?rr swj#9g}nU$   ?    '   >> I& r@target size_average ignore_indexreducec t|||rtt|||f||||||| S||tj||}t j jj|||tj||S)aCompute the negative log likelihood loss. See :class:`~torch.nn.NLLLoss` for details. Args: input: :math:`(N, C)` where `C = number of classes` or :math:`(N, C, H, W)` in case of 2D Loss, or :math:`(N, C, d_1, d_2, ..., d_K)` where :math:`K \geq 1` in the case of K-dimensional loss. `input` is expected to be log-probabilities. target: :math:`(N)` where each value is :math:`0 \leq \text{targets}[i] \leq C-1`, or :math:`(N, d_1, d_2, ..., d_K)` where :math:`K \geq 1` for K-dimensional loss. weight (Tensor, optional): A manual rescaling weight given to each class. If given, has to be a Tensor of size `C` size_average (bool, optional): Deprecated (see :attr:`reduction`). ignore_index (int, optional): Specifies a target value that is ignored and does not contribute to the input gradient. When :attr:`size_average` is ``True``, the loss is averaged over non-ignored targets. Default: -100 reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the sum of the output will be divided by the number of elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, and in the meantime, specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` Example:: >>> # input is of size N x C = 3 x 5 >>> input = torch.randn(3, 5, requires_grad=True) >>> # each element in target has to have 0 <= value < C >>> target = torch.tensor([1, 0, 4]) >>> output = F.nll_loss(F.log_softmax(input, dim=1), target) >>> output.backward() )rBrrrr) rrnll_lossrlegacy_get_stringr7r9r: nll_loss_ndr)rrrBrrrrs r?rr sV#5&&9$  FF #  %%  6#500vF 88<< # # vvz229=| r@ log_inputfullc  t||rtt||f|||||||| S||tj||}|dk7r|dk7r|dk7r|}t |dzt j|||||tj|}|S)a+Compute the Poisson negative log likelihood loss. See :class:`~torch.nn.PoissonNLLLoss` for details. Args: input: Expectation of underlying Poisson distribution. target: Random sample :math:`target \sim \text{Poisson}(input)`. log_input: If ``True`` the loss is computed as :math:`\exp(\text{input}) - \text{target} * \text{input}`, if ``False`` then loss is :math:`\text{input} - \text{target} * \log(\text{input}+\text{eps})`. Default: ``True`` full: Whether to compute full loss, i. e. to add the Stirling approximation term. Default: ``False`` :math:`\text{target} * \log(\text{target}) - \text{target} + 0.5 * \log(2 * \pi * \text{target})`. size_average (bool, optional): Deprecated (see :attr:`reduction`). eps (float, optional): Small value to avoid evaluation of :math:`\log(0)` when :attr:`log_input`\ =\ ``False``. Default: 1e-8 reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the sum of the output will be divided by the number of elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, and in the meantime, specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` )rrrrrrnonerNrT# is not a valid value for reduction)rrpoisson_nll_lossrrr2r7r) rrrrrrrrrys r?rrL sF#5&1$  FO  %  6#500vF FyF2yE7I%JJKK   vy$Z-@-@-K C Jr@varc t|||rtt|||f||||||St|tr)|dkr t d|t j|z}n#t j|dkr t d|j|jk7r|jdd|jk(rt j|d}nG|jdd|jddk(r|jddk(rn t d|dk7r|d k7r|d k7rt |d z|j}t j5|j| dddd t j|||z dz|z zz}|r,|d tjdtj zzz }|d k(r|j#S|d k(r|j%S|S#1swYxYw)aCompute the Gaussian negative log likelihood loss. See :class:`~torch.nn.GaussianNLLLoss` for details. Args: input: Expectation of the Gaussian distribution. target: Sample from the Gaussian distribution. var: Tensor of positive variance(s), one for each of the expectations in the input (heteroscedastic), or a single one (homoscedastic), or a positive scalar value to be used for all expectations. full (bool, optional): Whether to include the constant term in the loss calculation. Default: ``False``. eps (float, optional): Value added to var, for stability. Default: 1e-6. reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the output is the average of all batch member losses, ``'sum'``: the output is the sum of all batch member losses. Default: ``'mean'``. )rrrrzvar has negative entry/entriesNr*r+zvar is of incorrect sizerrNrT is not valid)min?r()rrgaussian_nll_lossrfloatr2r7 ones_likeanyr5rcloneno_gradclamp_r'mathpirNrT)rrrrrrlosss r?rr s4#5&#6$  FC     #u 7=> >EOOE** 37 9::  xxzUZZ\! ::<  *//#r*C JJL" CR 0SXXb\Q5F 78 8FyF2yE7I_455 ))+C  s  %))C.EFNq#83#>> ?D  dhhq477{+++Fyy{ e xxz s (HH  log_targetc t||rtt||f||||||S||tj||}nJ|dk(rt j d|dk(rtjd}ntj|}tj||||}|dk(r)|jdk7r||jdz }|S)aCompute the KL Divergence loss. Refer - The `Kullback-Leibler divergence Loss `__ See :class:`~torch.nn.KLDivLoss` for details. Args: input: Tensor of arbitrary shape in log-probabilities. target: Tensor of the same shape as input. See :attr:`log_target` for the target's interpretation. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'batchmean'`` | ``'sum'`` | ``'mean'``. ``'none'``: no reduction will be applied ``'batchmean'``: the sum of the output will be divided by the batchsize ``'sum'``: the output will be summed ``'mean'``: the output will be divided by the number of elements in the output Default: ``'mean'`` log_target (bool): A flag indicating whether ``target`` is passed in the log space. It is recommended to pass certain distributions (like ``softmax``) in the log space to avoid numerical issues caused by explicit ``log``. Default: ``False`` .. note:: :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, and in the meantime, specifying either of those two args will override :attr:`reduction`. .. warning:: :attr:`reduction` = ``'mean'`` doesn't return the true kl divergence value, please use :attr:`reduction` = ``'batchmean'`` which aligns with KL math definition. )rrrrrNzreduction: 'mean' divides the total loss by both the batch size and the support size.'batchmean' divides only by the batch size, and aligns with the KL div math definition.'mean' will be changed to behave the same as 'batchmean' in the next major release. batchmeanrT)rr) rrkl_divrlegacy_get_enumrrrr7r6r5)rrrrrrreduction_enumreduceds r?rr sR#5&1$  FO  %!  6#5#33L&I   MMf   #'007N'00;Nll5&.ZPGKEIIK1$4EJJLO+ Nr@label_smoothingc  t|||rtt|||f|||||||| S||tj||}t j jj|||tj|||S)a Compute the cross entropy loss between input logits and target. See :class:`~torch.nn.CrossEntropyLoss` for details. Args: input (Tensor) : Predicted unnormalized logits; see Shape section below for supported shapes. target (Tensor) : Ground truth class indices or class probabilities; see Shape section below for supported shapes. weight (Tensor, optional): a manual rescaling weight given to each class. If given, has to be a Tensor of size `C` size_average (bool, optional): Deprecated (see :attr:`reduction`). ignore_index (int, optional): Specifies a target value that is ignored and does not contribute to the input gradient. When :attr:`size_average` is ``True``, the loss is averaged over non-ignored targets. Note that :attr:`ignore_index` is only applicable when the target contains class indices. Default: -100 reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the sum of the output will be divided by the number of elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, and in the meantime, specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` label_smoothing (float, optional): A float in [0.0, 1.0]. Specifies the amount of smoothing when computing the loss, where 0.0 means no smoothing. The targets become a mixture of the original ground truth and a uniform distribution as described in `Rethinking the Inception Architecture for Computer Vision `__. Default: :math:`0.0`. Shape: - Input: Shape :math:`(C)`, :math:`(N, C)` or :math:`(N, C, d_1, d_2, ..., d_K)` with :math:`K \geq 1` in the case of `K`-dimensional loss. - Target: If containing class indices, shape :math:`()`, :math:`(N)` or :math:`(N, d_1, d_2, ..., d_K)` with :math:`K \geq 1` in the case of K-dimensional loss where each value should be between :math:`[0, C)`. If containing class probabilities, same shape as the input and each value should be between :math:`[0, 1]`. where: .. math:: \begin{aligned} C ={} & \text{number of classes} \\ N ={} & \text{batch size} \\ \end{aligned} Examples:: >>> # Example of target with class indices >>> input = torch.randn(3, 5, requires_grad=True) >>> target = torch.randint(5, (3,), dtype=torch.int64) >>> loss = F.cross_entropy(input, target) >>> loss.backward() >>> >>> # Example of target with class probabilities >>> input = torch.randn(3, 5, requires_grad=True) >>> target = torch.randn(3, 5).softmax(dim=1) >>> loss = F.cross_entropy(input, target) >>> loss.backward() )rBrrrrr) rr cross_entropyrrr7r9r:cross_entropy_lossr)rrrBrrrrrs r?rr/ sH#5&&9$  FF #  %%+  6#500vF 88<< * * I&  r@c t|||rtt|||f||||||S||tj||}ntj |}|j |j k7r.td|j d|j d|9t|j |j }|j|}tjjj||||S)a3Compute Binary Cross Entropy between the target and input probabilities. See :class:`~torch.nn.BCELoss` for details. Args: input: Tensor of arbitrary shape as probabilities. target: Tensor of the same shape as input with values between 0 and 1. weight (Tensor, optional): a manual rescaling weight if provided it's repeated to match input tensor shape size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the sum of the output will be divided by the number of elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, and in the meantime, specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` Examples:: >>> input = torch.randn(3, 2, requires_grad=True) >>> target = torch.rand(3, 2, requires_grad=False) >>> loss = F.binary_cross_entropy(torch.sigmoid(input), target) >>> loss.backward() rBrrrUsing a target size (') that is different to the input size (z7) is deprecated. Please ensure they have the same size.) rrbinary_cross_entropyrrrr5r2r expandr7r9r:)rrrBrrrrnew_sizes r?rr sB#5&&9$ FF #  %  6#5#33L&I#,,Y7 {{} $#FKKM?2YZ_ZdZdZfYgh5 5  v{{}fkkm<x( 88<< , ,UFFN SSr@ pos_weightc t||||rtt||||f||||||| S||tj||}ntj |}|j |j k(s.td|j d|j dtj|||||S)aCompute Binary Cross Entropy between target and input logits. See :class:`~torch.nn.BCEWithLogitsLoss` for details. Args: input: Tensor of arbitrary shape as unnormalized scores (often referred to as logits). target: Tensor of the same shape as input with values between 0 and 1 weight (Tensor, optional): a manual rescaling weight if provided it's repeated to match input tensor shape size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: ``'none'`` | ``'mean'`` | ``'sum'``. ``'none'``: no reduction will be applied, ``'mean'``: the sum of the output will be divided by the number of elements in the output, ``'sum'``: the output will be summed. Note: :attr:`size_average` and :attr:`reduce` are in the process of being deprecated, and in the meantime, specifying either of those two args will override :attr:`reduction`. Default: ``'mean'`` pos_weight (Tensor, optional): a weight of positive examples to be broadcasted with target. Must be a tensor with equal size along the class dimension to the number of classes. Pay close attention to PyTorch's broadcasting semantics in order to achieve the desired operations. For a target of size [B, C, H, W] (where B is batch size) pos_weight of size [B, C, H, W] will apply different pos_weights to each element of the batch or [C, H, W] the same pos_weights across the batch. To apply the same positive weight along all spatial dimensions for a 2D multi-class target [C, H, W] use: [C, 1, 1]. Default: ``None`` Examples:: >>> input = torch.randn(3, requires_grad=True) >>> target = torch.empty(3).random_(2) >>> loss = F.binary_cross_entropy_with_logits(input, target) >>> loss.backward() )rBrrrrz Target size (z") must be the same as input size (rs) rr binary_cross_entropy_with_logitsrrrr5r2r7)rrrBrrrrrs r?rr sT#5&&*E$ , FFJ /  %!  6#5#33L&I#,,Y7 KKMUZZ\ )FKKM?*LUZZ\NZ[ \    1 1 vvz> r@c lt||rtt||f||||||S|j|jk(s:t j d|jd|jdd||t j||}tj||\}}|dk(r>tjjj||t j|Stjjj||t j||S)aCompute the Smooth L1 loss. Function uses a squared term if the absolute element-wise error falls below beta and an L1 term otherwise. See :class:`~torch.nn.SmoothL1Loss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. beta (float, optional): Specifies the threshold at which to change from the squared term to the L1 term in the loss calculation. This value must be positive. Default: 1.0. Returns: Tensor: L1 loss (optionally weighted). )rrrrrri). This will likely lead to incorrect results due to broadcasting. Please ensure they have the same size.r(rr)rrsmooth_l1_lossr5rrrrr7broadcast_tensorsr9r:l1_lossr)rrrrrrexpanded_inputexpanded_targets r?rrs#>#5&1$  FO  %   KKMUZZ\ ) #FKKM?2YZ_ZdZdZfYgh5 5  6#500vF &+&=&=eV&L#NO s{xx||## OZ-@-@-K  xx||** OZ-@-@-KT  r@deltac "t|||rtt|||f|||||S|j|jk(s:t j d|jd|jddt j||\}}|?t jjj||tj||S|j|jk7r tdt jjj||tjd|}||z}|dk(r|S|d k(rt j|S|d k(r|jStd |d ) aICompute the Huber loss, with optional weighting. Function uses a squared term if the absolute element-wise error falls below delta and a delta-scaled L1 term otherwise. When delta equals 1, this loss is equivalent to SmoothL1Loss. In general, Huber loss differs from SmoothL1Loss by a factor of delta (AKA beta in Smooth L1). See :class:`~torch.nn.HuberLoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. delta (float, optional): The threshold at which to change between delta-scaled L1 and L2 loss. Default: 1.0. weight (Tensor, optional): Weights for each sample. Default: None. Returns: Tensor: Huber loss (optionally weighted). )rrrBrrrr(r*Weights and input must have the same size.rrTrNInvalid reduction mode: (. Expected one of 'none', 'mean', 'sum'.)rr huber_lossr5rrr7rr9r:rrr2rTrN) rrrrrBrrunweighted_loss weighted_losss r?rrNs<#5&&9$  FF #     KKMUZZ\ ) #FKKM?2YZ_ZdZdZfYgh5 5  ',&=&=eV&L#NO ~xx||&& OZ-@-@-KU   ;;=EJJL (IJ J ((,,11 OZ-@-@-H%  (&0   % 99]+ + &  %%' '*9+5]^ r@c :t||rtt|||f|||||S|j|jk(s:t j d|jd|jdd||t j||}tj||\}}||j|jk7r tdtj||z }||z} |dk(r| S|d k(rtj| S|d k(r+tj| tj|z Std |d tjjj||t j|S) a~Compute the L1 loss, with optional weighting. Function that takes the mean element-wise absolute value difference. See :class:`~torch.nn.L1Loss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. weight (Tensor, optional): Weights for each sample. Default: None. Returns: Tensor: L1 loss (optionally weighted). rrrrrrr(rrrrTrNrr)rrrr5rrrrr7rr2rrTr9r:r) rrrrrrBrrabsolute_errorsweighted_absolute_errorss r?rrs8#5&1$  FF #  %   KKMUZZ\ ) #FKKM?2YZ_ZdZdZfYgh5 5  6#500vF &+&=&=eV&L#NO  ;;=EJJL (IJ J))N_$DE#2V#;  + + % 9956 6 & 995669JJ J*9+5]^ xx||## OZ-@-@-K  r@c @t|||rtt|||f||||||S|j|jk(s:t j d|jd|jdd||t j||}tj||\}}||j|jk7r tdtj||z d}||z} |dk(r| S|d k(rtj| S|d k(r+tj| tj|z Std |d tjjj||t j|S) a]Compute the element-wise mean squared error, with optional weighting. See :class:`~torch.nn.MSELoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. weight (Tensor, optional): Weights for each sample. Default: None. Returns: Tensor: Mean Squared Error loss (optionally weighted). )rrrrBrrrr(rrrrTrNrr)rrmse_lossr5rrrrr7rr2rrTr9r:r) rrrrrrBrrsquared_errorsweighted_squared_errorss r?rrs4#5&&9$  FF #  %   KKMUZZ\ ) #FKKM?2YZ_ZdZdZfYgh5 5  6#500vF &+&=&=eV&L#NO  ;;=EJJL (IJ J>O#CQG"06"9  * * % 9945 5 & 9945 &8II I*9+5]^ xx||$$ OZ-@-@-K  r@input1input2marginc t|||rtt|||f||||||| S||tj||}ntj |}|j |j k7s!|j |j k7r?td|jd|jd|jdtj|||||S)a Compute the margin ranking loss. See :class:`~torch.nn.MarginRankingLoss` for details. Args: input1 (Tensor): Predicted values. input2 (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Margin ranking loss. rrrrzZmargin_ranking_loss : All input tensors should have same dimension but got sizes: input1: z , input2: z , target:  ) rrmargin_ranking_lossrrrr6rr5r7rrrrrrrrs r?rr2s6#666:$  VV $   %  6#5#33L&I#,,Y7 zz|vzz|#vzz|vzz|'C{{}oZ jWX Z    $ $VVVV^ TTr@c t||rtt||f||||||S||tj||}ntj |}t j||||S)a)Compute the hinge embedding loss. See :class:`~torch.nn.HingeEmbeddingLoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. margin (float, optional): Margin for hinge loss. Has a default value of 1. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Hinge embedding loss. r)rrhinge_embedding_lossrrrr7)rrrrrrrs r?rres}4#5&1$ FO  %  6#5#33L&I#,,Y7  % %eVV^ LLr@c t||rtt||f|||||S||tj||}ntj |}t jjj|||S)aCompute the multilabel margin loss. See :class:`~torch.nn.MultiLabelMarginLoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Mutilabel margin loss. r) rrmultilabel_margin_lossrrrr7r9r:rrrrrrs r?rrs0#5&1$ " FO  %  6#5#33L&I#,,Y7 88<< . .ufn MMr@c t||rtt||f|||||S||tj||}ntj |}t jjj|||S)aCompute the soft margin loss. See :class:`~torch.nn.SoftMarginLoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Soft margin loss. r) rrsoft_margin_lossrrrr7r9r:rs r?rrs0#5&1$  FO  %  6#5#33L&I#,,Y7 88<< ( ( GGr@c t|||rtt|||f||||||S||tj||}|t |zd|z t | zz }|||z}|j dz }|j|}|j||z }|dk(r|} | S|dk(r|j} | S|dk(r|j} | S|} t|dz)aCompute the multilabel soft margin loss. See :class:`~torch.nn.MultiLabelSoftMarginLoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Mutilabel soft margin loss. rr+rrrNrTr) rrmultilabel_soft_margin_lossrr logsigmoidr6r5rTrNr2) rrrBrrrr class_dimCrys r?rrs&2#5&&9$ ' FF #  %  6#500vF j' '1v:UF9K*K K LD f} aI 9A 88 8 "Q &DF J f iik J e hhj J_455r@c t|||rtt|||f||||||| S||tj||}ntj |}t j|||||S)a]Compute the cosine embedding loss. See :class:`~torch.nn.CosineEmbeddingLoss` for details. Args: input1 (Tensor): Predicted values. input2 (Tensor): Predicted values. target (Tensor): Ground truth values. margin (float, optional): Margin for cosine embedding. Has a default value of 0. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Cosine embedding loss. r)rrcosine_embedding_lossrrrr7rs r?rrs8#666:$ ! VV $   %  6#5#33L&I#,,Y7  & &vvvv~ VVr@c zt|||rtt|||f|||||||| S||tj||}ntj |}|dk7r|dk7r t d||jdk7r t dtjjj||||||S)aCompute the multi margin loss, with optional weighting. See :class:`~torch.nn.MultiMarginLoss` for details. Args: input (Tensor): Predicted values. target (Tensor): Ground truth values. p (int, optional): Has a default value of 1. 1 and 2 are the only supported values. margin (float, optional): Margin for multi margin loss. Has a default value of 1. weight (Tensor, optional): Weights for each sample. Default: None. size_average (bool, optional): Deprecated (see :attr:`reduction`). reduce (bool, optional): Deprecated (see :attr:`reduction`). reduction (str, optional): Specifies the reduction to apply to the output: 'none' | 'mean' | 'sum'. 'mean': the mean of the output is taken. 'sum': the output will be summed. 'none': no reduction will be applied. Default: 'mean'. Returns: Tensor: Multi margin loss (optionally weighted). )rrrBrrrr+r(z only p == 1 and p == 2 supportedzweight must be one-dimensional) rrmulti_margin_lossrrrr2r6r7r9r:) rrrrrBrrrrs r?rrNs<#5&&9$  FF #  %  6#5#33L&I#,,Y7Av!q&;<<  ::<1 => > 88<< ) ) vq&&. r@a- pixel_shuffle(input, upscale_factor) -> Tensor Rearranges elements in a tensor of shape :math:`(*, C \times r^2, H, W)` to a tensor of shape :math:`(*, C, H \times r, W \times r)`, where r is the :attr:`upscale_factor`. See :class:`~torch.nn.PixelShuffle` for details. Args: input (Tensor): the input tensor upscale_factor (int): factor to increase spatial resolution by Examples:: >>> input = torch.randn(1, 9, 4, 4) >>> output = torch.nn.functional.pixel_shuffle(input, 3) >>> print(output.size()) torch.Size([1, 1, 12, 12]) at pixel_unshuffle(input, downscale_factor) -> Tensor Reverses the :class:`~torch.nn.PixelShuffle` operation by rearranging elements in a tensor of shape :math:`(*, C, H \times r, W \times r)` to a tensor of shape :math:`(*, C \times r^2, H, W)`, where r is the :attr:`downscale_factor`. See :class:`~torch.nn.PixelUnshuffle` for details. Args: input (Tensor): the input tensor downscale_factor (int): factor to increase spatial resolution by Examples:: >>> input = torch.randn(1, 1, 12, 12) >>> output = torch.nn.functional.pixel_unshuffle(input, 3) >>> print(output.size()) torch.Size([1, 9, 4, 4]) a5 channel_shuffle(input, groups) -> Tensor Divide the channels in a tensor of shape :math:`(*, C , H, W)` into g groups and rearrange them as :math:`(*, C \frac g, g, H, W)`, while keeping the original tensor shape. See :class:`~torch.nn.ChannelShuffle` for details. Args: input (Tensor): the input tensor groups (int): number of groups to divide channels in and rearrange. Examples:: >>> input = torch.randn(1, 4, 2, 2) >>> print(input) [[[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]], [[13, 14], [15, 16]], ]] >>> output = torch.nn.functional.channel_shuffle(input, 2) >>> print(output) [[[[1, 2], [3, 4]], [[9, 10], [11, 12]], [[5, 6], [7, 8]], [[13, 14], [15, 16]], ]] a native_channel_shuffle(input, groups) -> Tensor Native kernel level implementation of the `channel_shuffle`. This function might become private in future releases, use with caution. Divide the channels in a tensor of shape :math:`(*, C , H, W)` into g groups and rearrange them as :math:`(*, C \frac g, g, H, W)`, while keeping the original tensor shape. See :class:`~torch.nn.ChannelShuffle` for details. Args: input (Tensor): the input tensor groups (int): number of groups to divide channels in and rearrange. Examples:: >>> input = torch.randn(1, 4, 2, 2) >>> print(input) [[[[1, 2], [3, 4]], [[5, 6], [7, 8]], [[9, 10], [11, 12]], [[13, 14], [15, 16]], ]] >>> output = torch.nn.functional.native_channel_shuffle(input, 2) >>> print(output) [[[[1, 2], [3, 4]], [[9, 10], [11, 12]], [[5, 6], [7, 8]], [[13, 14], [15, 16]], ]] scale_factor align_cornerscyrErr5rrPrs r?upsampler r@cyrErrs r?rrrr@cNtjddt|||||S)a@ Upsample input. Provided tensor is upsampled to either the given :attr:`size` or the given :attr:`scale_factor` .. warning:: This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. This is equivalent with ``nn.functional.interpolate(...)``. Note: {backward_reproducibility_note} The algorithm used for upsampling is determined by :attr:`mode`. Currently temporal, spatial and volumetric upsampling are supported, i.e. expected inputs are 3-D, 4-D or 5-D in shape. The input dimensions are interpreted in the form: `mini-batch x channels x [optional depth] x [optional height] x width`. The modes available for upsampling are: `nearest`, `linear` (3D-only), `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only) Args: input (Tensor): the input tensor size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]): output spatial size. scale_factor (float or Tuple[float]): multiplier for spatial size. Has to match input size if it is a tuple. mode (str): algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | ``'trilinear'``. Default: ``'nearest'`` align_corners (bool, optional): Geometrically, we consider the pixels of the input and output as squares rather than points. If set to ``True``, the input and output tensors are aligned by the center points of their corner pixels, preserving the values at the corner pixels. If set to ``False``, the input and output tensors are aligned by the corner points of their corner pixels, and the interpolation uses edge value padding for out-of-boundary values, making this operation *independent* of input size when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode` is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``. Default: ``False`` .. note:: With ``mode='bicubic'``, it's possible to cause overshoot, in other words it can produce negative values or values greater than 255 for images. Explicitly call ``result.clamp(min=0, max=255)`` if you want to reduce the overshoot when displaying the image. .. warning:: With ``align_corners = True``, the linearly interpolating modes (`linear`, `bilinear`, and `trilinear`) don't proportionally align the output and input pixels, and thus the output values can depend on the input size. This was the default behavior for these modes up to version 0.3.1. Since then, the default behavior is ``align_corners = False``. See :class:`~torch.nn.Upsample` for concrete examples on how this affects the outputs. zP`nn.functional.upsample` is deprecated. Use `nn.functional.interpolate` instead.r(rrr interpolaters r?rr%s.B MM 3 udL$ FFr@ct|ttjfrytt|tj ryt|t xr|j S)zType check the input number is an integer. Will return True for int, SymInt, Numpy integers and Tensors with integer elements. T)rr4r7SymIntnpintegerr rW)xs r? _is_integerrrsM !c5<<() ~*Q 3 a >)<)<)>%>>r@recompute_scale_factor antialiascyrErrr5rrPrrr s r?rr~ r@cyrErr s r?rrr r@cyrErr s r?rrr r@cyrErr s r?rrr r@c \t|rtt|f||||||| S|dvr |td|d}|j dz }| | td||Jd}t |t tfrt||k7r(tdt |jddd |d tjjs7td |Ds%td |D cgc] } t| c} |} nt!|D cgc]} |} } n{|n|Jd} t |t tfr9t||k7r(td t |jddd|d|}n$t!|D cgc]} |}} n td||r | td|dk(r| d}|u|rr|Jtjjstj"j%rt!|D cgc]n} tj&|j)| dzj+tj,|| tj.zj+p} } ntjjrVt!|D cgc]A} t1t3j&t+|j)| dz|| zC} } n9t!|D cgc]%} t5|j)| dz|| z'} } d}|r|dvr|j6dk(s td|j dk(r0|dk(r+tj"j8j;|| |S|j dk(r0|dk(r+tj"j8j=|| |S|j dk(r0|dk(r+tj"j8j?|| |S|j dk(r0|dk(r+tj"j8jA|| |S|j dk(r0|dk(r+tj"j8jC|| |S|j dk(r0|dk(r+tj"j8jE|| |S|j dk(r|dk(r| JtG|| S|j dk(r|dk(r| JtI|| S|j dk(r|dk(r| JtK|| S|j dk(r5|dk(r0|Jtj"j8jM|| ||S|j dk(r|dk(r|J|r,tj"j8jO|| ||StjjsG|jPs;tjRr'tUjVdjY|| ||Stj"j8j[|| ||S|j dk(r|dk(r|JtjjsG|jPs;tjRr'tUjVdjY|| ||Stj"j8j]|| ||S|j dk(rc|d k(r^|J|r,tj"j8j_|| ||Stj"j8ja|| ||S|j dk(r|dk(r tcd!|j dk(r|dk(r tcd"|j dk(r|dk(r tcd#|j dk(r|dk(r tcd$|j dk(r|dk(r tcd%|j dk(r|dk(r tcd&tcd'|j d(|d)cc} wcc} wcc} wcc} wcc} wcc} w)*aDown/up samples the input. Tensor interpolated to either the given :attr:`size` or the given :attr:`scale_factor` The algorithm used for interpolation is determined by :attr:`mode`. Currently temporal, spatial and volumetric sampling are supported, i.e. expected inputs are 3-D, 4-D or 5-D in shape. The input dimensions are interpreted in the form: `mini-batch x channels x [optional depth] x [optional height] x width`. The modes available for resizing are: `nearest`, `linear` (3D-only), `bilinear`, `bicubic` (4D-only), `trilinear` (5D-only), `area`, `nearest-exact` Args: input (Tensor): the input tensor size (int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int]): output spatial size. scale_factor (float or Tuple[float]): multiplier for spatial size. If `scale_factor` is a tuple, its length has to match the number of spatial dimensions; `input.dim() - 2`. mode (str): algorithm used for upsampling: ``'nearest'`` | ``'linear'`` | ``'bilinear'`` | ``'bicubic'`` | ``'trilinear'`` | ``'area'`` | ``'nearest-exact'``. Default: ``'nearest'`` align_corners (bool, optional): Geometrically, we consider the pixels of the input and output as squares rather than points. If set to ``True``, the input and output tensors are aligned by the center points of their corner pixels, preserving the values at the corner pixels. If set to ``False``, the input and output tensors are aligned by the corner points of their corner pixels, and the interpolation uses edge value padding for out-of-boundary values, making this operation *independent* of input size when :attr:`scale_factor` is kept the same. This only has an effect when :attr:`mode` is ``'linear'``, ``'bilinear'``, ``'bicubic'`` or ``'trilinear'``. Default: ``False`` recompute_scale_factor (bool, optional): recompute the scale_factor for use in the interpolation calculation. If `recompute_scale_factor` is ``True``, then `scale_factor` must be passed in and `scale_factor` is used to compute the output `size`. The computed output `size` will be used to infer new scales for the interpolation. Note that when `scale_factor` is floating-point, it may differ from the recomputed `scale_factor` due to rounding and precision issues. If `recompute_scale_factor` is ``False``, then `size` or `scale_factor` will be used directly for interpolation. Default: ``None``. antialias (bool, optional): flag to apply anti-aliasing. Default: ``False``. Using anti-alias option together with ``align_corners=False``, interpolation result would match Pillow result for downsampling operation. Supported modes: ``'bilinear'``, ``'bicubic'``. .. note:: With ``mode='bicubic'``, it's possible to cause overshoot. For some dtypes, it can produce negative values or values greater than 255 for images. Explicitly call ``result.clamp(min=0,max=255)`` if you want to reduce the overshoot when displaying the image. For ``uint8`` inputs, it already performs saturating cast operation. So, no manual `clamp` operation is needed. .. note:: Mode ``mode='nearest-exact'`` matches Scikit-Image and PIL nearest neighbours interpolation algorithms and fixes known issues with ``mode='nearest'``. This mode is introduced to keep backward compatibility. Mode ``mode='nearest'`` matches buggy OpenCV's ``INTER_NEAREST`` interpolation algorithm. .. note:: The gradients for the dtype ``float16`` on CUDA may be inaccurate in the upsample operation when using modes ``['linear', 'bilinear', 'bicubic', 'trilinear', 'area']``. For more details, please refer to the discussion in `issue#104157 `_. Note: {backward_reproducibility_note} )r5rrPrrr )nearestarea nearest-exactNzjalign_corners option can only be set with the interpolating modes: linear | bilinear | bicubic | trilinearFr(z2only one of size or scale_factor should be definedzkInput and output must have the same number of spatial dimensions, but got input with spatial dimensions of z and output size of zj. Please provide input tensor in (N, C, d1, d2, ...,dK) format and output size in (o1, o2, ...,oK) format.c32K|]}t|ywrE)r).0rs r? zinterpolate..*s8a;q>8szqexpected size to be one of int or Tuple[int] or Tuple[int, int] or Tuple[int, int, int], but got size with types zqInput and scale_factor must have the same number of spatial dimensions, but got input with spatial dimensions of z and scale_factor of shape zk. Please provide input tensor in (N, C, d1, d2, ...,dK) format and scale_factor in (s1, s2, ...,sK) format.z-either size or scale_factor should be definedz?recompute_scale_factor is not meaningful with an explicit size.rTr)bilinearbicubicrFz`Anti-alias option is restricted to bilinear and bicubic modes and requires a 4-D tensor as inputr,rrrlinearrtorch._decomp.decompositions trilinearrz.Got 3D input, but bilinear mode needs 4D inputz/Got 3D input, but trilinear mode needs 5D inputz,Got 4D input, but linear mode needs 3D inputz/Got 4D input, but trilinear mode needs 5D inputz,Got 5D input, but linear mode needs 3D inputz.Got 5D input, but bilinear mode needs 4D inputz=Input Error: Only 3D, 4D and 5D input Tensors supported (got z`D) for the modes: nearest | linear | bilinear | bicubic | trilinear | area | nearest-exact (got rs)2rrrr2r6rr]tupler3rXr7r[rYall TypeErrorr^rtr9_get_tracing_statefloorr5rtensorfloat32r4r_sym_intr r:upsample_nearest1dupsample_nearest2dupsample_nearest3d_upsample_nearest_exact1d_upsample_nearest_exact2d_upsample_nearest_exact3dadaptive_avg_pool1drrupsample_linear1d_upsample_bilinear2d_aais_cpu$are_deterministic_algorithms_enabled importlib import_module_upsample_linear_vecupsample_bilinear2dupsample_trilinear3d_upsample_bicubic2d_aaupsample_bicubic2dra) rr5rrPrrr r6 scale_factorsrr!rdris r?rrstZ &$  H %'#9   33  $O   !M ))+/C  L4MNN ### dT5M *4yC 88> 99))+8488#I[_I`VW$q'I`HacK).s4A44K4  !|| lT5M 2< C' <<@QRa\>M>HII * "   M   v~+-!%).D(((yy%%'EHH,G,G,Is KK!JJq1u-335#ll=+;5==QR%'  K YY # # %sDJJuUZZA%67-:JJKLK INc CDAE*]1-==>K $"99ejjAo n   yy{aDI-xx||..uk=QQ yy{aDI-xx||..uk=QQ yy{aDI-xx||..uk=QQ yy{aDO3xx||55e[-XX yy{aDO3xx||55e[-XX yy{aDO3xx||55e[-XX yy{aDFN&&&"5+66 yy{aDFN&&&"5+66 yy{aDFN&&&"5+66 yy{aDH,(((xx||-- ; }   yy{aDJ.((( 88<<77{M=  yy%%'<r:r@cyrErr8s r?r>r>r:r@cyrErr8s r?r>r>r:r@cyrErr8s r?r>r>!r:r@cPtjddt|||ddS)aiUpsamples the input, using bilinear upsampling. .. warning:: This function is deprecated in favor of :func:`torch.nn.functional.interpolate`. This is equivalent with ``nn.functional.interpolate(..., mode='bilinear', align_corners=True)``. Expected inputs are spatial (4 dimensional). Use `upsample_trilinear` for volumetric (5 dimensional) inputs. Args: input (Tensor): input size (int or Tuple[int, int]): output spatial size. scale_factor (int or Tuple[int, int]): multiplier for spatial size Note: {backward_reproducibility_note} zY`nn.functional.upsample_bilinear` is deprecated. Use `nn.functional.interpolate` instead.r(rrT)rPrrr8s r?r>r>*s.( MM 3 udLzQU VVr@)rrr)zerosborder reflectiongrid padding_modec nt||rtt||f|||||S|dk7r|dk7r|dk7rtd|d|dk7r|dk7r|d k7rtd |d|dk(rd }n |dk(rd }nd }|dk(rd }n |dk(rd }nd }|t j dd}t j|||||S)aCompute grid sample. Given an :attr:`input` and a flow-field :attr:`grid`, computes the ``output`` using :attr:`input` values and pixel locations from :attr:`grid`. Currently, only spatial (4-D) and volumetric (5-D) :attr:`input` are supported. In the spatial (4-D) case, for :attr:`input` with shape :math:`(N, C, H_\text{in}, W_\text{in})` and :attr:`grid` with shape :math:`(N, H_\text{out}, W_\text{out}, 2)`, the output will have shape :math:`(N, C, H_\text{out}, W_\text{out})`. For each output location ``output[n, :, h, w]``, the size-2 vector ``grid[n, h, w]`` specifies :attr:`input` pixel locations ``x`` and ``y``, which are used to interpolate the output value ``output[n, :, h, w]``. In the case of 5D inputs, ``grid[n, d, h, w]`` specifies the ``x``, ``y``, ``z`` pixel locations for interpolating ``output[n, :, d, h, w]``. :attr:`mode` argument specifies ``nearest`` or ``bilinear`` interpolation method to sample the input pixels. :attr:`grid` specifies the sampling pixel locations normalized by the :attr:`input` spatial dimensions. Therefore, it should have most values in the range of ``[-1, 1]``. For example, values ``x = -1, y = -1`` is the left-top pixel of :attr:`input`, and values ``x = 1, y = 1`` is the right-bottom pixel of :attr:`input`. If :attr:`grid` has values outside the range of ``[-1, 1]``, the corresponding outputs are handled as defined by :attr:`padding_mode`. Options are * ``padding_mode="zeros"``: use ``0`` for out-of-bound grid locations, * ``padding_mode="border"``: use border values for out-of-bound grid locations, * ``padding_mode="reflection"``: use values at locations reflected by the border for out-of-bound grid locations. For location far away from the border, it will keep being reflected until becoming in bound, e.g., (normalized) pixel location ``x = -3.5`` reflects by border ``-1`` and becomes ``x' = 1.5``, then reflects by border ``1`` and becomes ``x'' = -0.5``. Note: This function is often used in conjunction with :func:`affine_grid` to build `Spatial Transformer Networks`_ . Note: When using the CUDA backend, this operation may induce nondeterministic behaviour in its backward pass that is not easily switched off. Please see the notes on :doc:`/notes/randomness` for background. Note: NaN values in :attr:`grid` would be interpreted as ``-1``. Args: input (Tensor): input of shape :math:`(N, C, H_\text{in}, W_\text{in})` (4-D case) or :math:`(N, C, D_\text{in}, H_\text{in}, W_\text{in})` (5-D case) grid (Tensor): flow-field of shape :math:`(N, H_\text{out}, W_\text{out}, 2)` (4-D case) or :math:`(N, D_\text{out}, H_\text{out}, W_\text{out}, 3)` (5-D case) mode (str): interpolation mode to calculate output values ``'bilinear'`` | ``'nearest'`` | ``'bicubic'``. Default: ``'bilinear'`` Note: ``mode='bicubic'`` supports only 4-D input. When ``mode='bilinear'`` and the input is 5-D, the interpolation mode used internally will actually be trilinear. However, when the input is 4-D, the interpolation mode will legitimately be bilinear. padding_mode (str): padding mode for outside grid values ``'zeros'`` | ``'border'`` | ``'reflection'``. Default: ``'zeros'`` align_corners (bool, optional): Geometrically, we consider the pixels of the input as squares rather than points. If set to ``True``, the extrema (``-1`` and ``1``) are considered as referring to the center points of the input's corner pixels. If set to ``False``, they are instead considered as referring to the corner points of the input's corner pixels, making the sampling more resolution agnostic. This option parallels the ``align_corners`` option in :func:`interpolate`, and so whichever option is used here should also be used there to resize the input image before grid sampling. Default: ``False`` Returns: output (Tensor): output Tensor .. _`Spatial Transformer Networks`: https://arxiv.org/abs/1506.02025 .. warning:: When ``align_corners = True``, the grid positions depend on the pixel size relative to the input image size, and so the locations sampled by :func:`grid_sample` will differ for the same input given at different resolutions (that is, after being upsampled or downsampled). The default behavior up to version 1.2.0 was ``align_corners = True``. Since then, the default behavior has been changed to ``align_corners = False``, in order to bring it in line with the default for :func:`interpolate`. .. note:: ``mode='bicubic'`` is implemented using the `cubic convolution algorithm`_ with :math:`\alpha=-0.75`. The constant :math:`\alpha` might be different from packages to packages. For example, `PIL`_ and `OpenCV`_ use -0.5 and -0.75 respectively. This algorithm may "overshoot" the range of values it's interpolating. For example, it may produce negative values or values greater than 255 when interpolating input in [0, 255]. Clamp the results with :func:`torch.clamp` to ensure they are within the valid range. .. _`cubic convolution algorithm`: https://en.wikipedia.org/wiki/Bicubic_interpolation .. _`PIL`: https://github.com/python-pillow/Pillow/blob/4634eafe3c695a014267eefdce830b4a825beed7/src/libImaging/Resample.c#L51 .. _`OpenCV`: https://github.com/opencv/opencv/blob/f345ed564a06178670750bad59526cfa4033be55/modules/imgproc/src/resize.cpp#L908 )rPrGrrrrz_nn.functional.grid_sample(): expected mode to be 'bilinear', 'nearest' or 'bicubic', but got: 'rqrCrDrEzgnn.functional.grid_sample(): expected padding_mode to be 'zeros', 'border', or 'reflection', but got: 'rr+r(Default grid_sample and affine_grid behavior has changed to align_corners=False since 1.3.0. Please specify align_corners=True if the old behavior is desired. See the documentation of grid_sample for details.F)rr grid_sampler2rrr7 grid_sampler)rrFrPrGrrcpadding_mode_enums r?rJrJXsX#5$/$  DM  %'   zdi/DI4Emnrmsst u    H $ L ( %a )   z    w  !  @    eT96G WWr@thetac0t|rtt|f|||S|tjdd}|j st d|jt|dk(rY|jdk7s$|jdd k7s|jd dk7rt d |d |jd |dd}nvt|dk(rY|jdk7s$|jddk7s|jd dk7rt d|d |jd |dd}ntd|d |r$t|dk(rtjdnt|dkrt d|tj|||S)av Generate 2D or 3D flow field (sampling grid), given a batch of affine matrices :attr:`theta`. .. note:: This function is often used in conjunction with :func:`grid_sample` to build `Spatial Transformer Networks`_ . Args: theta (Tensor): input batch of affine matrices with shape (:math:`N \times 2 \times 3`) for 2D or (:math:`N \times 3 \times 4`) for 3D size (torch.Size): the target output image size. (:math:`N \times C \times H \times W` for 2D or :math:`N \times C \times D \times H \times W` for 3D) Example: torch.Size((32, 3, 24, 24)) align_corners (bool, optional): if ``True``, consider ``-1`` and ``1`` to refer to the centers of the corner pixels rather than the image corners. Refer to :func:`grid_sample` for a more complete description. A grid generated by :func:`affine_grid` should be passed to :func:`grid_sample` with the same setting for this option. Default: ``False`` Returns: output (Tensor): output Tensor of size (:math:`N \times H \times W \times 2`) .. _`Spatial Transformer Networks`: https://arxiv.org/abs/1506.02025 .. warning:: When ``align_corners = True``, the grid positions depend on the pixel size relative to the input image size, and so the locations sampled by :func:`grid_sample` will differ for the same input given at different resolutions (that is, after being upsampled or downsampled). The default behavior up to version 1.2.0 was ``align_corners = True``. Since then, the default behavior has been changed to ``align_corners = False``, in order to bring it in line with the default for :func:`interpolate`. .. warning:: When ``align_corners = True``, 2D affine transforms on 1D data and 3D affine transforms on 2D data (that is, when one of the spatial dimensions has unit size) are ill-defined, and not an intended use case. This is not a problem when ``align_corners = False``. Up to version 1.2.0, all grid points along a unit dimension were considered arbitrarily to be at ``-1``. From version 1.3.0, under ``align_corners = True`` all grid points along a unit dimension are considered to be at ``0`` (the center of the input image). )rNrIFz4Expected theta to have floating point type, but got rFr,r)r(r*z?Expected a batch of 2D affine matrices of shape Nx2x3 for size z. Got .rz?Expected a batch of 3D affine matrices of shape Nx3x4 for size r-zcaffine_grid only supports 4D and 5D sizes, for 2D and 3D affine transforms, respectively. Got size r+zSince version 1.3.0, affine_grid behavior has changed for unit-size grids when align_corners=True. This is not an intended use case of affine_grid. See the documentation of affine_grid for details.rz-Expected non-zero, positive output size. Got )rr affine_gridrrrWr2r/r3r6rXrarr7affine_grid_generator)rMr5r spatial_sizes r?rPrPsf &$ %5$m    @    " " $B5;;- P   4yA~ 99;! u{{2!3u{{2!7KQRVQWW]^c^i^i]jjkl BCy Ta 99;! u{{2!3u{{2!7KQRVQWW]^c^i^i]jjkl BCy ! vQ   \*a/  @ TaHOPP  & &udM BBr@rct|r4ttjjj |f||||Stj jsVtjrB|js |jr*|dk(r%tjdj||Stjjj ||||S)a pad(input, pad, mode="constant", value=None) -> Tensor Pads tensor. Padding size: The padding size by which to pad some dimensions of :attr:`input` are described starting from the last dimension and moving forward. :math:`\left\lfloor\frac{\text{len(pad)}}{2}\right\rfloor` dimensions of ``input`` will be padded. For example, to pad only the last dimension of the input tensor, then :attr:`pad` has the form :math:`(\text{padding\_left}, \text{padding\_right})`; to pad the last 2 dimensions of the input tensor, then use :math:`(\text{padding\_left}, \text{padding\_right},` :math:`\text{padding\_top}, \text{padding\_bottom})`; to pad the last 3 dimensions, use :math:`(\text{padding\_left}, \text{padding\_right},` :math:`\text{padding\_top}, \text{padding\_bottom}` :math:`\text{padding\_front}, \text{padding\_back})`. Padding mode: See :class:`torch.nn.CircularPad2d`, :class:`torch.nn.ConstantPad2d`, :class:`torch.nn.ReflectionPad2d`, and :class:`torch.nn.ReplicationPad2d` for concrete examples on how each of the padding modes works. Constant padding is implemented for arbitrary dimensions. Circular, replicate and reflection padding are implemented for padding the last 3 dimensions of a 4D or 5D input tensor, the last 2 dimensions of a 3D or 4D input tensor, or the last dimension of a 2D or 3D input tensor. Note: When using the CUDA backend, this operation may induce nondeterministic behaviour in its backward pass that is not easily switched off. Please see the notes on :doc:`/notes/randomness` for background. Args: input (Tensor): N-dimensional tensor pad (tuple): m-elements tuple, where :math:`\frac{m}{2} \leq` input dimensions and :math:`m` is even. mode: ``'constant'``, ``'reflect'``, ``'replicate'`` or ``'circular'``. Default: ``'constant'`` value: fill value for ``'constant'`` padding. Default: ``0`` Examples:: >>> t4d = torch.empty(3, 3, 4, 2) >>> p1d = (1, 1) # pad last dim by 1 on each side >>> out = F.pad(t4d, p1d, "constant", 0) # effectively zero padding >>> print(out.size()) torch.Size([3, 3, 4, 4]) >>> p2d = (1, 1, 2, 2) # pad last dim by (1, 1) and 2nd to last by (2, 2) >>> out = F.pad(t4d, p2d, "constant", 0) >>> print(out.size()) torch.Size([3, 3, 8, 4]) >>> t4d = torch.empty(3, 3, 4, 2) >>> p3d = (0, 1, 2, 1, 3, 3) # pad by (0, 1), (2, 1), and (3, 3) >>> out = F.pad(t4d, p3d, "constant", 0) >>> print(out.size()) torch.Size([3, 9, 7, 3]) )rPr replicater)rrr7nn functionalrr[rYr.is_cudais_xpur/r0_replication_padr9r:)rrrPrs r?rr]sD &$ HH   # #eXucE   99 ! ! #  5 5 7 MMU\\{"!..2""5#./ 88<<  E3e 44r@ztorch.nn.functionalzy pairwise_distance(x1, x2, p=2.0, eps=1e-6, keepdim=False) -> Tensor See :class:`torch.nn.PairwiseDistance` for details a pdist(input, p=2) -> Tensor Computes the p-norm distance between every pair of row vectors in the input. This is identical to the upper triangular portion, excluding the diagonal, of `torch.norm(input[:, None] - input, dim=2, p=p)`. This function will be faster if the rows are contiguous. If input has shape :math:`N \times M` then the output will have shape :math:`\frac{1}{2} N (N - 1)`. This function is equivalent to ``scipy.spatial.distance.pdist(input, 'minkowski', p=p)`` if :math:`p \in (0, \infty)`. When :math:`p = 0` it is equivalent to ``scipy.spatial.distance.pdist(input, 'hamming') * M``. When :math:`p = \infty`, the closest scipy function is ``scipy.spatial.distance.pdist(xn, lambda x, y: np.abs(x - y).max())``. Args: input: input tensor of shape :math:`N \times M`. p: p value for the p-norm distance to calculate between each vector pair :math:`\in [0, \infty]`. a cosine_similarity(x1, x2, dim=1, eps=1e-8) -> Tensor Returns cosine similarity between ``x1`` and ``x2``, computed along dim. ``x1`` and ``x2`` must be broadcastable to a common shape. ``dim`` refers to the dimension in this common shape. Dimension ``dim`` of the output is squeezed (see :func:`torch.squeeze`), resulting in the output tensor having 1 fewer dimension. .. math :: \text{similarity} = \dfrac{x_1 \cdot x_2}{\max(\Vert x_1 \Vert _2, \epsilon) \cdot \max(\Vert x_2 \Vert _2, \epsilon)} Supports :ref:`type promotion `. Args: x1 (Tensor): First input. x2 (Tensor): Second input. dim (int, optional): Dimension along which cosine similarity is computed. Default: 1 eps (float, optional): Small value to avoid division by zero. Default: 1e-8 Example:: >>> input1 = torch.randn(100, 128) >>> input2 = torch.randn(100, 128) >>> output = F.cosine_similarity(input1, input2) >>> print(output) aw one_hot(tensor, num_classes=-1) -> LongTensor Takes LongTensor with index values of shape ``(*)`` and returns a tensor of shape ``(*, num_classes)`` that have zeros everywhere except where the index of last dimension matches the corresponding value of the input tensor, in which case it will be 1. See also `One-hot on Wikipedia`_ . .. _One-hot on Wikipedia: https://en.wikipedia.org/wiki/One-hot Arguments: tensor (LongTensor): class values of any shape. num_classes (int, optional): Total number of classes. If set to -1, the number of classes will be inferred as one greater than the largest class value in the input tensor. Default: -1 Returns: LongTensor that has one more dimension with 1 values at the index of last dimension indicated by the input, and 0 everywhere else. Examples: >>> F.one_hot(torch.arange(0, 5) % 3) tensor([[1, 0, 0], [0, 1, 0], [0, 0, 1], [1, 0, 0], [0, 1, 0]]) >>> F.one_hot(torch.arange(0, 5) % 3, num_classes=5) tensor([[1, 0, 0, 0, 0], [0, 1, 0, 0, 0], [0, 0, 1, 0, 0], [1, 0, 0, 0, 0], [0, 1, 0, 0, 0]]) >>> F.one_hot(torch.arange(0, 6).view(3,2) % 3) tensor([[[1, 0, 0], [0, 1, 0]], [[0, 0, 1], [1, 0, 0]], [[0, 1, 0], [0, 0, 1]]]) anchorpositivenegativeswapc t|||rtt|||f||||||||||  S||tj||} ntj | } |dkrt d|tj|||||||| S)zCompute the triplet loss between given input tensors and a margin greater than 0. See :class:`~torch.nn.TripletMarginLoss` for details. )rrrr]rrrr#margin must be greater than 0, got )rrtriplet_margin_lossrrrr2r7) rZr[r\rrrr]rrrrs r?r`r`.s #68X>$  Xx (   %  6#5#33L&I#,,Y7 {>vhGHH  $ $(FAsD. r@distance_functionrr]rrbc tjjr tdt |||rt t |||f||||||| S|dvrt|d|dkrtd||j}|j}|j} ||k(r|| k(std|d|d | d |tj}|||} |||} |r|||} tj| | } tj|| z| z d} |d k(rtj| S|d k(rtj| S| S) zCompute the triplet margin loss for input tensors using a custom distance function. See :class:`~torch.nn.TripletMarginWithDistanceLoss` for details. zuF.triplet_margin_with_distance_loss does not support JIT scripting: functions requiring Callables cannot be scripted.ra)rNrTrrrr_zoThe anchor, positive, and negative tensors are expected to have the same number of dimensions, but got: anchor z D, positive zD, and negative zD inputsrTrN)r7r[rYrarr!triplet_margin_with_distance_lossr2r rpairwise_distanceminimum clamp_minrTrN)rZr[r\rbrr]ra_dimp_dimn_dimdist_posdist_neg dist_swaprs r?rdrdXs yy! @  #68X>$ - Xx (   /  //I;&IJKK{>vhGHH KKE MME MME UNu~>>CWEw.ugX ?   !33 2H 2H  %h9 ==95 ??6H,x7 ;DEyy f zz$ r@rc Pt||rtt||f|||||S|7|j||dj |j |}||z S|j||dj |j |}tj|||S)a\Perform :math:`L_p` normalization of inputs over specified dimension. For a tensor :attr:`input` of sizes :math:`(n_0, ..., n_{dim}, ..., n_k)`, each :math:`n_{dim}` -element vector :math:`v` along dimension :attr:`dim` is transformed as .. math:: v = \frac{v}{\max(\lVert v \rVert_p, \epsilon)}. With the default arguments it uses the Euclidean norm over vectors along dimension :math:`1` for normalization. Args: input: input tensor of any shape p (float): the exponent value in the norm formulation. Default: 2 dim (int or tuple of ints): the dimension to reduce. Default: 1 eps (float): small value to avoid division by zero. Default: 1e-12 out (Tensor, optional): the output tensor. If :attr:`out` is used, this operation won't be differentiable. )rr6rrTr!)r) rr normalizenormrg expand_as clamp_min_r7r)rrr6rrdenoms r?roros2#5#.$ s|UaScs   { 1c4 0::3?II%Pu} 1c4 0;;C@JJ5Qyy3//r@argrGmessagecnt|ts%t|dk(sJ|j|yy)Nr()rr4r3format)rtrGrus r?assert_int_or_pairrxs/ c3 3s8q=J'..2JJ 0= r@c t|rtt|f|||||Stjj j |t|t|t|t|S)aExtract sliding local blocks from a batched input tensor. .. warning:: Currently, only 4-D input tensors (batched image-like tensors) are supported. .. warning:: More than one element of the unfolded tensor may refer to a single memory location. As a result, in-place operations (especially ones that are vectorized) may result in incorrect behavior. If you need to write to the tensor, please clone it first. See :class:`torch.nn.Unfold` for details rVrUrT)rrunfoldr7r9r:im2colr)rr rVrUrTs r?r{r{si. &$  H     88<<   u[!5?E'NE&M r@c t|rtt|f||||||Stjj j |t|t|t|t|t|S)zCombine an array of sliding local blocks into a large containing tensor. .. warning:: Currently, only unbatched (3D) or batched (4D) image-like output tensors are supported. See :class:`torch.nn.Fold` for details rz)rrfoldr7r9r:col2imr)rr!r rVrUrTs r?r~r~sx &$  H      88<<    k k h g f  r@qvwbc|jd}||ur||urkt|||}|jdd|fjdj ddj dj }|d|d|dfS|j||dzg\}}|dx} } n|j||dzg\} } t||| } t||| } | jdd|fjdj ddj dj } | | d| dfS|jd\}} }|dx} x}}n|jd\} }}t||| t|| |t|||fS)aFPerform the in-projection step of the attention operation, using packed weights. Output is a triple containing projection tensors for query, key and value. Args: q, k, v: query, key and value tensors to be projected. For self-attention, these are typically the same tensor; for encoder-decoder attention, k and v are typically the same tensor. (We take advantage of these identities for performance if they are present.) Regardless, q, k and v must share a common embedding dimension; otherwise their shapes may vary. w: projection weights for q, k and v, packed into a single tensor. Weights are packed along dimension 0, in q, k, v order. b: optional projection biases for q, k and v, packed into a single tensor in q, k, v order. Shape: Inputs: - q: :math:`(..., E)` where E is the embedding dimension - k: :math:`(..., E)` where E is the embedding dimension - v: :math:`(..., E)` where E is the embedding dimension - w: :math:`(E * 3, E)` where E is the embedding dimension - b: :math:`E * 3` where E is the embedding dimension Output: - in output list :math:`[q', k', v']`, each output tensor will have the same shape as the corresponding input tensor. r*r,rr)r+r(N) r5r unflattenr transposerrMsplitchunk)rrrrrEprojw_qw_kvb_qb_kvq_projkv_projw_kw_vb_kb_vs r?_in_projection_packedrsD r AAv 6!Q?DrAq6*11b! 7DGT!W, ,AE +ICy!!dGGQAJ/ TAsC(FQd+G!!"q!f-11b! GAJ 3 3 S# 9" "C "#GGAJMCcac"F1c3$739LLLr@rrrrrrc |jd|jd|jd} } } |j| | fk(sJd| | fd|j|j| | fk(sJd| | fd|j|j| | fk(sJd| | fd|j|(|j| fk(sJd| fd|j|(|j| fk(sJd| fd|j|(|j| fk(sJd| fd|jt|||t|||t|||fS) aPerform the in-projection step of the attention operation. This is simply a triple of linear projections, with shape constraints on the weights which ensure embedding dimension uniformity in the projected outputs. Output is a triple containing projection tensors for query, key and value. Args: q, k, v: query, key and value tensors to be projected. w_q, w_k, w_v: weights for q, k and v, respectively. b_q, b_k, b_v: optional biases for q, k and v, respectively. Shape: Inputs: - q: :math:`(Qdims..., Eq)` where Eq is the query embedding dimension and Qdims are any number of leading dimensions. - k: :math:`(Kdims..., Ek)` where Ek is the key embedding dimension and Kdims are any number of leading dimensions. - v: :math:`(Vdims..., Ev)` where Ev is the value embedding dimension and Vdims are any number of leading dimensions. - w_q: :math:`(Eq, Eq)` - w_k: :math:`(Eq, Ek)` - w_v: :math:`(Eq, Ev)` - b_q: :math:`(Eq)` - b_k: :math:`(Eq)` - b_v: :math:`(Eq)` Output: in output triple :math:`(q', k', v')`, - q': :math:`[Qdims..., Eq]` - k': :math:`[Kdims..., Eq]` - v': :math:`[Vdims..., Eq]` r*z!expecting query weights shape of , but got zexpecting key weights shape of z!expecting value weights shape of zexpecting query bias shape of zexpecting key bias shape of zexpecting value bias shape of )r5rXr) rrrrrrrrrEqEkEvs r?_in_projectionrhsXQVVBZBB 99   K +B8*JsyykJK  99   I )"b *SYYKHI  99   K +B8*JsyykJK  ;#))u, ("z#))E , ;#))u, &ugZ {C , ;#))u, ("z#))E , !S# q#s 3VAsC5H HHr@aHscaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None, enable_gqa=False) -> Tensor: Computes scaled dot product attention on query, key and value tensors, using an optional attention mask if passed, and applying dropout if a probability greater than 0.0 is specified. The optional scale argument can only be specified as a keyword argument. .. code-block:: python # Efficient implementation equivalent to the following: def scaled_dot_product_attention(query, key, value, attn_mask=None, dropout_p=0.0, is_causal=False, scale=None, enable_gqa=False) -> torch.Tensor: L, S = query.size(-2), key.size(-2) scale_factor = 1 / math.sqrt(query.size(-1)) if scale is None else scale attn_bias = torch.zeros(L, S, dtype=query.dtype, device=query.device) if is_causal: assert attn_mask is None temp_mask = torch.ones(L, S, dtype=torch.bool).tril(diagonal=0) attn_bias.masked_fill_(temp_mask.logical_not(), float("-inf")) if attn_mask is not None: if attn_mask.dtype == torch.bool: attn_bias.masked_fill_(attn_mask.logical_not(), float("-inf")) else: attn_bias = attn_mask + attn_bias if enable_gqa: key = key.repeat_interleave(query.size(-3)//key.size(-3), -3) value = value.repeat_interleave(query.size(-3)//value.size(-3), -3) attn_weight = query @ key.transpose(-2, -1) * scale_factor attn_weight += attn_bias attn_weight = torch.softmax(attn_weight, dim=-1) attn_weight = torch.dropout(attn_weight, dropout_p, train=True) return attn_weight @ value .. warning:: This function is beta and subject to change. .. warning:: This function always applies dropout according to the specified ``dropout_p`` argument. To disable dropout during evaluation, be sure to pass a value of ``0.0`` when the module that makes the function call is not in training mode. For example: .. code-block:: python class MyModel(nn.Module): def __init__(self, p=0.5): super().__init__() self.p = p def forward(self, ...): return F.scaled_dot_product_attention(..., dropout_p=(self.p if self.training else 0.0)) Note: There are currently three supported implementations of scaled dot product attention: - `FlashAttention-2: Faster Attention with Better Parallelism and Work Partitioning`_ - `Memory-Efficient Attention`_ - A PyTorch implementation defined in C++ matching the above formulation The function may call optimized kernels for improved performance when using the CUDA backend. For all other backends, the PyTorch implementation will be used. All implementations are enabled by default. Scaled dot product attention attempts to automatically select the most optimal implementation based on the inputs. In order to provide more fine-grained control over what implementation is used, the following functions are provided for enabling and disabling implementations. The context manager is the preferred mechanism: - :func:`torch.nn.attention.sdpa_kernel`: A context manager used to enable or disable any of the implementations. - :func:`torch.backends.cuda.enable_flash_sdp`: Globally enables or disables FlashAttention. - :func:`torch.backends.cuda.enable_mem_efficient_sdp`: Globally enables or disables Memory-Efficient Attention. - :func:`torch.backends.cuda.enable_math_sdp`: Globally enables or disables the PyTorch C++ implementation. Each of the fused kernels has specific input limitations. If the user requires the use of a specific fused implementation, disable the PyTorch C++ implementation using :func:`torch.nn.attention.sdpa_kernel`. In the event that a fused implementation is not available, a warning will be raised with the reasons why the fused implementation cannot run. Due to the nature of fusing floating point operations, the output of this function may be different depending on what backend kernel is chosen. The c++ implementation supports torch.float64 and can be used when higher precision is required. For math backend, all intermediates are kept in torch.float if inputs are in torch.half or torch.bfloat16. For more information please see :doc:`/notes/numerical_accuracy` Grouped Query Attention (GQA) is an experimental feature. It currently works only for Flash_attention and math kernel on CUDA tensor, and does not support Nested tensor. Constraints for GQA: - number_of_heads_query % number_of_heads_key_value == 0 and, - number_of_heads_key == number_of_heads_value Note: {cudnn_reproducibility_note} a^ Args: query (Tensor): Query tensor; shape :math:`(N, ..., Hq, L, E)`. key (Tensor): Key tensor; shape :math:`(N, ..., H, S, E)`. value (Tensor): Value tensor; shape :math:`(N, ..., H, S, Ev)`. attn_mask (optional Tensor): Attention mask; shape must be broadcastable to the shape of attention weights, which is :math:`(N,..., L, S)`. Two types of masks are supported. A boolean mask where a value of True indicates that the element *should* take part in attention. A float mask of the same type as query, key, value that is added to the attention score. dropout_p (float): Dropout probability; if greater than 0.0, dropout is applied is_causal (bool): If set to true, the attention masking is a lower triangular matrix when the mask is a square matrix. The attention masking has the form of the upper left causal bias due to the alignment (see :class:`torch.nn.attention.bias.CausalBias`) when the mask is a non-square matrix. An error is thrown if both attn_mask and is_causal are set. scale (optional float, keyword-only): Scaling factor applied prior to softmax. If None, the default value is set to :math:`\frac{1}{\sqrt{E}}`. enable_gqa (bool): If set to True, Grouped Query Attention (GQA) is enabled, by default it is set to False. Returns: output (Tensor): Attention output; shape :math:`(N, ..., Hq, L, Ev)`. Shape legend: - :math:`N: \text{Batch size} ... : \text{Any number of other batch dimensions (optional)}` - :math:`S: \text{Source sequence length}` - :math:`L: \text{Target sequence length}` - :math:`E: \text{Embedding dimension of the query and key}` - :math:`Ev: \text{Embedding dimension of the value}` - :math:`Hq: \text{Number of heads of query}` - :math:`H: \text{Number of heads of key and value}` Examples: >>> # Optionally use the context manager to ensure one of the fused kernels is run >>> query = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") >>> key = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") >>> value = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") >>> with sdpa_kernel(backends=[SDPBackend.FLASH_ATTENTION]): >>> F.scaled_dot_product_attention(query,key,value) >>> # Sample for GQA for llama3 >>> query = torch.rand(32, 32, 128, 64, dtype=torch.float16, device="cuda") >>> key = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") >>> value = torch.rand(32, 8, 128, 64, dtype=torch.float16, device="cuda") >>> with sdpa_kernel(backends=[SDPBackend.MATH]): >>> F.scaled_dot_product_attention(query,key,value,enable_gqa=True) .. _FlashAttention-2\: Faster Attention with Better Parallelism and Work Partitioning: https://arxiv.org/abs/2307.08691 .. _Memory-Efficient Attention: https://github.com/facebookresearch/xformers .. _Grouped-Query Attention: https://arxiv.org/pdf/2305.13245 querykeykey_padding_mask attn_mask num_headsc|jdk(rd}|jdk(r|jdk(s*Jd|jd|jd|,|jdk(sJd|jd|+|jd vsJd |jd|S|jdk(rd }|jdk(r|jdk(s*Jd |jd|jd|,|jd k(sJd|jd||jd vsJd|jd|jdk(rE||jd|jdf}|j|k(sJd|d|j|Std|jd)Nr,TzJFor batched (3-D) `query`, expected `key` and `value` to be 3-D but found z-D and z-D tensors respectivelyr(zUFor batched (3-D) `query`, expected `key_padding_mask` to be `None` or 2-D but found z-D tensor insteadrzSFor batched (3-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D but found FzLFor unbatched (2-D) `query`, expected `key` and `value` to be 2-D but found r+zWFor unbatched (2-D) `query`, expected `key_padding_mask` to be `None` or 1-D but found zUFor unbatched (2-D) `query`, expected `attn_mask` to be `None`, 2-D or 3-D but found rz!Expected `attn_mask` shape to be z but got z?query should be unbatched 2D or batched 3D tensor but received z-D query tensor)r6rXAssertionError)rrrrrrrexpected_shapes r?_mha_shape_checkrMsU yy{a wwyA~%))+"2  '')GEIIK=8O Q 2  '#'')Q. .22455FH .  ==?f, 'mmo..?A ,B ;   wwyA~%))+"2  '')GEIIK=8O Q 2  '#'')Q. .22455FH .  ==?f, 'mmo..?A ,}}!#"+U[[^SYYq\!J .877GyQZQ`Q`Pab8  Meiik]Zi j  r@mask mask_name other_type other_name target_type check_othercB||j}tj|}|tjk7r|st d|d|r#|!||k7rt j d|d|d|s0tj||j|td}|S)Nz only bool and floating types of z are supportedzSupport for mismatched rrz/ is deprecated. Use same type for both instead.rz-inf) r/r7rWboolrrrr) masked_fill_r)rrrrrr _mask_dtype_mask_is_floats r?_canonical_maskrs jj 006 %** $^ 29+^L  :1j( -i[j\JEE##D <IIeFmD Kr@cj|yt|tjr |jSt d)Nz6input to _none_or_dtype() must be None or torch.Tensor)rr7r r/rr s r?_none_or_dtypers. } E5<< ({{ O PPr@src_lenbszctjtjdk(fdtjtjdk(fdy)Nrc.ddjdS)Nz(Expected key_padded_mask.shape[0] to be rrrX)rrsr?z)_check_key_padding_mask..s#:3%zJZJ`J`abJcIder@r+c.ddjdS)Nz(Expected key_padded_mask.shape[1] to be rr+r)rrsr?rz)_check_key_padding_mask..s#:7):N^NdNdefNgMhir@)r7 _check_withrrX)rrrs```r?_check_key_padding_maskrsU q!S(e  q!W,ir@embed_dim_to_checkin_proj_weight in_proj_biasbias_kbias_v add_zero_attn dropout_pout_proj_weight out_proj_bias need_weightsuse_separate_proj_weight q_proj_weight k_proj_weight v_proj_weightstatic_kstatic_vaverage_attn_weights is_causalc|||||||| | f }t|r)tt||||||||||| | | | f| |||||||||||d St||||||}|sF|j d}|j d}|j d}||j d}|j \}}}|j \}}}t |dt|d|j}|r | td|r||sd}nt |ddd |jd }|d }||k(s Jd |d |t|tjr|j|d} n||z} | |z|k(s Jd|d||rF|j dd|j ddk(saJd|j ddd|j dd|j |j k(s!Jd|j d|j |s|Jdt|||||\}!}"}#nP|Jd|Jd|Jd|dx}$x}%}&n|jd\}$}%}&t!|||||||$|%|& \}!}"}#||j#dk(rA||f}'|j |'k7rtd|j d|'d|j d}nc|j#dk(r3||z||f}(|j |(k7r9td|j d|(dtd |j#d!|||Jd"|Jd#tj$|"|j'd|dg}"tj$|#|j'd|dg}#| t)|d$}|t)|d$}n|J|J|!j+|||z| j-dd}!|4|"j+|"j d||z| j-dd}"nh|j/d||zk(sJd%||zd |j/d|j/d| k(sJd&| d |j/d|}"|4|#j+|#j d||z| j-dd}#nh|j/d||zk(sJd'||zd |j/d|j/d| k(sJd(| d |j/d|}#| r||zd| f})tj$|"tj0|)|"j|"j2)gd*}"tj$|#tj0|)|#j|#j2)gd*}#| t)|d$}| t)|d$}|"j/d}|tj4j7s+tj4j9s t;||||j+|dd|j=d+|d+d+j?||zd|}||}n||z}| sd,} |rv|!j \}*}+},|!tAjBd-tE|,z z}-|r |Jd.|(tjF||-|"j-d/d+}.n&tjH|-|"j-d/d+}.tK|.d+*}.| d,kDr tM|.| 0}.tjH|.|#}/|/j-ddjOj+||z|}/tQ|/| | }/|/j+|||/j/d}/|.j+||||}.|r|.jSd*}.|s"|/jUd}/|.jUd}.|/|.fS|M|j/ddk(r%|j#dk(r|j d}n|j+||d+|}|!j+|||| }!|"j+|||| }"|#j+|||| }#tW|!|"|#|| |}/|/jYddddjOj+||z|}/tQ|/| | }/|/j+|||/j/d}/|s|/jUd}/|/dfS)1aForward method for MultiHeadAttention. See :class:`torch.nn.MultiheadAttention` for details. Args: query, key, value: map a query and a set of key-value pairs to an output. See "Attention Is All You Need" for more details. embed_dim_to_check: total dimension of the model. num_heads: parallel attention heads. in_proj_weight, in_proj_bias: input projection weight and bias. bias_k, bias_v: bias of the key and value sequences to be added at dim=0. add_zero_attn: add a new batch of zeros to the key and value sequences at dim=1. dropout_p: probability of an element to be zeroed. out_proj_weight, out_proj_bias: the output projection weight and bias. training: apply dropout if is ``True``. key_padding_mask: if provided, specified padding elements in the key will be ignored by the attention. This is an binary mask. When the value is True, the corresponding value on the attention layer will be filled with -inf. need_weights: output attn_output_weights. Default: `True` Note: `needs_weight` defaults to `True`, but should be set to `False` For best performance when attention weights are not needed. *Setting needs_weights to `True` leads to a significant performance degradation.* attn_mask: 2D or 3D mask that prevents attention to certain positions. A 2D mask will be broadcasted for all the batches while a 3D mask allows to specify a different mask for the entries of each batch. is_causal: If specified, applies a causal mask as attention mask, and ignores attn_mask for computing scaled dot product attention. Default: ``False``. .. warning:: is_causal is provides a hint that the attn_mask is the causal mask.Providing incorrect hints can result in incorrect execution, including forward and backward compatibility. use_separate_proj_weight: the function accept the proj. weights for query, key, and value in different forms. If false, in_proj_weight will be used, which is a combination of q_proj_weight, k_proj_weight, v_proj_weight. q_proj_weight, k_proj_weight, v_proj_weight, in_proj_bias: input projection weight and bias. static_k, static_v: static key and value used for attention operators. average_attn_weights: If true, indicates that the returned ``attn_weights`` should be averaged across heads. Otherwise, ``attn_weights`` are provided separately per head. Note that this flag only has an effect when ``need_weights=True.``. Default: True Shape: Inputs: - query: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - key: :math:`(S, E)` or :math:`(S, N, E)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. - value: :math:`(S, E)` or :math:`(S, N, E)` where S is the source sequence length, N is the batch size, E is the embedding dimension. - key_padding_mask: :math:`(S)` or :math:`(N, S)` where N is the batch size, S is the source sequence length. If a FloatTensor is provided, it will be directly added to the value. If a BoolTensor is provided, the positions with the value of ``True`` will be ignored while the position with the value of ``False`` will be unchanged. - attn_mask: 2D mask :math:`(L, S)` where L is the target sequence length, S is the source sequence length. 3D mask :math:`(N*num_heads, L, S)` where N is the batch size, L is the target sequence length, S is the source sequence length. attn_mask ensures that position i is allowed to attend the unmasked positions. If a BoolTensor is provided, positions with ``True`` are not allowed to attend while ``False`` values will be unchanged. If a FloatTensor is provided, it will be added to the attention weight. - static_k: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. - static_v: :math:`(N*num_heads, S, E/num_heads)`, where S is the source sequence length, N is the batch size, E is the embedding dimension. E/num_heads is the head dimension. Outputs: - attn_output: :math:`(L, E)` or :math:`(L, N, E)` where L is the target sequence length, N is the batch size, E is the embedding dimension. - attn_output_weights: Only returned when ``need_weights=True``. If ``average_attn_weights=True``, returns attention weights averaged across heads of shape :math:`(L, S)` when input is unbatched or :math:`(N, L, S)`, where :math:`N` is the batch size, :math:`L` is the target sequence length, and :math:`S` is the source sequence length. If ``average_attn_weights=False``, returns attention weights per head of shape :math:`(num_heads, L, S)` when input is unbatched or :math:`(N, num_heads, L, S)`. ) rrrrrrrrrrrrr+Nrrr)rrrrrzNeed attn_mask if specifying the is_causal hint. You may use the Transformer module method `generate_square_subsequent_mask` to create this mask.F)rrrrrrz%was expecting embedding dimension of rtrunc) rounding_modez embed_dim z not divisible by num_heads r(zkey's sequence and batch dims z do not match value's z key shape z does not match value shape z<X1a( H (( H (( H (  " "C "#(..q1MCc           1a ==?a &0O/1"7 7HHXYhXiijk"++A.I ]]_ !"YAO/1"7 7HHXYhXiijk((99JK  f0F!FFH!HH IIq&--323 4 IIq&--323 4  Iv.I  '"#3V< ~~~~ wi2<%//2 %NN3 2wG FF3 7H 5 FF3 7H 5 FF3 7H 52 q!Y 9    1a + 6 6 8 = =cGmY W [/=I !&&w[5E5Ea5HI %--a0KD  r@r)NNFN)Nrr+FF)NrN)NF)F)rTF)rFF)r*)grF)rF)g{Gz?F)g?gUUUUUU?FF)Nr,N)r+Frr*)NN@FF) NNr(FrNFNFN)NNF皙?h㈵>)NNNNTrr)NNr)NN)g-C6?g?r)rrNF)NNNrN)TFNg:0yE>NrN)Fư>rN)NNrNF)NNrNrNr)NNNrN)NNNrNN)NNrNr)rNrN)NNrNN)rNNrN)rNNrN)NNrN)r+rNNNrN)NNrN)NNrNNF)rrCNrE)constantN)rr(rFNNrN)rr+g-q=N)r+rr+)NNN)T) TNTNFNNNNNTF)__doc__r/rrtypingrrrrr7rrr#r torch._Cr r torch._jit_internalr r rrrtorch._torch_docsrrrtorch.nnrrrtorch.nn.modules.utilsrrrrtorch.overridesrrrr torch.typesrDTyper4numpyrModuleNotFoundErrorconv1drwconv2dconv3dconv_transpose1dconv_transpose2dconv_transpose3dconv_tbcrr9r:rrrrrr1rD__name__r;rPrSrQrZrbrarfrjrirlrornr]r|rrrrrrrrrrrrrrrr*rrrrrrrrrrrrrrrrrrrrrrrrrprelurr log_sigmoidrgelu hardshrinkrr softplusr]rrrr#r1 softshrinkrr4r6rrr:r=r@rGrLrUrjrprurxr|r~rrrrrrrrrrrrrrrrrrrrr pixel_shufflepixel_unshufflechannel_shufflenative_channel_shufflerrrr9r>GRID_SAMPLE_INTERPOLATION_MODESGRID_SAMPLE_PADDING_MODESrJrPr __module__repdistcosine_similarityone_hotr`rdrorxr{r~rrrrrrrrrr@r?rs. ;; 22-VU3NN+ E  LL  F2#2'12 !,/ b  LL  F2#2'12 !.1 f  LL  F2#2'12 !-0 d    F 2# 2'1 2$'R    F 2# 2'1 2&)V    F 2# 2'1 2%(T  NN      F HHLL < HHLL D597; (, J J"3'J+C01J,U34 J  J f% J 66>J`597; (,   "3' +C01 ,U34    f%    0)   . #%597; (, N N"3'N+C01N,U34 N  N f% N 66>Nh597; (,   "3' +C01 ,U34    f%    0)   . #%04&''( 2 2"3'2 &s+ ,2s # 2  $ 2  22 66>2p04&''( V V"3'V &s+ ,Vs # V  $ V  VV V4   #  04&''( 2 2"3'2 &s+ ,2s # 2  $ 2  22 66>2p04&''( V V"3'V &s+ ,Vs # V  $ V  VV V4   #  04&''( 2 2"3'2 &s+ ,2s # 2  $ 2  22 66>2p04&''( V V"3'V &s+ ,Vs # V  $ V  VV V4   #  " "c" I"#Y " $s)$ "  #Y "R04&'48 $ $ $#3'$ &s+ , $ s # $ +C01 $ $V04&'48 B B B#3'B &s+ , B s # B +C01 B BJ04&'48 T T T#3'T &s+ , T s # T +C01 T TJ04 # #S%Z ##3'# &s+ , #  #  #T04 !V !VS%Z !V#3'!V &s+ , !V  !V  !VP04 ! !S%Z !! &s+ , !  !  !N!9 9"3'99 66> 9>! C  C"3' C C C '   , !#!@ @"3'@@ 66> @@! C  C"3' C C C '   , !#!@ @"3'@@ 66> @B! C  C"3' C C C '   , !#"    Av A4Ec4J Av A Av A4Ec4J Av A$      <      2 0 0 00 0  0j 8 8 88 8  8z 1 1 11 1  1l ( ( (( (  (^      0  NN    &   KK (v(C((:      2  HHLL    $ 6  v e D V  HHLL&&  KK    .  KK !  0 HHLL   KK .         <  LL  HHLL   HHLL&      %  HHLL   3 c s s !   # E?   F! ( ( #(( E? (  (Z G G G G  G G  GX!   # E?   D HHLL  +v+++,  HHLL $ F%#$"%# 0  NN .$$$&$2$$$&$()V)d)v)2I I II I  66> I"& $$rS rS rS#rSuo rS  rS  rS rS rSp!% $$+/ %!%S S Sf Suo S  S  S S S!(SS#S Sl8M1188Q;PQM T#Y 4 . $!) )6")&!) V  ) 6  )  )) ) )X tCy T &*$(#! ( (6"(&!( V  ( 6  (  (( ( (\ $!  3i V  6      : $ @ @3i@ V @ % @  @( $! # ## V # 6  #  #  #R ' ' ' '  '  '  'dII II I  I  II IX .x''..G1FGH $#'!; ; ; V ;4. ;  ; TN ;; ;B#'!9 9 99  9 4. 9  9 TN99 9@ X X X vu} X  X  X  X X|$(! I I I4.I TN I  I  I I^ $#'! Z Z Z V Z4. Z  Z TN ZZZ Z@ $#'! :T :T :T V :T4. :T TN :T  :T :T@ $#'!#'B B B V B4. B TN B  B B BP$(! = = = 4.=  TN =  =  =  = F# M M MM  M V  M  Mf$(!# F F F 4.F  TN F  F V  F  F X$(!# H H H 4.H  TN H  H V  H  H ^#'!0U 0U 0U 0U  0U 4. 0U TN 0U0U 0Ul#'! )M )M )M )M4. )M TN )M  )M )M^$(! &N &N &N4.&N TN &N  &N  &NX$(! &H &H &H4.&H TN &H  &H  &HX $#'! 9 9 9 V 94. 9 TN 9  9 9@#'!,W ,W ,W ,W  ,W 4. ,W TN ,W,W ,Wd##'!7 7 7 7  7 V  7 4. 7 TN77 7t  . 0 %(T%   (+\ $($(   3- 5/    D>       !%$($(   49  5/    D>         FGR .x''..G1FGH ?d ? *.$(-1    3-  4;'     D>  %TN           !%*.$(-1    49   4;'     D>  %TN           $($(-1    3-  5/     D>  %TN           !%$($(-1    49   5/     D>  %TN          *.$(-1[ [ 3-[4;'[  [ D> [ %TN [[ [|4+--44M7LMK $(  3- 5/      !%$(  49  5/     B8>/77>>WAVW $(  3- 5/      !%$(  49  5/      *.  3- 4;'      !%*.  49  4;'     W8 @ 1 9 9 @ @! ! #$( \X \X \X \X \X D> \X  \XD%)cC cC s)cCD>cC cCR! Q5 Q5 cQ5 Q5 E? Q5  Q5j'     KK 6  @  HHLL,/ l#'!' '''  '  '  ' '4.' TN'' '^GKJ JJJ  &&)96)A BC J  J JJ J^ "0 "0 "0 "0  "0 &  "0  "0JKDIKKsKtK()&'%& # #"3'# $#s # # c " #  #T()&'%& ! !"3'!#3'! $ ! s # ! c " ! !\ HM HM HMHM HM  HM  &\ HMd!  BI BI BIBI  BI  BI  BI & BI & BI & BI 666 !"BIJ + HHLL--ccF Gc(F'Gc(H6IZ] @; ; ; ;v& ;  ;  ;H  6      f8Q(6*QxQ ll -0 7:   :)-"&%*&*&*&*!%!%!%3e! e! e! e! e!  e! V$ e!6"e! V e! V e!e!e!e!F#e!e!v&e! !e!"#e!$#%e!&F#'e!(F#)e!*F#+e!,v-e!.v/e!01e!23e!4 68F# #$5e!s| Bs5AM,M,AM7M6AM7