L i# ddlZddlZddlZddlZddlmZmZddlmZddl Z ddl m Z m Z ddl m Z ddlmZddlmZdd lmZgd Ze j*e j,d Zd$d e d e dede fdZeded$d e d e dede fdZGddeZGddeZGddeZGddeZGddeZ Gdde Z!Gd d!e Z"Gd"d#e Z#y)%N)Optionaloverload) deprecated)_VFTensor)init) Parameter)PackedSequence)Module)RNNBaseRNNLSTMGRU RNNCellBaseRNNCellLSTMCellGRUCell)RNN_TANHRNN_RELUtensor permutationdimreturnc&|j||SN) index_selectrrrs Z/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/torch/nn/modules/rnn.py_apply_permutationr $s   sK 00z]`apply_permutation` is deprecated, please use `tensor.index_select(dim, permutation)` instead)categoryct|||Srr rs rapply_permutationr%(s fk3 77r!ceZdZUdZgdZdgZeed<eed<eed<eed<e ed<e ed <e ed <e ed <eed < d(dedededede d e d e d e d edd ffd Z d)dZ d)fd Z d)dZd*fd Zd)dZdedeedd fdZdedeedeeeeffdZ d+dedeeeefdedd fdZdZdededeedd fdZded eefd!Zdefd"Zd)d#Zd$Zfd%Zedeee fd&Z!fd'Z"xZ#S),r aBase class for RNN modules (RNN, LSTM, GRU). Implements aspects of RNNs shared by the RNN, LSTM, and GRU classes, such as module initialization and utility methods for parameter storage management. .. note:: The forward method is not implemented by the RNNBase class. .. note:: LSTM and GRU classes override some methods implemented by RNNBase. ) mode input_size hidden_size num_layersbias batch_firstdropout bidirectional proj_size all_weightsr'r(r)r*r+r,r-r.r/Nrc | | d} t|||_||_||_||_||_||_t||_ ||_ | |_ g|_ |rdnd} t|tjr(d|cxkrdkrn t#dt|t r t#d|dkDr |dk(rt%j&d|d|t|t(s!t+dt-|j.|dkr t#d |dkr t#d | dkr t#d | |k\r t#d |d k(rd|z}n)|dk(rd|z}n|dk(r|}n|dk(r|}nt#d|zg|_g|_t5|D]}t5| D]}| dkDr| n|}|dk(r|n|| z}t7t9j:||ffi| }t7t9j:||ffi| }t7t9j:|fi| }t7t9j:|fi| }d}|jdk(r|r||||f}n5||f}n0t7t9j:| |ffi| }|r|||||f}n|||f}|dk(rdnd}ddg}|r|ddgz }|jdkDr|dgz }|Dcgc]}|j=||}}t?||D]\}}tA||||j0jC||j2jE||jG|jIycc}w)Ndevicedtyper rzbdropout should be a number in range [0, 1] representing the probability of an element being zeroedzdropout option adds dropout after all but last recurrent layer, so non-zero dropout expects num_layers greater than 1, but got dropout=z and num_layers=z(hidden_size should be of type int, got: z%hidden_size must be greater than zeroz$num_layers must be greater than zerozEproj_size should be a positive integer or zero to disable projectionsz,proj_size has to be smaller than hidden_sizerrrrzUnrecognized RNN mode: _reverseweight_ih_l{}{}weight_hh_l{}{} bias_ih_l{}{} bias_hh_l{}{}weight_hr_l{}{})%super__init__r'r(r)r*r+r,floatr-r.r/_flat_weight_refs isinstancenumbersNumberbool ValueErrorwarningswarnint TypeErrortype__name___flat_weights_names _all_weightsranger torchemptyformatzipsetattrextendappend_init_flat_weightsreset_parameters)selfr'r(r)r*r+r,r-r.r/r3r4factory_kwargsnum_directions gate_sizelayer directionreal_hidden_sizelayer_input_sizew_ihw_hhb_ihb_hh layer_paramsw_hrsuffix param_namesxnameparam __class__s rrAzRNNBase.__init__Ts%+U;  $&$ &W~ *"SU+7GNN3$1$ '4(  Q;:? MM>>EYG(\+ +s+:4 ;L;U;U:VW  ! DE E ?CD D q=W   #KL L 6>KI U]KI Z #I Z #I6=> >#% :&+ 6E">2* 6 09A 9; "'1*J2B^2S!!KK,< =PP!KK,< =PP!Y!I.!IJ!Y!I.!IJ35 >>Q&(,dD$'? (,d| $ Y $<OOD(,dD$'E (,dD'9 '0A~202CD O_#EEK>>A%$5#66K@KL1qxxv6L L#&{L#A/KD%D$./((// <!!((5U* 6+ 6Z ! MsMc|jDcgc]}t||r t||ndc}|_|jDcgc]}|t j |ndc}|_|jycc}wcc}wr)rOhasattrgetattr _flat_weightsweakrefrefrCflatten_parameters)r[wnws rrYzRNNBase._init_flat_weightss.. ")r!2GD"  <  @D?Q?Q" :;amGKKN 5"  ! " s !A? Bct|dr8||jvr*|jj|}||j|<t|||y)NrO)rprOindexrrr@ __setattr__)r[attrvalueidxrns rrzzRNNBase.__setattr__sP 4. /DDH/H;/H8 4H;;IcVg|_t| ||}|j|Sr)rCr@_applyrY)r[fnrecurseretrns rrzRNNBase._applys.!#gnR) ! r!c|jdkDr"dtj|jz nd}|jD]}t j || |yNrg?r)mathsqrt parametersruniform_r[stdvweights rrZzRNNBase.reset_parameters'R484D4Dq4HsTYYt//00aoo' /F MM&4% . /r!input batch_sizesc,tjjsv|j|jdjk7rPtj j s2td|jdjd|j|dnd}|j|k7rtd|d|j|j|jdk7r*td |jd |jdy) Nrzinput must have the type z , got type r5r7zinput must have z dimensions, got z5input.size(-1) must be equal to input_size. Expected z, got ) rRjit is_scriptingr4rr_C_is_any_autocast_enabledrHr RuntimeErrorr(size)r[rrexpected_input_dims r check_inputzRNNBase.check_input,syy%%' t11!4:::99; /0B0B10E0K0K/LKX]XcXcWde#."9Qq 99;, ,"#5"66G }U  ??ejjn ,GGXX^_d_i_ijl_m^no  -r!c6|t|d}n.|jr|jdn|jd}|jrdnd}|jdkDr|j |z||jf}|S|j |z||j f}|SNrr r5)rKr,rr.r/r*r)r[rr mini_batchr]expected_hidden_sizes rget_expected_hidden_sizez RNNBase.get_expected_hidden_size?s  "[^,J*.*:*:A 1 J"00a >>A .0$ $# .0  $ $#r!hxrmsgc |j|k7r2t|j|t|jyr)rrrTlist)r[rrrs rcheck_hidden_sizezRNNBase.check_hidden_sizeUs9 779, ,szz*>RWWYPQ Q -r!cd}t|j|jD]3\}}t||r t ||nd}|#|&||us0d}|S|S)NFT)rUrCrOrprq)r[weights_changedrtrlrs r_weights_have_changedzRNNBase._weights_have_changed^so T33T5M5MN IC,3D$,?WT4(TF!co#%v:M"&   r!hiddencp|j|||j||}|j||yr)rrr)r[rrrrs rcheck_forward_argszRNNBase.check_forward_argsis8  ,#<>Q  * *A ??a  , ,A 99D  A   5 ( . .A <<1  & &A   U * 2 2Aqxx($--((r!ctjjs"|jr|j yyyr)rRrrrrY)r[s r_update_flat_weightszRNNBase._update_flat_weightss4yy%%'))+''),(r!c`|j|jj}|d=|S)NrC)rrcopy)r[states r __getstate__zRNNBase.__getstate__s. !!# ""$ % & r!c t ||d|vr |d|_d|vrd|_t |jddt s|j }|jrdnd}g|_g|_t|D]S}t|D]A}|dk(rdnd}gd}|Dcgc]}|j||}}|jry|jdkDr2|xj|gz c_|jj|}|xj|dd gz c_|jj|dd |jdkDrG|xj|ddg|d dgzz c_|jj|dd|d dgz |xj|ddgz c_|jj|ddDV|jD cgc]} t|| r t|| ndc} |_|jD cgc]} | t!j"| ndc} |_ycc}wcc} wcc} w) Nr0r/rr5r r9r:)r;r<r=r>r?r6r)r@ __setstate__rPr/rDstrr*r.rOrQrTr+rWrprqrrrsrtrC) r[dr*r]r_r`riweightsrkrvrwrns rrzRNNBase.__setstate__sS Q A  !- 0D  a DN$++A.q137J"&"4"4Q!N')D $ "D z* I!&~!6II+4>ZrFGAHH1qxxv6HGHyy>>A- --':- 44;;GD --'"1+>- 44;;GBQKH>>A- --'"1+'"#,1OO- 44;; ' wrs|n <!--'"1+>- 44;;GBQKH3I I:22"&-T2%6b!D@"D  @D?Q?Q" :;amGKKN 5" -I"" " s$I%!I  Ic |jDcgc]}|Dcgc]}t||c}c}}Scc}wcc}}wr)rPrq)r[rrs rr0zRNNBase.all_weightss? ,, 29 9vWT6 " 9  9 s :5 ::ctt|}|jdd|_|jdd|_|Sr)r@_replicate_for_data_parallelrrrO)r[replicarns rrz$RNNBase._replicate_for_data_parallels='68!( 5 5a 8&-&A&A!&D#r!r TFFrNNrN)T)zExpected hidden size {}, got {})$rN __module__ __qualname____doc__ __constants____jit_unused_properties__r__annotations__rKrGrBrArYrzrurrZrrrtuplerrrrrrrrrpropertyrr r0r __classcell__rns@rr r 0s   M"/ IOO J NN!#y y y  y  y  y y y y y  y v"):x / hv6F4&$$*26*:$ sC} $45 R R$CcM2R R  R ==%+=:B6:J= =33hv6F3 )C) * 1 f T$y/2  r!r cLeZdZdZe ddedededededed ed ed dfd Z edd Z fdZ ee jj dde dee d ee e ffdZee jj ddedee d eee ffdZddZxZS)ra7__init__(input_size,hidden_size,num_layers=1,nonlinearity='tanh',bias=True,batch_first=False,dropout=0.0,bidirectional=False,device=None,dtype=None) Apply a multi-layer Elman RNN with :math:`\tanh` or :math:`\text{ReLU}` non-linearity to an input sequence. For each element in the input sequence, each layer computes the following function: .. math:: h_t = \tanh(x_t W_{ih}^T + b_{ih} + h_{t-1}W_{hh}^T + b_{hh}) where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, and :math:`h_{(t-1)}` is the hidden state of the previous layer at time `t-1` or the initial hidden state at time `0`. If :attr:`nonlinearity` is ``'relu'``, then :math:`\text{ReLU}` is used instead of :math:`\tanh`. .. code-block:: python # Efficient implementation equivalent to the following with bidirectional=False rnn = nn.RNN(input_size, hidden_size, num_layers) params = dict(rnn.named_parameters()) def forward(x, hx=None, batch_first=False): if batch_first: x = x.transpose(0, 1) seq_len, batch_size, _ = x.size() if hx is None: hx = torch.zeros(rnn.num_layers, batch_size, rnn.hidden_size) h_t_minus_1 = hx.clone() h_t = hx.clone() output = [] for t in range(seq_len): for layer in range(rnn.num_layers): input_t = x[t] if layer == 0 else h_t[layer - 1] h_t[layer] = torch.tanh( input_t @ params[f"weight_ih_l{layer}"].T + h_t_minus_1[layer] @ params[f"weight_hh_l{layer}"].T + params[f"bias_hh_l{layer}"] + params[f"bias_ih_l{layer}"] ) output.append(h_t[-1].clone()) h_t_minus_1 = h_t.clone() output = torch.stack(output) if batch_first: output = output.transpose(0, 1) return output, h_t Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two RNNs together to form a `stacked RNN`, with the second RNN taking in outputs of the first RNN and computing the final results. Default: 1 nonlinearity: The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'`` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` batch_first: If ``True``, then the input and output tensors are provided as `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: ``False`` dropout: If non-zero, introduces a `Dropout` layer on the outputs of each RNN layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional RNN. Default: ``False`` Inputs: input, hx * **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. The input can also be a packed variable length sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or :func:`torch.nn.utils.rnn.pack_sequence` for details. * **hx**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the initial hidden state for the input sequence batch. Defaults to zeros if not provided. where: .. math:: \begin{aligned} N ={} & \text{batch size} \\ L ={} & \text{sequence length} \\ D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ H_{in} ={} & \text{input\_size} \\ H_{out} ={} & \text{hidden\_size} \end{aligned} Outputs: output, h_n * **output**: tensor of shape :math:`(L, D * H_{out})` for unbatched input, :math:`(L, N, D * H_{out})` when ``batch_first=False`` or :math:`(N, L, D * H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the RNN, for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, the output will also be a packed sequence. * **h_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the final hidden state for each element in the batch. Attributes: weight_ih_l[k]: the learnable input-hidden weights of the k-th layer, of shape `(hidden_size, input_size)` for `k = 0`. Otherwise, the shape is `(hidden_size, num_directions * hidden_size)` weight_hh_l[k]: the learnable hidden-hidden weights of the k-th layer, of shape `(hidden_size, hidden_size)` bias_ih_l[k]: the learnable input-hidden bias of the k-th layer, of shape `(hidden_size)` bias_hh_l[k]: the learnable hidden-hidden bias of the k-th layer, of shape `(hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` .. note:: For bidirectional RNNs, forward and backward are directions 0 and 1 respectively. Example of splitting the output layers when ``batch_first=False``: ``output.view(seq_len, batch, num_directions, hidden_size)``. .. note:: ``batch_first`` argument is ignored for unbatched inputs. .. include:: ../cudnn_rnn_determinism.rst .. include:: ../cudnn_persistent_rnn.rst Examples:: >>> rnn = nn.RNN(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> output, hn = rnn(input, h0) Nr(r)r* nonlinearityr+r,r-r.rc yrr8) r[r(r)r*rr+r,r-r.r3r4s rrAz RNN.__init__Zr!cyrr8r[argskwargss rrAz RNN.__init__i14r!c:d|vr tdt|dkDr|d|_|dd|ddz}n|jdd|_|jdk(rd}n+|jdk(rd }ntd |jd t ||g|i|y) Nr/=proj_size argument is only supported for LSTM, not RNN or GRUr7r6rtanhrrelurzUnknown nonlinearity 'z '. Select from 'tanh' or 'relu'.)rHrrpopr@rA)r[rrr'rns rrAz RNN.__init__ls & O  t9q= $QD 8d12h&D & >6 BD     &D   & (D():):(;;[\  ///r!rrcyrr8r[rrs rforwardz RNN.forward r!cyrr8rs rrz RNN.forwardrr!c |j|jrdnd}|}t|tri|\}}}}|d}|Gt j |j |z||j|j|j}n}|j||}nid}|jdvrtd|jd|jd k(} |jrdnd} | sU|j| }|t|jdk7rtd |jd |jd}n2|0|jd k7rtd |jd |jr|j!dn|j!d}d}d}|Ft j |j |z||j|j|j}n|j||}|J|j#||||j$d k(s|j$dk(sJ||j$d k(ret'j(|||j*|j,|j |j.|j0|j|j } n&t'j2|||j*|j,|j |j.|j0|j|j } n|j$d k(rZt'j(||||j*|j,|j |j.|j0|j } nYt'j2||||j*|j,|j |j.|j0|j } | d} | d} t|tr"t| |||}||j| |fS s"| j5 } | j5d} | |j| |fS)z( Runs the forward pass. r5r rNr4r3r5r7z(RNN: Expected input to be 2D or 3D, got zD tensor insteadr77For unbatched 2-D input, hx should also be 2-D but got -D tensor5For batched 3-D input, hx should also be 3-D but got rr)rr.rDr rRzerosr*r)r4r3rrrHr, unsqueezerrrr'rrnn_tanhrrr+r-trainingrnn_relusqueeze)r[rrr] orig_inputrsorted_indicesunsorted_indicesmax_batch_size is_batched batch_dimresultoutputr output_packeds rrz RNN.forwards !!#"00a j. 1CH @E;0@(^Nz[[OOn4"$$++ << ((^<Kyy{&( >uyy{mK[\)J!--1I 2>vvx1}*UVXV\V\V^U__hiaB>bffh!m&OPRPVPVPXzYbc/3.>.>UZZ]EJJqMN!N# z[[OOn4"$$++ << ((^<~~ r;7yyJ&$))z*AAA  yyJ&&&IIOOLLMM&&$$ &&IIOOLLMM&&$$ yyJ&&&IIOOLLMM&& &&IIOOLLMM&&  j. 1* ^5EM!$"5"5f>N"OO O^^I.F^^A&Ft**63CDDDr!)r rTFrFNNrr)rNrrrrrKrrGrBrArR _jit_internal_overload_methodrrrrr rrs@rrrsOAF "!#              440( ))48  !)&!1 vv~  *  ))<@ # )1&)9 ~v% & * AEr!rceZdZdZe ddedededededed ed ed dfd Zedd ZfdZde de e d e eeeffdZ de de e e fde e d dfdZ de e e fde e d e e e ffdZeej j" dde de e e e fd e e e e e fffdZeej j" ddede e e e fd e ee e e fffdZddZxZS)ra/%__init__(input_size,hidden_size,num_layers=1,bias=True,batch_first=False,dropout=0.0,bidirectional=False,proj_size=0,device=None,dtype=None) Apply a multi-layer long short-term memory (LSTM) RNN to an input sequence. For each element in the input sequence, each layer computes the following function: .. math:: \begin{array}{ll} \\ i_t = \sigma(W_{ii} x_t + b_{ii} + W_{hi} h_{t-1} + b_{hi}) \\ f_t = \sigma(W_{if} x_t + b_{if} + W_{hf} h_{t-1} + b_{hf}) \\ g_t = \tanh(W_{ig} x_t + b_{ig} + W_{hg} h_{t-1} + b_{hg}) \\ o_t = \sigma(W_{io} x_t + b_{io} + W_{ho} h_{t-1} + b_{ho}) \\ c_t = f_t \odot c_{t-1} + i_t \odot g_t \\ h_t = o_t \odot \tanh(c_t) \\ \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`c_t` is the cell state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{t-1}` is the hidden state of the layer at time `t-1` or the initial hidden state at time `0`, and :math:`i_t`, :math:`f_t`, :math:`g_t`, :math:`o_t` are the input, forget, cell, and output gates, respectively. :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. In a multilayer LSTM, the input :math:`x^{(l)}_t` of the :math:`l` -th layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous layer multiplied by dropout :math:`\delta^{(l-1)}_t` where each :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is :math:`0` with probability :attr:`dropout`. If ``proj_size > 0`` is specified, LSTM with projections will be used. This changes the LSTM cell in the following way. First, the dimension of :math:`h_t` will be changed from ``hidden_size`` to ``proj_size`` (dimensions of :math:`W_{hi}` will be changed accordingly). Second, the output hidden state of each layer will be multiplied by a learnable projection matrix: :math:`h_t = W_{hr}h_t`. Note that as a consequence of this, the output of LSTM network will be of different shape as well. See Inputs/Outputs sections below for exact dimensions of all variables. You can find more details in https://arxiv.org/abs/1402.1128. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two LSTMs together to form a `stacked LSTM`, with the second LSTM taking in outputs of the first LSTM and computing the final results. Default: 1 bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` batch_first: If ``True``, then the input and output tensors are provided as `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: ``False`` dropout: If non-zero, introduces a `Dropout` layer on the outputs of each LSTM layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional LSTM. Default: ``False`` proj_size: If ``> 0``, will use LSTM with projections of corresponding size. Default: 0 Inputs: input, (h_0, c_0) * **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. The input can also be a packed variable length sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or :func:`torch.nn.utils.rnn.pack_sequence` for details. * **h_0**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the initial hidden state for each element in the input sequence. Defaults to zeros if (h_0, c_0) is not provided. * **c_0**: tensor of shape :math:`(D * \text{num\_layers}, H_{cell})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{cell})` containing the initial cell state for each element in the input sequence. Defaults to zeros if (h_0, c_0) is not provided. where: .. math:: \begin{aligned} N ={} & \text{batch size} \\ L ={} & \text{sequence length} \\ D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ H_{in} ={} & \text{input\_size} \\ H_{cell} ={} & \text{hidden\_size} \\ H_{out} ={} & \text{proj\_size if } \text{proj\_size}>0 \text{ otherwise hidden\_size} \\ \end{aligned} Outputs: output, (h_n, c_n) * **output**: tensor of shape :math:`(L, D * H_{out})` for unbatched input, :math:`(L, N, D * H_{out})` when ``batch_first=False`` or :math:`(N, L, D * H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the LSTM, for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, the output will also be a packed sequence. When ``bidirectional=True``, `output` will contain a concatenation of the forward and reverse hidden states at each time step in the sequence. * **h_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{out})` containing the final hidden state for each element in the sequence. When ``bidirectional=True``, `h_n` will contain a concatenation of the final forward and reverse hidden states, respectively. * **c_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{cell})` for unbatched input or :math:`(D * \text{num\_layers}, N, H_{cell})` containing the final cell state for each element in the sequence. When ``bidirectional=True``, `c_n` will contain a concatenation of the final forward and reverse cell states, respectively. Attributes: weight_ih_l[k] : the learnable input-hidden weights of the :math:`\text{k}^{th}` layer `(W_ii|W_if|W_ig|W_io)`, of shape `(4*hidden_size, input_size)` for `k = 0`. Otherwise, the shape is `(4*hidden_size, num_directions * hidden_size)`. If ``proj_size > 0`` was specified, the shape will be `(4*hidden_size, num_directions * proj_size)` for `k > 0` weight_hh_l[k] : the learnable hidden-hidden weights of the :math:`\text{k}^{th}` layer `(W_hi|W_hf|W_hg|W_ho)`, of shape `(4*hidden_size, hidden_size)`. If ``proj_size > 0`` was specified, the shape will be `(4*hidden_size, proj_size)`. bias_ih_l[k] : the learnable input-hidden bias of the :math:`\text{k}^{th}` layer `(b_ii|b_if|b_ig|b_io)`, of shape `(4*hidden_size)` bias_hh_l[k] : the learnable hidden-hidden bias of the :math:`\text{k}^{th}` layer `(b_hi|b_hf|b_hg|b_ho)`, of shape `(4*hidden_size)` weight_hr_l[k] : the learnable projection weights of the :math:`\text{k}^{th}` layer of shape `(proj_size, hidden_size)`. Only present when ``proj_size > 0`` was specified. weight_ih_l[k]_reverse: Analogous to `weight_ih_l[k]` for the reverse direction. Only present when ``bidirectional=True``. weight_hh_l[k]_reverse: Analogous to `weight_hh_l[k]` for the reverse direction. Only present when ``bidirectional=True``. bias_ih_l[k]_reverse: Analogous to `bias_ih_l[k]` for the reverse direction. Only present when ``bidirectional=True``. bias_hh_l[k]_reverse: Analogous to `bias_hh_l[k]` for the reverse direction. Only present when ``bidirectional=True``. weight_hr_l[k]_reverse: Analogous to `weight_hr_l[k]` for the reverse direction. Only present when ``bidirectional=True`` and ``proj_size > 0`` was specified. .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` .. note:: For bidirectional LSTMs, forward and backward are directions 0 and 1 respectively. Example of splitting the output layers when ``batch_first=False``: ``output.view(seq_len, batch, num_directions, hidden_size)``. .. note:: For bidirectional LSTMs, `h_n` is not equivalent to the last element of `output`; the former contains the final forward and reverse hidden states, while the latter contains the final forward hidden state and the initial reverse hidden state. .. note:: ``batch_first`` argument is ignored for unbatched inputs. .. note:: ``proj_size`` should be smaller than ``hidden_size``. .. include:: ../cudnn_rnn_determinism.rst .. include:: ../cudnn_persistent_rnn.rst Examples:: >>> rnn = nn.LSTM(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> c0 = torch.randn(2, 3, 20) >>> output, (hn, cn) = rnn(input, (h0, c0)) Nr(r)r*r+r,r-r.r/rc yrr8) r[r(r)r*r+r,r-r.r/r3r4s rrAz LSTM.__init__rr!cyrr8rs rrAz LSTM.__init__rr!c,t|dg|i|y)Nrr@rAr[rrrns rrAz LSTM.__init__s 1$1&1r!rrc|t|d}n.|jr|jdn|jd}|jrdnd}|j|z||j f}|Sr)rKr,rr.r*r)rs rget_expected_cell_sizezLSTM.get_expected_cell_sizesn  "[^,J*.*:*:A 1 J"00a OOn ,      $#r!rc|j|||j|d|j||d|j|d|j||dy)Nrz"Expected hidden[0] size {}, got {}r z"Expected hidden[1] size {}, got {})rrrr)r[rrrs rrzLSTM.check_forward_argssf  ,  1I  ) )% = 0  1I  ' '{ ; 0 r!rrcF||St|d|t|d|fS)Nrr r$rs rrzLSTM.permute_hiddens8  I!"Q%57I qE;8   r!cyrr8rs rrz LSTM.forwardrr!cyrr8rs rrz LSTM.forwardrr!c & |j|}d}|jrdnd}|jdkDr |jn |j}t |t r|\}}}}|d} |t j|j|z| ||j|j} t j|j|z| |j|j|j} | | f}nY|j||}nE|jdvrtd|jd|jdk(} |jrdnd} | s|j| }|jr|j!dn|j!d} d}d}|t j|j|z| ||j|j} t j|j|z| |j|j|j} | | f}|j#|||n| rb|djdk7s|djdk7rd |djd |djd }t%||djdk7s|djdk7r6d |djd |djd }t%||djd|djdf}|j#||||j||}|dt'j(|||j*|j,|j|j.|j0|j|j }nYt'j(||||j*|j,|j|j.|j0|j }|d}|dd}t |t r"t ||||}||j||fS s9|j3 }|dj3d|dj3df}||j||fS) Nr5r rrrz)LSTM: Expected input to be 2D or 3D, got D insteadr7z=For batched 3-D input, hx and cx should also be 3-D but got (z-D, z -D) tensorsz?For unbatched 2-D input, hx and cx should also be 2-D but got ()rr.r/r)rDr rRrr*r4r3rrrHr,rrrrrlstmrrr+r-rr)r[rrrrr]rarrrh_zerosc_zerosrr rr r rr s rrz LSTM.forwardsI !!#  "00a-1^^a-?4>>TEUEU j. 1CH @E;0@(^Nz++OOn4"$++ <<  ++OOn4"$$++ << w'((^<yy{&( ? }IV)J!--1I 2.2.>.>UZZ]EJJqMN!N# z++OOn4"$++ <<  ++OOn4"$$++ << w'''r;?!uyy{a'2a599;!+;446qEIIK=RUYY[MQ\^+3//!uyy{a'2a599;!+;446qEIIK=RUYY[MQ\^+3//Q%//!,beooa.@AB''r;?((^<  XX""   ""   FXX""   "" F j. 1* ^5EM!$"5"5f>N"OO O 2 )++A.q 0A0A!0DE4..v7GHH Hr!rrr)rNrrrrrKrGrBrArrrrrrrRr rrr rrs@rrr s^@ !#              442 $ $*26*: $ sC}  $"  ffn% f%    &  &&. !  f%   vv~    ))CG  !)%*?!@ vuVV^,, - *  ))KO # )1%2G)H ~uVV^44 5 * uIr!rcFeZdZdZe ddedededededed ed dfd Zedd Zfd Zee jj dde de e d ee e ffdZee jj ddede e d eee ffdZddZxZS)ra__init__(input_size,hidden_size,num_layers=1,bias=True,batch_first=False,dropout=0.0,bidirectional=False,device=None,dtype=None) Apply a multi-layer gated recurrent unit (GRU) RNN to an input sequence. For each element in the input sequence, each layer computes the following function: .. math:: \begin{array}{ll} r_t = \sigma(W_{ir} x_t + b_{ir} + W_{hr} h_{(t-1)} + b_{hr}) \\ z_t = \sigma(W_{iz} x_t + b_{iz} + W_{hz} h_{(t-1)} + b_{hz}) \\ n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \\ h_t = (1 - z_t) \odot n_t + z_t \odot h_{(t-1)} \end{array} where :math:`h_t` is the hidden state at time `t`, :math:`x_t` is the input at time `t`, :math:`h_{(t-1)}` is the hidden state of the layer at time `t-1` or the initial hidden state at time `0`, and :math:`r_t`, :math:`z_t`, :math:`n_t` are the reset, update, and new gates, respectively. :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. In a multilayer GRU, the input :math:`x^{(l)}_t` of the :math:`l` -th layer (:math:`l \ge 2`) is the hidden state :math:`h^{(l-1)}_t` of the previous layer multiplied by dropout :math:`\delta^{(l-1)}_t` where each :math:`\delta^{(l-1)}_t` is a Bernoulli random variable which is :math:`0` with probability :attr:`dropout`. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` num_layers: Number of recurrent layers. E.g., setting ``num_layers=2`` would mean stacking two GRUs together to form a `stacked GRU`, with the second GRU taking in outputs of the first GRU and computing the final results. Default: 1 bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` batch_first: If ``True``, then the input and output tensors are provided as `(batch, seq, feature)` instead of `(seq, batch, feature)`. Note that this does not apply to hidden or cell states. See the Inputs/Outputs sections below for details. Default: ``False`` dropout: If non-zero, introduces a `Dropout` layer on the outputs of each GRU layer except the last layer, with dropout probability equal to :attr:`dropout`. Default: 0 bidirectional: If ``True``, becomes a bidirectional GRU. Default: ``False`` Inputs: input, h_0 * **input**: tensor of shape :math:`(L, H_{in})` for unbatched input, :math:`(L, N, H_{in})` when ``batch_first=False`` or :math:`(N, L, H_{in})` when ``batch_first=True`` containing the features of the input sequence. The input can also be a packed variable length sequence. See :func:`torch.nn.utils.rnn.pack_padded_sequence` or :func:`torch.nn.utils.rnn.pack_sequence` for details. * **h_0**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` or :math:`(D * \text{num\_layers}, N, H_{out})` containing the initial hidden state for the input sequence. Defaults to zeros if not provided. where: .. math:: \begin{aligned} N ={} & \text{batch size} \\ L ={} & \text{sequence length} \\ D ={} & 2 \text{ if bidirectional=True otherwise } 1 \\ H_{in} ={} & \text{input\_size} \\ H_{out} ={} & \text{hidden\_size} \end{aligned} Outputs: output, h_n * **output**: tensor of shape :math:`(L, D * H_{out})` for unbatched input, :math:`(L, N, D * H_{out})` when ``batch_first=False`` or :math:`(N, L, D * H_{out})` when ``batch_first=True`` containing the output features `(h_t)` from the last layer of the GRU, for each `t`. If a :class:`torch.nn.utils.rnn.PackedSequence` has been given as the input, the output will also be a packed sequence. * **h_n**: tensor of shape :math:`(D * \text{num\_layers}, H_{out})` or :math:`(D * \text{num\_layers}, N, H_{out})` containing the final hidden state for the input sequence. Attributes: weight_ih_l[k] : the learnable input-hidden weights of the :math:`\text{k}^{th}` layer (W_ir|W_iz|W_in), of shape `(3*hidden_size, input_size)` for `k = 0`. Otherwise, the shape is `(3*hidden_size, num_directions * hidden_size)` weight_hh_l[k] : the learnable hidden-hidden weights of the :math:`\text{k}^{th}` layer (W_hr|W_hz|W_hn), of shape `(3*hidden_size, hidden_size)` bias_ih_l[k] : the learnable input-hidden bias of the :math:`\text{k}^{th}` layer (b_ir|b_iz|b_in), of shape `(3*hidden_size)` bias_hh_l[k] : the learnable hidden-hidden bias of the :math:`\text{k}^{th}` layer (b_hr|b_hz|b_hn), of shape `(3*hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` .. note:: For bidirectional GRUs, forward and backward are directions 0 and 1 respectively. Example of splitting the output layers when ``batch_first=False``: ``output.view(seq_len, batch, num_directions, hidden_size)``. .. note:: ``batch_first`` argument is ignored for unbatched inputs. .. note:: The calculation of new gate :math:`n_t` subtly differs from the original paper and other frameworks. In the original implementation, the Hadamard product :math:`(\odot)` between :math:`r_t` and the previous hidden state :math:`h_{(t-1)}` is done before the multiplication with the weight matrix `W` and addition of bias: .. math:: \begin{aligned} n_t = \tanh(W_{in} x_t + b_{in} + W_{hn} ( r_t \odot h_{(t-1)} ) + b_{hn}) \end{aligned} This is in contrast to PyTorch implementation, which is done after :math:`W_{hn} h_{(t-1)}` .. math:: \begin{aligned} n_t = \tanh(W_{in} x_t + b_{in} + r_t \odot (W_{hn} h_{(t-1)}+ b_{hn})) \end{aligned} This implementation differs on purpose for efficiency. .. include:: ../cudnn_persistent_rnn.rst Examples:: >>> rnn = nn.GRU(10, 20, 2) >>> input = torch.randn(5, 3, 10) >>> h0 = torch.randn(2, 3, 20) >>> output, hn = rnn(input, h0) Nr(r)r*r+r,r-r.rc yrr8) r[r(r)r*r+r,r-r.r3r4s rrAz GRU.__init__sr!cyrr8rs rrAz GRU.__init__rr!cJd|vr tdt|dg|i|y)Nr/rr)rHr@rArs rrAz GRU.__init__ s4 & O  000r!rrcyrr8rs rrz GRU.forward'rr!cyrr8rs rrz GRU.forward.rr!c |j|}t|try|\}}}}|d}|W|jrdnd}t j |j |z||j|j|j}n|j||}nyd}|jdvrtd|jd|jdk(} |jrdnd} | sU|j| }|t|jdk7rtd |jd |jd}n2|0|jdk7rtd |jd |jr|j!dn|j!d}d}d}|V|jrdnd}t j |j |z||j|j|j}n|j||}|j#||||dt%j&|||j(|j*|j |j,|j.|j|j } nYt%j&||||j(|j*|j |j,|j.|j } | d} | d} t|tr"t| |||}||j| |fS s"| j1 } | j1d} | |j| |fS) Nrr5r rrz(GRU: Expected input to be 2D or 3D, got rr7rrr)rrDr r.rRrr*r)r4r3rrrHr,rrrrrgrurrr+r-rr)r[rrrrrrrr]rr r r rr s rrz GRU.forward5s) !!# j. 1CH @E;0@(^Nz&*&8&8a[[OOn4"$$++ << ((^<Kyy{&( >uyy{m9U)J!--1I 2>vvx1}*UVXV\V\V^U__hiaB>bffh!m&OPRPVPVPXzYbc/3.>.>UZZ]EJJqMN!N# z&*&8&8a[[OOn4"$$++ << ((^< r;7  WW""   ""   FWW""   "" F j. 1* ^5EM!$"5"5f>N"OO O 2*4..v7GHH Hr!)r TFrFNNrr)rNrrrrrKrGrBrArRr rrrrrr rrs@rrrsBB !#             441 ))48  !)&!1 vv~  *  ))<@ # )1&)9 ~v% & * bIr!rc eZdZUgdZeed<eed<eed<eed<eed< d dedededed df fd Zd e fd Z dd Z xZ S)r)r(r)r+r(r)r+ weight_ih weight_hhN num_chunksrc||d}t|||_||_||_t t j||z|ffi||_t t j||z|ffi||_ |rOt t j||zfi||_ t t j||zfi||_ n$|jdd|jdd|jy)Nr2bias_ihbias_hh)r@rAr(r)r+r rRrSr)r*r-r.register_parameterrZ) r[r(r)r+r+r3r4r\rns rrAzRNNCellBase.__init__s%+U; $& " KKk1:> Q. Q # KKk1;? R> R  $ J4GGDL% J4GGDL  # #It 4  # #It 4 r!cd}d|jvr|jdur|dz }d|jvr|jdk7r|dz }|jdi|jS) Nrr+Trrrz, nonlinearity={nonlinearity}r8)rr+rrTrs rrzRNNCellBase.extra_reprsd ) T]] "tyy'<  A T]] *t/@/@F/J 0 0Aqxx($--((r!c|jdkDr"dtj|jz nd}|jD]}t j || |yrrrs rrZzRNNCellBase.reset_parametersrr!)NNr) rNrrrrKrrGrrArrrZrrs@rrrss9MO J         B)C)/r!rc reZdZUdZgdZeed< d dededededdf fd Z dd e d e e de fd Z xZ S)rarAn Elman RNN cell with tanh or ReLU non-linearity. .. math:: h' = \tanh(W_{ih} x + b_{ih} + W_{hh} h + b_{hh}) If :attr:`nonlinearity` is `'relu'`, then ReLU is used in place of tanh. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` nonlinearity: The non-linearity to use. Can be either ``'tanh'`` or ``'relu'``. Default: ``'tanh'`` Inputs: input, hidden - **input**: tensor containing input features - **hidden**: tensor containing the initial hidden state Defaults to zero if not provided. Outputs: h' - **h'** of shape `(batch, hidden_size)`: tensor containing the next hidden state for each element in the batch Shape: - input: :math:`(N, H_{in})` or :math:`(H_{in})` tensor containing input features where :math:`H_{in}` = `input_size`. - hidden: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the initial hidden state where :math:`H_{out}` = `hidden_size`. Defaults to zero if not provided. - output: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the next hidden state. Attributes: weight_ih: the learnable input-hidden weights, of shape `(hidden_size, input_size)` weight_hh: the learnable hidden-hidden weights, of shape `(hidden_size, hidden_size)` bias_ih: the learnable input-hidden bias, of shape `(hidden_size)` bias_hh: the learnable hidden-hidden bias, of shape `(hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` Examples:: >>> rnn = nn.RNNCell(10, 20) >>> input = torch.randn(6, 3, 10) >>> hx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx = rnn(input[i], hx) ... output.append(hx) )r(r)r+rrNr(r)r+rcF||d}t||||fddi|||_y)Nr2r+r )r@rAr) r[r(r)r+rr3r4r\rns rrAzRNNCell.__init__s2%+U; [$W1WW(r!rrcp|jdvrtd|jd|/|jdvrtd|jd|jdk(}|s|jd}|Gtj|j d|j |j|j}n|s|jdn|}|jdk(rCtj|||j|j|j|j}nl|jd k(rCtj |||j|j|j|j}n|}t#d |j|s|j%d}|S) Nr r5z,RNNCell: Expected input to be 1D or 2D, got rz-RNNCell: Expected hidden to be 1D or 2D, got r5rrrrzUnknown nonlinearity: )rrHrrRrrr)r4r3rr rnn_tanh_cellr)r*r-r. rnn_relu_cellrrr[rrrrs rrzRNNCell.forwards 99;f $>uyy{m9U  >bffhf4?zS YY[A% OOA&E : 1 t//u{{5<<B)3aB    &##  C  & (##  CC!78I8I7JKL L++a.C r!)TrNNr)rNrrrrrrrKrGrArrrrrs@rrrsy4lJM " ) ) ) )  )  )-V-&)9-V-r!rc neZdZdZ d dedededdffd Z d ded ee eefde eeffd Z xZ S) ra2 A long short-term memory (LSTM) cell. .. math:: \begin{array}{ll} i = \sigma(W_{ii} x + b_{ii} + W_{hi} h + b_{hi}) \\ f = \sigma(W_{if} x + b_{if} + W_{hf} h + b_{hf}) \\ g = \tanh(W_{ig} x + b_{ig} + W_{hg} h + b_{hg}) \\ o = \sigma(W_{io} x + b_{io} + W_{ho} h + b_{ho}) \\ c' = f \odot c + i \odot g \\ h' = o \odot \tanh(c') \\ \end{array} where :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` Inputs: input, (h_0, c_0) - **input** of shape `(batch, input_size)` or `(input_size)`: tensor containing input features - **h_0** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the initial hidden state - **c_0** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the initial cell state If `(h_0, c_0)` is not provided, both **h_0** and **c_0** default to zero. Outputs: (h_1, c_1) - **h_1** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the next hidden state - **c_1** of shape `(batch, hidden_size)` or `(hidden_size)`: tensor containing the next cell state Attributes: weight_ih: the learnable input-hidden weights, of shape `(4*hidden_size, input_size)` weight_hh: the learnable hidden-hidden weights, of shape `(4*hidden_size, hidden_size)` bias_ih: the learnable input-hidden bias, of shape `(4*hidden_size)` bias_hh: the learnable hidden-hidden bias, of shape `(4*hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. Examples:: >>> rnn = nn.LSTMCell(10, 20) # (input_size, hidden_size) >>> input = torch.randn(2, 3, 10) # (time_steps, batch, input_size) >>> hx = torch.randn(3, 20) # (batch, hidden_size) >>> cx = torch.randn(3, 20) >>> output = [] >>> for i in range(input.size()[0]): ... hx, cx = rnn(input[i], (hx, cx)) ... output.append(hx) >>> output = torch.stack(output, dim=0) Nr(r)r+rc8||d}t||||fddi|y)Nr2r+r6rr[r(r)r+r3r4r\rns rrAzLSTMCell.__init__*%+U; [$W1WWr!rrc|jdvrtd|jd|Et|D]7\}}|jdvstd|d|jd|jdk(}|s|jd}|Kt j |j d|j|j|j}||f}n,|s(|djd|d jdfn|}tj|||j|j|j|j}|s(|dj!d|d j!df}|S) Nr5z-LSTMCell: Expected input to be 1D or 2D, got rzLSTMCell: Expected hx[z] to be 1D or 2D, got r5rrr )rrH enumeraterrRrrr)r4r3r lstm_cellr)r*r-r.r)r[rrr}r|rrrs rrzLSTMCell.forwardsk 99;f $? }IV  >'m  U99;f,$05KEIIK=Xab  YY[A% OOA&E :KK 1 t//u{{5<<EBAK"Q%//!$beooa&89QSBmm   NN NN LL LL  q6>>!$c!fnnQ&78C r!TNNr) rNrrrrKrGrArrrrrrs@rrrKs9~ X X X X  XDH$$!)%*?!@$ vv~ $r!rc XeZdZdZ d dedededdffd Zd ded eedefd Z xZ S) raz A gated recurrent unit (GRU) cell. .. math:: \begin{array}{ll} r = \sigma(W_{ir} x + b_{ir} + W_{hr} h + b_{hr}) \\ z = \sigma(W_{iz} x + b_{iz} + W_{hz} h + b_{hz}) \\ n = \tanh(W_{in} x + b_{in} + r \odot (W_{hn} h + b_{hn})) \\ h' = (1 - z) \odot n + z \odot h \end{array} where :math:`\sigma` is the sigmoid function, and :math:`\odot` is the Hadamard product. Args: input_size: The number of expected features in the input `x` hidden_size: The number of features in the hidden state `h` bias: If ``False``, then the layer does not use bias weights `b_ih` and `b_hh`. Default: ``True`` Inputs: input, hidden - **input** : tensor containing input features - **hidden** : tensor containing the initial hidden state for each element in the batch. Defaults to zero if not provided. Outputs: h' - **h'** : tensor containing the next hidden state for each element in the batch Shape: - input: :math:`(N, H_{in})` or :math:`(H_{in})` tensor containing input features where :math:`H_{in}` = `input_size`. - hidden: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the initial hidden state where :math:`H_{out}` = `hidden_size`. Defaults to zero if not provided. - output: :math:`(N, H_{out})` or :math:`(H_{out})` tensor containing the next hidden state. Attributes: weight_ih: the learnable input-hidden weights, of shape `(3*hidden_size, input_size)` weight_hh: the learnable hidden-hidden weights, of shape `(3*hidden_size, hidden_size)` bias_ih: the learnable input-hidden bias, of shape `(3*hidden_size)` bias_hh: the learnable hidden-hidden bias, of shape `(3*hidden_size)` .. note:: All the weights and biases are initialized from :math:`\mathcal{U}(-\sqrt{k}, \sqrt{k})` where :math:`k = \frac{1}{\text{hidden\_size}}` On certain ROCm devices, when using float16 inputs this module will use :ref:`different precision` for backward. Examples:: >>> rnn = nn.GRUCell(10, 20) >>> input = torch.randn(6, 3, 10) >>> hx = torch.randn(3, 20) >>> output = [] >>> for i in range(6): ... hx = rnn(input[i], hx) ... output.append(hx) Nr(r)r+rc8||d}t||||fddi|y)Nr2r+r7rr;s rrAzGRUCell.__init__r<r!rrcx|jdvrtd|jd|/|jdvrtd|jd|jdk(}|s|jd}|Gtj|j d|j |j|j}n|s|jdn|}tj|||j|j|j|j}|s|jd}|S)Nr5z,GRUCell: Expected input to be 1D or 2D, got rz-GRUCell: Expected hidden to be 1D or 2D, got r5rr)rrHrrRrrr)r4r3rgru_cellr)r*r-r.rr8s rrzGRUCell.forwards 99;f $>uyy{m9U  >bffhf4?zS YY[A% OOA&E : 1 t//u{{5<<B)3aBll   NN NN LL LL  ++a.C r!r@r) rNrrrrKrGrArrrrrs@rrrsc;B X X X X  X V &)9 V r!r)r )$rrErIrstypingrrtyping_extensionsrrRrrtorch.nnrtorch.nn.parameterr torch.nn.utils.rnnr moduler __all__rr _rnn_implsrKr FutureWarningr%r rrrrrrrr8r!rrNs  %( (-    1v1F11V1 c 8f8688F8 8cfcL yE'yET jI7jIZ JI'JIZ7/&7/ttktnk{k\ikir!