L ia$ddlZddlZddlZddlmZddlmZddlmZm Z m Z ddl m Z ddl Z ddlZ ddlmZmZddlmZddlmZgd ZGd d eZd e j2fd Zd e j2fdZdZdedee j2dffdZ d{dedee j2dffdZ d|de e j2ee j2fde dee j2fdZ!dZ"dZ# d}de$ee j2dffdZ%e de&d~d Z'd!Z( d|de$e j2fd"Z)d#e*e+e$e j2fdee j2dffd$Z, d|d%e j2d&e e j2de j2fd'Z-ddd(Z. ddee j2dffd)Z/ddd*deee j2dfdffd+Z0d,Z1d|d-Z2d.Z3d/Z4d0Z5 d|de$e j2fd1Z6d2Z7d3Z8deee j2dfe e ffd4Z9d5Z: ddee j2dffd6Z;d7Z<e d8e&dd:Z=d;Z>de$e$e e j2fd<Z?de$e$e e j2fd=Z@de fd>ZAdd?ZBde fd@ZCdAZDdBjZFdCZG d|dDZHde fdEZIde fdFZJde fdGZKdHZLde fdIZMdJZNdKZO ddePfdLZQdMZRdNZSdOZTdPZUdddddQdRZVdSZWdTZXdUZYd|dVZZdWZ[dXZ\dYjZ]dZZ^d[Z_d\Z`dd]d^ZadddddQd_Zbd`daddbd9dbdddddbdddc ddede efdeedfecdgecdhecdie djecdke dle dme dne doe dpe dqe dre e de f dsZddtZe dd`dadddbd9dbddddbdddu ddedefdeedve edfecdgecdhecdwe die djecdke dle dme dxe dye dqe dre de f"dzZfy)N)Iterable)product)CallableOptionalUnion) deprecated)_vmapvmap)is_tensor_like)_TensorOrTensors) gradcheck gradgradcheckGradcheckErrorget_numerical_jacobianget_analytical_jacobian)get_numerical_jacobian_wrt_specific_inputceZdZdZy)rzN>N0NNrc`t|xr"|jxs|jSr)r is_floating_point is_complexr$s r_is_float_or_complex_tensorr,/s' #  PC$9$9$;$Os~~?OPr input_tensorsreturn.c|Dcgc]J}t|r=|jr1|j|j|ftj L}}t |Scc}w)Nr)r, requires_grad new_zerosnumelrstridedtuple)r- numel_outputtouts r_allocate_jacobians_with_inputsr93s\ &q )aoo QWWY -emm DC : sAA!output_tensorsc||tjd}|Dcgc]0}t|r#|j||j ffi|2}}t |Scc}w)N)dtypedevicer)rr4r,r2r3r5)r: numel_inputr<r=optionsr7r8s r _allocate_jacobians_with_outputsr@Csg5==IG  &q )  [!''),88C : s5AFxonly_requiring_gradc#Kt|r|js|s|yyt|tjj r-t|t s|D]}t||Ed{yyy7 wr)r r1 isinstance collectionsabcrstr _iter_tensors)rArBelems rrHrHSsoa ??"5G#6 A{// 0As9K @D$T+>? ? ? @:L 0 ?sA(A6*A4+ A6ct|ttfrt|t t |St |r,|jtjtjhvr|S|jtjur|j}|jj}tj|j d|j#tj$|}|j'j)j+|}tj,|j/g|j |j#d|j|}|j1j3}|j/dkDrv|j5}|j7j9tj:|||j=dj?d}|jA||<tjB|||j jEdjG|jHStK|r|jtjLtjNhvr|jAj ddnd} |jtjPtjLhvr|jSn|jU} t |j1jWtjjW|j| } | jAj/tYd| jAj dz} | j/| j d z} | j/tYd| | zz}|| j[k7r+t]|jd |d | j[| jG|jHSt_|rta|j|S) Nr<r=r<rTr0)r blocksizez densify failed: expected nnz=z but got )1rDlistr5typemap_densifyr rrr4_mkldnnr'r=_indicesr<onesshape sparse_dimint8nonzeror7tozerosr3detachcoalescestrideindicesmultensor unsqueezesumvaluessparse_coo_tensor _coalesced_requires_grad_r1r%r"r#r crow_indices ccol_indices to_sparsemax_nnzAssertionErrorr(NotImplementedError)rAr= indices_dtypetmprarf x_coalescedr` flat_indicesrOcompressed_indicesr dense_numel batch_numel sparse_numels rrTrT_s]!dE]#tAws8Q'(( A !((u}}emm.L"L U%% % ** jj!11<<>2%**VT++-//#&&]&; YY[ 51771<<>#34 5QWWV hhj))+     "ZZ\F##%LL}VLVV Q $/#5#5#7F<  # #GVQWW = [  ^AOO , &a (xxE,,e.>.>?? HHJ  Qq ! xxE,,e.>.>?? NN !  QXXZ))1A1A)B C M M88y N hhj&&(C188:3C3CA3F,GG (..04F4L4LR4PP wwyC;+D$EE 1668 # 88*:<. RSRXRXRZQ[\ 00 q !!!((++ Hrc # Kt|rd}|j}t|j}|jt j ur0|jj}|j}n|jt jurRt j|j|jj}|j}nJ|jt jurTt j|j!|j#dj}|j}n|jt j$urC|j}|jdd}t j|j|jj'|d|dzdj)t j*||j,j/ddj1t j2t j4t j6||j,j9d|j}|j;dd}|jd}nz|jt j<urD|j}|jdd}t j|j!|j#dj'|d|dzdj)t j*||j,j/ddj1t j2t j4t j6||j,j9d|j}|j;dd}|jd}nt?d |jd |||j@}tC|D]}||} tE|jddD cgc] } tC| c} D]O} ||jGt| ztIfd tCtK|D} | | | fQy|jt jLk(rVtOtE|jD cgc] } tC| c} D]\} } |jQ} | | | fy|j@}tOtE|jD cgc] } tC| c} D] \} } || | fycc} wcc} wcc} ww) Ncxt|}d}dg|z}tt|D]}|||<|||z}|S)NrMr)lenreversedrange)sizedimrrr`is r get_stridez _iter_tensor..get_stridesPd)CCS3YFeCj) q tAw MrT) transposerMrNrr=z_iter_tensor for z inputc34K|]}||zywrr).0krax_strides r z_iter_tensor..sQGAJ!4Qs))r(rnrQrrrr'rVr7_valuesr _convert_indices_from_csr_to_coorj col_indicesrfr!rk row_indicesr"repeat_interleavemul_rcr=reshapeadd_stackwhererWrepeatflattenr#rpdatar~rtolistrer|rU enumerateto_dense)x_tensorrx_nnzx_size x_indicesx_valuesx_block_values x_blocksizerx_valuemx_idxd_idxx_tensor_denserars @@r _iter_tensorrs"X&  hmmo& ??e.. . ))+--/I'')H __ 0 0 0>>%%')=)=)?ac  (H __ 0 0 0>>%%')=)=)?4ac  (H __ 0 0 0%__.N(--/!4K66))+X-A-A-C#";q>KN#BAFell;xGOOPQSTUVKK EJJ{8??$STfQ&  &--a3HMM!$E __ 0 0 0%__.N(--/!4K66))+X-A-A-Ct#";q>KN#BAFell;xGOOPQSTUVKK EJJ{8??$STfQ&  &--a3HMM!$E%(9(//9J&&QR Rf%==u ,AqkG X]]_QR5H"I58"IJ ,#A,--/$u+=QeCK>PQQue++ , , EMM )%g(--//RQa/R&ST /LE5&..0N %. .  /==%g(--//RQa/R&ST )LE5E5( ( )#J 0S0Ss,Q>WV2 BW&V7 8AWV< 'WMbP?c zg}|t|t|}|std|Dr td||}t|Dcgc] \}}t |s|j s|"} }}tt t|d| D]\}\} } |t|| |||| |gz } |Scc}}w)aCompute the numerical Jacobian of `fn(inputs)` with respect to `target`. If not specified, targets are the input. Returns M * N Jacobians where N is the number of tensors in target that require grad and M is the number of non-integral outputs. Args: fn: the function to compute the jacobian for inputs: inputs to `fn` outputs: provide precomputed outputs to avoid one extra invocation of fn target: the Tensors wrt whom Jacobians are calculated (default=`inputs`) eps: the magnitude of the perturbation during finite differencing (default=`1e-3`) is_forward_ad: if this numerical jacobian is computed to be checked wrt forward AD gradients (this is used for error checking only) Returns: A list of M N-tuples of tensors Note that `target` may not even be part of `input` to `fn`, so please be **very careful** in this to not clone `target`. c3<K|]}|jywrr+ros rrz*_get_numerical_jacobian.. s AA AzsExpected output to be non-complex. get_numerical_jacobian no longer supports functions that return complex outputs.T)input is_forward_ad) _as_tupleany ValueErrorrr r1ziprHr) fninputsoutputstargetepsr jacobiansra inp_indicesinpinp_idxs r_get_numerical_jacobianrs213IB & 123 S A AA E  ~'a>!+<K's=+F 'TU  >C 5+      s B7! B7.B7aA`get_numerical_jacobian` was part of PyTorch's private API and not meant to be exposed. We are deprecating it and it will be removed in a future version of PyTorch. If you have a specific use for this or feature request for this to be a stable API, please file us an issue at https://github.com/pytorch/pytorch/issues/new)categorycp|dk7r tdfd}t||d||}td|DS)aCompute the numerical Jacobian for a given fn and its inputs. This is a Deprecated API. Args: fn: the function to compute the Jacobian for (must take inputs as a tuple) inputs: input to `fn` target: the Tensors wrt whom Jacobians are calculated (default=`input`) eps: the magnitude of the perturbation during finite differencing (default=`1e-3`) grad_out: defaults to 1.0. Returns: A list of Jacobians of `fn` (restricted to its first output) with respect to each input or target, if provided. Note that `target` may not even be part of `input` to `fn`, so please be **very careful** in this to not clone `target`. ?zaExpected grad_out to be 1.0. get_numerical_jacobian no longer supports values of grad_out != 1.0.c|Srr)inpsrs r fn_pack_inpsz,get_numerical_jacobian..fn_pack_inps]s $xrNc3&K|] }|d yw)rNr)rjacobian_for_each_outputs rrz)get_numerical_jacobian..bsW1I)!,Ws)rrr5)rrrrgrad_outrrs` rrr9sH: C 2  ( fdFCPI WYW WWrct|r|j|jk(sJ|j|jf|j|jk(s0J|j|j|jf|j }|j }|j }|j }|j||z |}|j||z|}|j|fdtfdt||DS)Ncj||||z dzz }|jjdS)NrrP)r^r)rbretnbhd_checks_fnnorm_vs rcomputez,_compute_numerical_gradient..compute|s6q!1uV$zz|##B''rc36K|]\}}||ywrr)rrrrs rrz._compute_numerical_gradient..s=6AqA=) r%rrnrXrfr^clonecopy_r5r) rentryvrrorigoutaoutbrs `` @r_compute_numerical_gradientres$E*||qxx'A%,,)AA'zz|qvvx'N%**,%++)NN'  HHJ  ;;=D KKq 4D KKq 4D KK( =St_= ==rctg}|t|tr|dn|}|rjt|tr||ddzn ||dz}t||D]0\}}|jrJ||dzz} |j | 2|S|D]'}|s|jrJ|j |)|S)NrrM?)rDr5rr+append) jvp_fndeltainput_is_complexrjvps ds_dx_tup ds_dy_tupds_dxds_dyconj_w_ds r*_compute_numerical_jvps_wrt_specific_inputrs "D:eU#;uQxGI%/u%=F58b= !6%RT*CU  95 "LE5'') ))urz)H KK !  " K E (8(8(: :: KK   Krjacobians_colsct|||jjr |jnd}t|D]%\}}|j D] \}}||||<'|SNrL)r@r<r+ritems) rrrr3rrjacobianrrs r_combine_jacobian_colsrsr 1U[[-C-CekkI!+ 8"((* DAqA$HQK  rrmaybe_perturbed_inputc|jtjk(r||jS|St |r|r||S|S|Sr)rrrU to_mkldnnr()rr fast_modes r_prepare_inputrsS  ||u}}$ ,(224 4L u % .:) (L rc |dnd}|j|jk(s(Jd|d|d|jd|jd |j|jk(s(Jd|d|d |jd|jd y) Nzon index {idx} zPExpected `func` to return outputs with the same shape when inputs are perturbed zby z, but got: shapes z and .zPExpected `func` to return outputs with the same dtype when inputs are perturbed z, but got: dtypes )rXr<)output1output2ridxon_indexs r#_check_outputs_same_dtype_and_shapers%(O H ==GMM ) &&.Zs3%8==/w}}oQ 8 ) ==GMM ) &&.Zs3%8==/w}}oQ 8 )rcLi}|||n|}|jsJt|D]b\}} } t||||} || } tjt | |} t | | || }t|||j||| <dt||||jS)N)rr) r1r_with_prepare_inputs functoolspartialr_get_numerical_jvp_fnrr+rr3)r input_idxrrrrr jacobian_colsrArr wrapped_fninput_to_perturbrrs rrrs46M!&F9 EE    %e,  3)"fiC S6"** /Sc ' (#~  J C e   "-% OOr)check_grad_dtypesall_uc 2tjj}td|D}t d|Dr t d|rtfd|D}ntfd|D}|j 5g}g} t|D]\} } t| r| jr~| jtjk(r t d|j| jtj| } |j|j!| d| j| |rtt#||D]\} \} } | j%| j'| t)|| }t+t,|}t|D]\}}|j!|\}}|r.|,|j/|j/k7r t1d || |j3d ||| |j5z|| |j%|j7d | j5nt|D]\} } tt9| j;Dcgc] }t=|c}D]\}}d | |<t)|| }t+t,|}t|D]\}}|j!|\}}|r.|,|j/|j/k7r t1d ||| ||j5f|| ||j%|j7d d | |< ddd|Scc}w#1swY|SxYw)aCompute the analytical Jacobian using forward mode AD of `fn(inputs)` using forward mode AD with respect to `target`. Return N * M Jacobians where N is the number of tensors in target that require grad and M is the number of non-integral outputs. Contrary to other functions here, this function requires "inputs" to actually be used by the function. The computed value is expected to be wrong if the function captures the inputs by side effect instead of using the passed ones (many torch.nn tests do this). Args: fn: the function to compute the jacobian for inputs: inputs to `fn` outputs: provide precomputed outputs to avoid one extra invocation of fn check_grad_dtypes: if True, will check that the gradient dtype are valid all_u (optional): if provided, the Jacobian will be right multiplied with this vector Returns: A tuple of M N-tuples of tensors c3RK|]}t|s|js|!ywr)r r1rrs rrz6_get_analytical_jacobian_forward_ad..sU~a/@Q__!Us ' ''c3<K|]}|jywrrrs rrz6_get_analytical_jacobian_forward_ad.. s 1a1<<> 1rzJExpected inputs to be non-complex for _get_analytical_jacobian_forward_ad.c36K|]}tdyw)rMN)r@rrrs rrz6_get_analytical_jacobian_forward_ad..s => ,Wa 8 rc3RK|]}t|j ywr)r@r3r s rrz6_get_analytical_jacobian_forward_ad..s$ EF ,Waggi @ $'7MKLDNN inputs are not support for forward AD gradcheck.rMNz'Forward AD gradient has dtype mismatch.rrPr)rautograd forward_adr5rr dual_levelrr r1rrU make_dualr^ zeros_liker unpack_dualrrview_asrfilterr,r+rsqueeze_zero_rrrr~)rrrrrfwAD tensor_inputsrfw_grads dual_inputsrrfw_gradu raw_outputs dual_outputsindex_od_ovalresrlin_idxgrad_idxs ` r#_get_analytical_jacobian_forward_adr's, >> $ $DUVUUM 1= 11 X    BO   JW    ?, ' $FAsc"s'8'8::.$QnnSZZ\53C3CC3HI 0 0 5a 89   s # $ $-S5-A#B >+  u %!;;= !:: rc"fd}|S)NctfdttD}tdt|DS)Nc3fK|](\}}t|rt||k(rndn|*ywr)r r)rrrrrrs rrz;_with_prepare_inputs..wrapped_fn..nsC 1a  1!y..dI V  s.1c3<K|]}|jywr)r)rrs rrz;_with_prepare_inputs..wrapped_fn..ts<1QWWY.wrapped_fnmsB ")F"34    < "c((;<<.jvp_fn{s* (%n  rr)rrrrrs```` rrrys Mrct|tr8t|ds(|dj||dj|fS|St|s|j|S|SNrrM)rDr5r(r)rrXs r_reshape_tensor_or_tupler2sb!U$QqT*aDLL'1e)<= = H%Q'99U# # HrcLt|tr||dz||dzfS||zSr1)rDr5)rrs r_mul_tensor_or_tupler4s0!UAaD!ad(##1u rc||}t|}t||||d}tjt|} t |||| } t ||j}t||}t| ||j|S)NT)r) r)rrrrrr2rXr4rr+) rrrrrrrrrrrs r%_get_numerical_jvp_wrt_specific_inputr6s 9 E,U3%b&)=MtTJ&&'JPSTN ":/?n UF $4$:$:;AQ$A 55##%} rc g}t||D]\} } t|| || ||} g} t|}t| t|k(sJt| |D]"\} }t |r| j | #$|Cg}t|| D] \}} |j t || "|j ||j | |Sr)rr6rr|r,r_dot_with_type_promotion)rrrfunc_outrall_vrrreduced_jacobiansrrall_Ju filtered_JuJuoutputjacobian_scalarsrs r_get_numerical_vJurAs35+u-2 6 C  X&6{c(m+++fh/ JB*62""2&    35 UK0 I2 ''(@B(GH I  $ $%5 6  $ $[ 1)2* rct||D]?\}}|jdk7s||z jj|kDs?yy)NrFT)rr3absrm)j1j2atolj1_xj2_xs r_check_jacobians_equalrIsP"bk d ::<1 $+!2!2!4!8!8!:T!A rcRt||}tt|d}d}d}t|D]\}}||} ||} t|D]\} } | $| j | j k7rd}n| | j | j k7rd}| | dd| fj c| jtjk(s| jn| } | dd| fj| jk(sJ| jd| dd| f<|||fS)NTFrP) r9rQrHrrr<rrrr4rr3r)list_of_list_of_tensorsr numel_outputs out_jacobiansdiff_input_listcorrect_grad_sizescorrect_grad_typesr tensor_listr out_jacobianjrcdenses r_stack_and_check_tensorsrUs9 4FMJM=67O#$;<7;a $Q' ";/ 7IAv!fkkmsxxz&A%*"#  (A%*"~QT"((*.4]]emm-KFOO%QW$AqD)//1U[[]BBB%*]]2%6 QT" 77 ,.@ @@ra NOTE: If your op relies on non-deterministic operations i.e., it is listed here: https://pytorch.org/docs/stable/generated/torch.use_deterministic_algorithms.html this failure might be expected. If you are adding a new operator, please file an issue and then use one of the workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. If the test - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck with `nondet_tol=` as a keyword argument. - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test to have `gradcheck_nondet_tol=`. - is a Module test (e.g., in common_nn.py), then modify the corresponding module_test entry to have `gradcheck_nondet_tol=` ctt|dfd}|r7t|j|}t|j|}n4t |j}t |j}|sj nd} t ||| \} } } t ||| \} }}t| | |}| s |r td| s td|std|dtz| S)NTcLtjj|ddSNT retain_graph allow_unusedrrgrad grad_outputrNr?s rvjp_fnz5_check_analytical_jacobian_attributes..vjp_fn+~~"" O[tRV#  rrMzGradient has dtype mismatchz&Analytical gradient has incorrect sizezBackward is not reentrant, i.e., running backward with same input and grad_output multiple times gives different values, although analytical gradient matches numerical gradient.The tolerance for nondeterminism was r) rQrH(_get_analytical_vjps_wrt_specific_outputr!_compute_analytical_jacobian_rowsr3rUrIrFAILED_NONDET_MSG)rr? nondet_tolrrrr`vjps1vjps2 output_numel jacobians1types_oksizes_ok jacobians2_ reentrantrNs ` @r%_check_analytical_jacobian_attributesros=67O 8QRS8QRS1&&,,.I1&&,,.I)26<<>L%= v|&"J(0v|LJ1&z:zJI ):;; EFF  45?.vjp_fnTrar) rr+rQrHrcrr3rUrI)rr?rerr`rfrgrhrirjrkrlrmrnrNs ` @rrr;s C 2   E  =67O .fflln EE -fflln EE<<>L%= v|&"J(0v|LJ1&z:zJI y(H 44rcBt|||tdd}||S)NinfF)rer)rofloat)rrr output_idxrs r_get_analytical_jacobianrgs.6 #e PUI Y rc tj|tj}|jd}g}t |j D]}|j d||<||}t|D]S\}}|dk(r|jg||xxt|tjr|jndgz cc<U|S)N memory_formatrPrr) rrlegacy_contiguous_formatviewr~r3rrrrDTensorr) r` sample_output grad_out_base flat_grad_outjacobians_rowsrS grad_inputsrd_xs rrcrcps$$U%C%CM"&&r*M9;N =&&( )  a]+  , FAsAv%%b) 1 )#u||< $"     rc||j|j}|Dcgc]/}t|tjr|j ndg1}}|Scc}wr)rrXrDrrr)r`rrrvjpvjpss rrbrbs[=#6#678KLW0EH 3 54@0D0 K0s4Ac dd}t|D]\}}t|s|js |jtj k(s6|jtj k(stjd|d|jr|j}nt|r|j}n|}|jtjurGtdt!|j#|j%Dst'd|dd}|s t)dy) NFzInput #z requires gradient and is not a double precision floating point or complex. This check will likely fail if all the inputs are not of double precision floating point or complex. c3:K|]\}}|dkDxs|dkyw)rrMNr)rstszs rrz _check_inputs..s,BF%bAg%szThe zth input has a dimension with stride 0. gradcheck only supports inputs that are non-overlapping to be able to compute the numerical gradients correctly. You should call .contiguous on the input before passing it to gradcheck.Tzngradcheck expects at least one input tensor to require gradient, but none of the them have requires_grad=True.)rr r1r<rfloat64 complex128warningswarn is_sparserr%rfrrUallrr`r RuntimeErrorr) tupled_inputsany_input_requiring_gradrrcontents r _check_inputsrs$m,,S # 3#4#4II.#))u?O?O2O cU#JJ }}++--c2**,~~U]]2"%gnn&6 "G'se$SS (, $9,< $ <   rcxtd|Dr tdtd|Dr tdy)Nc3hK|]*}t|tjst|,ywr)rDrrr(rr7s rrz!_check_outputs..s# T 1ell8S  # Ts22zySparse output is not supported at gradcheck yet. Please call to_dense(masked_grad=...) on the output of fn for gradcheck.c3K|]<}t|tjs|jtjk(>ywr)rDrrrrUrs rrz!_check_outputs..s* UAu||9T188u}} $ Us A"AzyMKLDNN output is not supported at gradcheck yet. Please call to_dense(masked_grad=...) on the output of fn for gradcheck.)rr)rs r_check_outputsrsN TW TT W   Ug UU W  Vrct|||||}|D];}|D]4}tj|djdkDs+t d=y)Nrrr3Numerical gradient for function expected to be zeroT)rrnerer)funcrr9rrjacobians_all_inputs_outputs%jacobians_all_outputs_and_fixed_inputrs r _check_no_differentiable_outputsrsm $; fhC}$ 2N-= Hxx!$((*Q.$I  rct||D]q\}}t|||||} | D]X} | jdk(r| tj| z j j |kDsOtdsy)NrrT)rr6r3rrrCrmr) rr9 all_inputsinputs_indicesrrrerrrjvps r%_check_no_differentiable_outputs_fastrs.%0 4T7JPQSVW Cyy{ae&&s++002668:E$I   ra gradcheck or gradgradcheck failed while testing batched gradient computation. This could have been invoked in a number of ways (via a test that calls gradcheck/gradgradcheck directly or via an autogenerated test). If you are adding a new operator, please file an issue and then use one of the workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. If the test - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck with `check_batched_grad=False` as a keyword argument. - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test to have `check_batched_grad=False` and/or `check_batched_gradgrad=False`. If you're modifying an existing operator that supports batched grad computation, or wish to make a new operator work with batched grad computation, please read the following. To compute batched grads (e.g., jacobians, hessians), we vmap over the backward computation. The most common failure case is if there is a 'vmap-incompatible operation' in the backward pass. Please see NOTE: [How to write vmap-compatible backward formulas] in the codebase for an explanation of how to fix this. a gradcheck failed while testing batched gradient computation with forward-mode AD. This test is enabled automatically when both `check_batched_grad=True` and `check_forward_ad=True`, but can be disabled in the following ways dependong on how the test was invoked (via a test that calls gradcheck directly or via an autogenerated test). If you are adding a new operator, please file an issue and then use one of the workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck. If the test - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck with `check_batched_forward_grad=False` as a keyword argument. - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test to have `check_batched_forward_grad=False` c Zd|d|d|rtntd|d|d jS)Nz For output z and input z: z Got: z Expected:  )FAILED_BATCHED_GRAD_MSG_FWD_ADFAILED_BATCHED_GRAD_MSGstrip)rrr$exprs r!_get_failed_batched_grad_test_msgrsT <{9+.#06MNO  EG  rc 0 tjj ttsJt D] \ t r jsdtjf fd }t sEtdDcgc]}tj }}|Dcgc] }|| }}t|Dcgc]}tj|}} t|tj|}t t||D]7\\} } tj$| | r t!t'| | d#ycc}wcc}wcc}w#t$r} t!d| dt"| d} ~ wwxYw)Ntangentc  j5 jj|t fdt D}t  |}g}|D]}| j |\}}||j|.|jtjg|j|jj|jt|cdddS#1swYyxYw)Nc3lK|]+\}}|k(rnt|r|jn|-ywr)r r^)rrrdualrs rrz=_test_batched_grad_forward_ad..jvp..1sA)!Si'*8*=#**,3H)s14rK)rrr^r5rrrrrr]r<r=expandrX) rinputs_with_dualr r dual_output primal_out tangent_outr current_inputrrrrs @rrz*_test_batched_grad_forward_ad..jvp.s" "~~m&:&:&.>{.K+J ". ;/ !KK "**:*::CTCT$fZ%5%56 Sz- " " "s CC<<Dr(While computing batched gradients, got:  Tr)rrrrDr5rr r1rr,r~ randn_likerrr rrrallcloser)rrrrmtangentsr7expectedshardsresultexr$rrrrs`` @@@r_test_batched_grad_forward_adr&s >> $ $D fe $$ $$-f$53 =}--2M2M  " " "2+=9 =B1XFE$$]3FF$,-qCF--698nEFEKK'EE U3Z H 56F&/s68/D%E  !IzS~~c3' 1y#s$  Y3h )G-E  :2$dCaBbc  s*E"1E' E,+$E11 F:FFc rtt|dtjtj j |ddfd}tdDcgc]}t j|}}|Dcgc] }|| }}t|Dcgc]}t j|}}tj5tjddtjdd t|t j|} dddt%t |D]5\} \} } t j&| | r t!t)|| | | ycc}wcc}wcc}w#t$r} t!d | d t"| d} ~ wwxYw#1swYxYw) NTrYcP|}tdt|D}|S)Nc3K|]P\}}||nDtjg|j|jj |j Ryw)NrK)rr]r<r=rrX)rr]rs rrz2_test_batched_grad..vjp..rsS c RsyyDKKCIIV W sAA)r5r)rresultsrNr]s rrz_test_batched_grad..vjpps3q' !/:    rrignorezThere is a performance drop)messagezPlease use `torch.vmap`rr)rQrHrrrrr]r~rrrrcatch_warningsfilterwarningsr rrrrrr)rr?rrrm grad_outputsgOrrrrrr$rrNr]s @@r_test_batched_gradras =56O      D7>#s #  -j)S# N   7@/A  !:2$dCZB[\     s<E7?E<F /F-<$F F*F%%F**F--F6c tt|d}|s tdtjj |||Dcgc]'}tj |tj)c}d}t||D]n\}}| t|tjrv|jtjk7rX|j|jk7r=tdt|jzdzt|jzdzt|rt|jjdd jd d }|j!|j!k7r0td |d |j!d |j!|j#|j#k7r0td |d|j#d |j#|j%}|j%}|r4tj&|tj |s5td|j)dj+s td|j,|j,k7r td|j.|j.k7r td|j1|j1k7sftdycc}w)NT(no Tensors requiring grad found in inputrr[zgrad is incorrect layout (z is not )ztorch.r_coozgrad is z& tensor, but has incorrect sparse_dim z , expected z% tensor, but has incorrect dense_dim z&backward not multiplied by grad_outputrzgrad is incorrect typezgrad is incorrect devicezgrad is incorrect size)rQrHrrrr]rrrrDrrr4rGr(replacerY dense_dimrreqrr<r=r) rrmaskedrNr grads_inputgidi sparse_kinds r!_test_backward_mul_by_grad_outputrs*.}VT/J*KO GHH..%%    Qe.L.L M &Kk?3$;B :  b%,, 'BII,FyyBII%$0"))n% !"))n% %R(!"))n44XrBJJ6SUV ==?bmmo5(";-0MMO,K 7HJ<<>R\\^3(";-0LLN+;r||~6FHBB >>"e&6&6r&:;$%MNNq !IJ J 88rxx  !9: : 99 ! !;< < 779 ! !9: :I$;J W s,K6 c4tjj}t|\}}t ||d\}}}|j 5g} g} t } t|D]\} } t| r| jr| jtjk(r td|j| jtj| } | j!|j#| d| j%| | j!| tt'| |D](\} \}}|j)|j+|*t|D]\}} || vr | |}|j| jtj| | |<t-|| }t/t0|}| j| |<t-|| }t/t0|}|| |<tt'||D]_\}\}}|j#|\}}|j#|\}}|4|7tj2||rNt5d|d| dd|d |  dddy#1swYyxYw) NTuse_forward_adr rMz2Mismatch in tangent values for output with index: z when input: z! has an undefined tangent value. z Got: z but expected: )rrr_get_inp_tensors _make_vectorsrsetrr r1rrUrrr^rrraddrrrrrr,rr)rrrr_inp_tensors_idx inp_tensors_all_vr _all_u_denserrtensor_indicesrrrrr dual_inp_objr dual_outputs1 dual_outputs2r!d_o1d_o2_val1res1_val2res2s r_test_undefined_forward_moders >> $ $D$4V$<!k"/WT#FE<  7 ' $FAsc"s'8'8::.$QnnSZZ\53C3CC3HI 0 0 5a 89""1%   s # $ )Xu)=> .OA| MM!))G, - ."&)" HC.(&s+L $~~cjjlE LM #zz|K #D+$67K"#> LM ,K )23}m3T)U %$"..t4 t"..t4 t   >>$5,P#+?$ -     %" +7p q7p s HJJJ-JJc  tt|d s tdd fdt||Dcgc]I}t |t j r-t jjj|Kc}g}t|ddkDrtt|D]o}t||}|jt|Dcgc]9\}}||k(r-t jjj|n|;c}}qtfd|DScc}wcc}}w)NTrc.tjdy)NaBackwards compatibility: New undefined gradient support checking feature is enabled by default, but it may break existing callers of this function. If this is true for you, you can call this function with "check_undefined_grad=False" to disable the feature)rrrrrwarn_bc_breakingz7_test_undefined_backward_mode..warn_bc_breakings  P rcz|Dcgc]'}tj|tj)}} tjj ||d}|D]6}||jdjr&t dycc}w#t $r}t d|d}~wwxYw)NrTrzExpected backward function to handle undefined output grads. Please look at "Notes about undefined output gradients" in "tools/autograd/derivatives.yaml"rzExpected all input grads to be undefined or zero when all output grads are undefined or zero. Please look at "Notes about undefined output gradients" in "tools/autograd/derivatives.yaml") rrrrr]rrrr)output_to_checkr grads_outputrerrNrs rcheck_undefined_grad_supportzC_test_undefined_backward_mode..check_undefined_grad_supports%    Qe.L.L M   ..--,T.K Bq "$8 1     4   s,B#B B:"B55B:rrMc3.K|] }|ywrr)rr?rs rrz0_test_undefined_backward_mode..PsS+F3Ss)rQrHr_differentiable_outputsrDrr_C _functions UndefinedGradr|r~rrr) rrrroutputs_to_checkundef_grad_idxrrrrNrs @@@r_test_undefined_backward_moders0*.}VT/J*KO GHH >-T6]; !U\\* HH   - - / 2  A !##CL1 N5dFmDO  # # #,O"< Qn,HH''557:   SBRS SS+ s AD< >E cbt|tr|St|tr t|S|fSr)rDr5rQrAs rrrSs+!U At Qxt rc8tdt|DS)Nc3:K|]}|js|ywr)r1rs rrz*_differentiable_outputs..]s.apply_to_c_outs..wrapped_fn..s"OQ1<<>Qq@Or )rr5)routsrr*s rrzB_real_and_imag_output..apply_to_c_outs..wrapped_fns"R[)DO$OO Orr)rr*rs`` rapply_to_c_outsz._real_and_imag_output..apply_to_c_outss Pr)rrimag)rr,s r_real_and_imag_outputr.s* 2uzz *OB ,K KKrcBfd}||d}||d}||fS)Ncfd}|S)Ncht|}D]}|||||<t|SrrQr)r new_inputsshould_be_complexcomplex_inp_indicesrr*rs rrzA_real_and_imag_input..apply_to_c_inps..wrapped_fnsNfJ%8 !0;01=AR3S1 ,- R_- -rr)rr*rr5rs`` rapply_to_c_inpsz-_real_and_imag_input..apply_to_c_inpss .rc&||jdzzSNr)r-rrs rz&_real_and_imag_input..sC$))b.4Hrc&|j|dzzSr8)rr9s rr:z&_real_and_imag_input..sDIIb4Hrr)rr5rr6real_fnimag_fns `` r_real_and_imag_inputr>s/ b"HIGb"HIG G rc t|D cgc]\} }|js| }} }tdt|D}| rf|rTt |\}}||}t |}|||||||||| |d ||}t |}|||||||||| | n|||||||||| | r>t|D cgc]"\} }t |r|jr| $}} }|rt|||\}}|Dcgc]+}t |r|jr |jn|-}}||}t |}|||||||||| |dd |Dcgc]+}t |r|jr |jn|-}}||}t |}|||||||||| |d | rt|||t|||yy|||||||||| d | rt|||yyycc}} wcc}} wcc}wcc}w)Nc3<K|]}|jywrrrs rrz'_gradcheck_real_imag..s MA MrT)rr)r)rrr)rrr) rr+rrr.r r r>r-rr) gradcheck_fnrr9rrrrtolrFrcheck_forward_adcheck_backward_adrecheck_undefined_gradrrcomplex_out_indiceshas_any_complex_outputr<r= imag_func_out imag_outputs real_func_out real_outputsrr5 imag_inputsdiff_imag_func_out real_inputsdiff_real_func_outs r_gradcheck_real_imagrPs*37);NAq||~1NN M89L MM !4T: GW#]3M2=AL ! 3 $]3M2=AL ! 3  ! $M2 3c"s~~'7   3)=  GW )+3/CNN4D#MK$[1M!8!G  "! 3# ")+3/CNN4D#MK$[1M!8!G  "! 3# $,Wm[Q,Wm[Q$ !# $,T7MJ$GcOd *sG4G4'G:0H0H)rrrrc  t|}|st||||| S| r|n t|} tt || ||| }t ||Dcgc]\}}|j s|}}}| rt||||}t|D]b\}}t|D]O\}}|||}t||j|j||r6tt||||| | ddyt|D]v\}}t||||}tt |||D]H\}\}}t||j|j||r1tt||||| | xycc}}w)Nr)rTr)rrrTr%rrr1r'r_allclose_with_type_promotionr\r=rr"ro)rr9rrrrBrFrrerrrrtupled_inputs_numericalrrnjanalytical_forwardr n_per_outrSnrrs r_slow_gradcheckrX*s "H / -s.  06m8M;R  # (  I"%Xy!9M2Q__MIM@ -=N &i0 LAy!), 1&q)!,4QQXXdS(,q!QRV  * g& DAq>q*.?J's:y|'DE  6Aq4QQXXdS(,Q1a)T   7Ns E8*E8cx|jdk(r|jdk(sJ||zjS)NrM)rre)rrs rr8r8ks1 557a>!Qd ++rc|tjk(rtjS|tjk(rtjS|Sr)rrr complex64float32rLs r_to_real_dtyper`ws5    }} %// !}} rc|jtjk(r|j}|rt |j n |j }tj |j|j||jj|j}||jz}tj|j||j|j}|St!|rK|jtj"tj$hvr!|j'|j)}}n |j+|j-}}|j/}|rt |j n |j }tj |j|j||jj|j}||jz}tj0||||j|j|j}|S|rt |j n |j }tj |j|j||j}||jz}|S)N) generatorrKr)rr=)rrr'rr`r<randr3r\r=rrXnormrgrVrr%r r"rjrrkrrfsparse_compressed_tensor) rArbdowncast_complexrr<rfvecru plain_indicess r_vec_from_tensorris  xx5###99;+;qww' JJx~~'9 = ReAHHR - T(.. !  &++-%%ajjlFAFFHQXXV8 J7 &a ( 88((%*:*:; ;010@!--/ 010@!--/ 88:+;qww' JJx~~'9 = ReAHHR - T(.. !  &++-,,    FFH8888   J ,<qww'jji8;;<  sxxz Jrct|Dcgc] \}}t|r|jr||f"}}}|Dcgc]}|d c}|Dcgc]}|d c}fScc}}wcc}wcc}wr1)rr r1)rrr7 inp_idx_tuptups rrrstm, Aq !  AK * *sCF *{,KSV,K KK  +,Ks%A A# A(ct|tr|dn|}|j}|dn|j}|t|zt|zS)Nrr)rDr5rer)rFrrsum_usum_vs r_adjusted_atolrpsL1e$!!A EEGE9C!%%'E %, u --ra Fast gradcheck failed but element-wise differences are small. This means that the test might've passed in slow_mode! If you are adding a new operator, please file an issue and then use one of the workarounds. The workaround depends on how your test invokes gradcheck/gradgradcheck: If the test - manually invokes gradcheck/gradgradcheck, then call gradcheck/gradgradcheck with `fast_mode=False` as a keyword argument. - is OpInfo-based (e.g., in test_ops_gradients.py), then modify the OpInfo for the test to have `gradcheck_fast_mode=False` - is a Module test (e.g., in common_nn.py), then modify the corresponding module_test entry to have `gradcheck_fast_mode=False` c Bt|||} |r$fd} t| f|fdd} nt|} | | z jj } t j | | ||} d| d| d| d}| r |tz }|S)NrcFt}||<t|Srr2)rr3rrrrs rnew_fnz,_run_slow_mode_and_get_error..new_fns,m,J$'Jy !T:./ ; ;rra The above quantities relating the numerical and analytical jacobians are computed in fast mode. See: https://github.com/pytorch/pytorch/issues/53876 for more background about fast mode. Below, we recompute numerical and analytical jacobians in slow mode: Numerical: z Analytical: z1 The max per-element difference (slow mode) is: z. )rr'rrCrmrrFAST_FAIL_SLOW_OK_MSG)rrrrrrBrFrrslow_numericalrsslow_analytical slow_max_diff slow_allclosemsgs`` `` r_run_slow_mode_and_get_errorrzs- mW#]N < > ]9-/'*2E1G  3 7Iz  $o5::<@@BMNN?ND$OM ''('()::G  M $$ JrcZt|r|jjdS|S)NrP)r(rr)rcs r_to_flat_dense_if_sparser|s'V$ ((,, rctj}d}g}g}|D]}|||d}t|} |jr<|||d} |j || ft| } |j | | fd|j ||j | |rdn|D cgc] } || | c} } | ||fScc} w)Nchtjd5t|cdddS#1swYyxYw)Ncpu)rr=ri)argss r_vec_from_tensor_cpuz+_make_vectors.._vec_from_tensor_cpu s-\\%  +#T* + + +s(1T)r Generatorr|r+r)rrrg_cpurr all_u_denserurur_denseuiui_denser8r:s rrrs OO E+ EK ) !#ud 3+B/ >> %c5$7B LL"b "/3H   (3 4 LL    x ( )  :A B3"3. B % $$Cs.Crc  t|D]\} }t|D]\}}| r || |}n||| }|j|j}t| || |r||nd}t ||j|j||rtt |||| ||| | | }t t|||| || | |zy)Nr)rr\r=rprRrzrr")all_analytical all_numericalrrrrr:rrBrFrrrrall_numerical_for_input_irSrWr updated_atol jacobians_strs r!_check_analytical_numerical_equalr%s )2-(@$ $78 DAq"1%a("1%a(AHH%A)$ae%(QUVL0ADDND,W <-!QdC! %(1aOY $$ rc Rt|\} }t||| \}}}| r|||fn t|||f\}}}t||| ||||| }| r|Jt ||t |||}n#|st |||| |||t||||||}t||| ||||||||| |  y)Nrr)rrT) rrrTrAr'rrr|r)rr9rrrrBrFrrerrrrinp_tensors_idxrr:rrinputs_numericalall_u_numericalall_v_numerical numerical_vJuanalytical_vJus r_fast_gradcheckrJs"$4F#; O[!.W^!E5+ #)hu7M.N7o'  $ M}}<   h /   1hZ ; GZ):E; &      $ rgư>gh㈵>T) rrFrBraise_exceptionrerErcheck_batched_gradcheck_batched_forward_gradrCrDrrrrrrFrBrrerErrrrCrDrrc | s | sJd| r | sJd| r | sJdtj}|jd|s tdi|Stdi|S#t$rYywxYw)aCheck gradients computed via small finite differences against analytical gradients wrt tensors in :attr:`inputs` that are of floating point or complex type and with ``requires_grad=True``. The check between numerical and analytical gradients uses :func:`~torch.allclose`. For most of the complex functions we consider for optimization purposes, no notion of Jacobian exists. Instead, gradcheck verifies if the numerical and analytical values of the Wirtinger and Conjugate Wirtinger derivatives are consistent. Because the gradient computation is done under the assumption that the overall function has a real-valued output, we treat functions with complex output in a special way. For these functions, gradcheck is applied to two real-valued functions corresponding to taking the real components of the complex outputs for the first, and taking the imaginary components of the complex outputs for the second. For more details, check out :ref:`complex_autograd-doc`. .. note:: The default values are designed for :attr:`input` of double precision. This check will likely fail if :attr:`input` is of less precision, e.g., ``FloatTensor``. .. note:: Gradcheck may fail when evaluated on non-differentiable points because the numerically computed gradients via finite differencing may differ those computed analytically (not necessarily because either is incorrect). For more context, see :ref:`non-differentiable-func-grad`. .. warning:: If any checked tensor in :attr:`input` has overlapping memory, i.e., different indices pointing to the same memory address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail because the numerical gradients computed by point perturbation at such indices will change values at all other indices that share the same memory address. Args: func (function): a Python function that takes Tensor inputs and returns a Tensor or a tuple of Tensors inputs (tuple of Tensor or Tensor): inputs to the function eps (float, optional): perturbation for finite differences atol (float, optional): absolute tolerance rtol (float, optional): relative tolerance raise_exception (bool, optional): indicating whether to raise an exception if the check fails. The exception gives more information about the exact nature of the failure. This is helpful when debugging gradchecks. nondet_tol (float, optional): tolerance for non-determinism. When running identical inputs through the differentiation, the results must either match exactly (default, 0.0) or be within this tolerance. check_undefined_grad (bool, optional): if ``True``, check if undefined output grads are supported and treated as zeros, for ``Tensor`` outputs. check_batched_grad (bool, optional): if ``True``, check if we can compute batched gradients using prototype vmap support. Defaults to False. check_batched_forward_grad (bool, optional): if ``True``, checks if we can compute batched forward gradients using forward ad and prototype vmap support. Defaults to ``False``. check_forward_ad (bool, optional): if ``True``, check that the gradients computed with forward mode AD match the numerical ones. Defaults to ``False``. check_backward_ad (bool, optional): if ``False``, do not perform any checks that rely on backward mode AD to be implemented. Defaults to ``True``. fast_mode (bool, optional): Fast mode for gradcheck and gradgradcheck is currently only implemented for R to R functions. If none of the inputs and outputs are complex a faster implementation of gradcheck that no longer computes the entire jacobian is run; otherwise, we fall back to the slow implementation. masked (bool, optional): if ``True``, the gradients of unspecified elements of sparse tensors are ignored. Defaults to ``False``. Returns: ``True`` if all differences satisfy allclose condition zIExpected at least one of check_forward_ad or check_backward_ad to be TruezESetting check_batched_grad=True requires check_backward_ad to be TruezLSetting check_batched_forward_grad=True requires check_forward_ad to be TruerFr)localscopypop_gradcheck_helperr)rrrrFrBrrerErrrrCrDrrrs rr r sj 0S 0#+<O =+3CV D 8==?DHH   $,t, ,!(4((  s A$$ A0/A0ct|}t|||}t|}t|t j | rt nt| }t|||||||||| | || | r t||| syt|D]\}}|s t|||t||| |r| r t|||y)N)r)rCrDrerET)rrr rrrrrXrPrrrrr)rrrrFrBrerErrrrCrDrrrr9rrArrs rrr s f%M- ]#H%h/G7$$$/&L    )+1 "%dM: '"41  }a 34&g}fE 1%dG]C r) rrFrBgen_non_contig_grad_outputsrrerErrcheck_fwd_over_revcheck_rev_over_revrrrrrrc | s | sJd| r | sJd| r | sJdt|}|#t|}tfd|D}n t|}t|t |Dchc] \}}t |s|j s|"c}}t |Dchc]\}}|j s|c}}fd}t|||z|||||| | | || | |Scc}}wcc}}w)a+ Check gradients of gradients computed via small finite differences against analytical gradients wrt tensors in :attr:`inputs` and :attr:`grad_outputs` that are of floating point or complex type and with ``requires_grad=True``. This function checks that backpropagating through the gradients computed to the given :attr:`grad_outputs` are correct. The check between numerical and analytical gradients uses :func:`~torch.allclose`. .. note:: The default values are designed for :attr:`input` and :attr:`grad_outputs` of double precision. This check will likely fail if they are of less precision, e.g., ``FloatTensor``. .. warning:: If any checked tensor in :attr:`input` and :attr:`grad_outputs` has overlapping memory, i.e., different indices pointing to the same memory address (e.g., from :func:`torch.Tensor.expand`), this check will likely fail because the numerical gradients computed by point perturbation at such indices will change values at all other indices that share the same memory address. Args: func (function): a Python function that takes Tensor inputs and returns a Tensor or a tuple of Tensors inputs (tuple of Tensor or Tensor): inputs to the function grad_outputs (tuple of Tensor or Tensor, optional): The gradients with respect to the function's outputs. eps (float, optional): perturbation for finite differences atol (float, optional): absolute tolerance rtol (float, optional): relative tolerance gen_non_contig_grad_outputs (bool, optional): if :attr:`grad_outputs` is ``None`` and :attr:`gen_non_contig_grad_outputs` is ``True``, the randomly generated gradient outputs are made to be noncontiguous raise_exception (bool, optional): indicating whether to raise an exception if the check fails. The exception gives more information about the exact nature of the failure. This is helpful when debugging gradchecks. nondet_tol (float, optional): tolerance for non-determinism. When running identical inputs through the differentiation, the results must either match exactly (default, 0.0) or be within this tolerance. Note that a small amount of nondeterminism in the gradient will lead to larger inaccuracies in the second derivative. check_undefined_grad (bool, optional): if True, check if undefined output grads are supported and treated as zeros check_batched_grad (bool, optional): if True, check if we can compute batched gradients using prototype vmap support. Defaults to False. fast_mode (bool, optional): if True, run a faster implementation of gradgradcheck that no longer computes the entire jacobian. masked (bool, optional): if True, the gradients of unspecified elements of sparse tensors are ignored (default, False). Returns: True if all differences satisfy allclose condition zLExpected at least one of check_fwd_over_rev or check_rev_over_rev to be TruezHSetting check_undefined_grad=True requires check_rev_over_rev to be TruezFSetting check_batched_grad=True requires check_rev_over_rev to be Truec 3K|]x}tjj|j|j s|j r |j ntj|jdddzyw)rPrMT)r<r=lowhighr1 noncontiguousN) rtesting make_tensorrXr*r+r<doubler=)rrArs rrz gradgradcheck..sm $  MM % %&&(ALLNgg\\xx"9 &  $ sA>BcNtfdt|d D}t|}tfdt| dD}tfdt|D}tjj |||dd}td|D}|S)Nc3PK|]\}}|vr|jn|ywrrirrrAdiff_input_args_indicess rrz2gradgradcheck..new_func..s3 1#$'>">A   A E #&c3PK|]\}}|vr|jn|ywrr)rrrAdiff_grad_output_indicess rrz2gradgradcheck..new_func..s3 1#$'?"?A   Q F rc32K|]\}}|vs |ywrrrs rrz2gradgradcheck..new_func..s" !Q18O3OA s T) create_graphr[c3&K|] }|| ywrr)rgs rrz2gradgradcheck..new_func..sD!amADs)r5rr rrr]) r input_argsrrdiff_input_argsrrrr num_outputss rnew_funczgradgradcheck..new_funcs !$} "56  *$ *;< !$ |}"56    #J/  nn)) _lTX* D{DD r) rrFrBrrerErrrrCrDr)rr r5r|rr r1r )rrrrrFrBrrrerErrrrrrrrtupled_grad_outputsrrArrrrs` ` @@@rrrGsER !3V 3%-?R @#+=P > f%M*$ *>?# $  $  ( 5)*K  .a.2C 34 a ( ++   '1+-+, 7 s;C3 C3C32C9C9)NN)F)NNrF)Nrrr)r.N)NF)FN)rr)FF)grErrcollections.abcr itertoolsrtypingrrrtyping_extensionsrr torch.testingtorch._vmap_internalsr r torch.overridesr torch.typesr __all__rrrr%r(r,r5r9r@boolrHrTrrQr FutureWarningrrrdictintrrrrr'r)rrr2r4r6rArIrUrdror|rrrcrbrrrrrrrrrrrrrrr rGr"r%r.r>rPrXr8rRr`rirrprtrzr|rrrrr rrrrrrsm$,,( -*(  H\HellOu||OQ   5<<  "<@   5<<  "QV @ U\\8ELL1 12 @IM @ ell @; |d)PDI2 % c! "#2j C  !X!XH>@49 %,,6 d5<<001  5<<  SX <<080F \\0 "DIP 5<< P6/44i 5s" #S ()iX   27  %,, :A 5s" #T4 /0A8$GK) 5<< )X0 C  !5!5H  $x % &'6 $x % &'&D&R   " , EG-0"$49  848v4T4n3$3l@F?TD?TD= 8) L*DKd >B ,+\L . EG"%P%Z"` Ox !%#$',""!!f) 3.// 0f) f)  f)  f)  f)f)f)f)f)f)!%f)f)f)f) TN!f)" #f)R9~04Z (- !%#$$##Z 3(( )Z Z+,Z  Z  Z Z"&ZZZZZZZZ !Z" #Z$ %Zr