L i/y UddlZddlZddlZddlZddlZddlmZmZddlm Z m Z m Z m Z m Z ddlmZmZmZmZmZmZmZmZddlmZddlmZmZddlZddlmZddlm Z dd l!m"Z"erdd l#m$Z$gd Z%ejLe'Z(Gd d ejRZ*deejVdfde*fdZ,GddeZ-dejVde-fdZ.deejVe ejVfddfdZ/GddZ0Gdde0Z1ejdde3de dfdZ4Gdde"Z5dd d!e ejVd"eee eejVgdfeejVgdffd#ed$de"fd%Z6d&a7e8e9d'<e:e;e;e;fZdejVde=fd+Z?Gd,d-Z@Gd.d/e0ZAGd0d1e ZBGd2d3ZCejdde eCddffd4ZDd5e eejVe-fdegdffd6ZEd5e eejVe-fd7ed8ede:ejVd9ffd:ZFy);N) defaultdictdeque) GeneratorIterableIteratorMutableMappingSequence)AnyCallablecastLiteral NamedTupleOptional TYPE_CHECKINGUnion) TypeAlias)WeakKeyDictionaryWeakValueDictionary)Variable)TorchDispatchMode)RemovableHandle) OpOverload) saved_tensors_hooks save_on_cpudisable_saved_tensors_hooksregister_multi_grad_hookallow_mutation_on_saved_tensorsNode GradientEdgeget_gradient_edgeincrement_versionceZdZejdefdZeejdeee de fdffdZ ejde fdZ eejdeefdZejdej$ddfd Zejd edefdefd Zejd edefdefd Zed edefdZy)rreturnct)a6Return the name. Example:: >>> import torch >>> a = torch.tensor([0., 0., 0.], requires_grad=True) >>> b = a.clone() >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) >>> print(b.grad_fn.name()) CloneBackward0 NotImplementedErrorselfs Z/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/torch/autograd/graph.pynamez Node.name0s "!.ctNr%r's r)next_functionszNode.next_functions? "!r+ct)zReturn the metadata.r%r's r)metadataz Node.metadataDr/r+ctr-r%r's r)_input_metadatazNode._input_metadataIr/r+tensorNctr-r%)r(r4s r)_register_hook_dictzNode._register_hook_dictNs!!r+fnct)aRegister a backward hook. The hook will be called every time a gradient with respect to the Node is computed. The hook should have the following signature:: hook(grad_inputs: Tuple[Tensor], grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None The hook should not modify its argument, but it can optionally return a new gradient which will be used in place of :attr:`grad_inputs`. This function returns a handle with a method ``handle.remove()`` that removes the hook from the module. .. note:: See :ref:`backward-hooks-execution` for more information on how when this hook is executed, and how its execution is ordered relative to other hooks. .. note:: In the rare case where the hook is registered while the Node has already begun execution, there is no longer any guarantee on :attr:`grad_outputs` content (it might be as usual or empty depending on other factors). The hook can still optionally return a new gradient to be used in place of :attr:`grad_inputs` independent of :attr:`grad_outputs`. Example:: >>> import torch >>> a = torch.tensor([0., 0., 0.], requires_grad=True) >>> b = a.clone() >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) >>> handle = b.grad_fn.register_hook(lambda gI, gO: (gO[0] * 2,)) >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([2., 2., 2.]) >>> handle.remove() # Removes the hook >>> a.grad = None >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([1., 1., 1.]) r%r(r7s r) register_hookzNode.register_hookRs V"!r+ct)aRegister a backward pre-hook. The hook will be called every time a gradient with respect to the Node is computed. The hook should have the following signature:: hook(grad_outputs: Tuple[Tensor]) -> Tuple[Tensor] or None The hook should not modify its argument, but it can optionally return a new gradient which will be used in place of :attr:`grad_outputs`. This function returns a handle with a method ``handle.remove()`` that removes the hook from the module. .. note:: See :ref:`backward-hooks-execution` for more information on how when this hook is executed, and how its execution is ordered relative to other hooks. Example:: >>> a = torch.tensor([0., 0., 0.], requires_grad=True) >>> b = a.clone() >>> assert isinstance(b.grad_fn, torch.autograd.graph.Node) >>> handle = b.grad_fn.register_prehook(lambda gI: (gI[0] * 2,)) >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([2., 2., 2.]) >>> handle.remove() >>> a.grad = None >>> b.sum().backward(retain_graph=True) >>> print(a.grad) tensor([1., 1., 1.]) r%r9s r)register_prehookzNode.register_prehooks D"!r+subclassc|turb|1|ttjj|j dus.t |tjjjrytS)NT) rgetattrtorch_C _functions__name__ issubclassautogradfunctionBackwardCFunctionNotImplemented)clsr=s r)__subclasshook__zNode.__subclasshook__sT $;$(;(;X=N=NPT UU(ENN$;$;$M$MNr+)rC __module__ __qualname__abcabstractmethodstrr*propertytuplerintr.dictr1listr r3r@Tensorr6r rr:r< classmethodtypeboolrJr+r)rr/sc "c " ""eHV, Any unpack_hook(Any) -> Tensor where the return value of ``pack_hook`` is a valid input to ``unpack_hook``. In general, you want ``unpack_hook(pack_hook(t))`` to be equal to ``t`` in terms of value, size, dtype and device. Example:: >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) >>> def pack_hook(x): ... print("Packing", x) ... return x.detach() >>> >>> def unpack_hook(x): ... print("Unpacking", x) ... return x >>> >>> a = torch.ones(5, requires_grad=True) >>> b = torch.ones(5, requires_grad=True) * 2 >>> with torch.autograd.graph.saved_tensors_hooks(pack_hook, unpack_hook): ... y = a * b Packing tensor([1., 1., 1., 1., 1.], requires_grad=True) Packing tensor([2., 2., 2., 2., 2.], grad_fn=) >>> y.sum().backward() Unpacking tensor([1., 1., 1., 1., 1.], requires_grad=True) Unpacking tensor([2., 2., 2., 2., 2.], grad_fn=) .. warning :: Performing an inplace operation on the input to either hooks may lead to undefined behavior. .. warning :: Only one pair of hooks is allowed at a time. When recursively nesting this context-manager, only the inner-most pair of hooks will be applied. .. warning :: To avoid reference cycle, the return value of ``pack_hook`` cannot hold a reference to the input tensor. For example, use `lambda x: x.detach()` instead of `lambda x: x` as the pack hook. pack_hook unpack_hookr#Nc ||_||_yr-)rprq)r(rprqs r)__init__zsaved_tensors_hooks.__init__=s #&r+ctjjj|j|j yr-)r@rA _autograd!_push_saved_tensors_default_hooksrprqr's r) __enter__zsaved_tensors_hooks.__enter__Es) << NND,, r+argscTtjjjyr-)r@rAru _pop_saved_tensors_default_hooks)r(rxs r)__exit__zsaved_tensors_hooks.__exit__Js ;;=r+r#N) rCrKrLrgr r@rUr rsrwobjectr{rYr+r)rrsb@D'U\\NC/0'seU\\12'  ' >f>>r+rc2eZdZdZddededdffd ZxZS)razContext manager under which tensors saved by the forward pass will be stored on cpu, then retrieved for backward. When performing operations within this context manager, intermediary results saved in the graph during the forward pass will be moved to CPU, then copied back to the original device when needed for the backward pass. If the graph was already on CPU, no tensor copy is performed. Use this context-manager to trade compute for GPU memory usage (e.g. when your model doesn't fit in GPU memory during training). Args: pin_memory (bool): If ``True`` tensors will be saved to CPU pinned memory during packing and copied to GPU asynchronously during unpacking. Defaults to ``False``. Also see :ref:`cuda-memory-pinning`. Example:: >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_CUDA) >>> # xdoctest: +REQUIRES(env:TORCH_DOCTEST_AUTOGRAD) >>> a = torch.randn(5, requires_grad=True, device="cuda") >>> b = torch.randn(5, requires_grad=True, device="cuda") >>> c = torch.randn(5, requires_grad=True, device="cuda") >>> >>> def f(a, b, c): ... prod_1 = a * b # a and b are saved on GPU ... with torch.autograd.graph.save_on_cpu(): ... prod_2 = prod_1 * c # prod_1 and c are saved on CPU ... y = prod_2 * a # prod_2 and a are saved on GPU ... return y >>> >>> y = f(a, b, c) >>> del a, b, c # for illustration only >>> # the content of a, b, and prod_2 are still alive on GPU >>> # the content of prod_1 and c only live on CPU >>> y.sum().backward() # all CPU tensors are moved back to GPU, for backward >>> # all intermediary tensors are released (deleted) after the call to backward pin_memory device_typer#Nc^tt|tjdtjdttj tjfffd }dttj tjfdtjffd }t |||y)Nr4r#c.s|j|jfStj|j |j |j jxr |j }|j||j|fS)N)dtypelayoutr) devicecpur@emptysizerr is_available is_sparsecopy_)r4packed device_modulers r) pack_to_cpuz)save_on_cpu.__init__..pack_to_cpuzsw vzz|44[[ ll}})668QAQAQ=Q F LL MM6* *r+rc4|\}}|j|S)N) non_blocking)to)rrr4rs r)unpack_from_cpuz-save_on_cpu.__init__..unpack_from_cpus#NFF99V*9= =r+)r?r@cudarUrQrsuperrs)r(rrrrr __class__s ` @r)rszsave_on_cpu.__init__wsx{EJJ?  + +u||U\\7Q1R + >E%,, *D$E >%,, > o6r+)Fr)rCrKrLrgrXrOrs __classcell__rs@r)rrNs(&P747c7t77r+r error_message)NNNc#Kd} tjjj}tjjj |d|)tjjj ytjjj |y#|)tjjj wtjjj |wxYww)aContext-manager that disables the saved tensors default hooks feature. Useful for if you are creating a feature that does not work with saved tensors default hooks. Args: error_message (str): When saved tensors default hooks are used when they have been are disabled, a RuntimeError with this error message gets raised. Example:: >>> # xdoctest: +SKIP(failing) >>> message = "saved tensors default hooks are disabled" >>> with torch.autograd.graph.disable_saved_tensors_hooks(message): ... # Raises RuntimeError: saved tensors default hooks are disabled ... with torch.autograd.graph.save_on_cpu(): ... pass N)r@rAru/_saved_tensors_hooks_get_disabled_error_message_saved_tensors_hooks_disable_saved_tensors_hooks_enable)rmaybe_prev_messages r)rrs* P HH   N N P  77 F   % HH   : : < HH   ; ;>> import torch >>> >>> a = torch.rand(2, 3, requires_grad=True) >>> b = torch.rand(2, 3, requires_grad=True) >>> c = a * b >>> d = a * b >>> >>> def fn(grads): ... print([g is not None for g in grads]) ... >>> torch.autograd.graph.register_multi_grad_hook((a, b, c, d), fn) >>> >>> c.sum().backward(retain_graph=True) [True, True, True, False] >>> c.sum().backward(inputs=(a,), retain_graph=True) [True, False, True, False] >>> rzExpects mode to be one of z but got rNidxr#c Hdtjddffd }|S)Ngradr#ctjj}|dk7sJdj|d|<j|dgz|< 5||dzc}|<|dk(r-t t tjj  ddd||< J dz k(rDtttttjgdf||=|=yy#1swYbxYw)N6expected this hook to be called inside a backward callr) r@rA_current_graph_task_idgetsummap_will_engine_execute_noder r r rrU) rid curr_countbuffercountr7grad_fnsr len_tensorslocknb_callss r) inner_hookzDregister_multi_grad_hook..get_inner_hook..inner_hook s"XX446RxLx"IIb!,b #ZZTF[,@Ar ,1"IuRy1})Jb !Q#& B BHM$ #'r 3+++A-h%,,1G(H'I4'OPRTUBvbzNb r .s ADD )r@rU) rrrrr7rrrrs ` r)get_inner_hookz0register_multi_grad_hook..get_inner_hooks' # #$ # #6 r+c3RK|]\}}|j| ywr-)r:).0irZrs r) z+register_multi_grad_hook..&s( 371aAOON1- . s$'rrctjj}|dk7sJd5|dc}|<dddry|y#1swYxYw)NrrT)r@rAr)rrprevr7rran_hooks r) wrapped_fnz,register_multi_grad_hook..wrapped_fn-sb002B8 UU U8 8%-b\4"hrl 8 tH  8 8s AAc3XK|]!}|jr|j#ywr-)r_r:)rr4rs r)rz+register_multi_grad_hook..8s- ##   , s'*) threadingLock ValueErrorrTrrclenrRr r@rUrQ enumerater rrX functoolswrapsr)rr7rsupported_modesrrrrrrrrrrs ` @@@@@@@@@r)rrsAp%O >> D ?"5o5FiPTvVWW u} ":<4g>?'l   %,,1E(F  < ;DW;M  0   +  (ELL>4/0" 5$/$5   U\\ d    !     r+F(_allow_mutation_on_saved_tensors_enabled_TID_SIDct|tjjjtjj j frd}n|j}t|||jfSr\) r]r@ _subclasses fake_tensor FakeTensorfunctional_tensorFunctionalTensordata_ptrr_versionr4rs r)_get_tidrUsd    ) ) 4 4    / / @ @ ??$ vJ&// 22r+ct|tjjjtjj j frd}n|j}||jfSr\) r]r@rrrrrrrrs r)_get_sidrds^    ) ) 4 4    / / @ @ ??$ foo &&r+c eZdZy)_HandleN)rCrKrLrYr+r)rrssr+rc eZdZdfd ZxZS)_swap_with_clonedcdtjdtffd }dtdtjffd }t|||y)Nr4r#ct|}t|}d}j|j||jvr*t }|j|<|j |<|Sj|}|Sr-)rr sid_to_tidaddtid_to_weakhandleroriginal)r4tidsidrctxs r)rpz-_swap_with_cloned.__init__..pack_hookys6"C6"C(,F NN3  # #C (#/// -3%%c*'- V$M..s3Mr+rcd}tsJ||jvrj|}|S|jvsJ|j|}|S)NzvTrying to backward outside of the 'allow_mutation_on_saved_tensors' contextin which the graph was originally recorded.)rclonedr)r error_msgresrs r)rqz/_swap_with_cloned.__init__..unpack_hooksj> < FY F;#jj(J-8y8-ll6*Jr+)r@rUrrrs)r(rrprqrs ` r)rsz_swap_with_cloned.__init__xsB ell w &  ELL  K0r+r_AllowMutationOnSavedContextr#N)rCrKrLrsrrs@r)rrws !1!1r+rc VeZdZd dZ d dddeedeedfd ee eefdef d Z y) _CloneArgBeforeMutateModer#Nc||_yr-r)r(rs r)rsz"_CloneArgBeforeMutateMode.__init__s r+funcrtypesrx.kwargscx|xsi}dtjddffd }t|jjD]r\}}|j |j j s*|jr ||dBt||tr||D] }|| h|||t||i|S)NrZr#cXt|}t|}j}||jvry|j|D]f}||jvr|j|}||j vr0|j |j|j |<|j |=hyyr-)rrrrrrrclone)rZrrrrr(s r) maybe_clonezA_CloneArgBeforeMutateMode.__torch_dispatch__..maybe_clones1+C1+C((Ccnn$>>#.-C#"7"77! 2237F+ ),f)=)C)C)ECJJv& V,#-%r+out) r@rUr_schema arguments alias_infois_writeis_outr]rT) r(rrrxrrrargrZs ` r)__torch_dispatch__z,_CloneArgBeforeMutateMode.__torch_dispatch__s2 -5<< -D -0"$,,"8"89 +HC~~)cnn.E.E::u .S 40"#Y'#A' S * +T$V$$r+r)rYN) rCrKrLrsrrWrQr rrSr rYr+r)rrs]!#+/ -%-%~-%CHo -% c3h( -% -%r+rceZdZddZddZy)rNct|_t|_t|_t t |_yr-)rrrrrrsetrr's r)rsz%_AllowMutationOnSavedContext.__init__s-=N=P ?P?R @S@U1>> import torch >>> with torch.autograd.graph.allow_mutation_on_saved_tensors(): ... # forward ... a = torch.ones(2, 3, requires_grad=True) ... b = a.clone() ... out = (b**2).sum() ... b.sin_() ... # backward ... out.sum().backward() ... tensor([[0.8415, 0.8415, 0.8415], [0.8415, 0.8415, 0.8415]], grad_fn=) z9allow_mutation_on_saved_tensors contexts cannot be nestedTFN)rrrrrjrrs r)rrsF ' (C 3  =!:3!? = =7"O8< 4I IIK7< 4 = = = IIK7< 4 = = = =sIB BA8A!A8B B!A55A88B =BB  B t_outputscfttt|}dttdttfd}dt t jdtfddtt t jddffd }||Dcgc]}|j|c}d fd }|Scc}w) Nrootsr#c3@K|syt}t}|D]'}||j||j|)|rU|j }|j D].\}}||vs| |j||j|0||rTyywr-)rrrappendpopleftr.)rseenqr^r7_s r) iter_graphz:_register_logging_hooks_on_whole_graph..iter_graphs % D  99;D,, A:    JsBA9BBrZcddlm}|y||jddjt t |j dS)Nr) dtype_abbrsNone[z, ])torch.utils._dtype_abbrsrrjoinrrOshape)rZrs r)fmtz3_register_logging_hooks_on_whole_graph..fmt)s?8 9agg&'q3sAGG3D)E(FaHHr+ grad_outputsctjj}ddjfd|Dd}d|d|}tj |y)Nr ,c3.K|] }|ywr-rY)rrZr%s r)rzJ_register_logging_hooks_on_whole_graph..prehook..3s'E1A'Esr!z Executing: z with grad_outputs: )r@rA_current_autograd_noder#logdebug)r&r^grad_outputs_strlog_strr%s r)prehookz7_register_logging_hooks_on_whole_graph..prehook1sUxx..0sxx'E 'EEFaHv%9:J9KL 'r+c4D]}|jyr-)r)rrs r)unregister_hooksz@_register_logging_hooks_on_whole_graph..unregister_hooks9s F MMO r+r|) rTrrcrrrr@rUrOr r<)rrrr/r^r1r%rs @@r)&_register_logging_hooks_on_whole_graphr2sC0)<=H$t*$(Ix %I#Ihx '=>4 ;EX:NO$t$$W-OG  Ps B.rxr.ctjtjk}|r t |} t j j|g|i||rSS#|rwwxYwr-)r+getEffectiveLevelloggingDEBUGr2r_execution_engine run_backward)rrxrattach_logging_hooksr1s r)_engine_run_backwardr:@sv 002gmmCA)L))66   &       s "A!! A-)GrM contextlibrr5r collectionsrrcollections.abcrrrrr typingr r r r rrrrtyping_extensionsrweakrefrrr@torch.autograd.variablertorch.utils._python_dispatchrtorch.utils.hooksr torch._opsr__all__ getLoggerrCr+ABCrrUrcrr r!rrcontextmanagerrOrrrrrXrhrQrRrrrrrrrrrr2r:rYr+r)rIs *SS   (: ,:-% g!~377~B ellN&B C   +: +JellJ|J6(eELL(5<<2H$HI(d(*Q>Q>h<7%<7~  Ps PyAQ7R P PF?0#(|! ell #|! (8ELL123T9:%,,%& ( |! , |!|!X27($6S# &i&S/i! 3U\\ 3d 3 'U\\ 'd '  "1+"1J1% 11%h    .= $,*.=.=b-ellL89:- b$h-`ellL89:  5<<  r+