L i UdZddlZddlZddlmZmZddlmZmZm Z ddl m Z ddl m Z ddlmZmZmZmZmZmZmZddlmZmZmZddlZddlmcmZdd lmZm Z m!Z!m"Z"m#Z#dd lm$Z$ed Z%ed Z&e'ed fZ(ee)d<e*e+efZ,ee)d<e*e+efZ-ee)d<e*eej\ej^fZ0ee)d<e*ee'ej\ejbfej^fZ2ee)d<ede(e,gee'e(e,ffZ3ee)d<ede(e,gdfZ4ee)d<gdZ5eZ6e*e7e3fe)d<eZ8e*e7e4fe)d<ej^ejrjtjvgZ<GddZ=e=Z>dee&e%fdee&e%ffdZ?dZ@dZA d;d eed eBfdeee&e%fgee&e%fffd!ZC dzBase optimizer.N) defaultdict OrderedDict)HashableIterableSequence)deepcopy)chain)AnyCallablecastOptionaloverloadTypeVarUnion) ParamSpecSelf TypeAlias)&_get_foreach_kernels_supported_devices$_get_fused_kernels_supported_devices"_group_tensors_by_device_and_dtypeIndicesTensorListList)RemovableHandle_T_P.ArgsKwargs StateDict DeviceDictDeviceDtypeDict OptimizerGlobalOptimizerPreHookGlobalOptimizerPostHook)r! register_optimizer_step_pre_hook!register_optimizer_step_post_hook_global_optimizer_pre_hooks_global_optimizer_post_hooksceZdZdZdefdZy)_RequiredParameterzCSingleton class representing a required parameter for an Optimizer.returncy)Nzselfs [/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/torch/optim/optimizer.py__repr__z_RequiredParameter.__repr__6s%N)__name__ __module__ __qualname____doc__strr0r,r1r/r)r)3sM&#&r1r)funcr*cdtjdtjdtffd }t j ||S)Nargskwargsr*cddl}tt|d}|j} |j|j d|j j|i|}|j j|j||S#|j j|j|wxYw)Nrdifferentiable) torch._dynamor r!is_grad_enabledset_grad_enableddefaults_dynamo graph_break)r9r:torchr. prev_gradretr7s r/ _use_gradz/_use_grad_for_differentiable.._use_grad>sItAw')E))+  . #E " "4==1A#B C MM % % '''C MM % % ' "E " "9 -  MM % % ' "E " "9 -s AB.C)rr9r:r functoolsupdate_wrapper)r7rFs` r/_use_grad_for_differentiablerI=s=BII"4Y- r1ctjjs tjj r|St |tj r|jS|SN)rCjit is_scriptingcompiler is_compiling isinstanceTensoritemxs r/ _get_valuerU\sD 99 ! ! #(C(C(E%a6qvvx=A=r1ctjjs3tjj rtj |S|SrK)rCrLrMrNrOstackrSs r/_stack_if_compilingrXds4 99 ! ! #(C(C(E{{1~r1single_tensor_fnc|r|t|j<dtttfdtttffd}|S)Nr7r*cpddl}tj|jj}d t |j jdtjdtjdtjffd }|S#t$rdYPwxYw)NrT state_stepsFr9r:c(tjjrl|jdds(r&|x}rt |t r|dj s*d|vr.|dx}r't |t r|dj r|i|S|i|S)N capturableFrr\)rCrNrOgetrPris_cuda)r9r:argkwarg disabled_funcr7has_state_stepsstate_steps_inds r/maybe_fallbackzG_disable_dynamo_if_unsupported..wrapper..maybe_fallbacks~~**,JJ|U3# 11S1sH-FNN!V+"("777"5(3a((%d5f55T,V,,r1)inspectrC_disable_dynamo signature parameterslistkeysindex ValueErrorrGwrapsrr9r:)r7rgpsrfrcrdres` @@@r/wrapperz/_disable_dynamo_if_unsupported..wrapperus--d3   t $ / / $"2779o33MBO   -"'' -RYY -  -$7 $#O $s(B'' B54B5)globalsr2r rr)rYrqs r/_disable_dynamo_if_unsupportedrsksG/? "++,#hr2v&#8BF+;#J Nr1paramsr< use_fusedctjjs|rytt |xrt fd|D}| xrt fd|D}||fS)N)FFc3K|]L}|duxsBt|tvxr/|jjvxrtj|NywrK)type_foreach_supported_typesdevicerCis_floating_point).0pfused_supported_devicess r/ z/_default_to_fused_or_foreach..s_  T  G/ / + !88 +''*  sAAc3K|]5}|duxs+t|tvxr|jjv7ywrK)rxryrz)r|r}foreach_supported_devicess r/rz/_default_to_fused_or_foreach..sL   T  G/ / ; !::  s;>)rCrLrMrrall)rtr<rufusedforeachrr~s @@r/_default_to_fused_or_foreachrs| yy>BD F H  #EiC    G '>r1r}cuda_unsupportedct}|r|jd|jj|vrt j |s2t d|d|jd|jjy)NcudazX`fused=True` requires all the params to be floating point Tensors of supported devices: z but z and )rremoverzrxrCr{ RuntimeErrordtype)r}rr~s r/_device_dtype_check_for_fusedrszCD&&v. HHMM4 49P9PQR9S ""9!:%yahhmm_ ^  :Tr1ct|D]X\}}tj|stj||||<|D]}tj||||<ZyrK) enumeraterC is_complex view_as_real)rtstate_and_gradsir}ss r/ _view_as_realrsg&!01   A **6!95F1I$ 0))!A$/! 00r1c|rtjStjtjk(rtjStjSrK)rCfloat32get_default_dtypefloat64)is_fuseds r/_get_scalar_dtypers;}}002emmC INr1 supports_xlacgd}tjjs-|jtjj |r|jd|S)z?Return the device type list that supports capturable optimizer.)rxpuhpuxla)rCrLrMappend_C_get_privateuse1_backend_name)rcapturable_supported_devicess r/!_get_capturable_supported_devicesrsJ#9 99 ! ! #$++EHH,R,R,TU$++E2 ''r1ct|tjr#|jdk7r|j S|S)aThis function converts a hyperparameter to a 0-dimension (scalar) tensor if it is a nonzero-dimensions 1-element tensor. If it is not a tensor, it is kept as is. Args: x (float or Tensor): A hyperparameter of the optimizer. If it is Tensor, it is needed to be 1-element. Returns: float or Tensor: a scalar tensor if x is Tensor otherwise Python scalar (float) value. r)rPrCrQdimsqueezerSs r/ _to_scalarrs/!U\\"quuw!|yy{r1zparams (iterable): iterable of parameters or named_parameters to optimize or iterable of dicts defining parameter groups. When using named_parameters, all parameters in all groups should be namedacforeach (bool, optional): whether foreach implementation of optimizer is used. If unspecified by the user (so foreach is None), we will try to use foreach over the for-loop implementation on CUDA, since it is usually significantly more performant. Note that the foreach implementation uses ~ sizeof(params) more peak memory than the for-loop version due to the intermediates being a tensorlist vs just one tensor. If memory is prohibitive, batch fewer parameters through the optimizer at a time or switch this flag to False (default: None)afused (bool, optional): whether the fused implementation is used. Currently, `torch.float64`, `torch.float32`, `torch.float16`, and `torch.bfloat16` are supported. (default: None) .. note:: The foreach and fused implementations are typically faster than the for-loop, single-tensor implementation, with fused being theoretically fastest with both vertical and horizontal fusion. As such, if the user has not specified either flag (i.e., when foreach = fused = None), we will attempt defaulting to the foreach implementation when the tensors are all on CUDA. Why not fused? Since the fused implementation is relatively new, we want to give it sufficient bake-in time. To specify fused, pass True for fused. To force running the for-loop implementation, pass False for either foreach or fused. acapturable (bool, optional): whether this instance is safe to capture in a graph, whether for CUDA graphs or for torch.compile support. Tensors are only capturable when on supported :ref:`accelerators`. Passing True can impair ungraphed performance, so if you don't intend to graph capture this instance, leave it False (default: False)a]differentiable (bool, optional): whether autograd should occur through the optimizer step in training. Otherwise, the step() function runs in a torch.no_grad() context. Setting to True can impair performance, so leave it False if you don't intend to run autograd through this instance (default: False)zmaximize (bool, optional): maximize the objective with respect to the params, instead of minimizing (default: False)hookc^tjt}|t|j<|S)aRegister a pre hook common to all optimizers. The hook should have the following signature:: hook(optimizer, args, kwargs) -> None or modified args and kwargs Args: hook (Callable): A user defined hook which is registered on all optimizers. Returns: :class:`torch.utils.hooks.RemovableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` )hooksrr&idrhandles r/r$r$#s) " "#> ?F-1 * Mr1c^tjt}|t|j<|S)aRegister a post hook common to all optimizers. The hook should have the following signature:: hook(optimizer, args, kwargs) -> None Args: hook (Callable): A user defined hook which is registered on all optimizers. Returns: :class:`torch.utils.hooks.RemovableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` )rrr'rrs r/r%r%7s) " "#? @F.2 + Mr1ParamsTRTc4eZdZUdZeeeegee eeffZ e e d<eeeegdfZ e e d<eee fe d<eee fe d<de d<d e d <d e d <de d <d edeeefddfdZdeeeffdZdeeefddfdZdefdZd7dZd7dZedeeefdeeeffdZe d8dededeee de ee ffee e!jDe!jFfe ee ffffdZ$d7dZ%de de&fdZ'de de&fd Z( d8dedgdfd!ede&fd"Z) d8dede*gee*fd!ede&fd#Z+e!jXde*fd$Z-e d9d%e!j\d&e!j\d'ed(e/eeefd)e0de!j\f d*Z1 d8dede*gee*fd!ede&fd+Z2 d8dedgdfd!ede&fd,Z3e!jXd-e*ddfd.Z4e!jXd:d/eddfd0Z5e6d9d;d2Z7e6d1ege8fde8fd3Z7d9d1eege8fdee8fd4Z7e!jXd5eeefddfd6Z9y)CnnV>TU  6A5F 24F| |  !DE E,q/40%|45L' :K  dK!8 9 : 591r1cJ|j|j|jdS)Nr@rrrr-s r/ __getstate__zOptimizer.__getstate__s# ZZ --  r1rc|jj|d|jvrt|_d|jvrt|_d|jvrt|_d|jvrt|_d|jvrt|_d|jvrt|_|j|jjddy) Nrrrrrrr<F) __dict__updaterrrrrrrrr@ setdefault)r.rs r/ __setstate__zOptimizer.__setstate__s U# &dmm ;-8]D * 't}} <.9mD + ,DMM A3>=D 0 -T]] B4?MD 1 1 F8C D 5 2$-- G9DD 6 !!#   !159r1c|jjdz}t|jD]F\}}|dz }|d|dz }t |j D]}|dk7s |d|d||dz }H|dz }|S)Nz ( zParameter Group rtz z: )) __class__r2rrsortedrl)r. format_stringrgroupkeys r/r0zOptimizer.__repr__s//$6 !$"3"34 BHAu T !M /s"5 5Mejjl+ B(?!tC55:,b%AAM B B  r1c(tjjstjjj rtjj rtjj}|rAtd|jDs%td|jjzdzt|dds=td|jDr |stjdd|_yyyyyyy) Nc3&K|] }|d ywr^Nr,r|rs r/rz=Optimizer._cuda_graph_capture_health_check..s%(-l#%z;Attempting CUDA graph capture of step() for an instance of z' but param_groups' capturable is False.rFc3&K|] }|d ywrr,rs r/rz=Optimizer._cuda_graph_capture_health_check..sKl+Kra This instance was constructed with capturable=True or some of all the param_groups came with capturable=True, but step() is running without CUDA graph capture. If you never intend to graph-capture this instance, capturable=True can impair performance, and you should set capturable=False.T)rCrNrObackendsris_built is_availableis_current_stream_capturingrrrrr2getattrwarningswarnr)r. capturings r/ _cuda_graph_capture_health_checkz*Optimizer._cuda_graph_capture_health_checks++-##,,. '') >>@I%151B1B%"#Qnn--.?@T#I5QK9J9JKK" m =A9#LR*/.r1cy)aEntry point for `torch.profile.profiler`. When python tracing is enabled the profiler will hook into this function at the CPython level to inspect the optimizer's parameters and param groups. It is called it after `step()` since many optimizers lazily initialize state. This is a workaround due to lack of a proper step hook on the optimizer, and will be removed if it exists. Nr,r-s r/_optimizer_step_codezOptimizer._optimizer_step_codesr1r7ctjdtjdtjdt ffd }|S)Nr9r:r*c|^}}tt|}d|jjd}tj j j|5ttj|jjD]C}||||}|t|trt|dk(r|\}}4t d|d |i|}|j!t|j"jt$jD] }|||||cdddS#1swYyxYw)NzOptimizer.step#z.stepz@ must return None or a tuple of (new_args, new_kwargs), but got .)r r!rr2rCautogradprofilerrecord_functionr r&valuesrrPtuplerrrrr') r9r:r._ profile_namepre_hookresultout post_hookr7s r/rqz,Optimizer.profile_hook_step..wrappersIHD1 4(D,T^^-D-D,EUKL((88F  %/6682299;! H&dD&9F)%fe4V9I+1LD&".#'&(hiohppq r# D+F+))+"'33::<0779"2IdD&1 2 3   sAD> BD>>E)rGrorr9r:r)r7rqs` r/profile_hook_stepzOptimizer.profile_hook_stepsB   277 bii A   >r1tensorlistlist with_indicesNNc tjjr$d|tt t |dfiSt ||S)zGroup a list of lists of tensors by device and dtype. Skips this step if we are compiling since this will occur during inductor lowering. rr)rCrNrOrkrangerr)rrs r/rz,Optimizer._group_tensors_by_device_and_dtypesF >> & & ( >4c.QRBS>T8U3V"WX X5nlS Sr1c(d|jjd|_t|jjdd}|sP|j |jj|j_d|jj_yy)NzOptimizer.zero_grad#z .zero_gradhookedT)rr2_zero_grad_profile_namersteprr)r.rs r/rzOptimizer._patch_step_function$sv"4>>#:#:";: F $,,h="&"8"89L9L"MDNN )-DNN   &r1rcvtj|j}||j|j<|S)aRegister an optimizer step pre hook which will be called before optimizer step. It should have the following signature:: hook(optimizer, args, kwargs) -> None or modified args and kwargs The ``optimizer`` argument is the optimizer instance being used. If args and kwargs are modified by the pre-hook, then the transformed values are returned as a tuple containing the new_args and new_kwargs. Args: hook (Callable): The user defined hook to be registered. Returns: :class:`torch.utils.hooks.RemovableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` )rrrrr.rrs r/register_step_pre_hookz Optimizer.register_step_pre_hook-s3&&&t'E'EF48&&vyy1 r1cvtj|j}||j|j<|S)aRegister an optimizer step post hook which will be called after optimizer step. It should have the following signature:: hook(optimizer, args, kwargs) -> None The ``optimizer`` argument is the optimizer instance being used. Args: hook (Callable): The user defined hook to be registered. Returns: :class:`torch.utils.hooks.RemovableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` )rrrrrs r/register_step_post_hookz!Optimizer.register_step_post_hookDs3"&&t'F'FG59'' 2 r1prependctj|j}||j|j<|r'|jj |jd|S)a&Register a state dict pre-hook which will be called before :meth:`~torch.optim.Optimizer.state_dict` is called. It should have the following signature:: hook(optimizer) -> None The ``optimizer`` argument is the optimizer instance being used. The hook will be called with argument ``self`` before calling ``state_dict`` on ``self``. The registered hook can be used to perform pre-processing before the ``state_dict`` call is made. Args: hook (Callable): The user defined hook to be registered. prepend (bool): If True, the provided pre ``hook`` will be fired before all the already registered pre-hooks on ``state_dict``. Otherwise, the provided ``hook`` will be fired after all the already registered pre-hooks. (default: False) Returns: :class:`torch.utils.hooks.RemoveableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` Flast)rrrr move_to_endr.rrrs r/register_state_dict_pre_hookz&Optimizer.register_state_dict_pre_hookYsS4&&t'K'KL:>,,VYY7   0 0 < state_dict or None The hook will be called with arguments ``self`` and ``state_dict`` after generating a ``state_dict`` on ``self``. The hook may modify the state_dict inplace or optionally return a new one. The registered hook can be used to perform post-processing on the ``state_dict`` before it is returned. Args: hook (Callable): The user defined hook to be registered. prepend (bool): If True, the provided post ``hook`` will be fired before all the already registered post-hooks on ``state_dict``. Otherwise, the provided ``hook`` will be fired after all the already registered post-hooks. (default: False) Returns: :class:`torch.utils.hooks.RemoveableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` Fr )rrrrr r s r/register_state_dict_post_hookz'Optimizer.register_state_dict_post_hookysS8&&t'L'LM;?--fii8   1 1 = =fiie = T r1c |jjD] }|| i d dtttfdtttff fd }|j Dcgc] }|| }}|j jDcic]0\}}t|tjr t|n||2}}}||d}|jjD]} | ||} | | }|Scc}wcc}}w)aReturn the state of the optimizer as a :class:`dict`. It contains two entries: * ``state``: a Dict holding current optimization state. Its content differs between optimizer classes, but some common characteristics hold. For example, state is saved per parameter, and the parameter itself is NOT saved. ``state`` is a Dictionary mapping parameter ids to a Dict with state corresponding to each parameter. * ``param_groups``: a List containing all parameter groups where each parameter group is a Dict. Each parameter group contains metadata specific to the optimizer, such as learning rate and weight decay, as well as a List of parameter IDs of the parameters in the group. If a param group was initialized with ``named_parameters()`` the names content will also be saved in the state dict. NOTE: The parameter IDs may look like indices but they are just IDs associating state with param_group. When loading from a state_dict, the optimizer will zip the param_group ``params`` (int IDs) and the optimizer ``param_groups`` (actual ``nn.Parameter`` s) in order to match state WITHOUT additional verification. A returned state dict might look something like: .. code-block:: text { 'state': { 0: {'momentum_buffer': tensor(...), ...}, 1: {'momentum_buffer': tensor(...), ...}, 2: {'momentum_buffer': tensor(...), ...}, 3: {'momentum_buffer': tensor(...), ...} }, 'param_groups': [ { 'lr': 0.01, 'weight_decay': 0, ... 'params': [0] 'param_names' ['param0'] (optional) }, { 'lr': 0.001, 'weight_decay': 0.5, ... 'params': [1, 2, 3] 'param_names': ['param1', 'layer.weight', 'layer.bias'] (optional) } ] } rrr*c r|jDcic]\}}|dk7s ||}}}jt|dDcic]\}}t|vr t|| c}}|dDcgc]}t|c}|d<t |dz |Scc}}wcc}}wcc}w)Nrt)itemsrrrr)rkvpackedrr}param_mappings start_indexs r/ pack_groupz(Optimizer.state_dict..pack_groups',{{}Ftq!X adFFF  ! !!*%/; G1!uN2qE1H @EXO!r!u 5OF8  3vh/0 0KMG Ps B(B(#B. :B4rr) rrrr6r rrrrPrCrQrr) r.rrgrrr packed_state state_dictr hook_resultrrs @@r/rzOptimizer.state_dictsl<<CCE H TN *,  d38n c3h 04/@/@A! 1 A A ((* 1'1ELL&A^BqE "q1 L  "( >>EEG )I#D*5K&(  )!B s )D5Dparamvalueparam_idrrcpd}d}|J|D]!}||dvs d|vr|dnd}d|vr|dnd}n|dk(r1|s|r+|jtj|jS|S|j r'|j|j |jS|j|jS)NFrtrr^r)rrz)rz)torCrrzr{r)rr r!rrrr^pgs r/(_process_value_according_to_param_policyz2Optimizer._process_value_according_to_param_policys ''' B2h<''."}7 %1=1CR -    &=UxxemmELLxII &&(xxekk%,,xGGxxu||x44r1ctj|j}||j|j<|r'|jj |jd|S)aRegister a load_state_dict pre-hook which will be called before :meth:`~torch.optim.Optimizer.load_state_dict` is called. It should have the following signature:: hook(optimizer, state_dict) -> state_dict or None The ``optimizer`` argument is the optimizer instance being used and the ``state_dict`` argument is a shallow copy of the ``state_dict`` the user passed in to ``load_state_dict``. The hook may modify the state_dict inplace or optionally return a new one. If a state_dict is returned, it will be used to be loaded into the optimizer. The hook will be called with argument ``self`` and ``state_dict`` before calling ``load_state_dict`` on ``self``. The registered hook can be used to perform pre-processing before the ``load_state_dict`` call is made. Args: hook (Callable): The user defined hook to be registered. prepend (bool): If True, the provided pre ``hook`` will be fired before all the already registered pre-hooks on ``load_state_dict``. Otherwise, the provided ``hook`` will be fired after all the already registered pre-hooks. (default: False) Returns: :class:`torch.utils.hooks.RemoveableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` Fr )rrrrr r s r/!register_load_state_dict_pre_hookz+Optimizer.register_load_state_dict_pre_hooksUB&&t'P'PQ?C11&))<   5 5 A A&))RW A X r1ctj|j}||j|j<|r'|jj |jd|S)a^Register a load_state_dict post-hook which will be called after :meth:`~torch.optim.Optimizer.load_state_dict` is called. It should have the following signature:: hook(optimizer) -> None The ``optimizer`` argument is the optimizer instance being used. The hook will be called with argument ``self`` after calling ``load_state_dict`` on ``self``. The registered hook can be used to perform post-processing after ``load_state_dict`` has loaded the ``state_dict``. Args: hook (Callable): The user defined hook to be registered. prepend (bool): If True, the provided post ``hook`` will be fired before all the already registered post-hooks on ``load_state_dict``. Otherwise, the provided ``hook`` will be fired after all the already registered post-hooks. (default: False) Returns: :class:`torch.utils.hooks.RemoveableHandle`: a handle that can be used to remove the added hook by calling ``handle.remove()`` Fr )rrrrr r s r/"register_load_state_dict_post_hookz,Optimizer.register_load_state_dict_post_hook>s[8&&t'Q'QR@D22699=   6 6 B B  C  r1rc |j}|jjD]}|||}||}|j}t |d}t |t |k7r t dd|D}d|D}tdt||Dr t dtttjd|Dtjd |D}dfd tt} |d jD]&\} } | |vr|| } | | | |d | | <"| | | <(d tttfdtttfdtttffd} t||Dcgc]\}}| ||}}}|j!| |d|j"jD] }|| ycc}}w)agLoad the optimizer state. Args: state_dict (dict): optimizer state. Should be an object returned from a call to :meth:`state_dict`. .. warning:: Make sure this method is called after initializing :class:`torch.optim.lr_scheduler.LRScheduler`, as calling it beforehand will overwrite the loaded learning rates. .. note:: The names of the parameters (if they exist under the "param_names" key of each param group in :meth:`state_dict`) will not affect the loading process. To use the parameters' names for custom cases (such as when the parameters in the loaded state dict differ from those initialized in the optimizer), a custom ``register_load_state_dict_pre_hook`` should be implemented to adapt the loaded dict accordingly. If ``param_names`` exist in loaded state dict ``param_groups`` they will be saved and override the current names, if present, in the optimizer state. If they do not exist in loaded state dict, the optimizer ``param_names`` will remain unchanged. Example: >>> # xdoctest: +SKIP >>> model = torch.nn.Linear(10, 10) >>> optim = torch.optim.SGD(model.parameters(), lr=3e-4) >>> scheduler1 = torch.optim.lr_scheduler.LinearLR( ... optim, ... start_factor=0.1, ... end_factor=1, ... total_iters=20, ... ) >>> scheduler2 = torch.optim.lr_scheduler.CosineAnnealingLR( ... optim, ... T_max=80, ... eta_min=3e-5, ... ) >>> lr = torch.optim.lr_scheduler.SequentialLR( ... optim, ... schedulers=[scheduler1, scheduler2], ... milestones=[20], ... ) >>> lr.load_state_dict(torch.load("./save_seq.pt")) >>> # now load the optimizer checkpoint after loading the LRScheduler >>> optim.load_state_dict(torch.load("./save_optim.pt")) Nrz.s71c!H+&7c38K|]}t|dywr,r-r.s r/rz,Optimizer.load_state_dict..s=1c!H+&=r/c3,K|] \}}||k7ywrKr,)r|p_lens_lens r/rz,Optimizer.load_state_dict..sN,%u~Nsz]loaded state dict contains a parameter group that doesn't match the size of optimizer's groupc3&K|] }|d ywr,r,r.s r/rz,Optimizer.load_state_dict..s#FAAhK#Frc3&K|] }|d ywr,r,r.s r/rz,Optimizer.load_state_dict..s#@AAhK#@rc Xt|tjrtj ||St|t r/|j Dcic]\}}|||c}}St|trt|fd|DS|Scc}}w)zBMake a deep copy of value, casting all tensors to device of param.)r!rrc36K|]}|yw)r!rNr,)r|r_castrrr!s r/rz;Optimizer.load_state_dict.._cast..s'#%XLQQ#s) rPrCrQr!r%rrrrx)rr r!rrrrr9s` `` r/r9z(Optimizer.load_state_dict.._casts%. II5(L#E4( !& 1uq8,TU E8,"tE{#"#  sB&rr8r new_groupr*c6|d|d<d|vr d|vr|d|d<|S)Nrt param_namesr,)rr:s r/ update_groupz/Optimizer.load_state_dict..update_groups6#(/Ih %-y*H+0+? -( r1r)NNN)copyrrrrrrnanyziprr from_iterablerrr6r rr)r.rrrgroups saved_groups param_lens saved_lensid_maprrrrr=rngrrr9s @r/load_state_dictzOptimizer.load_state_dictbsb __& AAHHJ )H"44K&(  ) "" > :; v;#l+ +N 87 = = N#j*2MN NC   ###F#FF###@#@@   0 .238n  #s(^ :=V\9RS2 Q+S S E<HICCJJL I dO TsG, set_to_nonec|jjddxs|jjdd}t|ds|j|r t d}nd}t j jj|j5|jD]}|dD]}|j|rd|_ |jj|jjn|jjd|r|jjr|jj!|J||jj"|jj$j'|j|rC|J|j)D],}|j)D]}t j*|.dddy#1swYyxYw)aReset the gradients of all optimized :class:`torch.Tensor` s. Args: set_to_none (bool, optional): Instead of setting to zero, set the grads to None. Default: ``True`` This will in general have lower memory footprint, and can modestly improve performance. However, it changes certain behaviors. For example: 1. When the user tries to access a gradient and perform manual ops on it, a None attribute or a Tensor full of 0s will behave differently. 2. If the user requests ``zero_grad(set_to_none=True)`` followed by a backward pass, ``.grad``\ s are guaranteed to be None for params that did not receive a gradient. 3. ``torch.optim`` optimizers have a different behavior if the gradient is 0 or None (in one case it does the step with a gradient of 0 and in the other it skips the step altogether). rFrrc ttSrK)rrkr,r1r/z%Optimizer.zero_grad..s [=Nr1Nrt)r@r_hasattrrrrCrrrrrgradgrad_fndetach_requires_grad_ is_sparsezero_rzrrr_foreach_zero_)r.rIrper_device_and_dtype_gradsrr}per_dtype_gradsgradss r/ zero_gradzOptimizer.zero_grads$--##Iu5 9J9J U: t67  % % ' )45N)O &)- & ^^ $ $ 4 4T5Q5Q R 4** 1x1Avv)&%)AF vv~~9 ! 0 ! 5 5e <#*aff.>.> ! 'A'M M'M :166== I$%FFLL!""(&.1 1"1==='A'H'H'J4O!0!7!7!94,,U344) 4 4 4s$G(DG((G1closurecyrKr,r.rYs r/rzOptimizer.steps25r1cyrKr,r[s r/rzOptimizer.step!s;>r1ct)zPerform a single optimization step to update parameter. Args: closure (Callable): A closure that reevaluates the model and returns the loss. Optional for most optimizers. )NotImplementedErrorr[s r/rzOptimizer.step$s "!r1rcpt|tstdt||d}t|tj r|g|d<n)t|t r tdt||d<g}g}|dD]N}t|tr+|d}|j||j|d>|j|P||d<t|dk7r(t|t|k(r||d<n td|dD]~}t|tj s!tdt j|z|jjd d r[|jrh|j rutd |jj#D]1\}}|t$ur||vrtd ||j'||3|d}t|tt |k7rt)j*d dt } |j,D]?} | j/t | dd|vd| vk7s*d|vrdnd} td| d| j1t |ds td|j,j|y )aAdd a param group to the :class:`Optimizer` s `param_groups`. This can be useful when fine tuning a pre-trained network as frozen layers can be made trainable and added to the :class:`Optimizer` as training progresses. Args: param_group (dict): Specifies what Tensors should be optimized along with group specific optimization options. z$param_group must be a dict, but got rtzoptimizer parameters need to be organized in ordered collections, but the ordering of tensors in sets will change between runs. Please use a list instead.rr<zOall optimizer params should be with/without names. Some param names are missingz>optimizer can only optimize Tensors, but one of the params is r<Nz can't optimize a non-leaf TensorzJparameter group didn't specify a value of required optimization parameter zoptimizer contains a parameter group with duplicate parameters; in future, this will cause an error; see github.com/pytorch/pytorch/issues/40967 for more information) stacklevelz with namesz without nameszPall optimizer param groups should be with/without names. cannot add param group z to the optimizerz7some parameters appear in more than one parameter group)rPrrrxrCrQsetrkrrrrnrr@r_is_leaf retains_gradrrequiredrrrrr isdisjoint) r.rrtextracted_param_tensorsextracted_param_namesr param_namenamedefault param_setrcurrent_group_txts r/rzOptimizer.add_param_group-s+t,B4 CTBUVW WX& fell +%+HK !  $g  %)LK !"$ " * 6E%'"1X %,,Z8'..uQx8'..u5  6!8 H $ % *()S1H-II-B M* e!* EEeU\\2027..2GH==$$%5t< !3!3 !CDD E"]]002 6MD'("t;'> `ae`fg&&tW5  6X& v;#c&k* * MMS  (+u && E   Sx1 2,-52HI$1[$@Lo"!..?-@@QS ##C H(=$>?VW W   -r1)r*NFrKT)rYNr*N):r2r3r4r5r rrrr rrr__annotations__rrintrr6r rrrr0rr staticmethodrrrrboolrrrCrzrrrrrrrrrrhrrQrkrr%r'r)rHrXrrfloatrr,r1r/r!r!Ss8 #+ tVtV|$% '#i$,T4,@$,F#GyG#C)9$9:: $S*;%; <<%VVS T :!9w!9$sCx.!9T!9F d38n :$sCx.:T:" # %AN  !Q!HRUO!!F#T&TT  U: ng&= > >? U5<<, -u^W5L/M MN P TT .+;.,=/,DIk]D01<@ F  Y/)1DDE    D ZIZZx  5||5||554S>* 5  5  55B% Y/)1DDE%%  %PDI"k]D01"<@" "H @)@@@D 64T64T6464p55 >HRY/>E>>"HXb%i%89"Xe_" V.4S>V.dV.V.r1rKrorp)Xr5rGr collectionsrrcollections.abcrrrr>r itertoolsr typingr r r r rrrtyping_extensionsrrrrCtorch.utils.hooksutilsrtorch.utils._foreach_utilsrrrrrrrrrrrqrr6rrrzrQrrr r"r#__all__r&rrr'nn parameter Parameterryr)rfrIrUrXobjectrsrkrtrrrrrr _params_doc _foreach_doc _fused_doc_capturable_doc_differentiable_doc _maximize_docr$r%rrrr!r,r1r/rs]088JJJ88 !!. T]t_S/i!cN "CH~ 9%Xell3U\\AB IB! U5<<, -. <%-$%f *=!>>% &.{D&.I4.O%PP  BMT#'="=>NCN=d3(?#?@P!LL%((*<*<*F*FG&&  xB'7HRV>9=/xV 45/ xB (2r6"223/rIN  04AE 4::/4   ||  '+     0(D(DI((< g  J F 6 > +Ao(,C( U\\HT#s(^4huS%,,EV?W6XX CL CLq .q .r1