L i; ddlZddlZddlZddlZddlZddlZddlZddlZddlm Z ddl m Z m Z m Z mZddlZddlmZddlZddlmZddlmZmZmZmZddlmZddlmZmZmZm Z m!Z!m"Z"gd Z#d Z$Gd d Z%e%Z&d5d e'de(e'e ffdZ)d5d e'de(e'e ffdZ* d5de'd e'de(e'e fde fdZ+de'de dede'fdZ,de(e'e fdefdZ-edde(e e fde'dej"j\fdZ/eddede(e e fde'dej"j\fdZ0Gd d!ej"j\Z1 d5de(e e fdej"j\fd"Z2d#ej"j\d$ej"j\d%e'fd&Z3d'e d$ej"j\d%e'fd(Z4d)ej"j\d*e'fd+Z5d)ej"j\d*e'fd,Z6d)ej"j\d-e7e'fd.Z8d)ej"j\d*e'fd/Z9 d6d0Z:Gd1d2Z;edGd3d4ej"j\Z||jvr|j|SyN)r)r module_names r! get_sourcez_EvalCacheLoader.get_sourceNs $// )??;/ /r#cNd|j}|xjdz c_|S)Nz.r )r)r r3s r!r.z_EvalCacheLoader._get_keySs% /   r#r8) r, __module__ __qualname__r"strdictrr5rr:r.r#r!rr+s7tCH~<# r#rr$r%c`tj|||}tt||d|y)Nexec)_loaderr5rBcompile)r$r%r2r3s r!_exec_with_sourcerE\s' --Wi 0Cc6 "G,r#c td|||S)Nforward) method_namer$r%r2)_method_from_src)r$r%r2s r!_forward_from_srcrJas 39 r#rHr6cP|j}t|||||}||=|Sr8)r/rE)rHr$r%r2r4fns r!rIrIgs2<<>Lc<3 k "B[! Ir#nameobjimporterc|tvrt|jSt|ry|j|\}}d|d|d|S)Nz import torchzfrom z import z as )r import_strrget_name)rMrNrOr9 attr_names r!_format_import_statementrTrsU %000d%..s3K ;-x {$tf ==r#c |jDchc]\}}t|||}}}djt|Scc}}w)N )itemsrTjoinsorted)r%rOrMrN import_strss r!_format_import_blockr[{sKGN}}:C$ sH5K 99VK( )) sATis_backward_compatiblebody import_blockch|jdxs|d}t||zi}t||S)N_codecode)getrJ_deserialize_graph_module)r^r_fn_srcrGs r!rrs9XXg  .$v,F v 5r:G $Wd 33r#generated_module_namecP|j|j}t||Sr8) import_modulerGrd)rOr^rfrGs r!rrs($$%:;CCG $Wd 33r#ceZdZfdZxZS)_CodeOnlyModulec0t|||_yr8)superr"__dict__)r r^ __class__s r!r"z_CodeOnlyModule.__init__s  r#)r,r<r=r" __classcell__rns@r!rjrjs r#rjcR|t_|jd}|ddlm}|}|jdd}|}Gdd|}t|}|jd i} |j |fi| } |jd d} | M|d =| j D];} | j| jd | | j| jd <=|| _ dd l m } | || || }|jD]\}}t||rt|||!|S)a Deserialize a GraphModule given the dictionary of the original module, using the code to reconstruct the graph. We delete the actual graph before saving the dictionary so that changes to the in-memory graph format do not get serialized. _tracer_clsNr )Tracer_graphmodule_cls_namercHeZdZdejj dedefdZy)._deserialize_graph_module..KeepModules___r6cyNTr@)r rwrxs r!is_leaf_modulez=_deserialize_graph_module..KeepModules.is_leaf_modulesr#N) r,r<r=torchnnModuler>boolr{r@r#r! KeepModulesrvs$ EHHOO   r#r_tracer_extras(_graphmodule_graph_node_meta_stack_trace stack_trace_make_graph_module) class_namegraph_module_cls)rjrGrc_symbolic_tracerstracenodesrMmetarr_lazy_graph_modulerrWhasattrsetattr)rGr^r tracer_clsrsgraphmodule_cls_name cls_tracerrcom tracer_extrasgraphnode_meta_stack_tracenodergmkvs r!rdrdsC&O-(J+ 88$;]K!Jj $ CHH-r2M KM   5} 5E!HH%OQUV( ; <KK LD$((D9E+@+K -( L #E6  U3FV B 1r1~ B1  Ir# from_module to_moduletargetc|jd^}}|D]R}t||}t||d}||ury|+tjj }t |||||}}Tt||}t |tjr7t |tjjs|j||yt |||yN. splitgetattrr|r}r~r isinstanceTensor Parameterregister_buffer) rrrprefixfielditemftorigs r! _copy_attrrs\\#&NVU & K & ItT * 6  9!A ItQ '!"AY  & ; &D$ %juxx?Q?Q.R!!%. 5$'r#from_objcn|jd^}}|D]>}t||d}|+tjj }t ||||}@t |tjr7t |tjjs|j||yt |||yrr)rrrrrrrs r! _assign_attrrs\\#&NVU ItT * 9!A ItQ ' (ELL)*%(($$3 !!%2 5(+r#modelrSc8t||jdSr)_get_attr_via_attr_listr)rrSs r! _get_attrrs "5)//#*> ??r#c`|jd}t||dd}t||dS)Nr)rrdelattr)rrS attr_namesrs r! _del_attrrs3%Jz#27A 1jn %%r# attr_listcxt|dk(r|S|^}}|}|D]}t||d}|Jt||Sr)lenr)rrrrrrs r!rrsW 9~ NVU A AtT "}} 1e r#cx|jd^}}|}|D]}t||}|dusyt||S)NrF)rr)rrSrrrrs r! _has_attrr&sM__S)NVU A At  : 1e r#c |j}|$t|tjjsJd|j dd||||}|j } | jd} d|d| z} t| d} d g} |jD]2\} } t| d s| jt| | d ||| 4dj| } t| d} | | z}|rt| | z|S) Nz4print_readable must be used on a module with a graphr T) root_moduleverboseinclude_strideinclude_devicecolored expanded_defrVzclass z(torch.nn.Module): rF) print_outputrrr)rrr|fxr python_coder$lstriprnamed_childrenrappend_print_readablerXprint)moduler9rrrrrrverbose_python_code module_codesubmodule_code_listsubmodule_name submodulesubmodule_codeoutputs r!rr1s. LLE  E588>>!B> B ++%%! ,&))K$$T*K;-';<{JK[!,K$%+%:%:%< ! 9g &  & &"!#1#   YY23N2N > )F kN*+ Mr#cReZdZdZedej defdZfdZ xZ S) _WrappedCallc ||_||_yr8)clscls_call)r rrs r!r"z_WrappedCall.__init__cs  r# frame_summaryr6c|j}|J|j}|Jt|}tj|j }t jjtjd}d|d}dj||dz |}d|zdz}d j|||dz} d j||||| gS) NzDdo not trace into traceback.format_exc when generating error message)reasonz%Call using an FX-traced Module, line z3 of the traced Module's generated forward function:r~z ~~~ <--- HERErV) linenolinerr0getlinesfilenamer|_dynamodisable traceback format_excrX) r err_linenor err_line_len all_src_linestb_repr custom_msg before_errmarkererr_and_after_errs r!_generate_error_messagez$_WrappedCall._generate_error_messageps#)) %%%!!4y !**=+A+AB --''  Y(    < * *  WW]:>JGH |#o5 IImJa&PQyy':z6CTUVVr#c |j|j|g|i|St|j||i|S#t$r}|j sJt jjt j|j d}d|jvr?ttj|tj|j!d|d}~wwxYw)Nr eval_with_key)file)rrlr__call__ Exception __traceback__r StackSummaryextractwalk_tbrrrrsysstderrwith_traceback)r rNargskwargsetopmost_framesummaryrns r!rz_WrappedCall.__call__s }}($t}}S:4:6::TXXs4dEfEE ?? "?&&..y/@/@/QRSUV !"6"?"?? 889MN&&t,, s!>> C(BC##C() r,r<r=r" staticmethodr FrameSummaryr>rrrorps@r!rrbs>!Wy/E/EW#WW4r#rceZdZdZd.fd Zed d/deejje e e ffde de ffd ZdgZed e fd Zej$d e d d fd Zedd0dee ej(fde fdZedde dejjd efdZedde d efdZedd1dZed e fdZedd efdZdefdZdZdZ dZ!dZ"ed d2dddd ed!efd"Z#d e ffd# Z$d$Z%e&jNd%Z(d&Z)d'Z*d(Z+d)Z,d*Z-d+Z.d,Z/d-Z0xZ1S)3ra GraphModule is an nn.Module generated from an fx.Graph. Graphmodule has a ``graph`` attribute, as well as ``code`` and ``forward`` attributes generated from that ``graph``. .. warning:: When ``graph`` is reassigned, ``code`` and ``forward`` will be automatically regenerated. However, if you edit the contents of the ``graph`` without reassigning the ``graph`` attribute itself, you must call ``recompile()`` to update the generated code. c|jD])}|jjdd}|dk7s'|}nGdd|}t||S)NrrGraphModuleImplc eZdZy),GraphModule.__new__..GraphModuleImplN)r,r<r=r@r#r!rr s r#)__mro__r=rrl__new__)rrrrcrrns r!r zGraphModule.__new__s_ A$$S)"-A%%    c w//r#Tr\rootrrct |||j_t |t j jrt|dr|j|_ t |tro|jD]\}}t||||jD]\}}t||||jD]\}}t||||jD]D}|j dvst |j"t$sJt|||j"Fnt |t&rg}|jD]}|j dvst |j"t$sJ|j"|vr*t)dt%|zdz|j"zdz|j+|j"|j-d|D]}t/||||nt)dt%|zd z||_d |_|j0j2r=d |j0j2j4vr|j0j2|_i|_|j0j6r|j0j6|_i|_g|_g|_g|_g|_ y ) a Construct a GraphModule. Args: root (Union[torch.nn.Module, Dict[str, Any]): ``root`` can either be an nn.Module instance or a Dict mapping strings to any attribute type. In the case that ``root`` is a Module, any references to Module-based objects (via qualified name) in the Graph's Nodes' ``target`` field will be copied over from the respective place within ``root``'s Module hierarchy into the GraphModule's module hierarchy. In the case that ``root`` is a dict, the qualified name found in a Node's ``target`` will be looked up directly in the dict's keys. The object mapped to by the Dict will be copied over into the appropriate place within the GraphModule's module hierarchy. graph (Graph): ``graph`` contains the nodes this GraphModule should use for code generation class_name (str): ``name`` denotes the name of this GraphModule for debugging purposes. If it's unset, all error messages will report as originating from ``GraphModule``. It may be helpful to set this to ``root``'s original name or a name that makes sense within the context of your transform. training)get_attr call_modulezNode z referenced target z. but that target was not provided in ``root``!c$|jdSr)count)rs r!z&GraphModule.__init__..sqwws|r#)r3zUnsupported type z passed for root!Nz)!rlr"rnr,rr|r}r~rrrjrr named_buffersnamed_parametersroprr>r? RuntimeErrorrsortrrrrr=rr_replace_hooks_create_node_hooks_erase_node_hooks_deepcopy_hooks) r r rrrrwrtargets_to_copytarget_to_copyrns r!r"zGraphModule.__init__s6 ", dEHHOO ,tZ( $  $0 //1.DAqtT1-.!..0.DAqtT1-.!113.DAqtT1-.  87799%dkk3777tT4;;7 8d # O 87799%dkk3777{{$.*#!$i(34#kk*O O$**4;;7 8$  %; <"1 IT.14H I2SY>ATTU U   JJ " "$**"8"8"E"EE#zz55D   :: $ $"&**";";D %' .02413/1r#r6c|jS)zF Return the ``Graph`` underlying this ``GraphModule`` )_graphrs r!rzGraphModule.graph,s {{r#gNct|tsJdt|||_||_|j y)z Set the underlying ``Graph`` for this ``GraphModule``. This will internally recompile the ``GraphModule`` so that the generated ``forward()`` function corresponds to ``g`` z#Expected a Graph instance, but got N)rrtyper! owning_module recompile)r r"s r!rzGraphModule.graph3s>!U#T'J4PQ7)%TT#  r#Ffolderr9c "t|}t|jdtj|j |dz d}dj t jDcgc]}|jc}}d|d|d}d td tjjd ttfd }g}|jD]\}} ||| } | g||dz } tj| | |j|| jj!ddj!dd} d| d| } ||dzd|d| dz }|j"j%D]9\} }| ||dzd| dt'|j(d|j*dz };|j,j%D]9\}}| ||dzd|dt'|j(d|j*dz };||dzd|dz }|t/|j0ddz }|dz } | j3||dz }|j3d t5|d!kDrt7j8d"|y y cc}w)#aaDumps out module to ``folder`` with ``module_name`` so that it can be imported with ``from import `` Args: folder (Union[str, os.PathLike]): The folder to write the code out to module_name (str): Top-level name to use for the ``Module`` while writing out the code T)exist_okz state_dict.ptz rVz import torch z from torch.nn import * class zF(torch.nn.Module): def __init__(self): super().__init__() r9rr6ctjtjtjtjtj tj tjg}t||vr|jSyr8) r}LinearConv1dConv2dConv3d BatchNorm1d BatchNorm2d BatchNorm3dr$__repr__)r9r safe_reprss r!_gen_model_reprz.GraphModule.to_folder.._gen_model_reprZs[    JF|z) //+,-r#Nz.pt  z torch.load(r'z', weights_only=False) # rzself.z = zself.register_buffer('z', torch.empty(z, dtype=z)) z" = torch.nn.Parameter(torch.empty(z"self.load_state_dict(torch.load(r'z/state_dict.pt')) rz module.pyz __init__.pyzfrom .module import *rz^Was not able to save the following children modules as reprs -saved as pickled files instead: )rmkdirr|save state_dictrXrvaluesrQr>r}r~rrrr2replace_buffersrWlistshapedtype _parametersrrb write_textrwarningswarn)r r'r9tabrcustom_builtins model_strr4blobified_modulesr module_str module_file module_repr buffer_namebuffer param_nameparam init_files r! to_folderzGraphModule.to_folder?sf V D) 4??$f&>?));K;R;R;T$UaQ\\$UVm    ehhoo (SV- #'#6#6#8 I K(f=J!$+c'::  6;/!((5$oo/77cBJJ4QTU $K=0I+W C!G9E+c*RH HI I$(==#6#6#8 F K~ C!G9$:;-W[\b\h\hWiVjjrsysssAAEF FI F "&!1!1!7!7!9 E J} C!G9E*5WX\]b]h]hXiWjjrsxs~s~r@DE EI E Qwi9&AT U   499a0144 {* y)]* 45  !A % MM33D2EG  &{%Vs1J rmc&|jd^}}|}|D]d}t||d}|+tjj }t |||t |tjjsy|}f|j||y)a% Adds the given submodule to ``self``. This installs empty Modules where none exist yet if they are subpaths of ``target``. Args: target: The fully-qualified string name of the new submodule (See example in ``nn.Module.get_submodule`` for how to specify a fully-qualified string.) m: The submodule itself; the actual object we want to install in the current Module Return: bool: Whether or not the submodule could be inserted. For this method to return True, each object in the chain denoted by ``target`` must either a) not exist yet, or b) reference an ``nn.Module`` (not a parameter or other attribute) rNFT)rrr|r}r~rr add_module)r rrQrrmodrsubmods r! add_submodulezGraphModule.add_submodules, c*# DS$-F~*T6*fehhoo6C  ua r#c\|jd}|dd|d}}|}|D]B}t||syt||}t|tj j rByt||sytt||tj j syt||y)ab Deletes the given submodule from ``self``. The module will not be deleted if ``target`` is not a valid target. Args: target: The fully-qualified string name of the new submodule (See example in ``nn.Module.get_submodule`` for how to specify a fully-qualified string.) Returns: bool: Whether or not the target string referenced a submodule we want to delete. A return value of ``False`` means that the ``target`` was not a valid reference to a submodule. rNrFT)rrrrr|r}r~r)r ratomspath target_submodrTrs r!delete_submodulezGraphModule.delete_submodules& S!#CRj%)m# D3%#t$Cc588??3 sM*'#}5uxxG]#r#cg}|jjD]}|jdk(s|jdk(s"|jj d}dt dt dt fd}|j tj|||jdk(s |j|j}|jD]7\}}|dk7s |jdj|j|g9|jDcgc] \}}||vs |} }}| D]}|j|y #t$rY.join_fns!88aQF99aS99r#rN)rrrrrr>extend itertools accumulate get_submodule named_modulesrrXAttributeErrorr[) r usedrfullpathr`rU submod_namerwrM to_deletes r!delete_all_unused_submodulesz(GraphModule.delete_all_unused_submodulessQJJ$$ Dww-'477j+@ ;;,,S1 :s:s:s: I007CD77m+ !%!3!3DKK!@.4.B.B.DRNK*b0 $ CHHdkk;5O,P QR3 B*.););)=RgdATAQTR R (D  ! !$ ' (* Ss$"6E .E  E)E EEcHt|ds td|jS)zn Return the Python code generated from the ``Graph`` underlying this ``GraphModule``. raz;Code has not been generated! Please report a bug to PyTorch)rrrars r!rbzGraphModule.codes* tW%M zzr#ct|jjtr^|jjjj |_|jjjj|_|jjd}|j|_ |j|_ t|}t|jdr|jjni}t!|j|j"||_dt'|vr |j(nd}dt'|vrt+|||_d}||_|S)z Recompile this GraphModule from its ``graph`` attribute. This should be called after editing the contained ``graph``, otherwise the generated code of this ``GraphModule`` will be out of date. r )r _co_fieldsrN _wrapped_callc.|j|g|i|Sr8)ro)r rrs r! call_wrappedz+GraphModule.recompile..call_wrappedDs%4%%d->x-HI ! * MDt}}$T4==t1Dd#KL M==vr!:DA )SXX 5#&88,J#K#Q#Q#S . 4Y- . 4* +,, S   r#c\ddlm}|||j}t|di|_|S)Nr rr)rrrrr)r rrs r!__copy__zGraphModule.__copy__s*: tzz24, r#)fast_sympy_printrrrc . tj}|5|rFddlm dtj j dtf fd }|jt|t||j|||||} | cdddS#1swYyxYw)zh Return the Python code generated for current GraphModule and its children GraphModules r) sympy_strexprr6c<|jjSr8)rr)rrs r! fast_reprz-GraphModule.print_readable..fast_reprs$TYY^^44r#N) contextlib ExitStacktorch._inductor.utilsrr|types PySymTyper> enter_contextrr _get_name) r rrrrrrctx_mgrrrrs @r!print_readablezGraphModule.print_readables &&(  ;5EKK$9$95c5%%&8&CD A%   s A)B  Bc`t|}d}dj||j|gS)NzD# To see more debug info, please use `graph_module.print_readable()`rV)rl__str__rXra)r orig_strprint_readable_reminderrns r!rzGraphModule.__str__s47?$ R yy(DJJ0GHIIr#c4|j}d|_|Srz)r _is_replica)r new_gms r!_replicate_for_data_parallelz(GraphModule._replicate_for_data_parallels! r#c#Kt|sJd|j| d|j|y#|j|wxYww)8 Takes a callable which will be called every time when we replace a node to a new node, or change the node's name. Callable takes three arguments: the old node we're changing, and NAME of the new node, followed by the user node which consumes the old node to be replaced. z Replace hook must be a callable.N)callable_register_replace_node_hook_unregister_replace_node_hookr rs r!_set_replace_hookzGraphModule._set_replace_hooksL{>>>{ ((+ 2   . .q 1D . .q 1s$A=AAAc^t|sJd|jj|y)r$create_node hook must be a callable.N)rrrrs r!rz'GraphModule._register_replace_node_hooks*{BBB{ ""1%r#c^t|sJd|jj|y)z Takes a callable which was previously registered to be called every time when we replace a node. This function will unregister that callable so it is no longer invoked on node replacement. rN)rrremovers r!rz)GraphModule._unregister_replace_node_hooks* {BBB{ ""1%r#c^t|sJd|jj|y)z Takes a callable which will be called after we create a new node. The callable takes the newly created node as input and returns None. rN)rrrrs r!_register_create_node_hookz&GraphModule._register_create_node_hook* {BBB{ &&q)r#c^t|sJd|jj|y)z Takes a callable which was previously registered to be called after we create a node. This function will unregister that callable so it is no longer invoked on node creation. rN)rrrrs r!_unregister_create_node_hookz(GraphModule._unregister_create_node_hookrr#c^t|sJd|jj|y)z Takes a callable which will be called after we erase a node. The callable takes the node that is being erased as input and returns None. #erase_node hook must be a callable.N)rrrrs r!_register_erase_node_hookz%GraphModule._register_erase_node_hook* {AAA{ %%a(r#c^t|sJd|jj|y)z Takes a callable which was previously registered to be called after we erase a node. This function will unregister that callable so it is no longer invoked on node erasure. rN)rrrrs r!_unregister_erase_node_hookz'GraphModule._unregister_erase_node_hook rr#c^t|sJd|jj|y)z Takes a callable which will be called when we deepcopy this graph module. The callable takes the resulting deepcopied graph module. !deepcopy hook must be a callable.N)rrrrs r!_register_deepcopy_hookz#GraphModule._register_deepcopy_hook* {???{ ##A&r#c^t|sJd|jj|y)z Takes a callable which was previously registered to be called after deepcopy. This function will unregister that callable so it is no longer invoked on deepcopy. rN)rrrrs r!_unregister_deepcopy_hookz%GraphModule._unregister_deepcopy_hookrr#)rztype[GraphModule])r)FxModule)r6N)TFFF)2r,r<r=__doc__r rrr|r}r~r?r>rrr"__jit_unused_properties__propertyrsetterosPathLikerPrrVr[rkrbrr&r rrrrrrrrrcontextmanagerrrrrrrrrrrorps@r!rrs^ 0&$/ ( a2EHHOOT#s(^34a2a2 a20a2N")  u  \\ u   %0Pc2;;&6 7PcP1Pd$/%C%EHHOO%%0%N$/'s't'0'R$/4(04(l c  $/!:!0!L ? 2 I$ <%0 ""'""""1"HJJ  2 2&&**))''r#rr8)TFFFF)=rr/rbr0rrrrBpathlibrtypingrrrrr|torch.nnr}torch.overridestorch.nn.modules.moduler torch.packager r r r _compatibilityrrrrrrrr__all__rrrCr>r?rErJrIrTr[r~rrrjrdrrrrr=rrrrrr@r#r!rs  11 .RR)  ">++\  -3-c3h- 3c3hDH)-c3h >3>S>H>>*$sCx.*H*d+4d38n4C4EHHOO4,4d+44%)#s(^4LO4 XX__4,4ehhoo59<S>< XX__<B(EHHOO((QT(8,3,588??,C,*@UXX__@@&UXX__&& 588?? tCy UXX__ .b;;|d+@ '%((//@ ',@ 'r#