L itXdZddlZddlZddlZddlZddlZddlZddlZddl Z ddl Z ddl Z ddl Z ddl Z ddlmZmZddlmZddlmZddlmZmZmZmZmZddlZddlZddlm Z ejBdd k(re"d d d l#m$Z$m%Z%d d l#m&Z'd dl(m)Z)m*Z*m+Z+m,Z,m-Z-m.Z.m/Z/e/j`e1Z2e-rd dl#m3Z3e.r ddl4Z4ddl5m6Z7dZ8 ejrjOdZ:dZ;ejNjye:ejNjye8k\Z=ddl>Z>e>j~jddZAndZAejjdduxsejjdduZJer3eJr1 ejrjOdZKe2jdeKdd dl#mMZMd dlNmOZOmPZPd dlQmRZRmSZSmTZTd dlUmVZVd dl(mWZWmXZXdZYd ZZd!Z[d"Z\d#Z]d$Z^d%Z_d&Z`d'Zad(Zbd)Zcd*Zdd+Zed,Zfd-Zgd.Zhd/Zid0Zjd1Zkd2eld3emd4eSfd5Znd2eld3emd4eSfd6Zod2eld3emd4eSfd7Zpd2eld3emd4eSfd8Zqd9Zrd:ZsGd;demfd?ZuGd@dAemeZvGdBdCePZwGdDdEePZxGdFdGePZyGdHdIePZzGdJdKePZ{GdLdMe{Z|GdNdOe}Z~GdPdQePZGdRdSePZGdTdUePZGdVdWePZGdXdYePZGdZd[ePZezeye{eetexeweee|eeed\ Zd]Zy#ejrjeCeDeEeFeGf$r dZ:dZ;dZ=dZAYwxYw#ejrj$rW ejrjOdZKe2jdeKdn!#ejrj$rdZJYnwxYwYwxYw)^z+ Integrations with other Python libraries. N)asdictfields)Enum)Path) TYPE_CHECKINGAnyLiteralOptionalUnion)_is_package_available WANDB_MODEofflineu%⚙️ Running in WANDB offline mode)PreTrainedModelTrainingArguments) __version__)PushToHubMixin flatten_dictis_datasets_availableis_pandas_availableis_tf_availableis_torch_availablelogging)TFPreTrainedModelz3.43.2comet_mlTz comet.api_keyFneptunezneptune-clientzNeptune version z available.zNeptune-client version ) modelcard)ProgressCallbackTrainerCallback)PREFIX_CHECKPOINT_DIRBestRunIntervalStrategy) ParallelMode)ENV_VARS_TRUE_VALUESis_torch_xla_availablectjddjtvrtj dyt jjdddl }t|dSy)NWANDB_DISABLEDzUsing the `WANDB_DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).Fwandbrrun) osgetenvupperr$loggerwarning importlibutil find_specr)hasattr)r)s q/mnt/ssd/data/python-lab/Trading/venv/lib/python3.12/site-packages/transformers/integrations/integration_utils.pyis_wandb_availabler5msa yy!2&,,.2FF t ~~(4ue$$cDtjjdduS)Ntrackior0r1r2r6r4is_trackio_availabler; >> # #I .d ::r6cDtjjdduS)Nclearmlr9r:r6r4is_clearml_availabler?r<r6c&tjddjdk(rtj dyt duryt dur%tj dtttytdurtj dyy) N COMET_MODEr(DISABLEDzUsing the `COMET_MODE=DISABLED` environment variable is deprecated and will be removed in v5. Use the --report_to flag to control the integrations used for logging result (for instance --report_to none).Fzcomet_ml version %s is installed, but version %s or higher is required. Please update comet_ml to the latest version to enable Comet logging with pip install 'comet-ml>=%s'.a#comet_ml is installed but the Comet API Key is not configured. Please set the `COMET_API_KEY` environment variable to enable Comet logging. Check out the documentation for other ways of configuring it: https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#set-the-api-keyT) r+r,r-r.r/_is_comet_installed_is_comet_recent_enough_comet_version_MIN_COMET_VERSION_is_comet_configuredr:r6r4is_comet_availablerHs yyr"((*j8 t e#%' t     u$ h  r6ctjjdduxs!tjjdduS)N tensorboard tensorboardXr9r:r6r4is_tensorboard_availablerLs7 >> # #M 2$ > v)..BZBZ[iBjrvBvvr6cDtjjdduS)Noptunar9r:r6r4is_optuna_availablerO >> # #H -T 99r6cDtjjdduS)Nrayr9r:r6r4is_ray_availablerSs >> # #E *$ 66r6cZtsytjjdduS)NFzray.tune)rSr0r1r2r:r6r4is_ray_tune_availablerUs%   >> # #J /t ;;r6cDtjjdduS)Nsigoptr9r:r6r4is_sigopt_availablerXrPr6ctjjdytjjdytjjdduS)NazuremlFz azureml.corezazureml.core.runr9r:r6r4is_azureml_availabler[sM~~ *2~~/7 >> # #$6 7t CCr6ctjddjdk(rytjj dduS)NDISABLE_MLFLOW_INTEGRATIONFALSETRUEFmlflow)r+r,r-r0r1r2r:r6r4is_mlflow_availableras< yy-w7==?6I >> # #H -T 99r6cdtjjdtjjdfvS)Ndagshubr`r9r:r6r4is_dagshub_availablerds1  00;Y^^=U=UV^=_` ``r6ctSN) _has_neptuner:r6r4is_neptune_availablerhs r6cDtjjdduS)N codecarbonr9r:r6r4is_codecarbon_availablerks >> # #L 1 ==r6cDtjjdduS)Nflytekitr9r:r6r4is_flytekit_availablerns >> # #J /t ;;r6cZtsytjjdduS)NFzflytekitplugins.deck)rnr0r1r2r:r6r4 is_flyte_deck_standard_availablerps& " >> # #$: ;4 GGr6cDtjjdduS)Ndvcliver9r:r6r4is_dvclive_availablersr<r6cDtjjdduS)Nswanlabr9r:r6r4is_swanlab_availablervr<r6cNtr0ddl}t||jjr |j St rt|tr|Strt|tr|Strt|tr|Std|j)NrzUnknown type for trial ) rOrN isinstancetrial BaseTrialparamsrUdictrXr5 RuntimeError __class__)ryrNs r4 hp_paramsrs eV\\33 4<<  eT "L eT "L eT "L 00AB CCr6n_trials directionreturnc &ddlddlmjjdk(r.ddj ffd }|j dd}|j dd}|j dd }t|tr|nd}|dn|}jd||d |} | j||||| | js@| j} tt| j| j | j"S| j$} | D cgc]6} tt| j| j&| j"8c} St)|D]} d_dg}jj,t.j0k7r t3d t4j6j9|d j;d|dt=ddj?}jA|_ycc} w)Nr)release_memoryrycnd}|rPtj|D]8}|jtstjj ||}:d _ jjdkDr jjtjk7r td j|jj|j |j"}|g}t$j&j)|d j+||n j+||t- dd& j/} j1| _ j2 j4\ _ _ j6j9 j S)NConly support DDP optuna HPO for ParallelMode.DISTRIBUTED currently.rsrcresume_from_checkpointry objective)r+listdir startswithr pathjoinrargs world_size parallel_moder# DISTRIBUTEDr}hp_spacery FixedTrialr{numbertorch distributedbroadcast_object_listtraingetattrevaluatecompute_objective model_wrappedmodel acceleratorclear) rycheckpoint_dir checkpointsubdir fixed_trialtrial_main_rank_listmetricsrNrtrainers r4 _objectivez(run_hp_search_optuna.._objectiveskJ jj8JF(()>?%'WW\\.&%I J!%G ||&&*<<--1I1II&'lmm  '$ll55ellELLQ (3}$!!778LRS7T Zu M Zu Mw T2:!**,$+$=$=g$F!4B'BWBWY`YfYf3g 0G !7=    % % '$$ $r6timeoutn_jobsrgc_after_trialF)r directions)rrrrrrrrrfr:)!rNaccelerate.utils.memoryrr process_indexTrialpoprxlist create_studyoptimize_is_multi_objective best_trialr!strrvaluer{ best_trialsvaluesrangerrr#rr}rrrrrrr)rrrkwargsrrrrrstudyrrbestirrrNrs` @@r4run_hp_search_optunars6||!!Q& %fll %8**Y-Ha($4e<",Y"=Y4 &2D ###YiJYRXY zHgfest((*))J3z001:3C3CZEVEVW W++KT_`DGC ,dkk4;;G` `x GA $G $(6 ||))\-E-EE"#hii    3 34Ha 3 P MM=QRS=TM Uw T2:!**,$+$=$=g$F! Gas;Hc 4ddlddldtffd }|jjs)ddlm}tjd|d|_|jt}d|_ d |vrad d i|d <|jjdkDrd |d d <d |jjdkDrdndz}tjd|d|d jd d}||j_d|vrddlm} | dg|d<d|vrddlm} m} m} m} t1|d| | | | frn|jj2r'|jj4t6j8k(r1t;dj=t?|dj@jBjE||tGjHfd}tKdrjL|_&jBjN|f|jQd|d|}|jSd|dd|jjT}tW|jX|jZd|j\|}||j_||S) Nrrycx ddlm}|j|r|jtd|_jj}|rld|_|j5}tt|jtdj}|j||dddn|j|t|dd|j!}|j#||_|j%|j ddt'j(5}|j+| jj,j/|}jj1|| dddyy#t $rYqwxYw#1swYxYw#1swYyxYw) Nr)NotebookProgressCallbackr*r)ryT)rdone)r)r)transformers.utils.notebookr pop_callback add_callbackrModuleNotFoundErrorrrget_checkpoint as_directorynextrglobr as_posixrrrupdatetempfileTemporaryDirectory_tune_save_checkpoint Checkpointfrom_directoryreport) ry local_trainerrrrcheckpoint_pathrtemp_checkpoint_dirrRs r4rz%run_hp_search_ray.._objectiveAs  L))*BC**+;<#' YY--/  '2M #((* Yn"&tN';'@'@DYCZZ[A\']"^"g"g"i##?RW#X Y Y   e  , =+t 4 <#,,.G&3&E&Eg&NM # NN)@)@$O P,,. A2E33CV3W YY11@@ATU    Z @ A A =+#    Y Y A As+,F)A F$5AF0 F! F!$F-0F9r)TrainerMemoryTrackerzMemory tracking for your Trainer is currently enabled. Automatically disabling the memory tracker since the memory tracker is not serializable.T)skip_memory_metricsresources_per_trialcpurgpuz1 CPUz and 1 GPUr(zgNo `resources_per_trial` arg was passed into `hyperparameter_search`. Setting it to a default value of z for each trial.progress_reporter) CLIReporterr)metric_columns scheduler) ASHASchedulerHyperBandForBOHBMedianStoppingRulePopulationBasedTrainingaYou are using {cls} as a scheduler but you haven't enabled evaluation during training. This means your trials will not report intermediate results to Ray Tune, and can thus not be stopped early or used to exploit other trials parameters. If this is what you want, do not use {cls}. If you would like to use {cls}, make sure you pass `do_eval=True` and `eval_strategy='steps'` in the Trainer `args`.)cls)rctrddl}tjj |j j d}tjjd|}tjj|}|tj|j<|jj||i|S)a" Wrapper around `tune.with_parameters` to ensure datasets_modules are loaded on each Actor. Without this, an ImportError will be thrown. See https://github.com/huggingface/transformers/issues/11565. Assumes that `_objective`, defined above, is a function. rNz __init__.pydatasets_modules)r datasets.loadr+rrloadinit_dynamic_modulesr0r1spec_from_file_locationmodule_from_specsysmodulesnameloader exec_module)rrdatasetsdynamic_modules_pathspecr trainables r4 dynamic_modules_import_trainablez;run_hp_search_ray..dynamic_modules_import_trainables ! " #%77<< 0R0R0TVc#d >>99:LNbcD(~~>>tD %5CKK " KK # #$4 5$)&))r6 __mixins__)config num_samples)metricmodescope)0rR ray.trainr|_memory_trackerr trainer_utilsrr.r/rTensorBoardCallbackrrn_gpuinfoget_n_gpuray.tunerray.tune.schedulersrrrrrxdo_eval eval_strategyr"NOr}formattype__name__tunewith_parameters functoolswrapsr3rr*rget_best_trial ray_scoper!trial_id last_resultrr)rrrrrr _tb_writer resource_msggpus_per_trialrrrrrranalysisrbest_runrRrs @@r4run_hp_search_rayr=s$A$$AL  " " 6 68 < #74"P%%&9:JGMF*). $% <<   !34F( )% 0',,2D2Dq2H,bQ   / 1 1266ua@N(GLL&((&1+&O"#ftt  ; -1CEUWn!o <<''7<<+E+EIYI\I\+\" #)&T&2E-F-O-O&"P  ((7(KI__Y* *(y,'6?6J6J(3sxx||(%  H (( )BQ-W^WcWcWmWm(nJz**J,B,B;,OQ[QbQbdlmHZ( Or6c r ddl}|jjdk(rUtjj ddk\r|j d|jdd|jdd|dd gd | }tjd |j|jD]8}|5d|_ |jjd kDr|jjt j"k7r t%d |j'|j(t*j,j/t1j2|jd|j5dn|j5d|j(t7|dd&|j9}|j;||_ |j=d|jddd;t?|jAd}tC|j|jDdjF|jH} | Sddlm%} | } |jMdd} | | jO| | jQjSd|jdd|dd gd |d}tjd |j|jTjV|jXkr| jQ|jj[jS} d|_ |jjd kDr|jjt j"k7r t%d |j'| t*j,j/t1j2|jd|j5dn|j5d| t7|dd&|j9}|j;||_ d|jdg}| jQ|jj]jS| j|}tjd| jd|jd| jQ|jj_}|jTjV|jXkrt?| jQ|jjaj_jcd}tC|j|jF|jH} | Ste|D](}d|_ t?t1j2|j}|jjt j"k7r t%d t*j,j/|dt1jfti|}tk|jmD]"\}}|dk7s to|j||$|j5dt7|dd|j9}|j;||_ +y#1swYxYw)NrrWz8.0.0 huggingfacezhuggingface-tunerrr)rrstrategyr)rr parametersrparallel_bandwidthbudgetz6created experiment: https://app.sigopt.com/experiment/zConly support DDP Sigopt HPO for ParallelMode.DISTRIBUTED currently.r)rr) Connectionproxies)rr!rr"observation_budgetprojectrr) suggestionrz"[suggestion_id, observation_id]: [, ] local_rank)8rWrrr0metadataversion set_projectcreate_experimentrr.ridlooprrrr#rr}_hp_search_setupr*rrrpickledumpsrrrr log_metricr get_best_runsr!rr assignmentsr$r set_proxies experimentscreateprogressobservation_countr& suggestions observationsfetchbest_assignments iterate_pagesrloadsbytesritemssetattr)rrrrrW experimentr*rrrr$connr%r)robsrargs_main_rankrkeyrs r4run_hp_search_sigoptrLs||!!Q&    % %h /7 :   } -11'"++D1"-IS]^_#$ 2J KKPQ[Q^Q^P_` a!( CC(,G%||..2"<<559Q9QQ"./t"uu009))?? W\\@Z`a?b T B T Qw T:B")"2"2"4,3,E,Eg,N)NN;0A0ABCC C" 0023A6Dtww K(@(F(FHXHXYHTQ *MMZMP7K6>%..0G(/(A(A'(JG%#.9J9JKL&&z}}5BBDKKWaWdWdmsKt @rRURXRXQYYZ[\!--jmm<BBD '%%77*:W:WW*((7HHJPPR``bcdefDtww D4D4DEHx GA $G !&,,w||"<=N||))\-E-EE"#hii    3 3N 3 J<<n 56D$Tl002 6 U,&GLL#u5 6 MMM 6w T2:!**,$+$=$=g$F! GYCCs 6D+Z,,Z6 c H ts tdddld}jjD]}t |t sd}n|sjt dgj_ dddd |jdd}|jdd}|jd d}|jd d} |jd d  jd} | d d < | d d <|r|| d <  fd} |sj| || }n._objective@sk 99eii%**,#&88   "?@  Tfh9O P 7K . 6&&(G ' 9 9' BG )'2N^+&vh/$$2$7$7$9#:< h  +J&$..K1HH j($..K1HH H-5#&66Jx &-&7&7J{ #,0LJ( )   r6)r'rQzwandb sweep id - )functioncountrNrrO)r5 ImportErrorr)callback_handler callbacksrx WandbCallbackrr report_torrsweep wandb.envenv set_entityr/r.ragentr!)rrrrreporting_to_wandbcallbackrPr'rrQ sweep_configrrrr)s` ` @@@r4run_hp_search_wandbrn%s  TUU,,66 h .!%   ]_-%YGLL tMJzz*d+HjjD)G ::fd #D ZZ$ 'F ZZ+ .F##D)L%.L6"%+L6" # V!!B ;;|WV;L  II  ( g& KK#H:./ KK:XK> :h'K)@*M^B_ai jjr6cg}trts|jdtr|jdt r|jdt r|jdtr|jdt r|jdtr|jdtr|jdtr|jd tr|jd tr|jd tr|jd |S) Nazure_mlrrcrrr`rrJr)rjr>rur8) r[raappendrHrdrsrhrLr5rkr?rvr;) integrationss r4$get_available_reporting_integrationsrspsL&9&;J'J'I&I&H%I&!M*G$ L)I&I&I& r6ci}d}t|}d}t|}|jD]G\}}|j|r ||d||dz<#|j|r ||d||dz<@||d|z<I|S)Neval_test_eval/ztest/ztrain/)lenrEr)dnew_d eval_prefixeval_prefix_len test_prefixtest_prefix_lenkvs r4r[r[s EK+&OK+&O $1 << $34E'Ao.// 0 \\+ &34E'Ao.// 0"#E(Q,  $ Lr6c4eZdZdZddZddZdZddZdZy) rz A [`TrainerCallback`] that sends the logs to [TensorBoard](https://www.tensorflow.org/tensorboard). Args: tb_writer (`SummaryWriter`, *optional*): The writer to use. Will instantiate one if not set. Nct}|s td|r ddlm}||_||_yd|_||_y#t $r/ ddlm}||_n#t $r d|_YnwxYwY||_ywxYw)NzuTensorBoardCallback requires tensorboard to be installed. Either update your PyTorch version or install tensorboardX.r) SummaryWriter)rLr}torch.utils.tensorboardr_SummaryWriterrarK tb_writer)selfrhas_tensorboardrs r4__init__zTensorBoardCallback.__init__s24)   /A&3###'D " //:*7D'"/*.D'/# /s3 ? A7 AA7A*'A7)A**A76A7cl|xs |j}|j|j||_yy)N)log_dir) logging_dirrr)rrrs r4_init_summary_writerz(TensorBoardCallback._init_summary_writers8-T--    *!000ADN +r6c |jsyd}|jr8|j}|*tjj |j |}|j|j|||j|jjd|jd|vrV|d}t|drD|j7|jj}|jjd|yyyyy)Nrrr model_config) is_world_process_zerois_hyper_param_searchrYr+rrrrradd_textto_json_stringr3r) rrrXcontrolrrrYrmodel_config_jsons r4on_train_beginz"TensorBoardCallback.on_train_begins**   & &))J%'',,t'7'7D >> !  % %dG 4 >> % NN # #FD,?,?,A B& w5(+ 0H(- (C(C(E%NN++N> !  % %d + >> %%D 1a#u.NN--aE4E4EF3'NN++Aq%2C2CDNN3ja A3?EE  NN " &r6c `|jr"|jjd|_yyrf)rcloserrrXrrs r4 on_train_endz TensorBoardCallback.on_train_ends% >> NN "!DN r6rf) r __module__ __qualname____doc__rrrrrr:r6r4rrs"#.B O,#."r6rr output_dirct|dd5t|trt|nt r(t|t rfd}|j |nMtrCt|tjjtfrt|dr t|dddy#1swYyxYw)Nz/model_architecture.txtzw+filec t|y)Nr)print)sfs r4 print_to_filez6save_model_architecture_to_file..print_to_files aa r6)print_fn base_model) openrxrrrrsummaryrrnnModulerr3)rrrrs @r4save_model_architecture_to_filers  34d ; !q e_ - %a  :e5F#G ! MM=M 1  ! uuxx? @WUT`Ea %a  ! ! !s BB88CcLeZdZdZdZdZdZedefdZ e de ddfdZ y ) WandbLogModelz)Enum of possible log model values in W&B.rendfalsercF|tjtjfvS)zYCheck if the value corresponds to a state where the `WANDB_LOG_MODEL` setting is enabled.)r CHECKPOINTENDrs r4 is_enabledzWandbLogModel.is_enabled s  00-2C2CDDDr6rct|tstdt||j t vr"t dtjddtjd|dtjS)Nz>Expecting to have a string `WANDB_LOG_MODEL` setting, but got zSetting `WANDB_LOG_MODEL` as WANDB_LOG_MODELzn is deprecated and will be removed in version 5 of transformers. Use one of `'end'` or `'checkpoint'` instead.z6Received unrecognized `WANDB_LOG_MODEL` setting value=z ; so disabling `WANDB_LOG_MODEL`)rxr TypeErrorrr-r$DeprecationWarningr+r,r.rrrr/r^)rrs r4 _missing_zWandbLogModel._missing_s%%\]abg]h\ijk k ;;=0 0$/ :K0L/MN[[  DUGKk l """r6N) rrrrrrr^propertyboolr classmethodrrr:r6r4rrsR3J C E EDEE #c #o # #r6rcFeZdZdZdZdZd dZd defdZd dZ d Z d Z y) rdz{ A [`TrainerCallback`] that logs metrics, media, model checkpoints to [Weight and Biases](https://www.wandb.com/). cNt}|satjjd7t j ddj tv}|r tdtd|r ddl }||_ d|_ tt j dd |_ y) Nr)r'r(a@You specified `report_to='wandb'` but also set the `WANDB_DISABLED` environment variable. This disables wandb logging, even though it was explicitly requested. - To enable wandb logging: unset `WANDB_DISABLED`. - To disable logging: use `report_to='none'`. Note: WANDB_DISABLED is deprecated and will be removed in v5.zFWandbCallback requires wandb to be installed. Run `pip install wandb`.rFrr)r5r0r1r2r+r,r-r$r}r)_wandb _initializedr _log_model)r has_wandbwandb_disabledr)s r4rzWandbCallback.__init__'s&( ~~''0 Setting `WANDB_LOG_MODEL` as `bool` will be deprecated in version 5 of 🤗 Transformers. - **WANDB_WATCH** (`str`, *optional* defaults to `"false"`): Can be `"gradients"`, `"all"`, `"parameters"`, or `"false"`. Set to `"all"` to log gradients and parameters. - **WANDB_PROJECT** (`str`, *optional*, defaults to `"huggingface"`): Set this to a custom string to store results in a different project. - **WANDB_DISABLED** (`bool`, *optional*, defaults to `False`): Whether to disable wandb entirely. Set `WANDB_DISABLED=true` to disable. NTr) ConfigErrorz`Automatic Weights & Biases logging enabled, to disable set os.environ["WANDB_DISABLED"] = "true"r peft_configrgroupzThe `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.F)repeatr' WANDB_PROJECTrallow_val_change define_metrictrain/global_stepr) step_metric step_sync WANDB_WATCHr)allr! gradientsd)loglog_freqtransformers_trainer)codemodel/num_parameterszZCould not log the number of model parameters in Weights & Biases due to an AttributeError.zbA ConfigError was raised whilst setting the number of model parameters in Weights & Biases config.model-r)rnum_parameters initial_modelrrr-wbrraliasesz[Visualize in Weights & Biases]() r:)3rrwandb.sdk.lib.config_utilrrr.rto_dictr3rrxr|rrYrun_namertermwarnr*rWr+r,rrrr%watchmax logging_steps_labelrAttributeErrorr/rrrrr1rArtifactrrrris_filenew_filewrite read_bytes log_artifacturlrAUTOGENERATED_TRAINER_COMMENT)rrrXrrWandbConfigError combined_dictrrrY init_args _watch_modeltemp_dir model_namemodel_artifactrfabadge_markdowns r4setupzWandbCallback.setup?sx8 ;;    N  & & KKr /t||~.Muh'ELL,D/9%,,/Mu||SXS_S_SgSgSi A< A= A um,1B1B1N#// QM;#? Q= Q ))JI%$. &!%)]]%Edoo '"*$(MM &!==DOO3KK((C$) {{&    IIo}E KK   % %m&9rD % Qt{{OT: ))*=> ))#;NZ^)_99]G " ? =B=Q=Q=S ""#9:))002Uh!MM1T]]doo5U!!3!3 45%dkkoo&:&:%;< &*[[%9%9'$FMeU]F^ELL,@,@,Bdh.2kk.@.@.D.DE[.\-1"&:&N4E8D!(^005999;!/!8!8d!8!K9r " 8999KKOO00,0X--1[[__-@-@,AD# ;;NCS?TT;=UU*u '`"  p$ x 699)UUsD'R:/DTT S;=A2T:S8S87S8;T TTNc |jy|j}|r(|jjd|_d|_|js|j |||fi|yyNF)rrfinishrrr )rrrXrrr hp_searchs r4rzWandbCallback.on_train_beginsa ;;  //  KK   %D  DM  DJJtUE 4V 4!r6rc |jy|jjr|jr|jruddlm}tj|}d|_ d|_ ||||dg} tj5} | j| |jslt|jj j#D cic]3\} } t%| t&j(r| j+ds| | 5c} } nKd|j,|j.d|j0d|jj2j5di} d | d <t6j9d |j:|j:|j<k(r#d |jj>j@n"d |jj>jB}tE|| |jjG|d | }tI| jKdD]X}|jMs|jO|jBd5}|jQ|jSdddZ|jj>jU|d gdddyyyycc} } w#1swYxYw#1swYyxYw)NrTrainerfakerrprocessing_class eval_dataset_rwztrain/total_flossrT final_modelzLogging model artifacts. ...rrrrrrr)+rrrrrrrcopydeepcopy deepspeeddeepspeed_pluginrr save_modelload_best_model_at_endr|rrErxnumbersNumberrmetric_for_best_model best_metric total_flosrrr.rrrr*r1rrrrrrrrrr)rrrXrrrrr args_for_fake fake_trainerrrrr-rartifactrr s r4rzWandbCallback.on_train_ends{ ;;   ?? % %$*;*;@[@[ ) MM$/M&*M #-1M *""%BRbhaiL,,. P(''1 66%))<)<$=$C$C$E Aq%a8cAR1  : :;O>O+U-=-=. 0B0B0F0FG]0^+/' :; -$//1QT[[__//01!$++//"6"6!78 0x@;;//ZgX`/ah,,S15Ayy{%..qvvD.A5RHHQ\\^4555 ,,X ,O= P PA\*; %2559 P Ps> AK 8J7D,K <K  J=92K 7K =K K  Kc gd}|jy|js|j||||jr|j D]-\}} ||vs | |jj j |</|j D cic] \}} ||vs || } }} t| } |jji| d|jiyycc} }w)N train_runtimetrain_samples_per_secondtrain_steps_per_second train_lossr#r) rrr rrEr*rr[rr rrrXrrrrsingle_value_scalarsrrnon_scalar_logss r4rzWandbCallback.on_logs  ;;    JJtUE *  & &  31,,12DKKOO++A. 315 ^1I]@]q!t^O^*?;O KKOOWW0CUEVEVW X '_s  CCc |jtjk(r|jr|jrt |j jjDcic]3\}}t|tjr|jds||5}}}|j jjd|d<d|j}t j"j%|j&|} t(j+d|d|j,|j,|j&k(r#d|j j.j0n"d|j j.j2} |j j5| d|} | j7| |j j9| d t;|j<d d |jg yyyycc}}w) Nrr checkpoint- Logging checkpoint artifacts in z. ...rrrepoch_rcheckpoint_global_step_r)rrrrrr|rrrErxrr rrrrr+rrrr.rrr*r1rradd_dirrroundepoch) rrrXrrrrcheckpoint_metadatackpt_dir artifact_pathcheckpoint_namer&s r4on_savezWandbCallback.on_save s ??m66 64;L;LQVQlQl!!4!45;;=#Aqa0c9J1# # ;?++:L:L:P:PQg:h  6 7$U%6%6$78HGGLL(CM KK:8*EJ KMM)T]]doo-M++,-dkkoo2234  {{++wYl+mH   ] + KK $ $VE%++q,A+B#CG^_d_p_p^qEr"s % %Rm;L 6#s)8G?c |jy|js|j||fi||jr't |}|jj |yyrf)rrr rr[rrrrXrrrs r4 on_predictzWandbCallback.on_predict sU ;;    DJJtU -f -  & &"7+G KKOOG $ 'r6rfNN rrrrrr rrrrr<r?r:r6r4rdrd"s:O0|U| 5*P!2*PXY*.%r6rdcFeZdZdZdZdZd dZd defdZd dZ d Z d Z y) TrackioCallbacka A [`TrainerCallback`] that logs metrics to Trackio. It records training metrics, model (and PEFT) configuration, and GPU memory usage. If `nvidia-ml-py` is installed, GPU power consumption is also tracked. **Requires**: ```bash pip install trackio ``` cZt}|s td|r ddl}||_d|_y)NzLTrackioCallback requires trackio to be installed. Run `pip install trackio`.rF)r;r}r8_trackior)r has_trackior8s r4rzTrackioCallback.__init__7s.*, mn n  #DM!r6c |jrtjdr+tj dtjd}n |j }tjdr+tj dtjd}n |j }i|j}t|drR|jFt|jtr |jn|jj}i||}t|dr |j|j} id| i|}|jj||j|d|j |jjj#|d  |j%|jjd <d |_yd |_y#t&$rtj)d Yd |_ywxYw)a Setup the optional Trackio integration. To customize the setup you can also set the arguments `project`, `trackio_space_id` and `hub_private_repo` in [`TrainingArguments`]. Please refer to the docstring of for more details. TRACKIO_PROJECTzThe `TRACKIO_PROJECT` environment variable is deprecated and will be removed in a future version. Use TrainingArguments.project instead.TRACKIO_SPACE_IDzThe `TRACKIO_SPACE_ID` environment variable is deprecated and will be removed in a future version. Use TrainingArguments.trackio_space_id instead.rNrallow)r'rspace_idresumeprivateTrrzQCould not log the number of model parameters in Trackio due to an AttributeError.)rr+r,r.r/r'trackio_space_idrr3rrxr|rrErWrhub_private_reporrrrr) rrrXrrr'rKrrrs r4r zTrackioCallback.setupAs  & &yy*+F))$56,,yy+,O99%7800.t||~.Muh'ELL,D/9%,,/Mu||SXS_S_SgSgSi A< A= A um,1B1B1N#// QM;#? Q= Q MM  ]]!--   MM ' '  ' M q?D?S?S?U $$%;<!D" q op  qs'GG=<G=Nc H|js|j|||fi|yyrfrr rrrXrrrs r4rzTrackioCallback.on_train_beginv'  DJJtUE 4V 4!r6rc l|jr(|jr|jjyyyrf)rrrErrrrXrrrrs r4rzTrackioCallback.on_train_endzs+  & &4+<+< MM ",= &r6c gd}trRtjjr3tjj }tjj |j } tjj|} d|d| dz d|d| | z i} tdr+tjj|} | dz | d|d<tjrqtjr]dgtjz} tj| | | Dcic]}|jD]\}}|| } }}}ni} |js|j!||||j"ra|jDcic] \}}||vs ||}}}t%|}|j&j)i|| d |j*iyycc}}}wcc}}w) Nr(zgpu/z/allocated_memoryi@z /memory_usagepynvmliz/powerr)rrcuda is_availablecurrent_deviceget_device_properties total_memorymemory_allocatedr power_drawdistis_initializedget_world_sizeall_gather_objectrErr rr[rErr)rrrXrrrrr. device_idxr\r]gpu_memory_logspower gathered_logsryrrr/s r4rzTrackioCallback.on_log~s   EJJ$;$;$=224J ::;;JGTTL$zz:::F zl"346F'6Rzl-02B\2QO%X. --j9=BT\$zl& 9:  "t':':' > &&}oF1>"U"UA1779"U41a1a4"U1"U"U O  JJtUE *  & &04 ^1I]@]q!t^O^*?;O MM  llOlEXZ_ZkZkl m ' #V_s;"G1" G8/G8c yrfr:rs r4r<zTrackioCallback.on_savesr6c |jy|js|j||fi||jr't |}|jj |yyrf)rErr rr[rr>s r4r?zTrackioCallback.on_predictsW ==   DJJtU -f -  & &"7+G MM  g & 'r6rfr@rAr:r6r4rCrC*s5 "3!j5#!2#!nF'r6rCc8eZdZdZdZdZd dZd dZdZdZ y) CometCallbackz_ A [`TrainerCallback`] that sends the logs to [Comet ML](https://www.comet.com/site/). ctdustdurtdtdtdd|_d|_d|_y)NFz!CometCallback requires comet-ml>=z- to be installed. Run `pip install comet-ml>=z`.)rCrDr}rFr _log_assets _experimentrs r4rzCometCallback.__init__s` % '+Be+K34F3GGtvHuIIKL " r6cd|_tjddj}|dvrd|_|j rtjd}d}d}|@|j }|dk(rd}n(|dk(rd }n |d vr|}n|rtjd |y|jr|tjd |d }ddl }|j|j} |j||| |_|jj|dd|j!i} t#|dr+|j$|j$j!} | | d<t#|dr|j&|j&} | | d<|jj)| ddd|jr8t+|dd} t+|dd}|jj-| |yyy)a Setup the optional Comet integration. Environment: - **COMET_MODE** (`str`, *optional*, default to `get_or_create`): Control whether to create and log to a new Comet experiment or append to an existing experiment. It accepts the following values: * `get_or_create`: Decides automatically depending if `COMET_EXPERIMENT_KEY` is set and whether an Experiment with that key already exists or not. * `create`: Always create a new Comet Experiment. * `get`: Always try to append to an Existing Comet Experiment. Requires `COMET_EXPERIMENT_KEY` to be set. * `ONLINE`: **deprecated**, used to create an online Experiment. Use `COMET_START_ONLINE=1` instead. * `OFFLINE`: **deprecated**, used to created an offline Experiment. Use `COMET_START_ONLINE=0` instead. * `DISABLED`: **deprecated**, used to disable Comet logging. Use the `--report_to` flag to control the integrations used for logging result instead. - **COMET_PROJECT_NAME** (`str`, *optional*): Comet project name for experiments. - **COMET_LOG_ASSETS** (`str`, *optional*, defaults to `TRUE`): Whether or not to log training assets (tf event logs, checkpoints, etc), to Comet. Can be `TRUE`, or `FALSE`. For a number of configurable items in the environment, see [here](https://www.comet.com/docs/v2/guides/experiment-management/configure-sdk/#explore-comet-configuration-options). TCOMET_LOG_ASSETSr^>1r_rANonlinerF)r get_or_creater;z:Invalid COMET_MODE env value %r, Comet logging is disabledzjHyperparameter Search is enabled, forcing the creation of new experiments, COMET_MODE value %r is ignoredr;r)r)rqrexperiment_config transformers) frameworkrrrmanual)rusourceflatten_nestedrY trial_params)optimization_idr!)rr+r,r-rlrlowerr.r/rrExperimentConfigrstartrm!__internal_api__set_model_graph__rr3rr __internal_api__log_parameters__rlog_optimization)rrrXr log_assetscomet_old_moderrqrrsr{rrrzoptimization_paramss r4r zCometCallback.setups<!YY17;AAC  &#D   & &YY|4NDF)!/!5!5!7!X-!F#y0"F#'II)D#NN#_aop**#NNE&   9 9 9t}} M -x~~V$ZklD     > >uP^ > _dlln-Fuh'ELL,D$||335 #/x um,1B1B1N#// (3}%    = =.RV > **")%t"D&-e^T&J#  11/^q1r +a 'r6Nc D|js|j|||yyrfrQrRs r4rzCometCallback.on_train_begin   JJtUE *!r6c |js|j||||jrL|j?t |}|jj ||j |jdyyy)Nrtstepr7rurr rrmr[__internal_api__log_metrics__rr7)rrrXrrrrrewritten_logss r4rzCometCallback.on_logsq  JJtUE *  & &+!-d!3  >>"):):%++Yg?, 'r6c X|jr|jr|jV|jdurHtj d|jj |jdd|j|jr"|jjd|_yyyy)NTz(Logging checkpoints. This may take time.) recursive log_file_namerF) rrrmrlr.rlog_asset_folderrrrcleanrs r4rzCometCallback.on_train_end"s   !1a#u.$$((A1(= >!< r6rf)rrrrrrrr:r6r4rr:s' 1 >r6rc>eZdZdZdZdZd dZd dZdZdZ d Z y) MLflowCallbackz A [`TrainerCallback`] that sends the logs to [MLflow](https://www.mlflow.org/). Can be disabled by setting environment variable `DISABLE_MLFLOW_INTEGRATION = TRUE`. cts tdddl}|jjj |_|jjj|_d|_ d|_ d|_ ||_ y)NzIMLflowCallback requires mlflow to be installed. Run `pip install mlflow`.rF) rar}r`utils validationMAX_PARAM_VAL_LENGTH_MAX_PARAM_VAL_LENGTHMAX_PARAMS_TAGS_PER_BATCH_MAX_PARAMS_TAGS_PER_BATCHr _auto_end_run_log_artifacts_ml_flow)rr`s r4rzMLflowCallback.__init__Wsf"$jk k%+\\%<%<%Q%Q"*0,,*A*A*[*['!"# r6c  tjddjtv|_tjddjtv|_tjdd|_tjdd|_tjddjtv|_tjdd|_ tjd d|_ tjj|jjtjjd k\|_t"j%d |jd |j&d |j d|j |j(r|jj+sj|j rH|jj-|j t"j%d|j nFt"j%dn0t"j%d|jj/|jj1|j s |jr|jr%|jj3|j|jj5|j&|j t"j%d|jj1j6j8d|_|j=}t?|dr,|j@ |j@j=}i||}|jr tC|n|}tE|jGD]F\}}tItK||jLkDs(t"jOd|d|d||=HtE|jG}|jrg|jjQrMtS|j} | tI|kr*t"j%dtI|d| d|d| }tUdtI||jVD]y} |j r7|jjYt[|| | |jVzdF|jjYt[|| | |jVz{tjdd} | r0t]j^| } |jja| d|_1y)a Setup the optional MLflow integration. Environment: - **HF_MLFLOW_LOG_ARTIFACTS** (`str`, *optional*): Whether to use MLflow `.log_artifact()` facility to log artifacts. This only makes sense if logging to a remote server, e.g. s3 or GCS. If set to `True` or *1*, will copy each saved checkpoint on each save in [`TrainingArguments`]'s `output_dir` to the local or remote artifact storage. Using it without a remote storage will just copy the files to your artifact location. - **MLFLOW_TRACKING_URI** (`str`, *optional*): Whether to store runs at a specific path or remote server. Unset by default, which skips setting the tracking URI entirely. - **MLFLOW_EXPERIMENT_NAME** (`str`, *optional*, defaults to `None`): Whether to use an MLflow experiment_name under which to launch the run. Default to `None` which will point to the `Default` experiment in MLflow. Otherwise, it is a case sensitive name of the experiment to be activated. If an experiment with this name does not exist, a new experiment with this name is created. - **MLFLOW_TAGS** (`str`, *optional*): A string dump of a dictionary of key/value pair to be added to the MLflow run as tags. Example: `os.environ['MLFLOW_TAGS']='{"release.candidate": "RC1", "release.version": "2.2.0"}'`. - **MLFLOW_NESTED_RUN** (`str`, *optional*): Whether to use MLflow nested runs. If set to `True` or *1*, will create a nested run inside the current run. - **MLFLOW_RUN_ID** (`str`, *optional*): Allow to reattach to an existing run which can be useful when resuming training from a checkpoint. When `MLFLOW_RUN_ID` environment variable is set, `start_run` attempts to resume a run with the specified run ID and other parameters are ignored. - **MLFLOW_FLATTEN_PARAMS** (`str`, *optional*, defaults to `False`): Whether to flatten the parameters dictionary before logging. - **MLFLOW_MAX_LOG_PARAMS** (`int`, *optional*): Set the maximum number of parameters to log in the run. HF_MLFLOW_LOG_ARTIFACTSr^MLFLOW_NESTED_RUNMLFLOW_TRACKING_URINMLFLOW_EXPERIMENT_NAMEMLFLOW_FLATTEN_PARAMS MLFLOW_RUN_IDMLFLOW_MAX_LOG_PARAMSz2.8.0zMLflow experiment_name=z , run_name=z , nested=z, tracking_uri=zMLflow tracking URI is set to zdEnvironment variable `MLFLOW_TRACKING_URI` is not provided and therefore will not be explicitly set.)rnestedzMLflow run started with run_id=Trrz " for key "z" as a parameter. MLflow's log_param() only accepts values no longer than 250 characters so we dropped this attribute. You can use `MLFLOW_FLATTEN_PARAMS` environment variable to flatten the parameters and avoid this message.z.Reducing the number of parameters to log from z to .rF) synchronous MLFLOW_TAGS)2r+r,r-r$r _nested_run _tracking_uri_experiment_name_flatten_params_run_id_max_log_params packagingr.parserr _async_logr.debugrris_tracking_uri_setset_tracking_uriget_tracking_uri active_runset_experiment start_runrrNrrr3rrrrErxrrr/isdigitrrr log_paramsr|jsonrCset_tagsr) rrrXrrrrrcombined_dict_itemsmax_log_paramsr mlflow_tagss r4r zMLflowCallback.setupdss@!ii(A7KQQSWkk99%8'BHHJNbbYY'\>\>^=_`a}}'')1T5E5E((MM001F1FG ''tGWGW'X >t}}?W?W?Y?^?^?e?e>fgh%)" LLNMuh'ELL,D$||335 A< A= A ;?;O;OL7UbM#M$7$7$9: , es5z?T%?%??NNCE7+VZU[\// &d+ ,#'}':':'<"= ##(<(<(D(D(F!$T%9%9!:!C(;$<<LLHM`IaHbbfgufvvwx+>o~*N'1c"568W8WX q??MM,,0Q9X9X5XYZhm-MM,,T2Ea!dNmNmJm2n-op  q))M48K"jj5  &&{3 r6Nc D|js|j|||yyrfrQrRs r4rzMLflowCallback.on_train_beginrr6c |js|j||||jr4i}|jD]\}} t | t t fr| ||<"t | tjr'| jdk(r| j||<ctjd| dt| d|d|jD cic]\}} tjdd|| } }} |j r)|j"j%| |j&d y|j"j%| |j& yycc} }w) Nrrrrzc" as a metric. MLflow's log_metric() only accepts float and int types so we dropped this attribute.z[^0-9A-Za-z_\-\.\ :/]rF)rrr)rr)rr rrErxrrrTensornumelitemr.r/rresubrr log_metricsr) rrrXrrrrrrrsanitized_metricss r4rzMLflowCallback.on_logsE  JJtUE *  & &G 1a#u.!"GAJ5<<0QWWY!^!"GAJNNCA3jQUVWQXPYYcdecfgoo ZaYfYfYh iQUQRTU(@#q!I1!L i  i ))2C%J[J[in)o ))2C%J[J[)\' '!js"E"c |jrP|jrC|jr6|jj r|jj yyyyyrf)rrrrrend_runrs r4rzMLflowCallback.on_train_endsL   !&>&@ %%''A!"= r6c |jr|jr|jrd|j}tj j |j|}tjd|d|jjj|d|i|jjjyyyy)Nr1r2z. This may take time. model_path) artifacts python_model)rrrrr+rrrr.rrpyfunc log_model PythonModelrrrXrrr9r:s r4r<zMLflowCallback.on_saves   ! Properties. If None (default), the value of the `NEPTUNE_PROJECT` environment variable is used. name (`str`, *optional*): Custom name for the run. base_namespace (`str`, *optional*, defaults to "finetuning"): In the Neptune run, the root namespace that will contain all of the metadata logged by the callback. log_parameters (`bool`, *optional*, defaults to `True`): If True, logs all Trainer arguments and model parameters provided by the Trainer. log_checkpoints (`str`, *optional*): If "same", uploads checkpoints whenever they are saved by the Trainer. If "last", uploads only the most recently saved checkpoint. If "best", uploads the best checkpoint (among the ones saved by the Trainer). If `None`, does not upload checkpoints. run (`Run`, *optional*): Pass a Neptune run object if you want to continue logging to an existing run. Read more about resuming runs in the [docs](https://docs.neptune.ai/logging/to_existing_object). **neptune_run_kwargs (*optional*): Additional keyword arguments to be passed directly to the [`neptune.init_run()`](https://docs.neptune.ai/api/neptune#init_run) function when a new run is created. For instructions and examples, see the [Transformers integration guide](https://docs.neptune.ai/integrations/transformers) in the Neptune documentation. z%source_code/integrations/transformersmodel_parametersryrytrainer_parametersz train/epochN finetuningT) api_tokenr'rbase_namespacer*log_parameterslog_checkpointsrr'rrr r c ts td ddlm} ddlm} | d|ttdf| d|ttdf| d|ttdf| d|t| d || tdf| d |t| d |ttdf||_ ||_ ||_||_d|_d |_d|_d |_|||d ||_d|_|jdu|_d|_|jdvrd|j|_d|_yd|_d |_y#t $rddlm} ddlm} YLwxYw)NzwNeptuneCallback requires the Neptune client library to be installed. To install the library, run `pip install neptune`.rr) verify_typerr'rrr*r r F)rr'r>rlastz checkpoints/T checkpoints)rh ValueErrorrrneptune.internal.utilsr raneptune.new.internal.utils#neptune.new.metadata_containers.runrrr_base_namespace_path_log_parameters_log_checkpoints _initial_run_run_is_monitoring_runr_force_reset_monitoring_run_init_run_kwargs_volatile_checkpoints_dir_should_upload_checkpoint_recent_checkpoint_path_target_checkpoints_namespace*_should_clean_recently_uploaded_checkpoint) rrr'rrr*r r neptune_run_kwargsrr s r4rzNeptuneCallback.__init__fs$%E   @ # : KS$t*,=>Iwd4j(9:FD3T "34$nc:E3d4j 12$nd;%d4j8IJ$2!- /+. "' +0(.7GUY p]o p)-&)-)>)>d)J&'+$  $4 43?@U@U?V1WD .>BD ;1>D .>CD ;A @ > ? ? @s EE0/E0cd|jr$|jj|`d|_yyrf)rstoprs r4_stop_run_if_existsz#NeptuneCallback._stop_run_if_existss( 99 IINN  DI r6c f ddlm}ddlm}m}|j |j}|j|j|di||_ |jdj|_y#t $rddlm}ddlm}m}YwxYw#||f$r}t|d}~wwxYw)Nr)init_run)NeptuneMissingApiTokenException"NeptuneMissingProjectNameExceptionsys/idr:)rr%neptune.exceptionsr&r'ra neptune.newneptune.new.exceptionsr#rrrrr@rr)radditional_neptune_kwargsr%r&r' run_paramses r4_initialize_runzNeptuneCallback._initialize_runs s ( n   " 72779J   d33 4 .:.DI99X.446DL s , r r s34ST 7-/Q 6 7s)A<AB<BBB0 B++B0c|j|_d|_|jdj|_d|_y)NTr()rrrr@rrs r4_use_initial_runz NeptuneCallback._use_initial_runs8%% "&yy*002  r6c.|j|jy|js |jry|jr<|js0|js$|j |j d|_y|j d|_y)N)with_idTF)rr1rrrr/rrs r4_ensure_run_with_monitoringz+NeptuneCallback._ensure_run_with_monitoringsv    (  ! ! #338O8Oyy!8!8AaAa$$T\\$:*.'$$&380r6c|j|jy|js(|j|jddddd|_yy)NF)r3capture_stdoutcapture_stderrcapture_hardware_metricscapture_traceback)rr1rr/rrrs r4'_ensure_at_least_run_without_monitoringz7NeptuneCallback._ensure_at_least_run_without_monitoringsV    (  ! ! #99$$ LL#(#(-2&+ %+0'r6cR|j|j|jSrf)rr:rs r4r*zNeptuneCallback.runs" 99   8 8 :yyr6c4|j|jSrf)r*rrs r4_metadata_namespacez#NeptuneCallback._metadata_namespacesxx1122r6cFt|jtj<yrf)r.r*rintegration_version_keyrs r4_log_integration_versionz(NeptuneCallback._log_integration_versions|j 2|j|jj#|j ||_y#t$r#}tjd|dYd}~d}~wwxYw)Nz..r(zONeptuneCallback was unable to made a copy of checkpoint due to I/O exception: 'z'. Could fail trying to upload.)r+rrrreplacelstriprshutilcopytreeOSErrorr.r/r=r upload_filesrr delete_files) rrNr target_path relative_pathconsistent_checkpoint_path cpkt_path copy_pathr.s r4_log_model_checkpointz%NeptuneCallback._log_model_checkpoints1&(ggll3CZ&PP m  ) ) 5)+d6T6TV`)a & )11$;BB277;;O GGLL)CYO  y98    !C!CDQQR]^  : :t?[?[?g  $ $T%G%G H U UVZVrVr s'4$ efgehi33 sA1E E1E,,E1c d|_|jr;|js |j#t j j |_|jdk(r|js tdyy)NrzWTo save the best model checkpoint, the load_best_model_at_end argument must be enabled.) rroverwrite_output_dirsave_total_limitrrrrrrs r4rzNeptuneCallback.on_init_endsh)-&  d&?&?4CXCXCd-5-H-H-J-O-OD *  F *43N3Nvw w4O *r6c |jsy|jd|_|j|jr"|j ||j ||jr|j|yy)NT) rr4rr@rrDrIrrMrRs r4rzNeptuneCallback.on_train_beginso**  ((*+/( %%'     ( ( .  & &u -  & &  3 3E : 'r6c $|jyrf)r#rs r4rzNeptuneCallback.on_train_end%s   "r6c~|j!tj|jd|jy)NT) ignore_errors)rrRrmtreer#rs r4rzNeptuneCallback.__del__(s.  ) ) 5 MM$88 M   "r6c p|jr*|j|jd|jyy)Nr1)rr\rrrs r4r<zNeptuneCallback.on_save.s2  ) )  & &t+eFWFWEX8Y Z *r6c 2|jdk(r|j}|jdsd|}|j|}|jrt j nt j}|jduxs|||j|_ yy)Nrru) rr!rrgreater_is_betternpgreaterlessr"r) rrrXrrrbest_metric_name metric_valueoperators r4 on_evaluatezNeptuneCallback.on_evaluate2s  F *#99 #..w7%*+;*<#= ";;'78L%)%;%;rzzH-2->->$-F-s(S_afararJsD * +r6c|jjD]}t||s|jcSt d)Nz6The trainer doesn't have a NeptuneCallback configured.)rbrcrxr* Exception)rrrls r4get_runzNeptuneCallback.get_run>s?00:: $H(C(||# $PQQr6rc ,|jsy|t|jD]h\}}t|tt fs|t jvr||j|<?|j|j||jjyy)Nr) rr[rErxrrr flat_metricsr=rr)rrrXrrrrrs r4rzNeptuneCallback.on_logFs**   +D1779 Z eec5\2;;;9>006006::5uGXGX:Y  Z r6rf)&rrrrr?rHrKrLrCrtr rrrr#r/r1r4r:rr*r=r@rDrIrMr\rrrrr<rnrrqr|rrr:r6r4rrBs\8F-N%1!?L $(!%"* #)-5DC=5D# 5D sm 5D  5D5D"#5Dn 7$! 9 0 33Dd\5c5s50x ;## [ tRR Z$sEz:J1K Zr6rc*eZdZdZdZdZddZdZy)CodeCarbonCallbackzI A [`TrainerCallback`] that tracks the CO2 emission of training. cts tdtjjr tdddl}||_d|_y)NzWCodeCarbonCallback requires `codecarbon` to be installed. Run `pip install codecarbon`.aZCodeCarbonCallback requires `codecarbon` package, which is not compatible with AMD ROCm (https://github.com/mlco2/codecarbon/pull/490). When using the Trainer, please specify the `report_to` argument (https://huggingface.co/docs/transformers/v4.39.3/en/main_classes/trainer#transformers.TrainingArguments.report_to) to disable CodeCarbonCallback.r)rkr}rr.hiprj _codecarbontracker)rrjs r4rzCodeCarbonCallback.__init__XsN&(i ]]  m  % r6c |j9|jr,|jj|j|_yyy)N)r)rzis_local_process_zeroryEmissionsTrackerrrs r4rzCodeCarbonCallback.on_init_endgs< << E$?$?++<<<XDL%@ r6Nc l|jr(|jr|jjyyyrf)rzr|r}rRs r4rz!CodeCarbonCallback.on_train_beginls( <_clearmlr}r _clearml_taskr_checkpoints_saved)rr>s r4rzClearMLCallback.__init__s=  ! #DMop p!!"$r6c Z |jy|jrytxjdz c_txjdz c_tjdk(rdndt tjzt_|jrtjd|jtj_|jjjr$|jjjr dt_ n dt_ |jjjr|jjjr|jjj|_ tj dtj"sdnd j%t'j(d hv|_tjd n|jjj-tj d d tj d ddddd|_ tj dd j%t'j(d hv|_dt_tjdd|_tj.tj z}|dztj0z}|jjjr6|j3|||jj5|dt6dnl|jj9|dds|jj;||n1|j3|tj.tj zt=|dd|dztj>z}tj@jCtj} tjtjk7r| dtjDzz } |jjjry|jj5|dt6d|jjGtjHtj z|jJjM| y|jj9|ddsY|jJjO|jjQtjHtj z|_%y|jjGtjHtj z|jJjM| yyy)Nrr(rz"Automatic ClearML logging enabled.FTCLEARML_LOG_MODELr^r_z)External ClearML Task has been connected.CLEARML_PROJECTzHuggingFace Transformers CLEARML_TASKr)rJpytorch) project_name task_nameauto_connect_frameworks output_uriz"ClearML Task has been initialized./zIf True, ignore Transformers hyperparameters overrides done in the UI/backend when running remotely. Otherwise, the overrides will be applied when running remotely)rr value_typer)defaultcastr zIf True, ignore Transformers model configuration overrides done in the UI/backend when running remotely. Otherwise, the overrides will be applied when running remotely)r config_dictr))rrr_train_run_counter_model_connect_counterr log_suffixrr.rr_should_close_on_train_endTaskrunning_locally current_taskr+r,_task_created_in_callbackr-r$unionrrW_hparams_section_ignore_hparams_overrides_copy_training_args_as_hparams set_parameterr get_parameterconnectr_ignoge_model_config_overrides_model_config_descriptionr _model_config_description_noteset_configuration_object_model_config_sectionrr from_dict get_configuration_object_as_dict) rrrXrrrsuffixed_hparams_sectionignore_hparams_config_sectionignore_model_config_section configuration_object_descriptions r4r zClearMLCallback.setups ==     **a/*..!3.!449BsSIkIkEl?l "  & & KK< =!!)"==E==--==?4==CUCUCbCbCdEJBEIB==%%557DMMl>l8ll4==%%557&&448"#'v 5&&??,BB_E_E__$)LL$8$8$:$D@ ++99:U_cjn9o#(<<#9#9**KK+AAOD^D^^$EL &&??,BB_E_E__$)LL$8$8$:$D@?:i 'r6c |jyg|_|jrd|_|js|j||||fi|yyr )rrrrr rUs r4rzClearMLCallback.on_train_beginsP == "$  & & %D   DJJtUE+; Fv F!r6c ptjr&|jjdt_yy)Nr)rrrrrrs r4rzClearMLCallback.on_train_ends*  5 5    $ $ &12O . 6r6c |jy|js|j||||fi||jrd}t |} d} t | } gd} |j D]\} }t |ttfrF| | vr=|jjj| tjz|_| j|rL|jjjdtjz| | d||j | j| rM|jjjdtjz| | d||j |jjjdtjz| ||j dt"j%d |d t'|d | d yy) Nrurv)r)r*r+r,r#r7r(eval)titleseriesr iterationtestrrrrzn" as a scalar. This invocation of ClearML logger's report_scalar() is incorrect so we dropped this attribute.)rrr rrxrErxrrr get_loggerreport_single_valuerrr report_scalarrr.r/r)rrrXrrrrrr{r|r}r~r.rrs r4rzClearMLCallback.on_logs ==   DJJtUE+; Fv F  & &!K!+.O!K!+.O$  ! 1a#u.00**557KK!"_%?%?!?qLk2**557EE"(?+E+E"E#$_%5#6"#&+&7&7 F k2**557EE"(?+E+E"E#$_%5#6"#&+&7&7 F**557EE")O,F,F"F#$"#&+&7&7 FNN3ja A3?EE9!  'r6c |jr|jr|jrd|j}tj j |j|}|tjz}tjd|d|jj|j|}|j|j||j|||jd|j j#||j$r|j$t'|j kr |jj(j*j-|j dddd |j d d|_|j$r$|j$t'|j kryyyyyyy#t.$r<} tj1d |j dj2d | Yd} ~ yd} ~ wwxYw) Nr1zLogging checkpoint artifact `z`. This may take some time.)taskrF) weights_pathtarget_filenamerauto_delete_filerT)delete_weights_fileforceraise_on_errorszCould not remove checkpoint `z5` after going over the `save_total_limit`. Error is: r)rrrrr+rrrrrr.rr OutputModelrupdate_weights_packagerrqr_rxrModelremoverpr/r) rrrXrrr9r:r output_modelr.s r4r<zClearMLCallback.on_saveHs ??t11e6Q6Q$U%6%6$78HGGLL(CMo888D KK7v=XY Z==44$:L:LSW4XL  d&8&8t D  / /* (++!& 0   # # * *< 8''D,A,ACH_H_D`,` MM''--44//2,0"(, 5+/*A*A!"*E'''D,A,ACH_H_D`,`',`'7R1?,!NN78O8OPQ8R8W8W7XYNOPNQR s:AG H  2HH ct|Dcic]J}|jr<|jjds!|jt ||jL}}|j j jj|jDcic]\}}t||}}}|jjj||ycc}wcc}}w)N_token)prefix)rrWrendswithrr utilities proxy_objectflatten_dictionaryrErr _argumentscopy_from_dict)r training_argsrfieldas_dictrr flat_dicts r4rz.ClearMLCallback._copy_training_args_as_hparamsfs . zz%**"5"5h"? JJ uzz: :  ,0==+B+B+O+O+b+bcj+k+q+q+st41aSVQYt t %%44Yv4N  us AC C%r@)NNN)rrrrrrrrrrrrrrrrr rrrr<rr:r6r4rrusy J%1 ?%I" G 9#  %!% %aFG3 3jF<Or6rc:eZdZdZddedeffd ZdZdZxZS) FlyteCallbacka^A [`TrainerCallback`] that sends the logs to [Flyte](https://flyte.org/). NOTE: This callback only works within a Flyte task. Args: save_log_history (`bool`, *optional*, defaults to `True`): When set to True, the training logs are saved as a Flyte Deck. sync_checkpoints (`bool`, *optional*, defaults to `True`): When set to True, checkpoints are synced with Flyte and can be used to resume training in the case of an interruption. Example: ```python # Note: This example skips over some setup steps for brevity. from flytekit import current_context, task @task def train_hf_transformer(): cp = current_context().checkpoint trainer = Trainer(..., callbacks=[FlyteCallback()]) output = trainer.train(resume_from_checkpoint=cp.restore()) ``` save_log_historysync_checkpointsct|ts tdt r t st jdd}ddlm }|j|_ ||_ ||_ y)NzLFlyteCallback requires flytekit to be installed. Run `pip install flytekit`.zSyncing log history requires both flytekitplugins-deck-standard and pandas to be installed. Run `pip install flytekitplugins-deck-standard pandas` to enable this feature.Fr)current_context)rrrnrarprr.r/rmrrcprr)rrrrr~s r4rzFlyteCallback.__init__si $&lm m/19L9N NNa  % ,!#.. 0 0r6c |jr{|jrnd|j}tjj |j |}tjd|d|jj|yyy)Nr1zSyncing checkpoint in z to Flyte. This may take time.) rrrr+rrrr.rrrrs r4r<zFlyteCallback.on_savesn  U%@%@$U%6%6$78HGGLL(CM KK0 :XY Z GGLL ' &A r6c |jrIddl}ddlm}ddlm}|j |j}|d|j|yy)Nr)Deck) TableRendererz Log History) rpandasrmrflytekitplugins.deck.rendererr DataFrame log_historyto_html) rrrXrrpdrrlog_history_dfs r4rzFlyteCallback.on_train_endsD   % C\\%*;*;Y>Y II " "4?? 3?Z(9 #r6c |jr|jrddlm}|jdur|||j d|j ddg}|j rdnd }tjj|j|}|j||jj||dd |jjyyy) NrrTrrrrrr )rrr)rrtransformers.trainerrrrrr+rrrrrrr) rrrXrrrr%rrs r4rzDVCLiveCallback.on_train_end s   !r6c d|_|jrtjdi|j }t |drR|j Ft|j tr |j n|j j }i||}t |dr |j|j}id|i|}|j}i} |!|j|jd|| d<n#|j|j| d<n||| d<tjdd| d <|jj|jj di| d |jj d <|jj j#| |jj j#d |j%it'|j(d k(st'|j(dk(ra|j+\} } |jj j#d| i|jj j#d| i|j.ntj1dd|jjj2j4j6d} t8xj:d| z c_yyy#t,$rtjdYwxYw)a Setup the optional SwanLab (*swanlab*) integration. One can subclass and override this method to customize the setup if needed. Find more information [here](https://docs.swanlab.cn/guide_cloud/integration/integration-huggingface-transformers.html). You can also override the following environment variables. Find more information about environment variables [here](https://docs.swanlab.cn/en/api/environment-variable.html#environment-variables) Environment: - **SWANLAB_API_KEY** (`str`, *optional*, defaults to `None`): Cloud API Key. During login, this environment variable is checked first. If it doesn't exist, the system checks if the user is already logged in. If not, the login process is initiated. - If a string is passed to the login interface, this environment variable is ignored. - If the user is already logged in, this environment variable takes precedence over locally stored login information. - **SWANLAB_PROJECT** (`str`, *optional*, defaults to `None`): Set this to a custom string to store results in a different project. If not specified, the name of the current running directory is used. - **SWANLAB_LOG_DIR** (`str`, *optional*, defaults to `swanlog`): This environment variable specifies the storage path for log files when running in local mode. By default, logs are saved in a folder named swanlog under the working directory. - **SWANLAB_MODE** (`Literal["local", "cloud", "disabled"]`, *optional*, defaults to `cloud`): SwanLab's parsing mode, which involves callbacks registered by the operator. Currently, there are three modes: local, cloud, and disabled. Note: Case-sensitive. Find more information [here](https://docs.swanlab.cn/en/api/py-init.html#swanlab-init) - **SWANLAB_LOG_MODEL** (`str`, *optional*, defaults to `None`): SwanLab does not currently support the save mode functionality.This feature will be available in a future release - **SWANLAB_WEB_HOST** (`str`, *optional*, defaults to `None`): Web address for the SwanLab cloud environment for private version (its free) - **SWANLAB_API_HOST** (`str`, *optional*, defaults to `None`): API address for the SwanLab cloud environment for private version (its free) TzYAutomatic SwanLab logging enabled, to disable set os.environ["SWANLAB_MODE"] = "disabled"rNr-experiment_nameSWANLAB_PROJECTr'u🤗transformers FRAMEWORKmodel_num_parameters PeftModelPeftMixedModelpeft_model_trainable_paramspeft_model_all_paramzQCould not log the number of model parameters in SwanLab due to an AttributeError.sSwanLab does not currently support the save mode functionality. This feature will be available in a future release.z[Visualize in SwanLab](rrr:)rrr.rrr3rrxr|rrYrr+r,rrqrWrrrrget_nb_trainable_parametersrrr/publiccloudexperiment_urlrr) rrrXrrrrrrYrtrainable_params all_paramr s r4r zSwanLabCallback.setup& sV!  & & KKs t.t||~.Muh'ELL,D/9%,,/Mu||SXS_S_SgSgSi A< A= A um,1B1B1N#// QM;#? Q= Q ))JI%$--*C26--*/N +,*/3}} +,'/9 +,#%99->#EIi }}$$&." ""1CDMM  - MM ' ' 6 q $$++-CUEYEYE[,\];'';6$u+:N:NRb:b272S2S2U/$iMM((//1NP`0abMM((//1G0ST *J ))-)>)>)@)G)G)M)M)\)\(]]^` 77R?O;PP7+S 'J" q op qs(CK**L  L Nc H|js|j|||fi|yyrfrQrRs r4rzSwanLabCallback.on_train_begin rSr6c ||j0|jr#|jrtj dyyyyNr rrrr.r/rUs r4rzSwanLabCallback.on_train_end ; ?? &4+<+<A\A\ NNF B]+< &r6c gd}|js|j||||jr|jD]6\}} ||vs |jj d|| i|j 8|jD cic] \}} ||vs || } }} t| } |jj i| d|j i|j yycc} }w)Nr(z single_value/rsr)rr rrErrrr[r-s r4rzSwanLabCallback.on_log s   JJtUE *  & &  X1,,MM%%qc':A&>UEVEV%W X15 ^1I]@]q!t^O^*?;O MM  YY2EuGXGXY`e`q`q  r '_s C%C%c ||j0|jr#|jrtj dyyyyrrrs r4r<zSwanLabCallback.on_save rr6c |js|j||fi||jr't|}|jj |yyrf)rr rr[rrr>s r4r?zSwanLabCallback.on_predict sJ  DJJtU -f -  & &"7+G MM  g & 'r6rfr@) rrrrrr rrrr<r?r:r6r4rr s.?aQF5s&'r6r) rprr`rrJr8r)rjr>rcflyterrruc |gSt|trd|k(rgSd|k(r t}n|g}|D]<}|tvs t |ddj tj d|Dcgc] }t| c}Scc}w)Nnonerz is not supported, only r*z are supported.)rxrrsINTEGRATION_TO_CALLBACKrrr\)re integrations r4#get_reporting_integration_callbacksr  s )S! Y I i <>I" I  5 5-7 BYB^B^B`8a7bbqr  EN N[ #K 0 NN Ns4B)rrrimportlib.metadatar0importlib.utilrrr+r4rrRrr dataclassesrrenumrpathlibrtypingrrr r r numpyrhpackaging.versionrtransformers.utils.import_utilsr r,rr(rrrr.rrrrrrrrrrr.rrtorch.distributedrr_rFr-rErCrrDrr get_configrGPackageNotFoundErrorrarrrKeyErrorr1r2rg_neptune_versionrrtrainer_callbackrrrr r!r"rr#r$r%r5r;r?rHrLrOrSrUrXr[rardrhrkrnrprsrvrrrrrrLrnrsr[rrrrdrCrjrrrrprrrvrrrrrr r:r6r4r0sp   &??A 299\i' 121%   H %$$!''// ;N'//55nEIZIZI`I`asItt!!/2>#$NNY't3my~~7O7OP`7aim7m \!$--55i@ &'7&8 DE@LL(@$;;Bw:7< :D: a><H ;;D*;C;C;g;|CCC7CL_C_C_g_DHk3Hk3HkWHkV:  U"/U"p !3 !C ! #C#:E%OE%PA'oA'HIOIX>o>.r$_r$j.In.Ib ) NZoNZb  DxOoxOv<IO<I~fofR['o['~ & $ "OmK //j)Ucemn!N#  !    2 2! !(1199:JK  KK12B1C;O P!!66 ! L ! !sIA8L 0L9 'L65L69N)0NN)N"N)!N""N)(N)