&gj2HdZddlZddlZddlZddlZddlZddlmZddlmZddl m Z m Z m Z ddl mZddlZddlmZddlmZGd d ZdS) z Job Executor for Python Scheduler Handles job execution in background threads with timeout support Based on DataHub web_panel/utils/job_manager.py pattern N)datetime)Path)OptionalDictAny)logger)SchedulerDatabase)SchedulerConfigceZdZdZddedefdZdefdZdd ed e d e efd Z dd ede e e fd e fdZde e e fd efdZde e d e fdZd ede d efdZddeded efdZd ede e e ffdZd efdZd ed efdZdS) JobExecutorz8Executes jobs in background threads with proper trackingNdbconfigc||_||_||_i|_t j|_tjddS)NzJobExecutor initialized) r rnotification_manager active_jobs threadingLocklockrinfo)selfr rrs KC:\Users\Mauro Gagliardi\Dropbox\adrian\PYTHON\Frank\daemon\job_executor.py__init__zJobExecutor.__init__sF $8!N$$  -.....pidc tj|}|d}|D]]} tjd|jd||I#tj$rYZwxYw tjd|jd||n#tj$rYnwxYwtj |g|zd\}}|D]]} tj d|jd||I#tj$rYZwxYwd S#tj$rtj d |d Yd St$r%}tj d |d|Yd }~d Sd }~wwxYw) z| Terminate process and all its children recursively Args: pid: Process ID to terminate T) recursivezKilling child process : zKilling parent process timeoutzForce killing process zProcess z already terminatedzError killing process tree N) psutilProcesschildrenrdebugrnamekill NoSuchProcess wait_procswarning Exceptionerror) rrparentr#childgonealivepes r_kill_process_treezJobExecutor._kill_process_tree!s3! C^C((F66H"  L!U%)!U!Uuzz||!U!UVVVJJLLLL+D  TvzTTV[[]]TTUUU '    !+VHx,?KKKKD%  N#OAE#O#OQVVXX#O#OPPPFFHHHH+D   # @ @ @ N>c>>> ? ? ? ? ? ? C C C LAsAAaAA B B B B B B B B B Cs.E'AA76E'7B E'B  E' ACE'C%"E'$C%%$E' AEE'E"E'!E""E'''F? F?F::F? schedulerjob_id triggered_byreturnc |j|}|stjd|ddS|ds#tjd|dd|ddS|jjd|d |t jd |}tjd |dd |d tj |j |||fd}| |S)a. Execute a job asynchronously Args: job_id: ID of job to execute triggered_by: Who/what triggered execution **kwargs: Additional execution parameters Returns: Execution ID for tracking, or None if job not found/disabled zJob z not foundNenabledJob 'r%z' (ID z) is disabled, skippingqueued)r4statusr5 start_timez Queued job 'z' (execution ID )F)targetargsdaemon) r get_jobrr+r)create_executionrnowrrThread_run_jobstart)rr4r5kwargsjob execution_idthreads r execute_jobzJobExecutor.execute_jobKs!goof%%  L2222 3 3 349~  NU3v;UUfUUU V V V40tw/ %|~~        O3v;OO OOOPPP!=\2     rrJrIc |d}|||} |j|dtjt |t jd|d|d||}tj}| dd}|rt jd |d t|d d 5} | d| d|d| d|d| d| dd| dt|t r|nd|d| d|dd| dd| d| ddddn #1swxYwYt jd |dt|t rddl} | |}t|dd 5} t#j||dr|dnd| t"jd !} dddn #1swxYwY|j5| |j|<dddn #1swxYwY|d"} | | #} | dkrd$nd%}nn#t"j$r\t jd |d&| d't jd(|d)| jd|| jd*} d+}YnwxYw|j5|j|ddddn #1swxYwYnBt|d d 5} | d| d|d| d|d| d| dd| dd|d| d|dd| dd| t#j|| t"j|dr|dndd ,} |j5| |j|<dddn #1swxYwY|d"} | | #} | dkrd$nd%}nn#t"j$r\t jd |d&| d't jd(|d)| jd|| jd*} d+}YnwxYw|j5|j|ddddn #1swxYwYdddn #1swxYwYtj}||z }||\}}|j||||| ||-|d$krt jd |d.|d/d0n<|d+krt jd |d&| d0nt jd |d1| |d2dk}|j r'|d%kr|s|j !||||| 3|d%kr|r|"||dSdSdS#tF$r}t j$d4|d5||j|d%tjt |6 t|dd 5} | ddd| d7|d| dddddn #1swxYwYn#Yn xYwYd}~dSYd}~dSd}~wwxYw)8z Internal: Execute job in subprocess Args: execution_id: Execution record ID job: Job definition dict r%running)r;r<log_filezStarting job 'z ' (execution r= show_consoleFr9zW' configured to show console window (debug mode) - output will be visible in CMD windowwutf-8)encodingz=== Job Execution Log === zJob:  zExecution ID: z Start Time: z%Y-%m-%d %H:%M:%Sz Command:  zWorking Directory: working_directoryz2==================================================z z6NOTE: Output displayed in console window (debug mode) z6Log file not captured - see console window for output Nz:' has show_console enabled - log can be monitored from GUIraT)cwdstdoutstderrtexttimeout_secondsr completedfailedz' timed out after z secondsz"Terminating process tree for job 'z' (PID: r )rYrZrXr[)r;end_timeduration_seconds exit_codestdout_previewstderr_previewz' completed successfully in z.1fsz' failed with exit code retry_count)rJjob_namer;durationrbzError executing job 'z': )r;r` error_messagezEXECUTION ERROR: )%_create_log_file_pathr update_executionrrDstrrr_build_commandgetopenwritestrftime isinstancejoinshlexsplit subprocessPopenSTDOUTrrwaitTimeoutExpiredr)rr2popflush total_seconds_read_log_previewsr+rnotify_job_result _handle_retryr* exception)rrJrIr5rg log_file_pathcmdr<rPrOrtprocessr rbr;r`rhrcrd has_retriesr1s rrFzJobExecutor._run_jobus v;22<JJ p  G $ $ #<>>]++ %    KOOO OOO P P P%%c**C"J77>599L^ A FHFFFGGG-w??? _8NN#ABBBNN#78#7#7#7888NN#DL#D#D#DEEENN#^*2E2EFY2Z2Z#^#^#^___NN#ajc6J6J/]ssPSPXPXY\P]P]#a#a#abbbNN#U=P9Q#U#U#UVVVNNh#4#4#4555NN#]^^^NN#]^^^ _ _ _ _ _ _ _ _ _ _ _ _ _ _ _ hHhhhiiic3''+ LLL++c**C-w???8(.8;>H.2-D-D]-S-S *NN G $ $!!)#-- %   $$ YHYY(YYYYZZZZ9$$ KXKKKKKLLLL RXRRyRRSSS m,q0K( &H2D2D2D);;!-%!%' <!!k!""<55555"!!!     EXEE!EE F F F G $ $!!!ff %    -w???48NN#6(#6#6#6777NN#O(>(>(>?  ! !--cgg6I.J.JKKJ/CC  ( (--cgg6I.J.JKKJt_5CC<(<<== = {  - -z#k"233 4    ' - - - 3{+,,,,, - s/D*D-,D-rVcddl}|r;t|dz dz dz }|rt|S|jdi}|r|D]\}}||ddrQ|d}|r:t|dz dz }|rt|cS|jS) z3Get Python executable, preferring venv if availablerNvenvScriptsz python.exeprojectsroot) sysrexistsrlrrnitems startswith executable)rrVr venv_pythonr project_nameproject_config venv_paths rrzJobExecutor._get_python_exe\s  (011F:YFUK!!## (;''';??:r22  4080@0@ 4 4, n$//0B0B620N0NOO4 . 2 26 : :I 4&*9oo &AL&P &--//4#&{#3#3333~rrgc@t|jj}|ddt jd}|dddd}|d|d}||z S)z#Create log file path with timestampT)parentsexist_okz %Y%m%d_%H%M%SrU_/z.log) rrexecution_log_dirmkdirrrDrqrreplace)rrJrglog_dir timestampsanitized_name log_filenames rrjz!JobExecutor._create_log_file_pathtst{455 dT 222LNN++O<< !))11#s;;CCCMM(::9::: %%rr preview_linesc, t|ddd5}|}dddn #1swxYwY|rd|| dnd}|dfS#t$r"}t jd|Yd}~dSd}~wwxYw) z+Read last N lines from log file for previewrrRr)rSerrorsNrzCould not read log preview: )NN)ro readlinesrsr*rr))rrrflinespreviewr1s rr~zJobExecutor._read_log_previewss mS79MMM &QR  & & & & & & & & & & & & & & &:?Fbgge]NOO4555BGD=     N=!== > > >::::: s2A'6 A':A':(A'' B1BBc jdd}td|D}|dkrldtjddd d |d zd dd zd fd}t j|ddStjddddd|dj r$j d|dSdS)zHandle job retry logic id)limitr4c32K|]}|ddkdVdS)r;r^NrA).0exs r z,JobExecutor._handle_retry..s0UUBxLH.retry_jobs6 ;'''  T AAAAArT)r>r@r9z' exhausted all z retries (z failures))rg attempt_countN) r get_recent_executionssumrrrrErGr+rnotify_final_failure)rrJrIrecent_executions failed_countrrs` ` @rrzJobExecutor._handle_retrys!G993t99UUUU'8UUUUU 3}- - -34K KGVGG;GGZfijZjGGmpq~mCDnDGGG H H H B B B B B B B  Id ; ; ; A A C C C C C LtVttc->Ptt\httt u u u( )>> [".?  rc|j5t|jcdddS#1swxYwYdS)z(Get list of currently running executionsN)rlistrkeys)rs rget_active_executionsz!JobExecutor.get_active_executionss Y 1 1(--//00 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1 1s &;??c|j5||jvr ddddS|j|}| | |dn)#t j$r|YnwxYw|j |dtj |j |dtjd| ddddS#t$r0}tjd |d |Yd}~ddddSd}~wwxYw dddn #1swxYwYdS) z Stop a running execution Args: execution_id: Execution ID to stop Returns: True if stopped successfully NFrr cancelled)r;r`zStopped execution TzError stopping execution r)rrpoll terminateryrvrzr&r rkrrDr{rrr*r+)rrJrr1s rstop_executionzJobExecutor.stop_executions)Y! !! !4#333! !! !! !! !! !! !! !! !&|4G||~~%!%%'''' Q ////%4''' ' G,,$*!)-$((t<<<K C\ C CDDD;! !! !! !! !! !! !! !! !>!!!!L!P\!P!PQ!P!PQQQ 555C! !! !! !! !! !! !! !! !>!3& ! !! !! !! !! !! !! !! !! !! !! !! !! !! !! !Fus_ E"EDA/.D/#BDBA#D ED;*E;EEEE)N)r3)r)__name__ __module__ __qualname____doc__r r rintr2rlrrLrrrFrrmrrrjtupler~rrboolrrArrr r sBB//,/o////(Cc(C(C(C(CT((#(S(U]^aUb((((T{{S{tCH~{S{{{{z($sCx.(T((((T#30 &# & & & & & &SRW #DcN21t1111 -3-4------rr )rrvrrrrrpathlibrtypingrrrlogururr! core.databaser core.configr r rArrrs  &&&&&&&&&& ++++++''''''DDDDDDDDDDr