Zj6R*dZddlZddlZddlZddlZddlmZmZmZddl m Z ddl m Z ddl m Z ddlmZddlmZdd lmZdd lmZdd lmZdd lmZdd lmZddlmZddlmZddl m!Z!ddl"m#Z#GddZ$dZ%e&dkr e%dSdS)zk Automation Daemon - Unified daemon for Frank Manages scheduling and execution for jobs and backup modules N)OptionalDictAny)datetime)BackgroundScheduler) CronTrigger)IntervalTrigger) DateTrigger)logger)Database)SchedulerConfig) setup_logging)convert_unix_to_iso_dayofweek) JobExecutor)NotificationManager)DependencyManager)process_pending_requestsceZdZdZdZdZdZdZdZdZ de fd Z d e d e fd Z d efdZdefdZdedefdZdefdZdZdZdZdZdZdefdZdefdZd edeefdZdedee effdZde fdZ d S)!AutomationDaemonz3Unified daemon managing multiple automation modulescZtdtjdtj|_tjd|jt |_t|jj |jj |_ t|j |j|_ t|j |j|j |_t!|j |j|_i|_|t)ddddd |_d |_d |_d|_d |_d |_t7jt6j|jt7jt6j|jtjd dS)Ndaemonz$Initializing Frank Automation DaemonzRunning on host: z Europe/RomeTi)coalesce max_instancesmisfire_grace_time)timezone job_defaultsFg<zAutomation Daemon initialized)rr infosocket gethostnamehostnamer configr database_pathlocal_database_pathdbrnotification_managerrexecutorrdependency_managermodules _load_modulesr schedulerrunning _stealth_mode_remote_config_version_last_version_check_version_check_intervalsignalSIGINT_signal_handlerSIGTERMselfs PC:\Users\Mauro Gagliardi\Dropbox\adrian\PYTHON\Frank\daemon\automation_daemon.py__init__zAutomationDaemon.__init__sh :;;;*,,  7 77888&'' 4;4dk6UVV%8$M$M!$DGT[$:STT "3DGT]"K"K(*  -" !"&*    #6:#*- ,.$  fmT%9::: fnd&:;;; 344444ctjdd|j|jd|jd<tjd ddlm}d||jd |jd <tjd n.#t$r!}tj d |Yd }~nd }~wwxYwtjdt|jdd S)z&Load and initialize automation moduleszLoading automation modules... Scheduler)namer(r)r,u ✓ Scheduler module loadedr) BackupModuleBackup)r=instancebackupu ✓ Backup module loadedu# ✗ Backup module not available: NLoaded z module(s)) r rr(r)r*modules.backupr>r& Exceptionwarninglen)r7r>es r8r+zAutomationDaemon._load_modulesTs 3444  "&"9% %  [!  3444 F 3 3 3 3 3 3 (L11&&DL " K4 5 5 5 5 F F F NDDD E E E E E E E E F  ;c$,//;;;<<<<>HHH2c|jsdStjdd|_|jjr/|jdtjd|j}|rtjdt|dd }tj}|rctj|z |krItj d |j}|rtj|z |kI|r$tj t|d |j D]\}}d |vr{t|d d re |d tjd|dS#t$r$}tjd|d|Yd}~|d}~wwxYwtjddS)zShutdown the daemon gracefullyNzShutting down Automation DaemonFT)waitzAPScheduler stoppedz Waiting for z active jobs to complete...rrz! jobs still running after timeoutr@cleanupzModule 'z' cleanup completedzError cleaning up module '': zAutomation Daemon stopped)r-r rr,rIr(get_active_executionsrFrQrRrEr*itemshasattrr_rDerror)r7activetimeoutrO module_name module_datarGs r8rIzAutomationDaemon.shutdowns&|  F 5666  > ! / N # # # . . . K- . . .4466  R KOs6{{OOO P P PGIKKE ?dikkE1W<< 1 <<>> ?dikkE1W<< R#f++PPPQQQ)- (:(:(<(< S S $K[((W[5Li-X-X(S +33555K K; K K KLLLL SSSL!Qk!Q!Qa!Q!QRRRRRRRRS  /00000s92F,, G6GGc*|jd}tjdt |d|D]O} ||#t $r*}tjd|dd|Yd}~Hd}~wwxYwdS) z:Load all enabled schedules from database and schedule themT) enabled_onlyzLoading z enabled schedule(s)zError scheduling 'r=r`N)r&get_all_schedulesr rrF_schedule_itemrDrd)r7 schedulesschedulerGs r8rNz AutomationDaemon._load_schedulessG--4-@@  Cs9~~CCCDDD! L LH L##H---- L L L J(62BJJqJJKKKKKKKK L L LsA B& B  Brnc ddl}|d}|d}|d}|d}|d} ||d}n#|d}YnxYw|d krtjd |d dS|||} | stjd |d dS|dkry|j|} | rR| ddr<| d|j kr+tjd|d| dd|j d dS|j } |g} n|dkr||j |} | D]T}|dd}|r:||j kr/tjd|d|dd|d|j d dSU|j } |g} nN|dkr+d|j vrtjddS|j} ||g} ntjd|d|d dS|j| | | d||d tjd!|d"|d#|d$|d%| dS)&zz Schedule an item with APScheduler Args: schedule: Schedule definition from database rNidr= target_type target_id schedule_typeschedule_configmanualzSkipping manual schedule ''z&Could not build trigger for schedule 'z ', skippingjobr"zSkipping schedule 'z': job hostname='z ', current='groupzSkipping group schedule 'z': job 'job_namez ' hostname='backup_profilerAz0Cannot schedule backup: backup module not loadedzUnknown target_type 'z' in schedule ' schedule_T)functriggerargsrpr=replace_existingz Scheduled 'z' (ID z):  z with )jsonloadsr debug_build_triggerrEr&get_jobrSr"_execute_scheduled_jobget_job_group_members_execute_scheduled_groupr*rd_execute_scheduled_backupr,add_jobr)r7rnr schedule_id schedule_namerqrrrsrtr~rw exec_func exec_args group_jobsgj job_hostnames r8rlzAutomationDaemon._schedule_itemse  tn  ( }- [)  1  :"jj2C)DEEOO :&'89OOO H $ $ LFmFFF G G G F%%m_EE  N^M^^^ _ _ _ F %  '//),,C swwz2.. 3z?dm3S3S A=AASVWaSbAAptp}AAABBB3I" II G # #66yAAJ   !vvj"55 LDM$A$AL"[]"["[TVWaTb"["[p|"["[KOKX"["["[\\\FF5I" II , , ,t|++ PQQQ6I$i0II L]]]]]]] ^ ^ ^ F (;((!      u-uu{uu{uuU^uufsuuvvvvvs A Arsrtc|dkr;|d}|r#t|}tj|Sn?|dkr t di|S|dkr'|d}|rt |SdS)z Build APScheduler trigger from schedule configuration Args: schedule_type: 'cron', 'interval', or 'date' schedule_config: Configuration dict Returns: APScheduler trigger object cronintervaldaterun_date)rN)rSrr from_crontabr r )r7rsrt cron_expr cron_expr_isors r8rzAutomationDaemon._build_trigger*s F " "'++F33I ? =i H H "/ >>> ? j ( ("55_55 5 f $ $&**:66H 6"H5555tr:job_idc@tjd| |j|d}|rtjd|d|dd Stjd|dd S#t $r%}tjd|d |Yd }~d Sd }~wwxYw) zv Execute a scheduled job (called by APScheduler) Args: job_id: Job ID to execute z(Triggered scheduled execution of job ID r, triggered_byzJob ID z$ queued for execution (execution ID )z could not be queuedzError executing scheduled job : N)r rr( execute_jobrErDr[)r7r execution_idrGs r8rz'AutomationDaemon._execute_scheduled_jobIs  GvGGHHH M=44V+4VVL G afaaR^aaabbbbbEEEEFFFFF M M M  KfKKKK L L L L L L L L L Ms9A.A.. B8BBgroup_idctjd| |j|d}|drtjd|dd St |dg}tjd|d|d d S#t$r%}tjd |d |Yd }~d Sd }~wwxYw) z| Execute a scheduled group (called by APScheduler) Args: group_id: Group ID to execute z*Triggered scheduled execution of group ID r,rsuccessz Group ID  completed successfully failed_jobsz completed with z failed job(s)z Error executing scheduled group rN) r rr)execute_job_grouprSrFrErDr[)r7rresult failed_countrGs r8rz)AutomationDaemon._execute_scheduled_group]s  KKKLLL Q,>>(?F zz)$$ c IIIIJJJJJ"6::mR#@#@AA a8aa\aaabbbbb Q Q Q  OOOAOO P P P P P P P P P QsA B$$>B$$ C.CCr profile_idctjd| |jdd}|||}|drtjd|dd S|dd}tjd|d |d S#t $r%}tjd |d |Yd }~d Sd }~wwxYw) z Execute a scheduled backup (called by APScheduler) Args: schedule_id: Schedule ID that triggered profile_id: Backup profile ID to execute z3Triggered scheduled execution of backup profile ID rAr@rzBackup profile ID rrdz Unknown errorz failed: z!Error executing scheduled backup rN)r rr* schedule_itemrSrdrDr[)r7rr backup_modulerrdrGs r8rz*AutomationDaemon._execute_scheduled_backupvs  V*VVWWW T L2:>M"00jIIFzz)$$ P TTTTUUUUU 7O<< N*NNuNNOOOOO T T T  RRRqRR S S S S S S S S S TsAB#10B## C-C  Creturnctj}|dkrdS|jdd}|jdd}||jcxko|kncS)zBReturns True if current time is within active window: Mon-Fri 8-20Fdaemon.active_hours_startzdaemon.active_hours_end)rr\weekdayr#rShour)r7r\rOends r8rTz"AutomationDaemon._is_active_windowsulnn ;;==A  5  ;Q??koo7<<&&&&3&&&&&r:cd|_|jjr|j|jdd}t jd|ddtj ddd S) uJPause all activity outside active hours — no DB access, no job executionTrru0Stealth mode ON — nessuna attività fino alle 02dz:00 (ora corrente: %H:%MrN) r.r,r-pauser#rSr rrr\strftime)r7rOs r8rUzAutomationDaemon._enter_stealths! > ! # N " " "  ;Q?? IuIIIemeqesese|e|~EfFfFIII J J J J Jr:cd|_|jjr|jt jdt jdddS)z/Resume normal activity at start of active hoursFu-Stealth mode OFF — ripresa attività (ora: rrN) r.r,r-resumer rrr\rr6s r8rVzAutomationDaemon._exit_stealthsb" > ! $ N ! ! # # # gHLNND[D[\cDdDdggghhhhhr:c|j}|D]}tj|d}tj|z }|j|d}|rb||ddzkrStjd|dd|j |ddtjd d S) z!Periodic health check and cleanup start_timertimeout_secondsz Execution rpz! appears stuck, marking as failedfailedz@Execution exceeded 2x timeout - marked as failed by health check)statusend_time error_messageN) r&rar fromisoformatr\ total_secondsrr rEupdate_execution)r7re executionr running_timerws r8rWzAutomationDaemon._health_checks..00  I!/ ,0GHHJ$LNNZ7FFHHL'//)H"566C |s+<'='ABB^IdO^^^___((dO#%\^^"d )  r:c|j}|dS|jZ||jkrOtjd|jd|d|j}|r|||_dS)u Poll remote B for config_version changes. If version changed → re-sync config tables + reload schedules (no restart needed). If B unreachable → skip silently (no crash). Nz$Config version changed on remote B (u → u) — resyncing)r&get_remote_config_versionr/r r_sync_config_from_remote_reload_schedules)r7vsynceds r8rYz-AutomationDaemon._check_remote_config_versions G - - / / 9 F  & 2qD**,,  C JJLLLL  R3t~/F/F/H/H+I+IRRRSSSSSr:c|j|}|r=|dr7||tjd|dddSdSdS)z} Add or update a schedule in the scheduler Args: schedule_id: Schedule ID to add/update enabledzAdded/updated schedule 'r=z' in schedulerN)r& get_schedulerlr r)r7rrns r8 add_schedulezAutomationDaemon.add_schedules7'' 44  U+ U    ) ) ) KS8F3CSSS T T T T T U U U Ur:c |jd|tjd|ddS#t$r%}tjd|d|Yd}~dSd}~wwxYw)zt Remove a schedule from the scheduler Args: schedule_id: Schedule ID to remove r|zRemoved schedule ID z from schedulerzCould not remove schedule z from scheduler: N)r, remove_jobr rrDrE)r7rrGs r8remove_schedulez AutomationDaemon.remove_schedules [ N % %&?+&?&? @ @ @ KK{KKK L L L L L [ [ [ NY YYVWYY Z Z Z Z Z Z Z Z Z [s59 A(A##A(chtjd||j|dS)z Execute a job manually (outside of schedule) Args: job_id: Job ID to execute Returns: Execution ID z&Manual execution requested for job ID rur)r rr(r)r7rs r8execute_job_manuallyz%AutomationDaemon.execute_job_manuallys8  EVEEFFF}((h(GGGr:ctjd|d|jvrd}tj|d|dS|jdd}||S)z Execute a backup manually (outside of schedule) Args: profile_id: Backup profile ID to execute Returns: Result dict with success status and details z1Manual execution requested for backup profile ID rAzBackup module not loadedF)rrdr@)r rr*rdexecute_backup)r7r error_msgrs r8execute_backup_manuallyz(AutomationDaemon.execute_backup_manuallysr  T TTUUU 4< ' '2I L # # #$y99 9 X.z: ++J777r:cf|j}|j}i}|jD]!\}}d|d|d||<"|jt|t|||rt|dj nddS)zGet daemon status informationTr=)loadedr=rN)r-scheduled_jobsactive_executionsr* next_run_time) r,rPr&rar*rbrSr-rFstrr)r7jobsrmodules_statusrgrhs r8get_daemon_statusz"AutomationDaemon.get_daemon_status s~&&(( G99;;(, (:(:(<(<   $K# <<++N; ' ' |!$ii!$%6!7!7%;?ISa!6777T    r:N)!__name__ __module__ __qualname____doc__r9r+r4rOrIrNdictrlrrintrrrboolrTrUrVrWrYrrrrrrrrrrr:r8rrsK==353535j===@ 333j$1$1$1L L L LIwtIwIwIwIwVC$>MSMMMM(QQQQQ2TSTcTTTT.'4''''JJJiii*((( TTT U U U U U [3 [ [ [ [ H3 H8C= H H H H8#8$sCx.8888( 4      r:rctjdtjdtjdt} |na#t$rtjdYnAt $r5}tjd|tjdYd}~nd}~wwxYwtjddS)zMain entry point for daemonz<============================================================z#Python Automation Daemon - StartingzReceived KeyboardInterruptzFatal error in daemon: rNr) r rrrOrZrDr[sysexit)rrGs r8mainr"s K K5666 K   F  222 011111 6166777  HQKKKKKs A!!B? B? +B::B?__main__)'rrQr2r rtypingrrrr!apscheduler.schedulers.backgroundrapscheduler.triggers.cronrapscheduler.triggers.intervalr apscheduler.triggers.dater logurur core.databaser core.configr core.loggerrcore.cron_utilsrdaemon.job_executorrdaemon.notification_managerrdaemon.dependency_managerrdaemon.relay_handlerrrrrrr:r8rs  &&&&&&&&&&AAAAAA111111999999111111""""""''''''%%%%%%999999++++++;;;;;;777777999999C C C C C C C C L& zDFFFFFr: