54 lines
2.9 KiB
Python
54 lines
2.9 KiB
Python
"""Bounded bridge process exit hooks; failure to report stays visible on disk."""
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import uuid
|
|
from control import Client
|
|
from worker import atomic_json
|
|
|
|
|
|
def report_process(cfg,root,role,process,records,started_at=None):
|
|
record=records.get(role)
|
|
code=process.poll()
|
|
if code is not None and code!=0 and not record:
|
|
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing'}
|
|
records[role]=record
|
|
health_path=Path(cfg.get('notification_state_file',str(root/'notifications.json'))) if role=='notifier' else root/(role+'-health.json')
|
|
try:
|
|
data=json.loads(health_path.read_text(encoding='utf-8'))
|
|
health=data.get('health',data)
|
|
healthy=health.get('status')=='ok' and time.time()-health.get('updated',0)<60
|
|
except (OSError,ValueError):healthy=False
|
|
if code is None and role=='notifier' and not healthy and not record and started_at is not None and time.time()-started_at>30:
|
|
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing'}
|
|
records[role]=record
|
|
if record and record.get('pending')=='resolved' and not (code is None and healthy):
|
|
# A failed transport may already have accepted recovery. Do not announce
|
|
# recovery after health is lost or restart the same incident blindly.
|
|
record.pop('pending',None)
|
|
record['needs_reconciliation']=True
|
|
record['delivery']='uncertain_recovery'
|
|
if record and record.get('needs_reconciliation'):
|
|
atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})
|
|
return
|
|
if record:
|
|
if code is None and healthy:
|
|
record.setdefault('healthy_since',time.time())
|
|
if record['state']=='firing' and not record.get('pending') and time.time()-record['healthy_since']>=30:
|
|
record['pending']='resolved'
|
|
else:record.pop('healthy_since',None)
|
|
if record.get('pending'):
|
|
try:
|
|
Client(cfg['control']).call('/api/hooks',{
|
|
'event_id':record['event_id'],'source':'bridge-'+role,'severity':'critical',
|
|
'status':record['pending'],'summary':'Local bridge '+role+' process or functional health failed',
|
|
'impact':'Conversation delivery or synchronization may be delayed',
|
|
'evidence':['Private executor log: '+role+'.log'],
|
|
'action':'Bounded supervised restart; unknown model executions are never replayed',
|
|
'next_step':'Inspect private executor health and original thread before repair'})
|
|
record['state']=record.pop('pending')
|
|
record['delivery']='accepted'
|
|
if record['state']=='resolved':records.pop(role,None)
|
|
except Exception:record['delivery']='failed'
|
|
atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})
|