Add event-driven reporting and durable failure recovery hooks

2026-10-19 03:13:33 -07:00
parent 579dece104
commit a4cf70d82b
18 changed files with 711 additions and 8 deletions
@@ -0,0 +1,53 @@
+"""Bounded bridge process exit hooks; failure to report stays visible on disk."""
+import json
+from pathlib import Path
+import time
+import uuid
+from control import Client
+from worker import atomic_json
+
+
+def report_process(cfg,root,role,process,records,started_at=None):
+    record=records.get(role)
+    code=process.poll()
+    if code is not None and code!=0 and not record:
+        record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing'}
+        records[role]=record
+    health_path=Path(cfg.get('notification_state_file',str(root/'notifications.json'))) if role=='notifier' else root/(role+'-health.json')
+    try:
+        data=json.loads(health_path.read_text(encoding='utf-8'))
+        health=data.get('health',data)
+        healthy=health.get('status')=='ok' and time.time()-health.get('updated',0)<60
+    except (OSError,ValueError):healthy=False
+    if code is None and role=='notifier' and not healthy and not record and started_at is not None and time.time()-started_at>30:
+        record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing'}
+        records[role]=record
+    if record and record.get('pending')=='resolved' and not (code is None and healthy):
+        # A failed transport may already have accepted recovery. Do not announce
+        # recovery after health is lost or restart the same incident blindly.
+        record.pop('pending',None)
+        record['needs_reconciliation']=True
+        record['delivery']='uncertain_recovery'
+    if record and record.get('needs_reconciliation'):
+        atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})
+        return
+    if record:
+        if code is None and healthy:
+            record.setdefault('healthy_since',time.time())
+            if record['state']=='firing' and not record.get('pending') and time.time()-record['healthy_since']>=30:
+                record['pending']='resolved'
+        else:record.pop('healthy_since',None)
+        if record.get('pending'):
+            try:
+                Client(cfg['control']).call('/api/hooks',{
+                    'event_id':record['event_id'],'source':'bridge-'+role,'severity':'critical',
+                    'status':record['pending'],'summary':'Local bridge '+role+' process or functional health failed',
+                    'impact':'Conversation delivery or synchronization may be delayed',
+                    'evidence':['Private executor log: '+role+'.log'],
+                    'action':'Bounded supervised restart; unknown model executions are never replayed',
+                    'next_step':'Inspect private executor health and original thread before repair'})
+                record['state']=record.pop('pending')
+                record['delivery']='accepted'
+                if record['state']=='resolved':records.pop(role,None)
+            except Exception:record['delivery']='failed'
+    atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})