"""Bounded bridge process exit hooks; failure to report stays visible on disk.""" import json from pathlib import Path import time import uuid from control import Client from worker import atomic_json def report_process(cfg,root,role,process,records,started_at=None): record=records.get(role) code=process.poll() if code is not None and code!=0 and not record: record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing','first_seen':time.time(),'severity':'warning'} records[role]=record health_path=Path(cfg.get('notification_state_file',str(root/'notifications.json'))) if role=='notifier' else root/(role+'-health.json') try: data=json.loads(health_path.read_text(encoding='utf-8')) health=data.get('health',data) healthy=health.get('status')=='ok' and time.time()-health.get('updated',0)<60 except (OSError,ValueError):healthy=False if code is None and role in ('notifier','report_relay') and not healthy and not record and started_at is not None and time.time()-started_at>30: record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing','first_seen':time.time(),'severity':'warning'} records[role]=record if record and record.get('pending')=='resolved' and not (code is None and healthy): # A failed transport may already have accepted recovery. Do not announce # recovery after health is lost or restart the same incident blindly. record.pop('pending',None) record['needs_reconciliation']=True record['delivery']='uncertain_recovery' if record and record.get('needs_reconciliation'): atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()}) return if record: if code is None and healthy: record.setdefault('healthy_since',time.time()) if record['state']=='firing' and not record.get('pending') and time.time()-record['healthy_since']>=30: record['pending']='resolved' else:record.pop('healthy_since',None) if record['state']=='firing' and not record.get('pending') and not (code is None and healthy) and not record.get('escalated'): delay=cfg.get('fault_escalation_seconds',120) if not isinstance(delay,(int,float)) or isinstance(delay,bool) or not 60<=delay<=3600: raise ValueError('Fault escalation interval must be between 60 and 3600 seconds') if time.time()-record.get('first_seen',time.time())>=delay: try: Client(cfg['control']).call('/api/hooks/escalate',{ 'source':'bridge-'+role,'event_id':record['event_id'], 'reason':'Functional health has remained unavailable beyond the bounded recovery window', 'impact':'Conversation delivery or synchronization is persistently delayed'}) record['escalated']=True record['escalation_delivery']='accepted' except Exception:record['escalation_delivery']='failed' if record.get('pending'): try: Client(cfg['control']).call('/api/hooks',{ 'event_id':record['event_id'],'source':'bridge-'+role,'severity':record.get('severity','warning'), 'status':record['pending'],'summary':{'worker':'消息转发暂时中断','mirror':'聊天同步暂时中断','notifier':'事件通知暂时中断','report_relay':'完成上报暂时中断'}.get(role,'本机桥接暂时中断'), 'impact':'网页收发或历史同步可能延迟', 'evidence':['Private executor log: '+role+'.log'], 'action':'已启用有界重连;结果未知的模型输入不重复执行', 'next_step':'持续异常时核对执行器健康与原会话;短暂自动恢复无需用户操作'}) record['state']=record.pop('pending') record['delivery']='accepted' if record['state']=='resolved':records.pop(role,None) except Exception:record['delivery']='failed' atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})