67 lines
4.2 KiB
Python
67 lines
4.2 KiB
Python
"""Bounded bridge process exit hooks; failure to report stays visible on disk."""
|
|
import json
|
|
from pathlib import Path
|
|
import time
|
|
import uuid
|
|
from control import Client
|
|
from worker import atomic_json
|
|
|
|
|
|
def report_process(cfg,root,role,process,records,started_at=None):
|
|
record=records.get(role)
|
|
code=process.poll()
|
|
if code is not None and code!=0 and not record:
|
|
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing','first_seen':time.time(),'severity':'warning'}
|
|
records[role]=record
|
|
health_path=Path(cfg.get('notification_state_file',str(root/'notifications.json'))) if role=='notifier' else root/(role+'-health.json')
|
|
try:
|
|
data=json.loads(health_path.read_text(encoding='utf-8'))
|
|
health=data.get('health',data)
|
|
healthy=health.get('status')=='ok' and time.time()-health.get('updated',0)<60
|
|
except (OSError,ValueError):healthy=False
|
|
if code is None and role in ('notifier','report_relay') and not healthy and not record and started_at is not None and time.time()-started_at>30:
|
|
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing','first_seen':time.time(),'severity':'warning'}
|
|
records[role]=record
|
|
if record and record.get('pending')=='resolved' and not (code is None and healthy):
|
|
# A failed transport may already have accepted recovery. Do not announce
|
|
# recovery after health is lost or restart the same incident blindly.
|
|
record.pop('pending',None)
|
|
record['needs_reconciliation']=True
|
|
record['delivery']='uncertain_recovery'
|
|
if record and record.get('needs_reconciliation'):
|
|
atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})
|
|
return
|
|
if record:
|
|
if code is None and healthy:
|
|
record.setdefault('healthy_since',time.time())
|
|
if record['state']=='firing' and not record.get('pending') and time.time()-record['healthy_since']>=30:
|
|
record['pending']='resolved'
|
|
else:record.pop('healthy_since',None)
|
|
if record['state']=='firing' and not record.get('pending') and not (code is None and healthy) and not record.get('escalated'):
|
|
delay=cfg.get('fault_escalation_seconds',120)
|
|
if not isinstance(delay,(int,float)) or isinstance(delay,bool) or not 60<=delay<=3600:
|
|
raise ValueError('Fault escalation interval must be between 60 and 3600 seconds')
|
|
if time.time()-record.get('first_seen',time.time())>=delay:
|
|
try:
|
|
Client(cfg['control']).call('/api/hooks/escalate',{
|
|
'source':'bridge-'+role,'event_id':record['event_id'],
|
|
'reason':'Functional health has remained unavailable beyond the bounded recovery window',
|
|
'impact':'Conversation delivery or synchronization is persistently delayed'})
|
|
record['escalated']=True
|
|
record['escalation_delivery']='accepted'
|
|
except Exception:record['escalation_delivery']='failed'
|
|
if record.get('pending'):
|
|
try:
|
|
Client(cfg['control']).call('/api/hooks',{
|
|
'event_id':record['event_id'],'source':'bridge-'+role,'severity':record.get('severity','warning'),
|
|
'status':record['pending'],'summary':{'worker':'消息转发暂时中断','mirror':'聊天同步暂时中断','notifier':'事件通知暂时中断','report_relay':'完成上报暂时中断'}.get(role,'本机桥接暂时中断'),
|
|
'impact':'网页收发或历史同步可能延迟',
|
|
'evidence':['Private executor log: '+role+'.log'],
|
|
'action':'已启用有界重连；结果未知的模型输入不重复执行',
|
|
'next_step':'持续异常时核对执行器健康与原会话；短暂自动恢复无需用户操作'})
|
|
record['state']=record.pop('pending')
|
|
record['delivery']='accepted'
|
|
if record['state']=='resolved':records.pop(role,None)
|
|
except Exception:record['delivery']='failed'
|
|
atomic_json(root/'bridge-faults.json',{'roles':records,'updated':time.time()})
|