mayge/agent-workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add reliable execution reporting and coordinated dashboard views

Browse Source
This commit is contained in:
aiagent
2026-10-19 04:29:33 -07:00
parent 547f62fa53
commit c2dfe311f7
41 changed files with 3867 additions and 107 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/faults.py
+21 -8
View File
@@ -11,7 +11,7 @@ def report_process(cfg,root,role,process,records,started_at=None):
record=records.get(role)
code=process.poll()
if code is not None and code!=0 and not record:
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing'}
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing','first_seen':time.time(),'severity':'warning'}
records[role]=record
health_path=Path(cfg.get('notification_state_file',str(root/'notifications.json'))) if role=='notifier' else root/(role+'-health.json')
try:
@@ -19,8 +19,8 @@ def report_process(cfg,root,role,process,records,started_at=None):
health=data.get('health',data)
healthy=health.get('status')=='ok' and time.time()-health.get('updated',0)<60
except (OSError,ValueError):healthy=False
if code is None and role=='notifier' and not healthy and not record and started_at is not None and time.time()-started_at>30:
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing'}
if code is None and role in ('notifier','report_relay') and not healthy and not record and started_at is not None and time.time()-started_at>30:
record={'event_id':str(uuid.uuid4()),'state':'firing','pending':'firing','first_seen':time.time(),'severity':'warning'}
records[role]=record
if record and record.get('pending')=='resolved' and not (code is None and healthy):
# A failed transport may already have accepted recovery. Do not announce
@@ -37,15 +37,28 @@ def report_process(cfg,root,role,process,records,started_at=None):
if record['state']=='firing' and not record.get('pending') and time.time()-record['healthy_since']>=30:
record['pending']='resolved'
else:record.pop('healthy_since',None)
if record['state']=='firing' and not record.get('pending') and not (code is None and healthy) and not record.get('escalated'):
delay=cfg.get('fault_escalation_seconds',120)
if not isinstance(delay,(int,float)) or isinstance(delay,bool) or not 60<=delay<=3600:
raise ValueError('Fault escalation interval must be between 60 and 3600 seconds')
if time.time()-record.get('first_seen',time.time())>=delay:
try:
Client(cfg['control']).call('/api/hooks/escalate',{
'source':'bridge-'+role,'event_id':record['event_id'],
'reason':'Functional health has remained unavailable beyond the bounded recovery window',
'impact':'Conversation delivery or synchronization is persistently delayed'})
record['escalated']=True
record['escalation_delivery']='accepted'
except Exception:record['escalation_delivery']='failed'
if record.get('pending'):
try:
Client(cfg['control']).call('/api/hooks',{
'event_id':record['event_id'],'source':'bridge-'+role,'severity':'critical',
'status':record['pending'],'summary':'Local bridge '+role+' process or functional health failed',
'impact':'Conversation delivery or synchronization may be delayed',
'event_id':record['event_id'],'source':'bridge-'+role,'severity':record.get('severity','warning'),
'status':record['pending'],'summary':{'worker':'消息转发暂时中断','mirror':'聊天同步暂时中断','notifier':'事件通知暂时中断','report_relay':'完成上报暂时中断'}.get(role,'本机桥接暂时中断'),
'impact':'网页收发或历史同步可能延迟',
'evidence':['Private executor log: '+role+'.log'],
'action':'Bounded supervised restart; unknown model executions are never replayed',
'next_step':'Inspect private executor health and original thread before repair'})
'action':'已启用有界重连；结果未知的模型输入不重复执行',
'next_step':'持续异常时核对执行器健康与原会话；短暂自动恢复无需用户操作'})
record['state']=record.pop('pending')
record['delivery']='accepted'
if record['state']=='resolved':records.pop(role,None)