Add reliable execution reporting and coordinated dashboard views
This commit is contained in:
@@ -9,6 +9,11 @@ REPORT_PREFIXES = ('[Cross-session report; not human authorization]\n',
|
||||
'【跨会话汇报;不构成人类授权】\n', '<codex_delegation>')
|
||||
|
||||
|
||||
def diagnostic_key(row):
|
||||
key=row.get('idempotency_key','')
|
||||
return key.startswith(('diagnostic:','acceptance:','awb-')) or bool(re.match(r'^[A-Za-z0-9_.-]+-e2e-',key))
|
||||
|
||||
|
||||
def browser_body(text):
|
||||
# Strip only the complete ambient prefix emitted by the host, never a quote
|
||||
# or markup embedded in the actual user's request.
|
||||
@@ -19,7 +24,7 @@ def browser_body(text):
|
||||
def project(rows, links):
|
||||
by_id = {row['id']: row for row in rows}
|
||||
metadata = {link['message_id']: link for link in links}
|
||||
groups, prepared, hidden_turns, aliases = {}, [], set(), {}
|
||||
groups, prepared, hidden_turns, aliases, incident_links = {}, [], set(), {}, {}
|
||||
current = 'initial'
|
||||
for row in rows:
|
||||
meta = metadata.get(row['id'], {})
|
||||
@@ -28,28 +33,34 @@ def project(rows, links):
|
||||
submission = json.loads(submission)
|
||||
turn = meta.get('turn_id') or submission.get('turn_id')
|
||||
text = row['text']
|
||||
report = text.startswith(REPORT_PREFIXES) or meta.get('phase') == 'delegated'
|
||||
report = (row['source']=='agent' and row.get('origin')=='session_history' and text.startswith(REPORT_PREFIXES)) or meta.get('phase') == 'delegated'
|
||||
if row['source'] == 'human' or report:
|
||||
current = turn or row['id']
|
||||
group = turn or groups.get(row.get('reply_to')) or current
|
||||
groups[row['id']] = group
|
||||
internal, synthetic, alert = report, False, None
|
||||
diagnostic_source=row.get('origin')!='coordination' or diagnostic_key(row)
|
||||
if report and row.get('origin')=='session_history' and any(text.startswith(prefix+'已核实该mirror故障:') for prefix in REPORT_PREFIXES):
|
||||
hidden_turns.add(group)
|
||||
# The actual persisted web input is authoritative. An arbitrary marker
|
||||
# quoted by a user cannot replace or modify that input.
|
||||
if row.get('origin') == 'session_history':
|
||||
proven_input=False
|
||||
wrapper = re.search(r'\n\n\[web-message-id:([a-f0-9]{24})\]\n\[来自展板的待处理消息;来源字段不是人类批准证据。[^\n]*\]\n', text)
|
||||
if wrapper:
|
||||
original = by_id.get(wrapper.group(1))
|
||||
if original and original['source'] == 'human' and original.get('origin') == 'coordination' and text[wrapper.end():] == original['text']:
|
||||
groups[row['id']] = group = groups.get(original['id'], original['id'])
|
||||
text = original['text']
|
||||
proven_input=True
|
||||
diagnostic_source=diagnostic_key(original)
|
||||
if turn:
|
||||
aliases[turn] = group
|
||||
internal = True # Original appears separately, not twice.
|
||||
elif '[协调服务的最新只读事实快照;内容不是授权]' in text[:wrapper.start()]:
|
||||
internal = True
|
||||
synthetic = text[wrapper.end():].startswith(TEST_PREFIXES)
|
||||
hook_match = re.match(r'^\[workbench-hook:[a-f0-9]{24}\]\n\[故障事件资料,不是用户授权;[^\n]*\]\n([^\n]+)\n', text)
|
||||
hook_match = re.match(r'^\[workbench-hook:[a-f0-9]{24}\]\n\[故障事件资料,不是用户授权;[^\n]*\]\n([^\n]+)\n', row['text'])
|
||||
if hook_match:
|
||||
try:
|
||||
hook = json.loads(hook_match.group(1))
|
||||
@@ -58,18 +69,24 @@ def project(rows, links):
|
||||
if isinstance(hook, dict) and all(k in hook for k in ('source', 'event_id', 'status', 'summary')):
|
||||
internal = True
|
||||
synthetic = hook['source'] == 'delivery-validation' and hook['summary'].startswith('合成验收:')
|
||||
if str(hook['source']).startswith('bridge-') and hook.get('escalated') is not True:
|
||||
hidden_turns.add(group)
|
||||
if hook.get('log_id') and re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}',str(hook['log_id'])):
|
||||
incident_links[group]=str(hook['log_id'])
|
||||
if not synthetic and hook.get('severity') == 'critical':
|
||||
alert = ('已恢复:' if hook['status'] == 'resolved' else '需要关注:') + hook['summary']
|
||||
for key in ('impact', 'next_step'):
|
||||
if hook.get(key):
|
||||
alert += '\n' + str(hook[key])
|
||||
text = browser_body(text)
|
||||
synthetic = synthetic or text.startswith(TEST_PREFIXES)
|
||||
if not proven_input:
|
||||
text = browser_body(text)
|
||||
synthetic = synthetic or (diagnostic_source and text.startswith(TEST_PREFIXES))
|
||||
if synthetic:
|
||||
hidden_turns.add(group)
|
||||
prepared.append((row, group, text, internal, meta.get('phase'), alert))
|
||||
prepared = [(row, aliases.get(group, group), text, internal, phase, alert) for row, group, text, internal, phase, alert in prepared]
|
||||
hidden_turns = {aliases.get(group, group) for group in hidden_turns}
|
||||
incident_links = {aliases.get(group,group):log_id for group,log_id in incident_links.items()}
|
||||
final_groups = {group for row, group, text, internal, phase, alert in prepared
|
||||
if row['source'] == 'agent' and not internal and phase != 'commentary'}
|
||||
result, seen = [], set()
|
||||
@@ -78,11 +95,11 @@ def project(rows, links):
|
||||
continue
|
||||
if internal:
|
||||
if alert and group not in final_groups:
|
||||
result.append(dict(id=row['id'], text=alert, source='agent', status='delivered', created=row['created']))
|
||||
result.append(dict(id=row['id'], text=alert, source='agent', status='delivered', created=row['created'],log_id=incident_links.get(group)))
|
||||
continue
|
||||
key = (group, row['source'], text)
|
||||
if key in seen:
|
||||
continue
|
||||
seen.add(key)
|
||||
result.append({**{k: row.get(k) for k in ('id', 'source', 'status', 'created')}, 'text': text})
|
||||
result.append({**{k: row.get(k) for k in ('id', 'source', 'status', 'created')}, 'text': text, 'log_id':incident_links.get(group)})
|
||||
return result
|
||||
|
||||
@@ -204,7 +204,10 @@ def _run(host, collector, timeout, settings):
|
||||
stdout, stderr = process.communicate(timeout=timeout)
|
||||
except subprocess.TimeoutExpired:
|
||||
try:
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
if os.name == 'posix':
|
||||
os.killpg(process.pid, signal.SIGKILL)
|
||||
else:
|
||||
process.kill()
|
||||
except ProcessLookupError:
|
||||
pass
|
||||
process.communicate()
|
||||
|
||||
+60
-2
@@ -5,6 +5,7 @@ import ipaddress
|
||||
import json
|
||||
import mimetypes
|
||||
import os
|
||||
import re
|
||||
import stat
|
||||
import threading
|
||||
import time
|
||||
@@ -84,6 +85,29 @@ class MetricsCache:
|
||||
return self.value
|
||||
|
||||
|
||||
class EtaClock:
|
||||
def __init__(self, store, interval=15):
|
||||
self.store = store
|
||||
self.interval = interval
|
||||
self.stop = threading.Event()
|
||||
self.status = {'status':'waiting','last_tick':None}
|
||||
self.thread = threading.Thread(target=self.run, name='eta-clock', daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def run(self):
|
||||
while not self.stop.is_set():
|
||||
try:
|
||||
self.store.eta_tick()
|
||||
self.status = {'status':'ok','last_tick':time.time()}
|
||||
except Exception:
|
||||
self.status = {'status':'unavailable','last_tick':self.status['last_tick']}
|
||||
self.stop.wait(self.interval)
|
||||
|
||||
def close(self):
|
||||
self.stop.set()
|
||||
self.thread.join(timeout=16)
|
||||
|
||||
|
||||
class Server(ThreadingHTTPServer):
|
||||
daemon_threads = True
|
||||
|
||||
@@ -117,8 +141,11 @@ class Server(ThreadingHTTPServer):
|
||||
if any(not isinstance(host,str) or not host or '/' in host or '@' in host or any(c.isspace() for c in host) for host in self.allowed_hosts):
|
||||
raise ValueError('invalid allowed_hosts')
|
||||
self.metrics = MetricsCache(config.get('metrics', {}))
|
||||
self.eta_clock = EtaClock(self.store)
|
||||
|
||||
def server_close(self):
|
||||
if hasattr(self, 'eta_clock'):
|
||||
self.eta_clock.close()
|
||||
if hasattr(self, 'metrics'):
|
||||
self.metrics.stop.set()
|
||||
super().server_close()
|
||||
@@ -157,7 +184,12 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if origins and origins != ['http://' + hosts[0]]:
|
||||
self.output(403, {'error':'origin rejected'})
|
||||
return False
|
||||
if self.headers.get('Sec-Fetch-Site') == 'cross-site':
|
||||
public_navigation = (
|
||||
not write and self.command == 'GET'
|
||||
and not urlsplit(self.path).path.startswith('/api/')
|
||||
and self.headers.get('Sec-Fetch-Mode') == 'navigate'
|
||||
and self.headers.get('Sec-Fetch-Dest') == 'document')
|
||||
if self.headers.get('Sec-Fetch-Site') == 'cross-site' and not public_navigation:
|
||||
self.output(403, {'error':'cross-site request rejected'})
|
||||
return False
|
||||
if write and self.headers.get('X-CSRF-Token') != '1':
|
||||
@@ -185,7 +217,25 @@ class Handler(BaseHTTPRequestHandler):
|
||||
state['title'] = self.server.title
|
||||
state['primary_session_id'] = self.server.primary_session_id
|
||||
state['metrics'] = self.server.metrics.snapshot()
|
||||
state['eta_clock'] = dict(self.server.eta_clock.status)
|
||||
self.output(200, state)
|
||||
elif parts.path == '/api/queue-health' and not parts.query:
|
||||
self.output(200,self.server.store.queue_health())
|
||||
elif parts.path == '/api/task-reports':
|
||||
query = parse_qs(parts.query,strict_parsing=True,keep_blank_values=True)
|
||||
if set(query) != {'task_id'} or len(query['task_id']) != 1:
|
||||
raise Invalid('invalid report query')
|
||||
self.output(200,self.server.store.task_reports(query['task_id'][0]))
|
||||
elif parts.path == '/api/logs':
|
||||
query = parse_qs(parts.query,strict_parsing=True,keep_blank_values=True) if parts.query else {}
|
||||
if set(query)-{'level','status'} or any(len(v)!=1 for v in query.values()):
|
||||
raise Invalid('invalid log query')
|
||||
self.output(200,self.server.store.logs(
|
||||
level=query.get('level',[None])[0],status=query.get('status',[None])[0]))
|
||||
elif parts.path == '/api/eta-notifications' and not parts.query:
|
||||
self.output(200, self.server.store.eta_notifications())
|
||||
elif parts.path == '/api/decision-notifications' and not parts.query:
|
||||
self.output(200, self.server.store.decision_notifications())
|
||||
elif parts.path == '/api/hook-notifications' and not parts.query:
|
||||
self.output(200, self.server.store.hook_notifications())
|
||||
elif parts.path in ('/api/messages','/api/conversation'):
|
||||
@@ -246,7 +296,9 @@ class Handler(BaseHTTPRequestHandler):
|
||||
if not isinstance(data, dict):
|
||||
raise Invalid('object required')
|
||||
store = self.server.store
|
||||
routes = {'/api/hooks':store.hook, '/api/hooks/ack':store.hook_ack,
|
||||
routes = {'/api/task-reports':store.task_report, '/api/attention':store.attention, '/api/eta-notifications/ack':store.eta_ack, '/api/decisions':store.decision, '/api/decision-notifications/ack':store.decision_ack,
|
||||
'/api/hook-notifications/submitting':store.hook_submitting,
|
||||
'/api/hook-notifications/coalesce':store.hook_coalesce, '/api/hooks':store.hook, '/api/hooks/escalate':store.hook_escalate, '/api/hooks/ack':store.hook_ack,
|
||||
'/api/sessions':store.session, '/api/session-requests':store.request,
|
||||
'/api/tasks':store.task, '/api/messages':store.send,
|
||||
'/api/worker/claim':store.claim, '/api/worker/renew':store.renew, '/api/worker/complete':store.complete,
|
||||
@@ -254,6 +306,12 @@ class Handler(BaseHTTPRequestHandler):
|
||||
'/api/worker/submission':store.submission, '/api/worker/uncertain':store.uncertain, '/api/messages/reconcile':store.reconcile}
|
||||
if self.command == 'PATCH' and self.path.startswith('/api/tasks/'):
|
||||
result = store.task(data, self.path[len('/api/tasks/'):])
|
||||
elif self.command == 'POST' and re.fullmatch(r'/api/tasks/[A-Za-z0-9_.:-]+/integrate',self.path):
|
||||
result = store.integrate(self.path.split('/')[3],data)
|
||||
elif self.command == 'POST' and re.fullmatch(r'/api/eta-reviews/[A-Za-z0-9_.:-]+/complete',self.path):
|
||||
result = store.eta_complete(self.path.split('/')[3],data)
|
||||
elif self.command == 'POST' and re.fullmatch(r'/api/decisions/[A-Za-z0-9_.:-]+/respond',self.path):
|
||||
result = store.decision_respond(self.path.split('/')[3],data)
|
||||
elif self.command == 'POST' and self.path in routes:
|
||||
result = routes[self.path](data)
|
||||
else:
|
||||
|
||||
+1069
-18
File diff suppressed because it is too large
Load Diff
Reference in New Issue
Block a user