mayge/agent-workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add event-driven reporting and durable failure recovery hooks

Browse Source
This commit is contained in:
aiagent
2026-10-19 03:13:33 -07:00
parent 579dece104
commit a4cf70d82b
18 changed files with 711 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
agent_control/server.py
+4 -1
View File
@@ -186,6 +186,8 @@ class Handler(BaseHTTPRequestHandler):
state['primary_session_id'] = self.server.primary_session_id
state['metrics'] = self.server.metrics.snapshot()
self.output(200, state)
elif parts.path == '/api/hook-notifications' and not parts.query:
self.output(200, self.server.store.hook_notifications())
elif parts.path == '/api/messages':
query = parse_qs(parts.query, strict_parsing=True)
if set(query) != {'session_id'} or len(query['session_id']) != 1:
@@ -243,7 +245,8 @@ class Handler(BaseHTTPRequestHandler):
if not isinstance(data, dict):
raise Invalid('object required')
store = self.server.store
routes = {'/api/sessions':store.session, '/api/session-requests':store.request,
routes = {'/api/hooks':store.hook, '/api/hooks/ack':store.hook_ack,
'/api/sessions':store.session, '/api/session-requests':store.request,
'/api/tasks':store.task, '/api/messages':store.send,
'/api/worker/claim':store.claim, '/api/worker/renew':store.renew, '/api/worker/complete':store.complete,
'/api/events':store.event, '/api/mirror':store.mirror, '/api/worker/start':store.start,
agent_control/store.py
+117
View File
@@ -51,6 +51,17 @@ class Store:
lease_id TEXT, lease_until REAL, response_id TEXT,
UNIQUE(session_id,idempotency_key));
CREATE INDEX IF NOT EXISTS messages_session ON messages(session_id,created);
CREATE TABLE IF NOT EXISTS hooks(
id INTEGER PRIMARY KEY AUTOINCREMENT,
source TEXT NOT NULL, event_id TEXT NOT NULL, status TEXT NOT NULL,
data TEXT NOT NULL, UNIQUE(source,event_id,status));
CREATE TABLE IF NOT EXISTS hook_notifications(
seq INTEGER PRIMARY KEY AUTOINCREMENT,
id TEXT NOT NULL UNIQUE, hook_id INTEGER NOT NULL,
target TEXT NOT NULL, data TEXT NOT NULL,
delivered_at REAL, UNIQUE(hook_id,target));
CREATE INDEX IF NOT EXISTS hook_notifications_pending
ON hook_notifications(delivered_at,seq);
CREATE TABLE IF NOT EXISTS history_links(
session_id TEXT NOT NULL, external_id TEXT NOT NULL,
source TEXT NOT NULL, data TEXT NOT NULL,
@@ -455,12 +466,118 @@ class Store:
self._get(db, 'sessions', identifier(sid))
return self._event(db, kind, body, sid)
@staticmethod
def _hook_record(data):
allowed = {'event_id','source','task_id','session_id','severity','status',
'summary','impact','evidence','action','next_step'}
required = {'event_id','source','severity','status','summary'}
if not isinstance(data, dict) or set(data) - allowed or required - set(data):
raise Invalid('invalid hook fields')
if data['severity'] not in ('info','warning','critical') or data['status'] not in ('firing','resolved'):
raise Invalid('invalid hook severity or status')
record = dict(event_id=identifier(data['event_id'], 'event_id'),
source=identifier(data['source'], 'source'),
severity=data['severity'],status=data['status'],
summary=string(data['summary'], 'summary', 1000))
for field in ('task_id','session_id'):
record[field] = identifier(data[field], field) if data.get(field) is not None else None
for field in ('impact','action','next_step'):
value = data.get(field, '')
if not isinstance(value, str) or len(value) > 2000:
raise Invalid('invalid ' + field)
record[field] = value
evidence = data.get('evidence', [])
if not isinstance(evidence, list) or len(evidence) > 8:
raise Invalid('invalid evidence')
record['evidence'] = [string(value, 'evidence item', 1000) for value in evidence]
return record
@staticmethod
def _notification(row):
return {'notification_id': row['id'], 'target': row['target'],
'hook': json.loads(row['data']),
'status': 'delivered' if row['delivered_at'] is not None else 'pending',
'authorization': False}
def hook(self, data):
# Hooks are untrusted observations, never executable commands or approvals.
record = self._hook_record(data)
with self.db() as db:
db.execute('BEGIN IMMEDIATE')
if record['status'] == 'resolved':
firing = db.execute('SELECT data FROM hooks WHERE source=? AND event_id=? AND status=?',
(record['source'],record['event_id'],'firing')).fetchone()
if not firing:
raise Conflict('resolution requires a firing event')
original = json.loads(firing['data'])
record['severity'] = original['severity']
for field in ('task_id','session_id'):
if record[field] is not None and record[field] != original[field]:
raise Conflict('resolution object conflict')
record[field] = original[field]
for field, table in (('task_id','tasks'),('session_id','sessions')):
if record[field] is not None:
self._get(db, table, record[field])
old = db.execute('SELECT id,data FROM hooks WHERE source=? AND event_id=? AND status=?',
(record['source'],record['event_id'],record['status'])).fetchone()
if old:
saved = json.loads(old['data'])
if {k:v for k,v in saved.items() if k != 'created'} != record:
raise Conflict('hook idempotency conflict')
notifications = [self._notification(row) for row in db.execute(
'SELECT * FROM hook_notifications WHERE hook_id=? ORDER BY seq', (old['id'],))]
return {'hook':saved,'duplicate':True,'notifications':notifications}
record['created'] = time.time()
cursor = db.execute('INSERT INTO hooks(source,event_id,status,data) VALUES(?,?,?,?)',
(record['source'],record['event_id'],record['status'],json.dumps(record,ensure_ascii=False)))
hook_id = cursor.lastrowid
# Persist one event per transition, with bounded text rather than repeated evidence logs.
self._event(db, 'hook.' + record['status'],
json.dumps({k:record[k] for k in ('source','event_id','severity','status','summary')},
ensure_ascii=False), record['session_id'])
targets = ('control','human') if record['severity'] == 'critical' else ('control',)
notifications = []
for target in targets:
notification_id = secrets.token_hex(12)
db.execute('INSERT INTO hook_notifications(id,hook_id,target,data) VALUES(?,?,?,?)',
(notification_id,hook_id,target,json.dumps(record,ensure_ascii=False)))
row = db.execute('SELECT * FROM hook_notifications WHERE id=?',(notification_id,)).fetchone()
notifications.append(self._notification(row))
return {'hook':record,'duplicate':False,'notifications':notifications}
def hook_notifications(self):
with self.db() as db:
db.execute('BEGIN')
count = db.execute('SELECT count(*) FROM hook_notifications WHERE delivered_at IS NULL').fetchone()[0]
rows = db.execute('SELECT * FROM hook_notifications WHERE delivered_at IS NULL ORDER BY seq LIMIT 100').fetchall()
return {'notifications':[self._notification(row) for row in rows],
'pending_count':count,'has_more':count > len(rows)}
def hook_ack(self, data):
if not isinstance(data, dict) or set(data) != {'notification_id'}:
raise Invalid('invalid hook acknowledgment fields')
notification_id = identifier(data['notification_id'], 'notification_id')
with self.db() as db:
db.execute('BEGIN IMMEDIATE')
row = db.execute('SELECT * FROM hook_notifications WHERE id=?',(notification_id,)).fetchone()
if not row:
raise Invalid('unknown notification')
already = row['delivered_at'] is not None
if not already:
db.execute('UPDATE hook_notifications SET delivered_at=? WHERE id=?',(time.time(),notification_id))
return {'notification_id':notification_id,'status':'delivered','duplicate':already}
def state(self):
with self.db() as db:
db.execute('BEGIN')
result = {table: [json.loads(row['data']) for row in db.execute('SELECT data FROM ' + table + ' ORDER BY id')]
for table in ('sessions','tasks','requests')}
result['events'] = [dict(json.loads(row['data']),id=row['id']) for row in db.execute('SELECT * FROM events ORDER BY id DESC LIMIT 100')]
result['hooks'] = {
'pending_notifications': db.execute('SELECT count(*) FROM hook_notifications WHERE delivered_at IS NULL').fetchone()[0],
'active_firing': db.execute("SELECT count(*) FROM hooks f WHERE f.status='firing' AND NOT EXISTS (SELECT 1 FROM hooks r WHERE r.source=f.source AND r.event_id=f.event_id AND r.status='resolved')").fetchone()[0],
'recent': [json.loads(row['data']) for row in db.execute('SELECT data FROM hooks ORDER BY id DESC LIMIT 20')],
}
result['summary'] = {row['status']: row['n'] for row in db.execute('SELECT json_extract(data,\'$.status\') status,count(*) n FROM tasks GROUP BY status')}
now = time.time()
for session in result['sessions']: