mayge/agent-workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Add event-driven reporting and durable failure recovery hooks

Browse Source
This commit is contained in:
aiagent
2026-10-19 03:13:33 -07:00
parent 579dece104
commit a4cf70d82b
18 changed files with 711 additions and 8 deletions
Download Patch File Download Diff File Expand all files Collapse all files
tests/test_faults.py
+49
View File
@@ -0,0 +1,49 @@
from pathlib import Path
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import faults
class FaultTests(unittest.TestCase):
def test_repeat_exit_and_stable_recovery_report_once(self):
with tempfile.TemporaryDirectory() as root:
client=SimpleNamespace(call=lambda *args:None)
records={}
with patch.object(faults,'Client',return_value=client),patch.object(client,'call') as call,patch.object(faults.time,'time',return_value=100):
for _ in range(3):faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:2),records)
self.assertEqual(call.call_count,1)
incident=records['mirror']['event_id']
(Path(root)/'mirror-health.json').write_text('{"status":"ok","updated":100}')
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:None),records)
with patch.object(faults,'Client',return_value=client),patch.object(client,'call') as call,patch.object(faults.time,'time',return_value=131):
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:None),records)
self.assertEqual(call.call_args.args[1]['event_id'],incident)
self.assertEqual(call.call_args.args[1]['status'],'resolved')
self.assertEqual(records,{})
def test_recovery_delivery_failure_then_exit_never_sends_recovery(self):
with tempfile.TemporaryDirectory() as root:
c=SimpleNamespace(call=lambda *args:None)
records={'mirror':{'event_id':'incident','state':'firing','pending':'resolved','delivery':'failed'}}
with patch.object(faults,'Client',return_value=c),patch.object(c,'call') as call:
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:2),records)
call.assert_not_called()
self.assertTrue(records['mirror']['needs_reconciliation'])
self.assertEqual(records['mirror']['delivery'],'uncertain_recovery')
def test_hook_failure_remains_persistently_visible(self):
with tempfile.TemporaryDirectory() as root:
c=SimpleNamespace(call=lambda *args:None)
records={}
with patch.object(faults,'Client',return_value=c),patch.object(c,'call',side_effect=RuntimeError('offline')):
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:2),records)
self.assertEqual(records['mirror']['delivery'],'failed')
self.assertEqual(records['mirror']['pending'],'firing')
self.assertTrue((Path(root)/'bridge-faults.json').exists())
if __name__=='__main__':unittest.main()
tests/test_hook_cli.py
+22
View File
@@ -0,0 +1,22 @@
from pathlib import Path
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import hooks
class HookCliTests(unittest.TestCase):
def test_first_success_does_not_emit_nonexistent_recovery(self):
with tempfile.TemporaryDirectory() as root:
config=Path(root)/'private.json';config.write_text('{}')
args=['hooks.py','--config',str(config),'--source','check','--event-id','first','--summary','check','--','program']
c=SimpleNamespace(call=lambda *args:None)
with patch.object(sys,'argv',args),patch.object(hooks,'Client',return_value=c),patch.object(c,'call') as call,patch.object(hooks.subprocess,'run',return_value=SimpleNamespace(returncode=0)):
self.assertEqual(hooks.main(),0)
call.assert_not_called()
if __name__=='__main__':unittest.main()
tests/test_hooks.py
+188
View File
@@ -0,0 +1,188 @@
"""Synthetic hook transitions, durable outbox, and HTTP security regressions."""
import concurrent.futures
import http.client
import json
from pathlib import Path
import sys
import tempfile
import threading
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from agent_control.store import Store, Invalid, Conflict
from agent_control.server import Server
def sample(**overrides):
data = dict(event_id='failure-1', source='synthetic-test', severity='critical',
status='firing', summary='Synthetic worker failure', impact='Task paused',
evidence=['synthetic check result'], action='No action taken', next_step='Inspect task')
data.update(overrides)
return data
class HookStoreTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.path = Path(self.tmp.name) / 'test.sqlite3'
self.store = Store(self.path)
def tearDown(self):
self.tmp.cleanup()
def test_persistent_dedup_ack_and_no_queue_or_execution(self):
with patch('subprocess.run', side_effect=AssertionError('hook must not execute')):
first = self.store.hook(sample(action='shell text is data only'))
again = Store(self.path).hook(sample(action='shell text is data only'))
self.assertFalse(first['duplicate'])
self.assertTrue(again['duplicate'])
self.assertEqual(first['notifications'], again['notifications'])
self.assertEqual({x['target'] for x in first['notifications']}, {'control', 'human'})
self.assertTrue(all(x['authorization'] is False for x in first['notifications']))
self.assertEqual(len(self.store.state()['events']), 1)
self.assertEqual(self.store.state()['hooks']['pending_notifications'], 2)
self.assertEqual(self.store.state()['hooks']['active_firing'], 1)
nid = first['notifications'][0]['notification_id']
self.assertFalse(self.store.hook_ack({'notification_id':nid})['duplicate'])
self.assertTrue(Store(self.path).hook_ack({'notification_id':nid})['duplicate'])
self.assertEqual(self.store.hook_notifications()['pending_count'], 1)
self.assertTrue(self.store.hook(sample(action='shell text is data only'))['duplicate'])
self.assertEqual(self.store.hook_notifications()['pending_count'], 1)
self.assertEqual(len(self.store.state()['events']), 1)
self.assertEqual(self.store.state()['sessions'], [])
def test_info_warning_and_critical_routes(self):
for severity, count in [('info',1), ('warning',1), ('critical',2)]:
result = self.store.hook(sample(event_id=severity, severity=severity))
self.assertEqual(len(result['notifications']), count)
self.assertEqual(result['notifications'][0]['target'], 'control')
self.assertEqual(self.store.hook_notifications()['pending_count'], 4)
def test_resolution_requires_firing_and_inherits_severity_and_objects(self):
with self.assertRaises(Conflict): self.store.hook(sample(status='resolved'))
self.store.session(dict(id='session-a', name='Synthetic session'))
self.store.task(dict(id='task-a', name='Synthetic task', session_id='session-a'))
self.store.hook(sample(task_id='task-a', session_id='session-a'))
resolved = self.store.hook(sample(status='resolved', severity='info', summary='Recovered'))
self.assertEqual(resolved['hook']['severity'], 'critical')
self.assertEqual(resolved['hook']['task_id'], 'task-a')
self.assertEqual(resolved['hook']['session_id'], 'session-a')
self.assertEqual(len(resolved['notifications']), 2)
self.assertTrue(self.store.hook(sample(status='resolved', severity='warning', summary='Recovered'))['duplicate'])
self.assertEqual(self.store.state()['hooks']['active_firing'], 0)
self.assertEqual(sum(e['kind'].startswith('hook.') for e in self.store.state()['events']), 2)
self.assertTrue(self.store.hook(sample(task_id='task-a', session_id='session-a'))['duplicate'])
self.assertEqual(self.store.hook_notifications()['pending_count'], 4)
# Recurrence uses a new event ID and creates a new lifecycle.
self.store.hook(sample(event_id='failure-2'))
self.assertEqual(self.store.state()['hooks']['active_firing'], 1)
def test_conflicting_duplicates_and_resolution_objects_do_not_mutate(self):
self.store.hook(sample())
before = self.store.hook_notifications()
for update in ({'summary':'different'}, {'severity':'warning'}, {'evidence':['different']}):
with self.assertRaises(Conflict): self.store.hook(sample(**update))
with self.assertRaises(Conflict): self.store.hook(sample(status='resolved',session_id='other'))
self.assertEqual(self.store.hook_notifications(), before)
self.assertEqual(len(self.store.state()['events']), 1)
def test_schema_rejects_unknown_types_lengths_and_unregistered_references(self):
invalid = [sample(shell='anything'), sample(severity='urgent'), sample(status='done'),
sample(summary=''), sample(summary='x'*1001), sample(impact='x'*2001),
sample(source='bad source'), sample(event_id=[]), sample(evidence='not an array'),
sample(evidence=['x']*9), sample(evidence=['x'*1001]), sample(evidence=[{}]),
sample(task_id='missing'), sample(session_id='missing'), sample(action=None),
{k:v for k,v in sample().items() if k!='summary'}]
for payload in invalid:
with self.subTest(payload=payload):
with self.assertRaises(Invalid): self.store.hook(payload)
self.assertEqual(self.store.state()['hooks']['recent'], [])
self.assertEqual(self.store.hook_notifications()['pending_count'], 0)
for payload in ({}, {'notification_id':'missing'}, {'notification_id':'id','shell':'x'}):
with self.assertRaises(Invalid): self.store.hook_ack(payload)
def test_atomic_concurrent_duplicates_and_ack(self):
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: self.store.hook(sample()), range(16)))
self.assertEqual(sum(not r['duplicate'] for r in results), 1)
self.assertEqual(self.store.hook_notifications()['pending_count'], 2)
nid = results[0]['notifications'][0]['notification_id']
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
results = list(pool.map(lambda _: self.store.hook_ack({'notification_id':nid}), range(16)))
self.assertEqual(sum(not r['duplicate'] for r in results), 1)
self.assertEqual(len(self.store.state()['events']), 1)
def test_bounded_pending_and_recent_with_source_isolation(self):
for index in range(105):
self.store.hook(sample(event_id='event-'+str(index),severity='info'))
result = self.store.hook_notifications()
self.assertEqual(len(result['notifications']), 100)
self.assertEqual(result['pending_count'], 105)
self.assertTrue(result['has_more'])
self.assertEqual(len(self.store.state()['hooks']['recent']), 20)
self.assertEqual(result['notifications'][0]['hook']['event_id'], 'event-0')
self.store.hook(sample(event_id='event-0',source='different-source',severity='info'))
self.assertEqual(self.store.hook_notifications()['pending_count'], 106)
class HookHTTPTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
root = Path(self.tmp.name)
root.chmod(0o700)
token = root / 'token'
self.secret = 'h'*48
token.write_text(self.secret)
token.chmod(0o600)
self.metrics = patch('agent_control.server.MetricsCache')
mock = self.metrics.start()
mock.return_value.snapshot.return_value = {'status':'synthetic'}
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
self.thread.start()
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join()
self.metrics.stop()
self.tmp.cleanup()
def request(self,path,method='GET',payload=None,headers=None):
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
merged.update(headers or {})
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
response = conn.getresponse()
status, body = response.status,json.loads(response.read())
conn.close()
return status, body
def test_routes_auth_origin_csrf_validation_and_state(self):
self.assertEqual(self.request('/api/hooks','POST',sample(),{'Authorization':''})[0],401)
self.assertEqual(self.request('/api/hooks','POST',sample(),{'X-CSRF-Token':''})[0],403)
self.assertEqual(self.request('/api/hooks','POST',sample(),{'Origin':'https://untrusted.example'})[0],403)
self.assertEqual(self.request('/api/hooks','POST',sample(shell='x'))[0],400)
self.assertEqual(self.request('/api/hooks','POST',sample(status='resolved'))[0],409)
self.assertEqual(self.request('/api/hook-notifications',headers={'Authorization':''})[0],401)
status, first = self.request('/api/hooks','POST',sample())
self.assertEqual(status,200)
self.assertTrue(self.request('/api/hooks','POST',sample())[1]['duplicate'])
status, queue = self.request('/api/hook-notifications')
self.assertEqual(status,200)
self.assertEqual(len(queue['notifications']),2)
nid = first['notifications'][0]['notification_id']
self.assertEqual(self.request('/api/hooks/ack','POST',{'notification_id':nid},{'Authorization':''})[0],401)
self.assertEqual(self.request('/api/hooks/ack','POST',{'notification_id':nid,'target':'human'})[0],400)
self.assertEqual(self.request('/api/hooks/ack','POST',{'notification_id':nid})[1]['status'],'delivered')
self.assertTrue(self.request('/api/hooks/ack','POST',{'notification_id':nid})[1]['duplicate'])
self.assertEqual(self.request('/api/state')[1]['hooks']['pending_notifications'],1)
self.assertEqual(self.request('/api/hooks','POST',sample(status='resolved',severity='info'))[1]['hook']['severity'],'critical')
self.assertEqual(self.request('/api/state')[1]['hooks']['active_firing'],0)
self.assertEqual(self.request('/api/hook-notifications?target=human')[0],404)
if __name__ == '__main__':
unittest.main()
tests/test_notifier.py
+75
View File
@@ -0,0 +1,75 @@
import json
from pathlib import Path
import sys
import tempfile
from types import SimpleNamespace
import unittest
from unittest.mock import patch
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
import notifier
class NotifierTests(unittest.TestCase):
def fixture(self,root):
cfg={'control':{},'notification_targets':{'control':'11111111-1111-4111-8111-111111111111'},'codex':'codex','cwd':root}
n={'notification_id':'note1','target':'control','hook':{'source':'test','event_id':'incident','summary':'synthetic failure','severity':'warning','status':'firing'}}
return cfg,n,Path(root)/'state.json'
def test_ack_retry_never_resubmits_accepted_queue(self):
with tempfile.TemporaryDirectory() as root:
cfg,n,path=self.fixture(root)
class Client:
failed=False
def call(self,p,data=None):
if p.endswith('notifications'):return {'notifications':[n]}
if not self.failed:self.failed=True;raise RuntimeError('ack offline')
return {}
c=Client();state={}
result=SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['notification_targets']['control']+'.')
with patch.object(notifier.subprocess,'run',return_value=result) as run:
with self.assertRaises(RuntimeError):notifier.deliver(c,cfg,state,path)
notifier.deliver(c,cfg,state,path)
self.assertEqual(run.call_count,1)
self.assertTrue(state['deliveries']['note1']['acked'])
def test_uncertain_queue_never_blindly_replays(self):
with tempfile.TemporaryDirectory() as root:
cfg,n,path=self.fixture(root)
class Client:
def call(self,p,data=None):return {'notifications':[n]}
class Reader:
def __init__(self,*args):pass
def __enter__(self):return self
def __exit__(self,*args):pass
def history(self,thread):return {'id':thread},[]
state={}
with patch.object(notifier.subprocess,'run',return_value=SimpleNamespace(returncode=1,stdout='')) as run,patch.object(notifier,'AppServer',Reader):
with self.assertRaisesRegex(RuntimeError,'uncertain'):notifier.deliver(Client(),cfg,state,path)
with self.assertRaisesRegex(RuntimeError,'reconciliation'):notifier.deliver(Client(),cfg,state,path)
self.assertEqual(run.call_count,1)
def test_unknown_target_does_not_block_other_target(self):
with tempfile.TemporaryDirectory() as root:
cfg,n,path=self.fixture(root)
cfg['notification_targets']['human']='33333333-3333-4333-8333-333333333333'
other={**n,'notification_id':'note2','target':'human'}
class Client:
def call(self,p,data=None):return {'notifications':[n,other]} if p.endswith('notifications') else {}
results=[SimpleNamespace(returncode=1,stdout=''),SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['notification_targets']['human']+'.')]
state={}
with patch.object(notifier.subprocess,'run',side_effect=results) as run:
with self.assertRaises(RuntimeError):notifier.deliver(Client(),cfg,state,path)
self.assertEqual(run.call_count,2)
self.assertTrue(state['deliveries']['note2']['acked'])
self.assertEqual(state['deliveries']['note1']['state'],'submitting')
def test_no_events_means_no_conversation_input(self):
with tempfile.TemporaryDirectory() as root:
cfg,n,path=self.fixture(root)
c=SimpleNamespace(call=lambda *args:{'notifications':[]})
with patch.object(notifier.subprocess,'run') as run:
notifier.deliver(c,cfg,{},path)
run.assert_not_called()
if __name__=='__main__':unittest.main()