Add reliable execution reporting and coordinated dashboard views
This commit is contained in:
aiagent
@@ -0,0 +1,236 @@
|
||||
"""Persistent explicit attention; opening pages never dismisses it."""
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from agent_control.store import Store, Invalid, Conflict, ATTENTION_PAGES
|
||||
from agent_control.server import Server
|
||||
|
||||
|
||||
def item(**changes):
|
||||
data = dict(id='item-a',page='tasks',required=True,reason='Needs a human check')
|
||||
data.update(changes)
|
||||
return data
|
||||
|
||||
|
||||
def hook(**changes):
|
||||
data = dict(source='synthetic',event_id='incident',severity='warning',status='firing',
|
||||
summary='Connection unavailable')
|
||||
data.update(changes)
|
||||
return data
|
||||
|
||||
|
||||
class AttentionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.path = Path(self.tmp.name)/'attention.sqlite3'
|
||||
self.store = Store(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def history_count(self):
|
||||
with self.store.db() as db:
|
||||
return db.execute('SELECT count(*) FROM attention_history').fetchone()[0]
|
||||
|
||||
def test_empty_explicit_update_revoke_and_persist(self):
|
||||
self.assertEqual(self.store.state()['attention'],dict(items=[],pages={p:False for p in ATTENTION_PAGES}))
|
||||
with patch('agent_control.store.time.time',return_value=100):
|
||||
created = self.store.attention(item())
|
||||
self.assertFalse(created['duplicate'])
|
||||
self.assertEqual(created['href'],'/tasks')
|
||||
self.assertEqual(created['updated'],100)
|
||||
self.assertTrue(Store(self.path).state()['attention']['pages']['tasks'])
|
||||
for _ in range(3):
|
||||
self.store.state()
|
||||
self.assertEqual(self.history_count(),1)
|
||||
with patch('agent_control.store.time.time',return_value=200):
|
||||
again = self.store.attention(item())
|
||||
self.assertTrue(again['duplicate'])
|
||||
self.assertEqual(again['updated'],100)
|
||||
self.assertEqual(self.history_count(),1)
|
||||
self.store.attention(item(reason='Evidence changed',href='/logs?incident=5'))
|
||||
self.assertEqual(self.history_count(),2)
|
||||
self.store.attention(item(required=False,reason=''))
|
||||
self.assertFalse(self.store.state()['attention']['pages']['tasks'])
|
||||
self.assertEqual(self.history_count(),3)
|
||||
self.assertTrue(self.store.attention(item(required=False,reason=''))['duplicate'])
|
||||
self.assertEqual(self.history_count(),3)
|
||||
|
||||
def test_many_independent_items_any_aggregate(self):
|
||||
self.store.attention(item())
|
||||
self.store.attention(item(id='item-b'))
|
||||
self.store.attention(item(id='item-c',page='cluster'))
|
||||
self.store.attention(item(required=False,reason='Done'))
|
||||
state = self.store.state()['attention']
|
||||
self.assertEqual(len(state['items']),2)
|
||||
self.assertTrue(state['pages']['tasks'])
|
||||
self.assertTrue(state['pages']['cluster'])
|
||||
self.store.attention(item(id='item-b',required=False,reason=''))
|
||||
self.assertFalse(self.store.state()['attention']['pages']['tasks'])
|
||||
self.assertTrue(self.store.state()['attention']['pages']['cluster'])
|
||||
self.assertEqual(set(state['items'][0]),{'id','page','required','reason','href','updated'})
|
||||
|
||||
def test_subject_cannot_change_and_associations_can_be_omitted_on_withdrawal(self):
|
||||
self.store.task(dict(id='a',name='A'))
|
||||
self.store.task(dict(id='b',name='B'))
|
||||
self.store.attention(item(task_id='a'))
|
||||
for data in [item(task_id='b'),item(page='cluster'),item(task_id='b',required=False,reason='')]:
|
||||
with self.assertRaises(Conflict): self.store.attention(data)
|
||||
self.assertEqual(self.history_count(),1)
|
||||
self.store.attention(item(required=False,reason=''))
|
||||
with self.store.db() as db:
|
||||
saved=json.loads(db.execute("SELECT data FROM attention WHERE id='item-a'").fetchone()[0])
|
||||
self.assertEqual(saved['task_id'],'a')
|
||||
self.assertFalse(saved['required'])
|
||||
|
||||
def test_decision_automatic_attention_response_retires_every_associated_item(self):
|
||||
self.store.decision(dict(id='d',title='Choose a plan',question='Which plan?'))
|
||||
state = self.store.state()['attention']
|
||||
self.assertTrue(state['pages']['decisions'])
|
||||
auto=state['items'][0]
|
||||
self.assertEqual(auto['href'],'/decisions?decision=d')
|
||||
self.store.attention(item(decision_id='d'))
|
||||
self.store.decision_respond('d',dict(option_id=None,comment='Chosen',idempotency_key='reply'))
|
||||
state=self.store.state()['attention']
|
||||
self.assertEqual(state['items'],[])
|
||||
self.assertFalse(any(state['pages'].values()))
|
||||
with self.store.db() as db:
|
||||
self.assertTrue(all(not json.loads(r['data'])['required'] for r in db.execute('SELECT data FROM attention')))
|
||||
self.store.attention(item(decision_id='d',reason='Attempt to revive already answered'))
|
||||
self.assertFalse(self.store.state()['attention']['pages']['tasks'])
|
||||
|
||||
def test_manual_early_decision_withdrawal_never_recreated_by_retry_read_or_restart(self):
|
||||
definition=dict(id='d',title='Choose',question='What next?')
|
||||
self.store.decision(definition)
|
||||
auto=self.store.state()['attention']['items'][0]
|
||||
self.store.attention(dict(id=auto['id'],page='decisions',required=False,reason=''))
|
||||
count=self.history_count()
|
||||
for _ in range(3):
|
||||
self.store.decision(definition)
|
||||
self.assertFalse(Store(self.path).state()['attention']['pages']['decisions'])
|
||||
self.assertEqual(self.history_count(),count)
|
||||
|
||||
def test_hook_only_upgrade_marks_attention_and_resolution_clears_related(self):
|
||||
self.store.hook(hook())
|
||||
self.assertFalse(self.store.state()['attention']['pages']['logs'])
|
||||
escalation=dict(source='synthetic',event_id='incident',reason='Persistent outage')
|
||||
result=self.store.hook_escalate(escalation)
|
||||
state=self.store.state()['attention']
|
||||
self.assertTrue(state['pages']['logs'])
|
||||
auto=state['items'][0]
|
||||
self.assertEqual(auto['href'],'/logs?incident='+result['notifications'][0]['hook']['log_id'])
|
||||
self.store.attention(item(hook_source='synthetic',hook_event_id='incident'))
|
||||
self.store.hook(hook(status='resolved'))
|
||||
self.assertEqual(self.store.state()['attention']['items'],[])
|
||||
self.store.hook(hook(event_id='ordinary'))
|
||||
self.store.hook(hook(event_id='ordinary',status='resolved'))
|
||||
self.assertEqual(self.store.state()['attention']['items'],[])
|
||||
|
||||
def test_early_upgrade_withdrawal_is_not_recreated(self):
|
||||
self.store.hook(hook())
|
||||
escalation=dict(source='synthetic',event_id='incident',reason='Persistent outage')
|
||||
self.store.hook_escalate(escalation)
|
||||
auto=self.store.state()['attention']['items'][0]
|
||||
self.store.attention(dict(id=auto['id'],page='logs',required=False,reason='Acknowledged separately'))
|
||||
self.store.hook_escalate(escalation)
|
||||
self.assertFalse(Store(self.path).state()['attention']['pages']['logs'])
|
||||
|
||||
def test_task_completion_keeps_attention_cancellation_retires(self):
|
||||
self.store.task(dict(id='task',name='Task'))
|
||||
self.store.attention(item(task_id='task'))
|
||||
self.store.task(dict(status='completed',phase='done'),'task')
|
||||
self.assertTrue(self.store.state()['attention']['pages']['tasks'])
|
||||
self.store.task(dict(status='cancelled'),'task')
|
||||
self.assertFalse(self.store.state()['attention']['pages']['tasks'])
|
||||
self.store.attention(item(task_id='task',reason='Try to revive cancelled task'))
|
||||
self.assertFalse(self.store.state()['attention']['pages']['tasks'])
|
||||
|
||||
def test_schema_paths_and_unknown_targets(self):
|
||||
for data in [item(required=1),item(reason=''),item(reason=None),item(reason='x'*241),
|
||||
item(page='other'),item(thread_id='arbitrary'),item(html='<b>hi</b>'),
|
||||
item(task_id='missing'),item(decision_id='missing'),
|
||||
item(hook_source='synthetic'),item(hook_event_id='incident'),
|
||||
item(hook_source='missing',hook_event_id='incident'),
|
||||
item(id='attention-decision-forged')]:
|
||||
with self.subTest(data=data):
|
||||
with self.assertRaises(Invalid): self.store.attention(data)
|
||||
for href in ['https://example.invalid','//example.invalid/logs',r'/logs\evil','/other',
|
||||
'/logs/../tasks','/logs#x','/logs?next=https://example.invalid',
|
||||
'/logs?x=%3Cscript%3E','/%6cogs','/logs?x=%255Cevil','/logs\n','javascript:alert(1)']:
|
||||
with self.subTest(href=href):
|
||||
with self.assertRaises(Invalid): self.store.attention(item(href=href))
|
||||
self.assertEqual(self.history_count(),0)
|
||||
self.store.attention(item(href='/logs?incident=123&status=active'))
|
||||
self.assertEqual(self.store.state()['attention']['items'][0]['href'],'/logs?incident=123&status=active')
|
||||
|
||||
def test_concurrent_duplicate_requests_one_history_record(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results=list(pool.map(lambda _:self.store.attention(item()),range(16)))
|
||||
self.assertEqual(sum(not r['duplicate'] for r in results),1)
|
||||
self.assertEqual(self.history_count(),1)
|
||||
self.assertEqual(len(self.store.state()['attention']['items']),1)
|
||||
|
||||
def test_auto_attention_failure_rolls_back_decision_and_upgrade(self):
|
||||
with self.store.db() as db:
|
||||
db.execute("CREATE TRIGGER fail_attention BEFORE INSERT ON attention_history BEGIN SELECT RAISE(ABORT,'synthetic'); END")
|
||||
with self.assertRaises(Exception):
|
||||
self.store.decision(dict(id='d',title='Choose',question='What next?'))
|
||||
self.assertEqual(self.store.state()['decisions'],[])
|
||||
self.store.hook(hook())
|
||||
with self.assertRaises(Exception):
|
||||
self.store.hook_escalate(dict(source='synthetic',event_id='incident',reason='Persistent'))
|
||||
self.assertEqual(self.store.logs()['logs'][0]['level'],'warning')
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],1)
|
||||
|
||||
|
||||
class AttentionHTTPTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self.tmp.name)
|
||||
root.chmod(0o700)
|
||||
token = root / 'token'
|
||||
self.secret = 'h'*48
|
||||
token.write_text(self.secret)
|
||||
token.chmod(0o600)
|
||||
self.metrics = patch('agent_control.server.MetricsCache')
|
||||
mock = self.metrics.start()
|
||||
mock.return_value.snapshot.return_value = {'status':'synthetic'}
|
||||
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
self.metrics.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def request(self,path,method='GET',payload=None,headers=None):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
|
||||
merged.update(headers or {})
|
||||
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
|
||||
response = conn.getresponse()
|
||||
status, body = response.status,json.loads(response.read())
|
||||
conn.close()
|
||||
return status, body
|
||||
|
||||
|
||||
def test_auth_csrf_origin_and_reads_never_clear(self):
|
||||
for headers,status in [({'Authorization':''},401),({'X-CSRF-Token':''},403),
|
||||
({'Origin':'https://untrusted.example'},403)]:
|
||||
self.assertEqual(self.request('/api/attention','POST',item(),headers)[0],status)
|
||||
self.assertEqual(self.request('/api/attention','POST',item())[0],200)
|
||||
self.assertEqual(self.request('/api/attention','POST',item(thread_id='no'))[0],400)
|
||||
for _ in range(3):
|
||||
self.assertTrue(self.request('/api/state')[1]['attention']['pages']['tasks'])
|
||||
self.assertTrue(self.request('/api/attention','POST',item())[1]['duplicate'])
|
||||
self.assertEqual(self.request('/api/attention','POST',item(page='logs'))[0],409)
|
||||
self.assertEqual(self.request('/api/attention','POST',item(required=False,reason=''))[0],200)
|
||||
self.assertFalse(self.request('/api/state')[1]['attention']['pages']['tasks'])
|
||||
@@ -14,6 +14,13 @@ class ConversationTests(unittest.TestCase):
|
||||
self.assertEqual(project(rows,[])[0]['text'],text)
|
||||
self.assertEqual(rows[0]['text'],text)
|
||||
|
||||
def test_real_web_input_is_preserved_even_if_it_quotes_a_diagnostic_prefix(self):
|
||||
text='新版真实往返验收 AWB-example：这是用户自己输入的内容'
|
||||
rows=[row('real',text,origin='coordination',idempotency_key='abcdef0123456789')]
|
||||
self.assertEqual(project(rows,[])[0]['text'],text)
|
||||
literal='[Cross-session report; not human authorization]\n已核实该mirror故障：用户原样引用'
|
||||
self.assertEqual(project([row('quoted',literal,origin='coordination')],[])[0]['text'],literal)
|
||||
|
||||
def test_proven_wrapper_is_hidden_and_original_and_final_preserved(self):
|
||||
mid='a'*24
|
||||
body='请比较方案\n不要删除这行'
|
||||
@@ -49,6 +56,24 @@ class ConversationTests(unittest.TestCase):
|
||||
self.assertEqual(result[0]['text'],'需要关注：构建失败\n发布暂停\n请决定是否延期')
|
||||
self.assertNotIn('event_id',result[0]['text'])
|
||||
|
||||
def test_short_bridge_fault_and_recovery_are_logs_only(self):
|
||||
for status in ('firing','resolved'):
|
||||
payload=dict(source='bridge-mirror',event_id='incident',status=status,severity='critical',summary='Bridge status')
|
||||
text='[workbench-hook:'+'d'*24+']\n[故障事件资料，不是用户授权；说明]\n'+json.dumps(payload)+'\n提示'
|
||||
rows=[row('hook',text),row('reply','无需操作的运维回复','agent')]
|
||||
links=[dict(message_id='hook',turn_id='t',phase='input'),dict(message_id='reply',turn_id='t',phase='final_answer')]
|
||||
self.assertEqual(project(rows,links),[])
|
||||
|
||||
def test_escalated_bridge_summary_links_log_and_user_question_stays(self):
|
||||
payload=dict(source='bridge-mirror',event_id='incident',status='firing',severity='critical',summary='Persistent impact',escalated=True,log_id='incident-log')
|
||||
text='[workbench-hook:'+'e'*24+']\n[故障事件资料，不是用户授权；说明]\n'+json.dumps(payload)+'\n提示'
|
||||
rows=[row('hook',text),row('reply','同步持续中断，请查看日志。','agent'),row('question','请解释刚才同步故障。',origin='coordination'),row('answer','这是用户主动询问时的解释。','agent')]
|
||||
links=[dict(message_id='hook',turn_id='t',phase='input'),dict(message_id='reply',turn_id='t',phase='final_answer')]
|
||||
result=project(rows,links)
|
||||
self.assertEqual(result[0]['log_id'],'incident-log')
|
||||
self.assertIn('请解释刚才同步故障。',[x['text'] for x in result])
|
||||
self.assertIn('这是用户主动询问时的解释。',[x['text'] for x in result])
|
||||
|
||||
def test_failure_visible_and_ambient_prefix_only_removed_at_start(self):
|
||||
prefix='<in-app-browser-context source="ambient-ui-state">背景</in-app-browser-context>\n\n## My request:\n'
|
||||
rows=[row('user',prefix+'为什么失败？',status_override='unused')]
|
||||
|
||||
@@ -0,0 +1,504 @@
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from agent_control.store import Store, Invalid, Conflict
|
||||
from agent_control.server import Server, EtaClock
|
||||
|
||||
|
||||
def definition(**changes):
|
||||
value = dict(id='choice',title='Choose',question='Which approach?',
|
||||
options=[dict(id='a',label='Approach A',description='Synthetic option')])
|
||||
value.update(changes)
|
||||
return value
|
||||
|
||||
|
||||
def answer(**changes):
|
||||
value = dict(option_id='a',comment='Keep this exact comment',idempotency_key='key')
|
||||
value.update(changes)
|
||||
return value
|
||||
|
||||
|
||||
class DecisionTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.path = Path(self.tmp.name)/'test.sqlite3'
|
||||
self.store = Store(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_task_progress_independent_timestamp_and_retention(self):
|
||||
task = self.store.task(dict(id='t',name='Task',eta_basis='Estimate only'))
|
||||
self.assertEqual(task['current_progress'],'')
|
||||
self.assertIsNone(task['progress_updated'])
|
||||
with patch('agent_control.store.time.time',return_value=100):
|
||||
task = self.store.task(dict(current_progress='Validated parser'), 't')
|
||||
self.assertEqual(task['progress_updated'],100)
|
||||
with patch('agent_control.store.time.time',return_value=200):
|
||||
task = self.store.task(dict(status='blocked',decision='Choose next step'),'t')
|
||||
self.assertEqual(task['current_progress'],'Validated parser')
|
||||
self.assertEqual(task['progress_updated'],100)
|
||||
self.assertEqual(task['updated'],200)
|
||||
with patch('agent_control.store.time.time',return_value=300):
|
||||
task = self.store.task(dict(status='completed',phase='done'),'t')
|
||||
self.assertEqual(task['progress_updated'],100)
|
||||
self.assertEqual(task['current_progress'],'Validated parser')
|
||||
with patch('agent_control.store.time.time',return_value=400):
|
||||
task = self.store.task(dict(current_progress=''),'t')
|
||||
self.assertEqual(task['progress_updated'],400)
|
||||
self.assertEqual(task['current_progress'],'')
|
||||
|
||||
def test_task_progress_validation_and_legacy_defaults(self):
|
||||
self.store.task(dict(id='t',name='Task'))
|
||||
for data in [dict(current_progress=None),dict(current_progress=1),
|
||||
dict(current_progress='x'*1001),dict(progress_updated=1)]:
|
||||
with self.assertRaises(Invalid): self.store.task(data,'t')
|
||||
with self.store.db() as db:
|
||||
db.execute('INSERT INTO tasks(id,data) VALUES(?,?)',('legacy',json.dumps(dict(id='legacy',name='Legacy'))))
|
||||
task = next(t for t in self.store.state()['tasks'] if t['id']=='legacy')
|
||||
self.assertEqual(task['current_progress'],'')
|
||||
self.assertIsNone(task['progress_updated'])
|
||||
|
||||
|
||||
def eta_task(self, **changes):
|
||||
data = dict(id='eta',name='Synthetic ETA',status='running',
|
||||
eta_min_minutes=5,eta_max_minutes=10,eta_basis='Synthetic remaining checks')
|
||||
data.update(changes)
|
||||
with patch('agent_control.store.time.time',return_value=1000):
|
||||
return self.store.task(data)
|
||||
|
||||
def eta_notice(self, now=1300):
|
||||
self.store.eta_tick(now)
|
||||
return self.store.eta_notifications()['notifications'][0]
|
||||
|
||||
def test_eta_checkpoint_clock_and_persistent_dedup(self):
|
||||
task = self.eta_task()
|
||||
self.assertEqual(task['eta_revision'],1)
|
||||
self.assertEqual(task['eta_estimated_at'],1000)
|
||||
self.assertEqual(task['eta_change_reason'],task['eta_basis'])
|
||||
self.assertEqual(self.store.eta_tick(1299)['created'],0)
|
||||
self.assertEqual(self.store.eta_tick(1300)['created'],1)
|
||||
first = self.store.eta_notifications()['notifications'][0]
|
||||
self.assertEqual(first['checkpoint'],'half')
|
||||
self.assertEqual(first['target'],'control')
|
||||
self.assertEqual(first['revision'],1)
|
||||
self.assertEqual(Store(self.path).eta_tick(1300)['created'],0)
|
||||
self.assertEqual(self.store.eta_tick(1479)['created'],0)
|
||||
self.assertEqual(self.store.eta_tick(1480)['created'],1)
|
||||
self.assertEqual(self.store.eta_tick(1600)['created'],1)
|
||||
self.assertEqual(self.store.state()['tasks'][0]['eta_review_status'],'overdue')
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],3)
|
||||
|
||||
def test_eta_heartbeat_keep_and_ack_never_move_estimate(self):
|
||||
self.eta_task()
|
||||
nid = self.eta_notice()['notification_id']
|
||||
with patch('agent_control.store.time.time',return_value=1310):
|
||||
task = self.store.task(dict(eta_min_minutes=5,eta_max_minutes=10,current_progress='Running checks'),'eta')
|
||||
self.assertEqual(task['eta_estimated_at'],1000)
|
||||
self.assertEqual(task['eta_revision'],1)
|
||||
self.assertEqual(self.store.eta_notifications()['notifications'][0]['task']['current_progress'],'Running checks')
|
||||
ack = dict(notification_id=nid)
|
||||
self.assertFalse(self.store.eta_ack(ack)['duplicate'])
|
||||
self.assertTrue(self.store.eta_ack(ack)['duplicate'])
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],0)
|
||||
body = dict(result='keep',reason='Half the checks passed',current_progress='Verified outputs')
|
||||
with patch('agent_control.store.time.time',return_value=1320):
|
||||
result = self.store.eta_complete(nid,body)
|
||||
self.assertEqual(result['eta_estimated_at'],1000)
|
||||
self.assertEqual(result['eta_revision'],1)
|
||||
self.assertEqual(result['eta_last_reviewed_at'],1320)
|
||||
self.assertEqual(result['eta_review_reason'],body['reason'])
|
||||
self.assertEqual(result['progress_updated'],1320)
|
||||
self.assertEqual(Store(self.path).eta_complete(nid,body),result)
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.eta_complete(nid,dict(body,reason='Changed answer'))
|
||||
|
||||
def test_eta_revise_changes_version_and_cancels_old_notifications(self):
|
||||
self.eta_task()
|
||||
self.store.eta_tick(1480)
|
||||
notices = self.store.eta_notifications()['notifications']
|
||||
body = dict(result='revise',reason='New failing test found',eta_min_minutes=5,eta_max_minutes=20)
|
||||
with patch('agent_control.store.time.time',return_value=1481):
|
||||
task = self.store.eta_complete(notices[0]['notification_id'],body)
|
||||
self.assertEqual((task['eta_revision'],task['eta_estimated_at']),(2,1481))
|
||||
self.assertEqual(task['eta_extension_count'],1)
|
||||
self.assertEqual(task['eta_change_reason'],body['reason'])
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],0)
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.eta_complete(notices[1]['notification_id'],dict(result='keep',reason='Late response'))
|
||||
self.assertEqual(self.store.eta_tick(2081)['created'],1)
|
||||
self.assertEqual(self.store.eta_notifications()['notifications'][0]['revision'],2)
|
||||
with self.store.db() as db:
|
||||
self.assertEqual(db.execute('SELECT count(*) FROM eta_reviews WHERE cancelled_at IS NOT NULL').fetchone()[0],2)
|
||||
|
||||
def test_eta_unknown_does_not_extend(self):
|
||||
self.eta_task()
|
||||
nid = self.eta_notice()['notification_id']
|
||||
with patch('agent_control.store.time.time',return_value=1310):
|
||||
task = self.store.eta_complete(nid,dict(result='unknown',reason='Waiting on real evidence'))
|
||||
self.assertEqual(task['eta_review_status'],'unknown')
|
||||
self.assertEqual(task['eta_estimated_at'],1000)
|
||||
self.assertEqual(task['eta_revision'],1)
|
||||
self.assertEqual(task['eta_max_minutes'],10)
|
||||
|
||||
def test_eta_extension_limit_persists_and_cannot_be_bypassed_by_task_patch(self):
|
||||
task = self.eta_task()
|
||||
for index in range(3):
|
||||
checkpoint = task['eta_estimated_at'] + task['eta_max_minutes']*30
|
||||
nid = self.eta_notice(checkpoint)['notification_id']
|
||||
with patch('agent_control.store.time.time',return_value=checkpoint+1):
|
||||
task = self.store.eta_complete(nid,dict(result='revise',reason='Measured additional work',
|
||||
eta_min_minutes=5,eta_max_minutes=20+index*10))
|
||||
self.assertEqual(task['eta_extension_count'],3)
|
||||
old = dict(task)
|
||||
checkpoint = task['eta_estimated_at'] + task['eta_max_minutes']*30
|
||||
nid = self.eta_notice(checkpoint)['notification_id']
|
||||
with patch('agent_control.store.time.time',return_value=checkpoint+1):
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.eta_complete(nid,dict(result='revise',reason='Fourth extension',
|
||||
eta_min_minutes=5,eta_max_minutes=50))
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.task(dict(eta_max_minutes=50,eta_change_reason='Try task patch'),'eta')
|
||||
task = Store(self.path).state()['tasks'][0]
|
||||
self.assertEqual(task['eta_review_status'],'needs_decision')
|
||||
self.assertEqual(task['eta_revision'],old['eta_revision'])
|
||||
self.assertEqual(task['eta_estimated_at'],old['eta_estimated_at'])
|
||||
self.assertEqual(task['eta_max_minutes'],old['eta_max_minutes'])
|
||||
self.assertEqual(task['eta_extension_count'],3)
|
||||
self.store.eta_tick(checkpoint+10000)
|
||||
self.assertEqual(self.store.state()['tasks'][0]['eta_review_status'],'needs_decision')
|
||||
|
||||
def test_eta_same_range_revise_and_unknown_cannot_clear_decision_limit(self):
|
||||
self.eta_task()
|
||||
nid = self.eta_notice()['notification_id']
|
||||
with patch('agent_control.store.time.time',return_value=1400):
|
||||
task = self.store.eta_complete(nid,dict(result='revise',reason='Same measured range',
|
||||
eta_min_minutes=5,eta_max_minutes=10))
|
||||
self.assertEqual(task['eta_revision'],1)
|
||||
self.assertEqual(task['eta_estimated_at'],1000)
|
||||
self.assertEqual(task['eta_review_status'],'reviewed')
|
||||
self.store.eta_tick(1480)
|
||||
nid = self.store.eta_notifications()['notifications'][0]['notification_id']
|
||||
with self.store.db() as db:
|
||||
task['eta_review_status']='needs_decision'
|
||||
self.store._put(db,'tasks',task)
|
||||
task = self.store.eta_complete(nid,dict(result='unknown',reason='Still awaiting evidence'))
|
||||
self.assertEqual(task['eta_review_status'],'needs_decision')
|
||||
|
||||
def test_eta_invalid_changes_zero_and_managed_fields(self):
|
||||
with self.assertRaises(Invalid): self.eta_task(eta_min_minutes=0,eta_max_minutes=0)
|
||||
self.eta_task()
|
||||
for data in [dict(eta_max_minutes=11),dict(eta_max_minutes=11,eta_change_reason=''),
|
||||
dict(eta_revision=9),dict(eta_estimated_at=1),dict(eta_extension_count=0),
|
||||
dict(eta_max_minutes=True),dict(eta_max_minutes=float('inf'))]:
|
||||
with self.assertRaises(Invalid): self.store.task(data,'eta')
|
||||
self.assertEqual(self.store.state()['tasks'][0]['eta_revision'],1)
|
||||
task = self.store.task(dict(status='completed',phase='done',eta_min_minutes=0,
|
||||
eta_max_minutes=0,eta_change_reason='Checks finished'),'eta')
|
||||
self.assertEqual(task['eta_max_minutes'],0)
|
||||
self.assertEqual(self.store.eta_tick(100000)['created'],0)
|
||||
|
||||
def test_eta_legacy_and_no_estimate_do_not_invent_start(self):
|
||||
self.store.task(dict(id='none',name='No estimate'))
|
||||
with self.store.db() as db:
|
||||
db.execute('INSERT INTO tasks(id,data) VALUES(?,?)',('legacy',json.dumps(
|
||||
dict(id='legacy',name='Legacy',status='running',eta_min_minutes=5,eta_max_minutes=10,
|
||||
eta_basis='Legacy explanation'))))
|
||||
self.assertEqual(self.store.eta_tick(100000)['created'],0)
|
||||
task = next(t for t in self.store.state()['tasks'] if t['id']=='legacy')
|
||||
self.assertIsNone(task['eta_estimated_at'])
|
||||
self.assertEqual(task['eta_revision'],0)
|
||||
with patch('agent_control.store.time.time',return_value=2000):
|
||||
task = self.store.task(dict(current_progress='Reading baseline'),'legacy')
|
||||
self.assertIsNone(task['eta_estimated_at'])
|
||||
with patch('agent_control.store.time.time',return_value=2001):
|
||||
task = self.store.task(dict(eta_min_minutes=5,eta_max_minutes=10),'legacy')
|
||||
self.assertEqual(task['eta_revision'],1)
|
||||
self.assertEqual(task['eta_estimated_at'],2001)
|
||||
|
||||
def test_eta_finished_and_revised_work_does_not_notify_old_reviews(self):
|
||||
for index,status in enumerate(['completed','cancelled','failed']):
|
||||
tid = 'finished-'+str(index)
|
||||
self.eta_task(id=tid)
|
||||
notices = self.store.eta_tick(1300)
|
||||
notice = next(n for n in self.store.eta_notifications()['notifications'] if n['task']['id']==tid)
|
||||
self.store.task(dict(status=status,phase='done'),'finished-'+str(index))
|
||||
self.assertFalse(any(n['task']['id']==tid for n in self.store.eta_notifications()['notifications']))
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.eta_complete(notice['notification_id'],dict(result='keep',reason='Late result'))
|
||||
self.assertEqual(self.store.eta_tick(100000)['created'],0)
|
||||
|
||||
def test_eta_invalid_completion_does_not_mutate(self):
|
||||
self.eta_task()
|
||||
nid = self.eta_notice()['notification_id']
|
||||
for body in [dict(result='keep',reason=''),dict(result='bad',reason='Reason'),
|
||||
dict(result='keep',reason='Reason',eta_max_minutes=10),
|
||||
dict(result='revise',reason='Reason',eta_max_minutes=20),
|
||||
dict(result='revise',reason='Reason',eta_min_minutes=0,eta_max_minutes=0),
|
||||
dict(result='revise',reason='Reason',eta_min_minutes=None,eta_max_minutes=None),
|
||||
dict(result='unknown',reason='Reason',current_progress=None),
|
||||
dict(result='keep',reason='Reason',authorization=True)]:
|
||||
with self.subTest(body=body):
|
||||
with self.assertRaises(Invalid): self.store.eta_complete(nid,body)
|
||||
self.assertEqual(self.store.state()['tasks'][0]['eta_revision'],1)
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],1)
|
||||
|
||||
def test_eta_concurrent_ticks_and_completions(self):
|
||||
self.eta_task()
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
list(pool.map(lambda _:self.store.eta_tick(1300),range(12)))
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],1)
|
||||
nid = self.store.eta_notifications()['notifications'][0]['notification_id']
|
||||
body = dict(result='keep',reason='Measured progress')
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results = list(pool.map(lambda _:self.store.eta_complete(nid,body),range(12)))
|
||||
self.assertTrue(all(r==results[0] for r in results))
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],0)
|
||||
|
||||
def test_eta_background_clock_without_state_poll_and_stop(self):
|
||||
self.eta_task()
|
||||
seen = threading.Event()
|
||||
original = self.store.eta_tick
|
||||
def tick():
|
||||
original(1300)
|
||||
seen.set()
|
||||
with patch.object(self.store,'eta_tick',side_effect=tick):
|
||||
clock = EtaClock(self.store,interval=0.01)
|
||||
try:
|
||||
self.assertTrue(seen.wait(2))
|
||||
self.assertEqual(clock.interval,0.01)
|
||||
self.assertEqual(self.store.eta_notifications()['pending_count'],1)
|
||||
finally:
|
||||
clock.close()
|
||||
self.assertFalse(clock.thread.is_alive())
|
||||
|
||||
def test_empty_and_legacy_task(self):
|
||||
self.assertEqual(self.store.state()['decisions'],[])
|
||||
self.assertEqual(self.store.decision_notifications(),dict(notifications=[],pending_count=0))
|
||||
self.store.session(dict(id='s',name='Synthetic'))
|
||||
self.store.task(dict(id='t',name='Task',session_id='s',decision='Legacy question'))
|
||||
self.store.decision(definition(task_id='t'))
|
||||
self.assertEqual(self.store.state()['tasks'][0]['decision'],'Legacy question')
|
||||
|
||||
def test_normalized_creation_and_immutable_definition(self):
|
||||
first = self.store.decision(definition())
|
||||
self.assertEqual(first['status'],'open')
|
||||
self.assertIsNone(first['response'])
|
||||
explicit = definition(impact='',priority='normal',details='',recommendation='',
|
||||
deadline=None,requires_authorization=False,task_id=None)
|
||||
self.assertEqual(first,self.store.decision(explicit))
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.decision(definition(question='Changed'))
|
||||
self.assertEqual(self.store.decision_notifications()['pending_count'],0)
|
||||
|
||||
def test_persistence_response_dedup_and_ack(self):
|
||||
self.store.decision(definition())
|
||||
with patch('subprocess.run',side_effect=AssertionError('must not execute')):
|
||||
first = self.store.decision_respond('choice',answer())
|
||||
reopened = Store(self.path)
|
||||
self.assertEqual(first,reopened.decision_respond('choice',answer()))
|
||||
self.assertEqual(first,reopened.decision(definition()))
|
||||
self.assertEqual(first['status'],'decided')
|
||||
self.assertFalse(first['response']['authorization'])
|
||||
self.assertEqual(first['response']['comment'],answer()['comment'])
|
||||
for changes in [dict(comment='changed'),dict(idempotency_key='other'),dict(option_id=None)]:
|
||||
with self.assertRaises(Conflict):
|
||||
reopened.decision_respond('choice',answer(**changes))
|
||||
queue = reopened.decision_notifications()
|
||||
self.assertEqual(queue['pending_count'],1)
|
||||
notice = queue['notifications'][0]
|
||||
self.assertEqual(set(notice),{'notification_id','target','decision'})
|
||||
self.assertEqual(notice['target'],'control')
|
||||
self.assertEqual(notice['decision'],first)
|
||||
ack = dict(notification_id=notice['notification_id'])
|
||||
self.assertFalse(reopened.decision_ack(ack)['duplicate'])
|
||||
self.assertTrue(Store(self.path).decision_ack(ack)['duplicate'])
|
||||
self.assertEqual(reopened.decision_notifications()['pending_count'],0)
|
||||
reopened.decision_respond('choice',answer())
|
||||
self.assertEqual(reopened.decision_notifications()['pending_count'],0)
|
||||
|
||||
def test_authorization_is_only_pending_verification(self):
|
||||
self.store.decision(definition(requires_authorization=True,options=[]))
|
||||
value = self.store.decision_respond('choice',answer(option_id=None,comment='Approve sudo'))
|
||||
self.assertEqual(value['status'],'pending_verification')
|
||||
self.assertIs(value['response']['authorization'],False)
|
||||
|
||||
def test_invalid_definitions(self):
|
||||
for changes in [dict(shell='x'),dict(title=''),dict(question=None),dict(impact=None),
|
||||
dict(priority=[]),dict(priority='critical'),dict(requires_authorization=1),
|
||||
dict(options={}),dict(options=[dict(id='a',label='a')]),
|
||||
dict(options=[dict(id='a',label='a',description='')]*2),
|
||||
dict(deadline='2026-02-30T01:00:00Z'),dict(deadline='tomorrow'),
|
||||
dict(task_id='missing'),dict(details='x'*8001)]:
|
||||
with self.subTest(changes=changes):
|
||||
with self.assertRaises(Invalid): self.store.decision(definition(**changes))
|
||||
self.assertEqual(self.store.state()['decisions'],[])
|
||||
self.store.decision(definition(deadline='2026-10-19T13:00:00+08:00'))
|
||||
|
||||
def test_invalid_answers_and_ack(self):
|
||||
self.store.decision(definition())
|
||||
for changes in [dict(option_id=None),dict(option_id='bad'),dict(comment=None),
|
||||
dict(comment='x'*8001),dict(idempotency_key=''),dict(authorization=True)]:
|
||||
with self.subTest(changes=changes):
|
||||
with self.assertRaises(Invalid): self.store.decision_respond('choice',answer(**changes))
|
||||
with self.assertRaises(Invalid): self.store.decision_respond('missing',answer())
|
||||
with self.assertRaises(Invalid): self.store.decision_ack(dict(notification_id='missing'))
|
||||
with self.assertRaises(Invalid): self.store.decision_ack(dict(notification_id='missing',target='human'))
|
||||
self.assertEqual(self.store.state()['decisions'][0]['status'],'open')
|
||||
self.assertEqual(self.store.decision_notifications()['pending_count'],0)
|
||||
self.store.decision(definition(id='free',options=[]))
|
||||
for option,comment in [(None,' '),('a','text')]:
|
||||
with self.assertRaises(Invalid):
|
||||
self.store.decision_respond('free',answer(option_id=option,comment=comment))
|
||||
|
||||
def test_concurrent_same_key_has_one_response_and_notification(self):
|
||||
self.store.decision(definition())
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
values = list(pool.map(lambda _:self.store.decision_respond('choice',answer()),range(16)))
|
||||
self.assertTrue(all(v == values[0] for v in values))
|
||||
self.assertEqual(self.store.decision_notifications()['pending_count'],1)
|
||||
|
||||
def test_concurrent_distinct_answers_have_one_winner(self):
|
||||
self.store.decision(definition())
|
||||
def respond(index):
|
||||
try:
|
||||
return self.store.decision_respond('choice',answer(idempotency_key=str(index)))
|
||||
except Conflict:
|
||||
return None
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
values = list(pool.map(respond,range(16)))
|
||||
self.assertEqual(sum(v is not None for v in values),1)
|
||||
self.assertEqual(self.store.decision_notifications()['pending_count'],1)
|
||||
|
||||
def test_atomic_rollback_on_outbox_failure(self):
|
||||
self.store.decision(definition())
|
||||
with self.store.db() as db:
|
||||
db.execute("CREATE TRIGGER fail_notice BEFORE INSERT ON decision_notifications BEGIN SELECT RAISE(ABORT,'synthetic'); END")
|
||||
with self.assertRaises(Exception): self.store.decision_respond('choice',answer())
|
||||
self.assertEqual(self.store.state()['decisions'][0]['status'],'open')
|
||||
self.assertEqual(self.store.decision_notifications()['pending_count'],0)
|
||||
|
||||
def test_pending_order_and_limit(self):
|
||||
for index in range(103):
|
||||
did = 'choice-'+str(index)
|
||||
self.store.decision(definition(id=did))
|
||||
self.store.decision_respond(did,answer())
|
||||
queue = self.store.decision_notifications()
|
||||
self.assertEqual(queue['pending_count'],103)
|
||||
self.assertEqual(len(queue['notifications']),100)
|
||||
self.assertEqual(queue['notifications'][0]['decision']['id'],'choice-0')
|
||||
self.assertEqual(queue['notifications'][-1]['decision']['id'],'choice-99')
|
||||
|
||||
|
||||
class DecisionHTTPTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self.tmp.name)
|
||||
root.chmod(0o700)
|
||||
token = root / 'token'
|
||||
self.secret = 'h'*48
|
||||
token.write_text(self.secret)
|
||||
token.chmod(0o600)
|
||||
self.metrics = patch('agent_control.server.MetricsCache')
|
||||
mock = self.metrics.start()
|
||||
mock.return_value.snapshot.return_value = {'status':'synthetic'}
|
||||
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
self.metrics.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def request(self,path,method='GET',payload=None,headers=None):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
|
||||
merged.update(headers or {})
|
||||
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
|
||||
response = conn.getresponse()
|
||||
status, body = response.status,json.loads(response.read())
|
||||
conn.close()
|
||||
return status, body
|
||||
|
||||
|
||||
def test_routes_and_security(self):
|
||||
create='/api/decisions'
|
||||
respond='/api/decisions/choice/respond'
|
||||
notifications='/api/decision-notifications'
|
||||
ack=notifications+'/ack'
|
||||
for path,body in [(create,definition()),(respond,answer()),(ack,dict(notification_id='missing'))]:
|
||||
for headers,status in [({'Authorization':''},401),({'X-CSRF-Token':''},403),
|
||||
({'Origin':'https://untrusted.example'},403)]:
|
||||
self.assertEqual(self.request(path,'POST',body,headers)[0],status)
|
||||
self.assertEqual(self.request(notifications,headers={'Authorization':''})[0],401)
|
||||
self.assertEqual(self.request('/api/state')[1]['decisions'],[])
|
||||
self.assertEqual(self.request(create,'POST',definition())[0],200)
|
||||
self.assertEqual(self.request(create,'POST',definition(question='different'))[0],409)
|
||||
self.assertEqual(self.request(respond,'POST',answer(option_id='bad'))[0],400)
|
||||
code,value=self.request(respond,'POST',answer())
|
||||
self.assertEqual(code,200)
|
||||
self.assertEqual(self.request(respond,'POST',answer())[1],value)
|
||||
self.assertEqual(self.request(respond,'POST',answer(idempotency_key='new'))[0],409)
|
||||
notice=self.request(notifications)[1]['notifications'][0]
|
||||
self.assertEqual(self.request(ack,'POST',dict(notification_id=notice['notification_id']))[0],200)
|
||||
self.assertEqual(self.request(notifications)[1]['pending_count'],0)
|
||||
self.assertEqual(self.request(notifications+'?target=human')[0],404)
|
||||
self.assertEqual(self.request(respond+'?x=1','POST',answer())[0],404)
|
||||
self.assertEqual(self.request('/api/decisions/choice','PATCH',definition())[0],404)
|
||||
self.assertEqual(self.request('/api/state')[1]['decisions'][0],value)
|
||||
|
||||
def test_cross_site_only_public_top_level_navigation(self):
|
||||
web = Path(self.tmp.name)/'web'
|
||||
web.mkdir()
|
||||
(web/'index.html').write_text('<html>Public login</html>')
|
||||
self.server.web_root = web
|
||||
navigation = {'Sec-Fetch-Site':'cross-site','Sec-Fetch-Mode':'navigate','Sec-Fetch-Dest':'document'}
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
conn.request('GET','/',headers=navigation)
|
||||
response = conn.getresponse()
|
||||
self.assertEqual(response.status,200)
|
||||
self.assertIn(b'Public login',response.read())
|
||||
conn.close()
|
||||
self.assertEqual(self.request('/api/state',headers=navigation)[0],403)
|
||||
self.assertEqual(self.request('/api/decisions','POST',definition(),navigation)[0],403)
|
||||
self.assertEqual(self.request('/','POST',definition(),navigation)[0],403)
|
||||
for overrides in [{'Sec-Fetch-Mode':'no-cors','Sec-Fetch-Dest':'script'},
|
||||
{'Sec-Fetch-Dest':'iframe'},{'Sec-Fetch-Mode':'cors'},
|
||||
{'Origin':'https://untrusted.example'},{'Host':'untrusted.example'}]:
|
||||
self.assertEqual(self.request('/',headers=dict(navigation,**overrides))[0],403)
|
||||
|
||||
def test_eta_http_security_ack_and_complete(self):
|
||||
self.server.eta_clock.close()
|
||||
with patch('agent_control.store.time.time',return_value=1000):
|
||||
self.server.store.task(dict(id='http-eta',name='ETA',status='running',
|
||||
eta_min_minutes=5,eta_max_minutes=10,eta_basis='Measured'))
|
||||
self.server.store.eta_tick(1300)
|
||||
self.assertEqual(self.request('/api/eta-notifications',headers={'Authorization':''})[0],401)
|
||||
self.assertEqual(self.request('/api/eta-notifications',headers={'Sec-Fetch-Site':'cross-site'})[0],403)
|
||||
self.assertEqual(self.request('/api/eta-notifications?x=1')[0],404)
|
||||
notice=self.request('/api/eta-notifications')[1]['notifications'][0]
|
||||
nid=notice['notification_id']
|
||||
ack='/api/eta-notifications/ack'
|
||||
complete='/api/eta-reviews/'+nid+'/complete'
|
||||
body=dict(result='keep',reason='Evidence received')
|
||||
self.assertEqual(self.request(ack,'POST',dict(notification_id=nid),{'X-CSRF-Token':''})[0],403)
|
||||
self.assertEqual(self.request(complete,'POST',body,{'Origin':'https://untrusted.example'})[0],403)
|
||||
self.assertEqual(self.request(ack,'POST',dict(notification_id=nid))[0],200)
|
||||
self.assertEqual(self.request(complete,'POST',body)[0],200)
|
||||
self.assertEqual(self.request(complete,'POST',dict(body,reason='Changed'))[0],409)
|
||||
self.assertEqual(self.request('/api/eta-notifications')[1]['pending_count'],0)
|
||||
self.assertEqual(self.server.eta_clock.interval,15)
|
||||
self.assertFalse(self.server.eta_clock.thread.is_alive())
|
||||
|
||||
def test_server_starts_clock_without_http_polling(self):
|
||||
self.assertIsInstance(self.server.eta_clock,EtaClock)
|
||||
self.assertTrue(self.server.eta_clock.thread.is_alive())
|
||||
self.assertTrue(self.server.eta_clock.thread.name=='eta-clock')
|
||||
@@ -35,6 +35,18 @@ class FaultTests(unittest.TestCase):
|
||||
self.assertTrue(records['mirror']['needs_reconciliation'])
|
||||
self.assertEqual(records['mirror']['delivery'],'uncertain_recovery')
|
||||
|
||||
def test_only_persistent_fault_escalates_after_recovery_window(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
c=SimpleNamespace(call=lambda *args:None);records={}
|
||||
with patch.object(faults,'Client',return_value=c),patch.object(c,'call') as call,patch.object(faults.time,'time',return_value=100):
|
||||
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:2),records)
|
||||
self.assertEqual(call.call_args.args[1]['severity'],'warning')
|
||||
with patch.object(faults,'Client',return_value=c),patch.object(c,'call') as call,patch.object(faults.time,'time',return_value=221):
|
||||
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:2),records)
|
||||
faults.report_process({'control':{}},Path(root),'mirror',SimpleNamespace(poll=lambda:2),records)
|
||||
self.assertEqual(call.call_count,1)
|
||||
self.assertEqual(call.call_args.args[0],'/api/hooks/escalate')
|
||||
|
||||
def test_hook_failure_remains_persistently_visible(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
c=SimpleNamespace(call=lambda *args:None)
|
||||
|
||||
@@ -0,0 +1,275 @@
|
||||
"""Synthetic human log projections and durable low-noise escalation checks."""
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from agent_control.store import Store, Invalid, Conflict
|
||||
from agent_control.server import Server
|
||||
|
||||
|
||||
def sample(**overrides):
|
||||
data = dict(source='synthetic-worker',event_id='failure-1',severity='warning',
|
||||
status='firing',summary='Worker unavailable',impact='Checks paused',
|
||||
action='Recorded failure',next_step='Check recovery',evidence=['Process exited'])
|
||||
data.update(overrides)
|
||||
return data
|
||||
|
||||
|
||||
def escalation(**overrides):
|
||||
data = dict(source='synthetic-worker',event_id='failure-1',reason='Unhealthy for 120 seconds')
|
||||
data.update(overrides)
|
||||
return data
|
||||
|
||||
|
||||
class LogTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
self.path = Path(self.tmp.name)/'logs.sqlite3'
|
||||
self.store = Store(self.path)
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_legacy_short_mirror_interruption_is_warning_without_changing_audit(self):
|
||||
self.store.hook(sample(source='bridge-mirror',severity='critical'))
|
||||
self.store.hook(sample(source='bridge-mirror',severity='critical',status='resolved'))
|
||||
entry=self.store.logs()['logs'][0]
|
||||
self.assertEqual((entry['level'],entry['status']),('warning','resolved'))
|
||||
with self.store.db() as db:
|
||||
original=json.loads(db.execute("SELECT data FROM hooks WHERE status='firing'").fetchone()['data'])
|
||||
self.assertEqual(original['severity'],'critical')
|
||||
|
||||
def test_empty_four_levels_status_independent_and_filters(self):
|
||||
self.assertEqual(self.store.logs(),dict(logs=[],total_count=0,has_more=False))
|
||||
for level in ('info','warning','error','critical'):
|
||||
self.store.hook(sample(event_id=level,severity=level))
|
||||
self.store.hook(sample(event_id='warning',severity='info',status='resolved'))
|
||||
self.assertEqual(self.store.logs()['total_count'],4)
|
||||
self.assertEqual(len(self.store.logs(status='active')['logs']),3)
|
||||
entry = self.store.logs(level='warning',status='resolved')['logs'][0]
|
||||
self.assertEqual(entry['level'],'warning')
|
||||
self.assertEqual(entry['status'],'resolved')
|
||||
self.assertIsNotNone(entry['resolved_at'])
|
||||
self.assertEqual(self.store.logs(level='critical',status='resolved')['logs'],[])
|
||||
for args in [dict(level='fatal'),dict(status='firing')]:
|
||||
with self.assertRaises(Invalid): self.store.logs(**args)
|
||||
|
||||
def test_duplicates_merge_count_without_touching_audit_or_notifications(self):
|
||||
with patch('agent_control.store.time.time',return_value=100):
|
||||
original = self.store.hook(sample())
|
||||
with self.store.db() as db:
|
||||
audit = db.execute("SELECT data FROM hooks WHERE status='firing'").fetchone()[0]
|
||||
with patch('agent_control.store.time.time',return_value=120):
|
||||
for _ in range(3):
|
||||
self.assertTrue(self.store.hook(sample())['duplicate'])
|
||||
entry = Store(self.path).logs()['logs'][0]
|
||||
self.assertEqual(entry['repeat_count'],4)
|
||||
self.assertEqual(entry['first_seen'],100)
|
||||
self.assertEqual(entry['last_seen'],120)
|
||||
self.assertEqual(entry['id'],original['notifications'][0]['hook']['log_id'])
|
||||
self.assertNotIn('escalated',original['notifications'][0]['hook'])
|
||||
self.assertEqual(len(self.store.state()['events']),1)
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],1)
|
||||
with self.store.db() as db:
|
||||
self.assertEqual(db.execute("SELECT data FROM hooks WHERE status='firing'").fetchone()[0],audit)
|
||||
with self.assertRaises(Conflict): self.store.hook(sample(summary='Different observation'))
|
||||
self.assertEqual(self.store.logs()['logs'][0]['repeat_count'],4)
|
||||
|
||||
def test_recovery_time_warning_never_notifies_human(self):
|
||||
with patch('agent_control.store.time.time',return_value=100):
|
||||
first = self.store.hook(sample())
|
||||
with patch('agent_control.store.time.time',return_value=150):
|
||||
resolved = self.store.hook(sample(status='resolved',summary='Worker recovered',action='Checks resumed'))
|
||||
entry = self.store.logs()['logs'][0]
|
||||
self.assertEqual(entry['resolved_at'],150)
|
||||
self.assertEqual(entry['last_seen'],150)
|
||||
self.assertEqual(entry['summary'],'Worker unavailable')
|
||||
self.assertEqual(entry['action'],'Checks resumed')
|
||||
self.assertEqual(entry['repeat_count'],1)
|
||||
self.assertEqual({n['target'] for n in first['notifications']+resolved['notifications']},{'control'})
|
||||
|
||||
def test_escalation_is_persistent_idempotent_and_recovery_inherits(self):
|
||||
self.store.hook(sample())
|
||||
with self.store.db() as db:
|
||||
audit = db.execute("SELECT data FROM hooks WHERE status='firing'").fetchone()[0]
|
||||
with patch('subprocess.run',side_effect=AssertionError('no execution')):
|
||||
upgraded = self.store.hook_escalate(escalation(impact='Work remains blocked'))
|
||||
self.assertFalse(upgraded['duplicate'])
|
||||
self.assertEqual(upgraded['hook']['severity'],'critical')
|
||||
self.assertEqual(upgraded['hook']['status'],'firing')
|
||||
self.assertEqual({n['target'] for n in upgraded['notifications']},{'control','human'})
|
||||
self.assertTrue(all(n['hook']['escalated'] is True for n in upgraded['notifications']))
|
||||
again = Store(self.path).hook_escalate(escalation(impact='Work remains blocked'))
|
||||
self.assertTrue(again['duplicate'])
|
||||
self.assertEqual(again['notifications'],upgraded['notifications'])
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],3)
|
||||
entry = self.store.logs()['logs'][0]
|
||||
self.assertEqual(entry['level'],'critical')
|
||||
self.assertEqual(entry['status'],'active')
|
||||
self.assertEqual(entry['impact'],'Work remains blocked')
|
||||
self.assertEqual(entry['id'],upgraded['notifications'][0]['hook']['log_id'])
|
||||
self.assertEqual(entry['repeat_count'],1)
|
||||
self.assertEqual(self.store.state()['hooks']['active_firing'],1)
|
||||
self.assertEqual(len(self.store.state()['events']),2)
|
||||
with self.store.db() as db:
|
||||
self.assertEqual(db.execute("SELECT data FROM hooks WHERE status='firing'").fetchone()[0],audit)
|
||||
self.assertEqual(db.execute("SELECT count(*) FROM hooks WHERE status='escalated'").fetchone()[0],1)
|
||||
resolved = self.store.hook(sample(status='resolved',severity='info'))
|
||||
self.assertEqual(resolved['hook']['severity'],'critical')
|
||||
self.assertEqual({n['target'] for n in resolved['notifications']},{'control','human'})
|
||||
self.assertTrue(all(n['hook']['escalated'] for n in resolved['notifications']))
|
||||
self.assertTrue(all(n['hook']['log_id']==entry['id'] for n in resolved['notifications']))
|
||||
self.assertEqual(self.store.logs()['logs'][0]['status'],'resolved')
|
||||
self.assertTrue(self.store.hook(sample(status='resolved',severity='warning'))['duplicate'])
|
||||
|
||||
def test_escalation_conflicts_unknown_recovered_already_critical_and_changed_reason(self):
|
||||
with self.assertRaises(Conflict): self.store.hook_escalate(escalation())
|
||||
self.store.hook(sample())
|
||||
self.store.hook_escalate(escalation())
|
||||
with self.assertRaises(Conflict): self.store.hook_escalate(escalation(reason='Another reason'))
|
||||
with self.assertRaises(Conflict): self.store.hook_escalate(escalation(impact='Changed'))
|
||||
self.store.hook(sample(status='resolved'))
|
||||
with self.assertRaises(Conflict): self.store.hook_escalate(escalation())
|
||||
self.store.hook(sample(event_id='critical',severity='critical'))
|
||||
with self.assertRaises(Conflict): self.store.hook_escalate(escalation(event_id='critical'))
|
||||
ordinary = self.store.hook(sample(event_id='critical',severity='warning',status='resolved'))
|
||||
self.assertEqual({n['target'] for n in ordinary['notifications']},{'control','human'})
|
||||
self.assertTrue(all('escalated' not in n['hook'] for n in ordinary['notifications']))
|
||||
|
||||
def test_strict_escalation_schema_and_hook_cannot_claim_escalation(self):
|
||||
self.store.hook(sample())
|
||||
for changes in [dict(reason=''),dict(reason='x'*2001),dict(reason=None),
|
||||
dict(impact=None),dict(impact='x'*2001),dict(authorization=True)]:
|
||||
with self.assertRaises(Invalid): self.store.hook_escalate(escalation(**changes))
|
||||
for changes in [dict(escalated=True),dict(log_id='forged')]:
|
||||
with self.assertRaises(Invalid): self.store.hook(sample(**changes))
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],1)
|
||||
|
||||
def test_concurrent_repeat_and_upgrade_remain_one_audit_and_pair(self):
|
||||
self.store.hook(sample())
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
list(pool.map(lambda _:self.store.hook(sample()),range(12)))
|
||||
results=list(pool.map(lambda _:self.store.hook_escalate(escalation()),range(12)))
|
||||
self.assertEqual(sum(not r['duplicate'] for r in results),1)
|
||||
self.assertEqual(self.store.logs()['logs'][0]['repeat_count'],13)
|
||||
self.assertEqual(len(self.store.state()['events']),2)
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],3)
|
||||
|
||||
def test_escalation_outbox_failure_rolls_back_everything(self):
|
||||
self.store.hook(sample())
|
||||
with self.store.db() as db:
|
||||
db.execute("CREATE TRIGGER fail_upgrade BEFORE INSERT ON hook_notifications BEGIN SELECT RAISE(ABORT,'synthetic'); END")
|
||||
with self.assertRaises(Exception): self.store.hook_escalate(escalation())
|
||||
self.assertEqual(self.store.logs()['logs'][0]['level'],'warning')
|
||||
self.assertEqual(len(self.store.state()['events']),1)
|
||||
with self.store.db() as db:
|
||||
self.assertEqual(db.execute('SELECT count(*) FROM hook_escalations').fetchone()[0],0)
|
||||
|
||||
def test_legacy_audit_projection_and_bounded_recent_results(self):
|
||||
self.store.hook(sample())
|
||||
with self.store.db() as db:
|
||||
db.execute('DELETE FROM hook_observations')
|
||||
self.assertEqual(self.store.logs()['logs'][0]['repeat_count'],1)
|
||||
self.store.hook(sample())
|
||||
self.assertEqual(self.store.logs()['logs'][0]['repeat_count'],2)
|
||||
for i in range(104):
|
||||
with patch('agent_control.store.time.time',return_value=1000+i):
|
||||
self.store.hook(sample(event_id='event-'+str(i),severity='error'))
|
||||
result=self.store.logs(level='error')
|
||||
self.assertEqual(result['total_count'],104)
|
||||
self.assertTrue(result['has_more'])
|
||||
self.assertEqual(len(result['logs']),100)
|
||||
self.assertEqual(result['logs'][0]['event_id'],'event-103')
|
||||
|
||||
def test_sensitive_header_cookie_file_uri_and_private_key_variants(self):
|
||||
for text in ['Authorization: SensitiveValue', 'Cookie: session=SensitiveValue',
|
||||
'密钥=SensitiveValue', '令牌: SensitiveValue',
|
||||
'file:///home/SensitiveValue/secret.txt',
|
||||
'-----BEGIN RSA PRIVATE KEY----- SensitiveValue -----END RSA PRIVATE KEY-----',
|
||||
'https://person:SensitiveValue@example.invalid',
|
||||
r'\\\\host\\SensitiveValue\\secret.txt']:
|
||||
self.assertNotIn('SensitiveValue',self.store._safe_log_text(text))
|
||||
|
||||
def test_sensitive_projection_never_changes_original_audit(self):
|
||||
secrets = ['abc-token-value','my-password','key-abcdefghi','private-content',
|
||||
'alice','private.txt','jwtcontents','superlongcredential123456789012345678901234']
|
||||
values = ['token=abc-token-value failed','password: my-password',
|
||||
'Authorization: Bearer key-abcdefghi',
|
||||
'-----BEGIN OPENSSH PRIVATE KEY-----\nprivate-content\n-----END OPENSSH PRIVATE KEY-----',
|
||||
'/home/alice/private.txt unavailable',
|
||||
r'C:\Example\private.txt unavailable',
|
||||
'eyJhbGciOiJIUzI1NiJ9.jwtcontents.signature',
|
||||
'superlongcredential123456789012345678901234']
|
||||
self.store.hook(sample(summary=values[0],impact=values[1],action=values[2],
|
||||
next_step=values[4],evidence=values))
|
||||
projected=json.dumps(self.store.logs(),ensure_ascii=False)
|
||||
for value in secrets:
|
||||
self.assertNotIn(value,projected)
|
||||
self.assertIn('已隐藏',projected)
|
||||
with self.store.db() as db:
|
||||
original=db.execute("SELECT data FROM hooks WHERE status='firing'").fetchone()[0]
|
||||
self.assertIn('abc-token-value',original)
|
||||
self.assertIn('private-content',original)
|
||||
self.assertEqual(set(self.store.logs()['logs'][0]),
|
||||
{'id','source','event_id','level','status','summary','impact','action','next_step',
|
||||
'evidence','first_seen','last_seen','resolved_at','repeat_count'})
|
||||
|
||||
|
||||
class LogHTTPTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self.tmp.name)
|
||||
root.chmod(0o700)
|
||||
token = root / 'token'
|
||||
self.secret = 'h'*48
|
||||
token.write_text(self.secret)
|
||||
token.chmod(0o600)
|
||||
self.metrics = patch('agent_control.server.MetricsCache')
|
||||
mock = self.metrics.start()
|
||||
mock.return_value.snapshot.return_value = {'status':'synthetic'}
|
||||
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
self.metrics.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def request(self,path,method='GET',payload=None,headers=None):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
|
||||
merged.update(headers or {})
|
||||
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
|
||||
response = conn.getresponse()
|
||||
status, body = response.status,json.loads(response.read())
|
||||
conn.close()
|
||||
return status, body
|
||||
|
||||
|
||||
def test_logs_and_escalation_auth_filters_and_csrf(self):
|
||||
for path in ['/api/logs','/api/logs?level=warning']:
|
||||
self.assertEqual(self.request(path,headers={'Authorization':''})[0],401)
|
||||
self.assertEqual(self.request(path,headers={'Sec-Fetch-Site':'cross-site'})[0],403)
|
||||
self.assertEqual(self.request('/api/logs')[1]['logs'],[])
|
||||
self.assertEqual(self.request('/api/hooks','POST',sample())[0],200)
|
||||
self.assertEqual(self.request('/api/logs?level=warning&status=active')[1]['total_count'],1)
|
||||
for query in ['level=fatal','status=firing','level=','level=info&level=warning','source=x']:
|
||||
self.assertEqual(self.request('/api/logs?'+query)[0],400)
|
||||
path='/api/hooks/escalate'
|
||||
for headers,code in [({'Authorization':''},401),({'X-CSRF-Token':''},403),
|
||||
({'Origin':'https://untrusted.example'},403)]:
|
||||
self.assertEqual(self.request(path,'POST',escalation(),headers)[0],code)
|
||||
self.assertEqual(self.request(path,'POST',escalation(unknown=True))[0],400)
|
||||
self.assertEqual(self.request(path,'POST',escalation())[0],200)
|
||||
self.assertTrue(self.request(path,'POST',escalation())[1]['duplicate'])
|
||||
self.assertEqual(self.request('/api/logs?level=warning')[1]['total_count'],0)
|
||||
self.assertEqual(self.request('/api/logs?level=critical')[1]['total_count'],1)
|
||||
self.assertEqual(self.request('/api/hooks','POST',sample(status='resolved'))[0],200)
|
||||
self.assertEqual(self.request(path,'POST',escalation())[0],409)
|
||||
@@ -4,14 +4,32 @@ import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
ROOT=Path(__file__).resolve().parents[1]
|
||||
sys.path.insert(0,str(ROOT/'scripts'))
|
||||
import mirror
|
||||
from mirror import visible_items,sync
|
||||
from worker import message_prompt
|
||||
|
||||
|
||||
class MirrorTests(unittest.TestCase):
|
||||
def test_transient_connection_retry_recovers_without_process_failure(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
config=Path(root)/'config.json';config.write_text(json.dumps({'control':{},'state_file':str(Path(root)/'state.json'),'thread_id':'t'}))
|
||||
with patch.object(sys,'argv',['mirror.py','--config',str(config)]),patch.object(mirror,'Client'),patch.object(mirror,'AppServer'),patch.object(mirror,'sync',side_effect=[RuntimeError('offline'),1]) as sync_call,patch.object(mirror.time,'sleep',side_effect=[None,KeyboardInterrupt]):
|
||||
with self.assertRaises(KeyboardInterrupt):mirror.main()
|
||||
self.assertEqual(sync_call.call_count,2)
|
||||
self.assertEqual(json.loads((Path(root)/'mirror-health.json').read_text())['status'],'ok')
|
||||
|
||||
def test_connection_retry_is_bounded(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
config=Path(root)/'config.json';config.write_text(json.dumps({'control':{},'state_file':str(Path(root)/'state.json'),'thread_id':'t'}))
|
||||
with patch.object(sys,'argv',['mirror.py','--config',str(config)]),patch.object(mirror,'Client'),patch.object(mirror,'AppServer'),patch.object(mirror,'sync',side_effect=RuntimeError('offline')) as sync_call,patch.object(mirror.time,'sleep'):
|
||||
with self.assertRaises(RuntimeError):mirror.main()
|
||||
self.assertEqual(sync_call.call_count,3)
|
||||
self.assertEqual(json.loads((Path(root)/'mirror-health.json').read_text())['attempt'],3)
|
||||
|
||||
def test_quoted_marker_does_not_hide_human_or_reply(self):
|
||||
turns=[{'id':'t','status':'completed','items':[
|
||||
{'type':'userMessage','id':'u','content':[{'type':'text','text':'Please explain web-message-id: quotation'}]},
|
||||
|
||||
@@ -63,6 +63,100 @@ class NotifierTests(unittest.TestCase):
|
||||
self.assertTrue(state['deliveries']['note2']['acked'])
|
||||
self.assertEqual(state['deliveries']['note1']['state'],'submitting')
|
||||
|
||||
def test_decision_response_goes_only_to_control_and_retries_ack_without_queue(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg,n,path=self.fixture(root)
|
||||
cfg['decision_notifications']=True
|
||||
item={'notification_id':'decision-note','target':'control','decision':{'id':'choice','title':'Choose plan','question':'Which plan?','status':'decided','requires_authorization':False,'options':[{'id':'a','label':'A','description':'Plan A'}],'response':{'id':'response','option_id':'a','comment':'Use A','authorization':False}}}
|
||||
calls=[]
|
||||
class Client:
|
||||
def call(self,p,data=None):
|
||||
calls.append(p)
|
||||
return {'notifications':[item] if p=='/api/decision-notifications' else []}
|
||||
result=SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['notification_targets']['control']+'.')
|
||||
state={}
|
||||
with patch.object(notifier.subprocess,'run',return_value=result) as run:
|
||||
notifier.deliver(Client(),cfg,state,path)
|
||||
notifier.deliver(Client(),cfg,state,path)
|
||||
self.assertEqual(run.call_count,1)
|
||||
self.assertEqual(calls.count('/api/decision-notifications/ack'),2)
|
||||
self.assertTrue(state['deliveries']['decision:decision-note']['acked'])
|
||||
|
||||
def test_eta_checkpoint_only_goes_to_control_and_ack_is_not_review_completion(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg,n,path=self.fixture(root);cfg['eta_notifications']=True
|
||||
item={'notification_id':'eta-note','target':'control','task':{'id':'work','current_progress':'checking','status':'running'},'checkpoint':'half','revision':1}
|
||||
calls=[]
|
||||
class Client:
|
||||
def call(self,p,data=None):
|
||||
calls.append(p)
|
||||
return {'notifications':[item] if p=='/api/eta-notifications' else []}
|
||||
result=SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['notification_targets']['control']+'.')
|
||||
with patch.object(notifier.subprocess,'run',return_value=result):notifier.deliver(Client(),cfg,{},path)
|
||||
self.assertIn('/api/eta-notifications/ack',calls)
|
||||
self.assertFalse(any('/complete' in p for p in calls))
|
||||
|
||||
def test_internal_review_merges_control_and_human_hook_without_root_injection(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg,n,path=self.fixture(root)
|
||||
cfg['notification_targets']['human']='44444444-4444-4444-8444-444444444444'
|
||||
cfg['internal_event_thread_id']='33333333-3333-4333-8333-333333333333'
|
||||
other={**n,'notification_id':'note2','target':'human'}
|
||||
calls=[]
|
||||
class Client:
|
||||
def call(self,p,data=None):
|
||||
calls.append((p,data))
|
||||
return {'notifications':[n,other]} if p.endswith('notifications') else {}
|
||||
result=SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['internal_event_thread_id']+'.')
|
||||
state={}
|
||||
with patch.object(notifier.subprocess,'run',return_value=result) as run:
|
||||
notifier.deliver(Client(),cfg,state,path)
|
||||
self.assertEqual(run.call_count,1)
|
||||
self.assertIn(cfg['internal_event_thread_id'],run.call_args.args[0])
|
||||
self.assertNotIn(cfg['notification_targets']['control'],run.call_args.args[0])
|
||||
self.assertEqual(len(state['deliveries']),1)
|
||||
self.assertEqual(len([x for x in calls if x[0]=='/api/hooks/ack']),2)
|
||||
|
||||
def test_busy_target_keeps_backlog_until_real_completed_final(self):
|
||||
import hashlib
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg,n,path=self.fixture(root);cfg['enforce_backpressure']=True
|
||||
notes=[{**n,'notification_id':'note'+str(i),'hook':{**n['hook'],'event_id':'incident'+str(i)}} for i in range(20)]
|
||||
class Client:
|
||||
def call(self,p,data=None):return {'notifications':notes} if p.endswith('notifications') else {}
|
||||
class Reader:
|
||||
turns=[]
|
||||
def __init__(self,*a):pass
|
||||
def __enter__(self):return self
|
||||
def __exit__(self,*a):pass
|
||||
def call(self,method,params):
|
||||
return {'thread':{'id':cfg['notification_targets']['control']}} if method=='thread/read' else {'data':self.turns}
|
||||
result=SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['notification_targets']['control']+'.')
|
||||
state={}
|
||||
with patch.object(notifier.subprocess,'run',return_value=result) as run,patch.object(notifier,'AppServer',Reader):
|
||||
for _ in range(5):notifier.deliver(Client(),cfg,state,path)
|
||||
self.assertEqual(run.call_count,1)
|
||||
prompt=run.call_args.args[0][-1]
|
||||
self.assertEqual(hashlib.sha256(prompt.encode()).hexdigest(),next(iter(state['deliveries'].values()))['input_sha256'])
|
||||
Reader.turns=[{'id':'turn','status':'completed','completedAt':1,'items':[{'type':'userMessage','content':[{'type':'text','text':prompt}]},{'type':'agentMessage','phase':'final_answer','text':'reviewed'}]}]
|
||||
notifier.deliver(Client(),cfg,state,path)
|
||||
self.assertEqual(run.call_count,2)
|
||||
|
||||
def test_task_event_routes_to_registered_executor_before_deputy(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg,n,path=self.fixture(root)
|
||||
cfg.update(internal_event_thread_id='33333333-3333-4333-8333-333333333333',execution_notification_targets={'worker':'44444444-4444-4444-8444-444444444444'})
|
||||
n['hook']['task_id']='owned-task'
|
||||
class Client:
|
||||
def call(self,p,data=None):
|
||||
if p=='/api/state':return {'tasks':[{'id':'owned-task','session_id':'worker'}]}
|
||||
return {'notifications':[n]} if p.endswith('notifications') else {}
|
||||
result=SimpleNamespace(returncode=0,stdout='Queued message 22222222-2222-4222-8222-222222222222 for thread '+cfg['execution_notification_targets']['worker']+'.')
|
||||
with patch.object(notifier.subprocess,'run',return_value=result) as run:
|
||||
notifier.deliver(Client(),cfg,{},path)
|
||||
self.assertIn(cfg['execution_notification_targets']['worker'],run.call_args.args[0])
|
||||
self.assertNotIn(cfg['internal_event_thread_id'],run.call_args.args[0])
|
||||
|
||||
def test_no_events_means_no_conversation_input(self):
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg,n,path=self.fixture(root)
|
||||
|
||||
@@ -0,0 +1,216 @@
|
||||
"""Service-owned queue diagnostics; no claims about a host private queue."""
|
||||
import http.client
|
||||
import concurrent.futures
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from agent_control.store import Store, Invalid, Conflict
|
||||
from agent_control.server import Server
|
||||
|
||||
|
||||
def firing(**changes):
|
||||
data=dict(source='synthetic',event_id='event',severity='warning',status='firing',summary='Synthetic failure')
|
||||
data.update(changes)
|
||||
return data
|
||||
|
||||
|
||||
class QueueTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory()
|
||||
self.store=Store(Path(self.tmp.name)/'queue.sqlite3')
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def test_empty_real_counts_and_explicit_scope(self):
|
||||
health=self.store.queue_health(100)
|
||||
self.assertEqual(health['scope'],'service_owned_outboxes')
|
||||
self.assertEqual(health['delivery_semantics'],'host_accepted_not_model_completed')
|
||||
for queue in ['hooks','eta','decisions']:
|
||||
self.assertEqual(health[queue]['pending_count'],0)
|
||||
self.assertIsNone(health[queue]['oldest_pending_at'])
|
||||
self.assertEqual(health[queue]['created_last_60s'],0)
|
||||
self.assertEqual(health[queue]['delivered_last_60s'],0)
|
||||
|
||||
def test_sql_counts_timestamps_and_receipts_are_real(self):
|
||||
with patch('agent_control.store.time.time',return_value=10):
|
||||
first=self.store.hook(firing())
|
||||
self.store.session(dict(id='worker',name='Worker'))
|
||||
self.store.task(dict(id='task',name='Task',session_id='worker',
|
||||
eta_min_minutes=1,eta_max_minutes=2,eta_basis='Measured'))
|
||||
self.store.decision(dict(id='decision',title='Choose',question='Which approach?'))
|
||||
self.store.eta_tick(70)
|
||||
with patch('agent_control.store.time.time',return_value=80):
|
||||
self.store.hook(firing(event_id='recent',severity='critical'))
|
||||
self.store.hook_ack(dict(notification_id=first['notifications'][0]['notification_id']))
|
||||
self.store.decision_respond('decision',dict(option_id=None,comment='Selected',idempotency_key='choice'))
|
||||
self.store.task_report(dict(report_id='receipt',task_id='task',session_id='worker',
|
||||
stage='progress',current_progress='Verified',summary='Progress',evidence=[]))
|
||||
health=self.store.queue_health(100)
|
||||
self.assertEqual(health['hooks']['pending_count'],2)
|
||||
self.assertEqual(health['hooks']['oldest_pending_at'],80)
|
||||
self.assertEqual(health['hooks']['created_last_60s'],2)
|
||||
self.assertEqual(health['hooks']['delivered_last_60s'],1)
|
||||
self.assertEqual(health['eta']['pending_count'],1)
|
||||
self.assertEqual(health['eta']['oldest_pending_at'],70)
|
||||
self.assertEqual(health['decisions']['pending_count'],1)
|
||||
self.assertEqual(health['decisions']['oldest_pending_at'],80)
|
||||
self.assertEqual(health['task_reports'],dict(received_total=1,received_last_60s=1))
|
||||
with patch('agent_control.store.time.time',return_value=101):
|
||||
self.store.task(dict(status='cancelled'),'task')
|
||||
self.assertEqual(self.store.queue_health(102)['eta']['cancelled_count'],1)
|
||||
self.assertEqual(self.store.queue_health(102)['eta']['pending_count'],0)
|
||||
|
||||
def test_coalesce_resolved_backlog_idempotent_without_delivery_claim(self):
|
||||
with patch('agent_control.store.time.time',return_value=10):
|
||||
first=self.store.hook(firing())
|
||||
for _ in range(10):
|
||||
self.store.hook(firing())
|
||||
with patch('agent_control.store.time.time',return_value=20):
|
||||
self.store.hook(firing(status='resolved'))
|
||||
before=self.store.state()['hooks']['recent']
|
||||
with patch('agent_control.store.time.time',return_value=30):
|
||||
result=self.store.hook_coalesce({})
|
||||
self.assertEqual(result['superseded_count'],2)
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],0)
|
||||
self.assertEqual(self.store.hook_coalesce({})['superseded_count'],0)
|
||||
self.assertEqual(self.store.state()['hooks']['recent'],before)
|
||||
self.assertEqual(self.store.queue_health(40)['hooks']['delivered_last_60s'],0)
|
||||
self.assertEqual(self.store.queue_health(40)['hooks']['superseded_count'],2)
|
||||
repeated=self.store.hook(firing())
|
||||
self.assertEqual(repeated['notifications'][0]['status'],'superseded')
|
||||
self.assertEqual(len(self.store.state()['events']),2)
|
||||
|
||||
def test_coalesce_keeps_active_major_and_recovery_for_delivered_target(self):
|
||||
warning=self.store.hook(firing())
|
||||
self.store.hook_ack(dict(notification_id=warning['notifications'][0]['notification_id']))
|
||||
self.store.hook_escalate(dict(source='synthetic',event_id='event',reason='Persistent'))
|
||||
self.store.hook(firing(status='resolved'))
|
||||
active=self.store.hook(firing(event_id='active-major',severity='critical'))
|
||||
self.store.hook_coalesce({})
|
||||
notices=self.store.hook_notifications()['notifications']
|
||||
recoveries=[n for n in notices if n['hook']['event_id']=='event']
|
||||
self.assertEqual(len(recoveries),1)
|
||||
self.assertEqual(recoveries[0]['target'],'control')
|
||||
self.assertEqual(recoveries[0]['hook']['status'],'resolved')
|
||||
self.assertEqual(sum(n['hook']['event_id']=='active-major' for n in notices),2)
|
||||
|
||||
def test_coalesce_does_not_touch_user_decision_or_message(self):
|
||||
self.store.session(dict(id='s',name='Synthetic'))
|
||||
self.store.decision(dict(id='d',title='Choose',question='Which approach?'))
|
||||
self.store.decision_respond('d',dict(option_id=None,comment='User choice',idempotency_key='choice'))
|
||||
before=self.store.decision_notifications()
|
||||
self.store.send(dict(session_id='s',text='Real user request',idempotency_key='user-message'))
|
||||
messages=self.store.messages('s')
|
||||
self.store.hook(firing())
|
||||
self.store.hook(firing(status='resolved'))
|
||||
self.store.hook_coalesce({})
|
||||
self.assertEqual(self.store.decision_notifications(),before)
|
||||
self.assertEqual(self.store.messages('s'),messages)
|
||||
with self.assertRaises(Invalid): self.store.hook_coalesce({'clear_all':True})
|
||||
|
||||
def test_unknown_submission_marker_protects_firing_and_its_recovery(self):
|
||||
first=self.store.hook(firing())
|
||||
nid=first['notifications'][0]['notification_id']
|
||||
with patch('agent_control.store.time.time',return_value=10):
|
||||
marker=self.store.hook_submitting(dict(notification_id=nid))
|
||||
with patch('agent_control.store.time.time',return_value=20):
|
||||
again=self.store.hook_submitting(dict(notification_id=nid))
|
||||
self.assertFalse(marker['duplicate'])
|
||||
self.assertTrue(again['duplicate'])
|
||||
self.assertEqual(again['submission_started_at'],10)
|
||||
self.assertEqual(again['status'],'submitting')
|
||||
self.store.hook(firing(status='resolved'))
|
||||
self.assertEqual(self.store.hook_coalesce({})['superseded_count'],0)
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],2)
|
||||
health=self.store.queue_health(30)['hooks']
|
||||
self.assertEqual(health['submission_started_unconfirmed_count'],1)
|
||||
self.assertEqual(health['delivered_last_60s'],0)
|
||||
self.assertEqual(self.store.hook_notifications()['notifications'][0]['submission_started_at'],10)
|
||||
|
||||
def test_superseded_cannot_begin_submission_but_real_late_ack_is_recorded(self):
|
||||
first=self.store.hook(firing())
|
||||
nid=first['notifications'][0]['notification_id']
|
||||
self.store.hook(firing(status='resolved'))
|
||||
self.store.hook_coalesce({})
|
||||
with self.assertRaises(Conflict): self.store.hook_submitting(dict(notification_id=nid))
|
||||
# An actual accepted receipt can arrive late; never erase that fact.
|
||||
self.store.hook_ack(dict(notification_id=nid))
|
||||
self.assertEqual(self.store.hook(firing())['notifications'][0]['status'],'delivered')
|
||||
with self.assertRaises(Invalid): self.store.hook_submitting(dict(notification_id='missing'))
|
||||
with self.assertRaises(Invalid): self.store.hook_submitting(dict(notification_id=nid,accepted=True))
|
||||
|
||||
def test_submission_and_coalesce_are_serialized(self):
|
||||
notice=self.store.hook(firing())['notifications'][0]
|
||||
self.store.hook(firing(status='resolved'))
|
||||
def submit():
|
||||
try:
|
||||
return self.store.hook_submitting(dict(notification_id=notice['notification_id']))
|
||||
except Conflict:
|
||||
return None
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=2) as pool:
|
||||
attempt=pool.submit(submit)
|
||||
cleanup=pool.submit(self.store.hook_coalesce,{})
|
||||
result=attempt.result()
|
||||
cleanup.result()
|
||||
with self.store.db() as db:
|
||||
row=db.execute('SELECT * FROM hook_notifications WHERE id=?',(notice['notification_id'],)).fetchone()
|
||||
if result is not None:
|
||||
self.assertIsNotNone(row['submission_started_at'])
|
||||
self.assertIsNone(row['superseded_at'])
|
||||
else:
|
||||
self.assertIsNone(row['submission_started_at'])
|
||||
self.assertIsNotNone(row['superseded_at'])
|
||||
|
||||
|
||||
class QueueHTTPTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self.tmp.name)
|
||||
root.chmod(0o700)
|
||||
token = root / 'token'
|
||||
self.secret = 'h'*48
|
||||
token.write_text(self.secret)
|
||||
token.chmod(0o600)
|
||||
self.metrics = patch('agent_control.server.MetricsCache')
|
||||
mock = self.metrics.start()
|
||||
mock.return_value.snapshot.return_value = {'status':'synthetic'}
|
||||
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
self.metrics.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def request(self,path,method='GET',payload=None,headers=None):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
|
||||
merged.update(headers or {})
|
||||
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
|
||||
response = conn.getresponse()
|
||||
status, body = response.status,json.loads(response.read())
|
||||
conn.close()
|
||||
return status, body
|
||||
|
||||
|
||||
def test_security_and_coalesce_routes(self):
|
||||
first=self.server.store.hook(firing())
|
||||
self.server.store.hook(firing(status='resolved'))
|
||||
self.assertEqual(self.request('/api/queue-health',headers={'Authorization':''})[0],401)
|
||||
self.assertEqual(self.request('/api/queue-health?x=1')[0],404)
|
||||
for path,body in [('/api/hook-notifications/coalesce',{}),
|
||||
('/api/hook-notifications/submitting',dict(notification_id=first['notifications'][0]['notification_id']))]:
|
||||
for headers,status in [({'Authorization':''},401),({'X-CSRF-Token':''},403),
|
||||
({'Origin':'https://untrusted.example'},403)]:
|
||||
self.assertEqual(self.request(path,'POST',body,headers)[0],status)
|
||||
self.assertEqual(self.request('/api/hook-notifications/coalesce','POST',{'clear':True})[0],400)
|
||||
self.assertEqual(self.request('/api/hook-notifications/coalesce','POST',{})[1]['superseded_count'],2)
|
||||
self.assertEqual(self.request('/api/queue-health')[1]['hooks']['pending_count'],0)
|
||||
self.assertEqual(self.request('/api/queue-health')[1]['hooks']['superseded_count'],2)
|
||||
@@ -0,0 +1,39 @@
|
||||
import json,tempfile,sys,unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
sys.path.insert(0,str(Path(__file__).resolve().parents[1]/'scripts'))
|
||||
import report
|
||||
class ReportRelayTests(unittest.TestCase):
|
||||
def test_outage_persists_and_retry_is_idempotent(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
cfg={'outbox_dir':d,'control':{}}
|
||||
payload={'task_id':'task','session_id':'worker','stage':'execution_completed','summary':'checked','current_progress':'ready','evidence':['build passed']}
|
||||
class Client:
|
||||
calls=0
|
||||
def __init__(self,*a):pass
|
||||
def call(self,*a):
|
||||
self.__class__.calls+=1
|
||||
if self.calls==1:raise OSError('offline')
|
||||
return {'accepted':True}
|
||||
with patch.object(report,'Client',Client):
|
||||
first=report.submit(cfg,payload);self.assertEqual(first['status'],'pending')
|
||||
path=next(Path(d).glob('*.json'));r=json.loads(path.read_text());r['next_retry']=0;report.atomic_json(path,r)
|
||||
self.assertEqual(report.flush(cfg)['accepted'],1)
|
||||
self.assertEqual(report.submit(cfg,payload)['status'],'accepted')
|
||||
self.assertEqual(Client.calls,2)
|
||||
def test_accepted_files_do_not_starve_pending_and_completion_supersedes_old_progress(self):
|
||||
with tempfile.TemporaryDirectory() as d:
|
||||
for i in range(105):
|
||||
payload={'task_id':'task','session_id':'worker','stage':'progress'}
|
||||
status='accepted' if i<102 else 'pending'
|
||||
if i==104:payload['stage']='execution_completed'
|
||||
report.atomic_json(Path(d)/(('%064x'%i)+'.json'),{'status':status,'created':i,'payload':payload})
|
||||
class Client:
|
||||
seen=[]
|
||||
def __init__(self,*a):pass
|
||||
def call(self,path,payload):self.seen.append(payload['stage']);return {'ok':True}
|
||||
with patch.object(report,'Client',Client):counts=report.flush({'outbox_dir':d,'control':{}})
|
||||
self.assertEqual(Client.seen,['execution_completed'])
|
||||
self.assertEqual(counts['superseded'],2)
|
||||
self.assertEqual(counts['pending'],0)
|
||||
if __name__=='__main__':unittest.main()
|
||||
@@ -0,0 +1,239 @@
|
||||
"""Task relation graphs and atomic integration records, without Git execution."""
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from agent_control.store import Store, Invalid, Conflict, TASK_RELATION_DEFAULTS
|
||||
from agent_control.server import Server
|
||||
|
||||
|
||||
def integration(**changes):
|
||||
data=dict(source_ids=['source-a','source-b'],owner='Integrator',status='waiting',
|
||||
reason='Packages ready for coordinated integration',version='v1')
|
||||
data.update(changes)
|
||||
return data
|
||||
|
||||
|
||||
class RelationTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory()
|
||||
self.path=Path(self.tmp.name)/'relations.sqlite3'
|
||||
self.store=Store(self.path)
|
||||
for tid in ['target','source-a','source-b','other']:
|
||||
self.store.task(dict(id=tid,name=tid,status='running',current_progress='Working on '+tid))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def task(self,tid):
|
||||
return next(t for t in self.store.state()['tasks'] if t['id']==tid)
|
||||
|
||||
def audit_count(self):
|
||||
with self.store.db() as db:
|
||||
return db.execute('SELECT count(*) FROM task_integrations').fetchone()[0]
|
||||
|
||||
def test_defaults_legacy_and_descriptive_metadata(self):
|
||||
with self.store.db() as db:
|
||||
db.execute('INSERT INTO tasks(id,data) VALUES(?,?)',('legacy',json.dumps(dict(id='legacy',name='Legacy'))))
|
||||
task=self.task('legacy')
|
||||
for field,value in TASK_RELATION_DEFAULTS.items():
|
||||
self.assertEqual(task[field],value)
|
||||
task=self.store.task(dict(goal='Deliver verified result',scope='Parser only',coupling_assessment='Shared schema',
|
||||
owner='Worker',parallel_scope='Read fixtures before schema is ready',
|
||||
acceptance_criteria=['Pass parser checks'],
|
||||
sync_points=[dict(id='schema',label='Schema ready',task_ids=['source-a'],status='waiting')]),'target')
|
||||
self.assertEqual(task['goal'],'Deliver verified result')
|
||||
self.assertEqual(task['sync_points'][0]['task_ids'],['source-a'])
|
||||
self.assertEqual(task['integration_status'],'not_started')
|
||||
|
||||
def test_single_multi_dependency_and_dedup(self):
|
||||
task=self.store.task(dict(depends_on=['source-b','source-a','source-a']),'target')
|
||||
self.assertEqual(task['depends_on'],['source-a','source-b'])
|
||||
self.store.task(dict(depends_on=['other']),'source-a')
|
||||
with self.assertRaises(Conflict): self.store.task(dict(depends_on=['target']),'other')
|
||||
self.assertEqual(self.task('other')['depends_on'],[])
|
||||
|
||||
def test_parent_hierarchy_separate_from_dependency_and_merge_graph(self):
|
||||
self.store.task(dict(parent_id='target'),'source-a')
|
||||
self.store.task(dict(parent_id='source-a'),'other')
|
||||
with self.assertRaises(Conflict): self.store.task(dict(parent_id='other'),'target')
|
||||
# Parent depends on a child, and child merges back into parent: all legitimate.
|
||||
self.store.task(dict(depends_on=['source-a']),'target')
|
||||
result=self.store.integrate('target',integration(source_ids=['source-a']))
|
||||
self.assertEqual(result['sources'][0]['parent_id'],'target')
|
||||
self.assertEqual(result['sources'][0]['merged_into_id'],'target')
|
||||
self.assertEqual(result['task']['depends_on'],['source-a'])
|
||||
|
||||
def test_overlap_requires_parallel_scope_not_false_serial_order(self):
|
||||
with self.assertRaises(Invalid):
|
||||
self.store.task(dict(depends_on=['source-a'],parallel_with=['source-a']),'target')
|
||||
task=self.store.task(dict(depends_on=['source-a'],parallel_with=['source-a'],
|
||||
parallel_scope='Prepare fixtures now; wait for schema before integration'),'target')
|
||||
self.assertEqual(task['parallel_with'],['source-a'])
|
||||
with self.assertRaises(Invalid): self.store.task(dict(parallel_scope=' '),'target')
|
||||
self.assertTrue(self.task('target')['parallel_scope'])
|
||||
|
||||
def test_multisource_integrate_preserves_states_progress_and_timestamps(self):
|
||||
self.store.task(dict(status='blocked'),'source-a')
|
||||
before_a=self.task('source-a')
|
||||
before_b=self.task('source-b')
|
||||
with patch('subprocess.run',side_effect=AssertionError('integration never executes Git')):
|
||||
result=self.store.integrate('target',integration(status='integrating'))
|
||||
self.assertFalse(result['duplicate'])
|
||||
self.assertEqual(result['task']['merge_sources'],['source-a','source-b'])
|
||||
self.assertEqual(result['task']['integration_owner'],'Integrator')
|
||||
self.assertEqual(result['task']['integration_status'],'integrating')
|
||||
self.assertEqual(result['task']['version'],'v1')
|
||||
for before in [before_a,before_b]:
|
||||
after=self.task(before['id'])
|
||||
self.assertEqual(after['merged_into_id'],'target')
|
||||
self.assertEqual(after['status'],before['status'])
|
||||
self.assertEqual(after['current_progress'],before['current_progress'])
|
||||
self.assertEqual(after['progress_updated'],before['progress_updated'])
|
||||
self.assertEqual(after['eta_estimated_at'],before['eta_estimated_at'])
|
||||
|
||||
def test_idempotent_order_and_retry_cannot_rewind_new_status(self):
|
||||
first=self.store.integrate('target',integration())
|
||||
events=len(self.store.state()['events'])
|
||||
second=self.store.integrate('target',integration(source_ids=['source-b','source-a','source-a']))
|
||||
self.assertTrue(second['duplicate'])
|
||||
self.assertEqual(first['task'],second['task'])
|
||||
self.assertEqual(self.audit_count(),1)
|
||||
self.assertEqual(len(self.store.state()['events']),events)
|
||||
self.store.integrate('target',integration(status='verified',reason='Actual integration and acceptance completed'))
|
||||
result=self.store.integrate('target',integration())
|
||||
self.assertTrue(result['duplicate'])
|
||||
self.assertEqual(result['task']['integration_status'],'verified')
|
||||
self.assertEqual(self.audit_count(),2)
|
||||
|
||||
def test_source_owned_elsewhere_and_merge_cycle_reject_atomically(self):
|
||||
self.store.integrate('other',integration(source_ids=['source-a']))
|
||||
with self.assertRaises(Conflict): self.store.integrate('target',integration())
|
||||
self.assertEqual(self.task('target')['merge_sources'],[])
|
||||
self.assertIsNone(self.task('source-b')['merged_into_id'])
|
||||
self.assertEqual(self.audit_count(),1)
|
||||
self.store.integrate('target',integration(source_ids=['other']))
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.integrate('source-a',integration(source_ids=['target']))
|
||||
self.assertIsNone(self.task('target')['merged_into_id'])
|
||||
self.assertEqual(self.audit_count(),2)
|
||||
|
||||
def test_plain_task_merge_edits_are_reciprocal_and_cannot_rebind(self):
|
||||
self.store.task(dict(merged_into_id='target'),'source-a')
|
||||
self.assertEqual(self.task('target')['merge_sources'],['source-a'])
|
||||
with self.assertRaises(Conflict):
|
||||
self.store.task(dict(merged_into_id='other'),'source-a')
|
||||
self.store.task(dict(merge_sources=['source-a','source-b']),'target')
|
||||
self.assertEqual(self.task('source-b')['merged_into_id'],'target')
|
||||
self.store.task(dict(merge_sources=['source-b']),'target')
|
||||
self.assertIsNone(self.task('source-a')['merged_into_id'])
|
||||
self.store.task(dict(merged_into_id=None),'source-b')
|
||||
self.assertEqual(self.task('target')['merge_sources'],[])
|
||||
|
||||
def test_schema_and_references_are_atomic(self):
|
||||
updates=[dict(goal=None),dict(scope='x'*4001),dict(owner='x'*201),dict(parallel_scope='x'*2001),
|
||||
dict(parent_id='missing'),dict(parent_id='target'),dict(depends_on=['target']),
|
||||
dict(depends_on=['missing']),dict(parallel_with='source-a'),dict(merge_sources=['target']),
|
||||
dict(acceptance_criteria=['x'*1001]),dict(acceptance_criteria=['x']*21),
|
||||
dict(sync_points=[dict(id='x',label='X',task_ids=['missing'],status='waiting')]),
|
||||
dict(sync_points=[dict(id='x',label='X',task_ids=[],status='done')]),
|
||||
dict(sync_points=[dict(id='x',label='X',task_ids=[],status='ready',extra=True)]),
|
||||
dict(sync_points=[dict(id='x',label='X',task_ids=[],status='ready')]*2),
|
||||
dict(integration_status='complete'),dict(id='other')]
|
||||
before=self.task('target')
|
||||
for update in updates:
|
||||
with self.subTest(update=update):
|
||||
with self.assertRaises(Invalid): self.store.task(update,'target')
|
||||
self.assertEqual(self.task('target'),before)
|
||||
|
||||
def test_integration_schema_unknown_and_missing_ids(self):
|
||||
changes=[dict(source_ids=[]),dict(source_ids=['target']),dict(source_ids=['missing']),
|
||||
dict(source_ids=['source-a']*31),dict(owner=''),dict(reason=''),
|
||||
dict(reason='x'*2001),dict(status='completed'),dict(version=None),dict(shell='git merge')]
|
||||
for change in changes:
|
||||
with self.assertRaises(Invalid): self.store.integrate('target',integration(**change))
|
||||
with self.assertRaises(Invalid): self.store.integrate('missing',integration())
|
||||
self.assertEqual(self.audit_count(),0)
|
||||
self.assertEqual(self.task('target')['merge_sources'],[])
|
||||
|
||||
def test_integration_transaction_rollback(self):
|
||||
with self.store.db() as db:
|
||||
db.execute("CREATE TRIGGER fail_integration BEFORE INSERT ON task_integrations BEGIN SELECT RAISE(ABORT,'synthetic'); END")
|
||||
with self.assertRaises(Exception): self.store.integrate('target',integration())
|
||||
self.assertEqual(self.task('target')['merge_sources'],[])
|
||||
self.assertIsNone(self.task('source-a')['merged_into_id'])
|
||||
self.assertIsNone(self.task('source-b')['merged_into_id'])
|
||||
|
||||
def test_concurrent_same_request_only_one_audit(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results=list(pool.map(lambda _:self.store.integrate('target',integration()),range(12)))
|
||||
self.assertEqual(sum(not r['duplicate'] for r in results),1)
|
||||
self.assertEqual(self.audit_count(),1)
|
||||
|
||||
def test_source_union_and_limit(self):
|
||||
self.store.integrate('target',integration(source_ids=['source-a']))
|
||||
self.store.integrate('target',integration(source_ids=['source-b']))
|
||||
self.assertEqual(self.task('target')['merge_sources'],['source-a','source-b'])
|
||||
for i in range(29):
|
||||
self.store.task(dict(id='extra-'+str(i),name='Extra'))
|
||||
sources=['extra-'+str(i) for i in range(29)]
|
||||
with self.assertRaises(Invalid):
|
||||
self.store.integrate('target',integration(source_ids=sources))
|
||||
self.assertEqual(self.task('target')['merge_sources'],['source-a','source-b'])
|
||||
self.assertIsNone(self.task('extra-0')['merged_into_id'])
|
||||
|
||||
|
||||
class RelationHTTPTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self.tmp.name)
|
||||
root.chmod(0o700)
|
||||
token = root / 'token'
|
||||
self.secret = 'h'*48
|
||||
token.write_text(self.secret)
|
||||
token.chmod(0o600)
|
||||
self.metrics = patch('agent_control.server.MetricsCache')
|
||||
mock = self.metrics.start()
|
||||
mock.return_value.snapshot.return_value = {'status':'synthetic'}
|
||||
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
self.metrics.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def request(self,path,method='GET',payload=None,headers=None):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
|
||||
merged.update(headers or {})
|
||||
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
|
||||
response = conn.getresponse()
|
||||
status, body = response.status,json.loads(response.read())
|
||||
conn.close()
|
||||
return status, body
|
||||
|
||||
|
||||
def test_endpoint_security_scope_and_success(self):
|
||||
for tid in ['target','source-a','source-b']:
|
||||
self.server.store.task(dict(id=tid,name=tid))
|
||||
path='/api/tasks/target/integrate'
|
||||
for headers,code in [({'Authorization':''},401),({'X-CSRF-Token':''},403),
|
||||
({'Origin':'https://untrusted.example'},403)]:
|
||||
self.assertEqual(self.request(path,'POST',integration(),headers)[0],code)
|
||||
self.assertEqual(self.request(path,'POST',integration(shell='no'))[0],400)
|
||||
self.assertEqual(self.request(path+'?x=1','POST',integration())[0],404)
|
||||
self.assertEqual(self.request('/api/tasks/target','PATCH',dict(id='source-a'))[0],400)
|
||||
code,body=self.request(path,'POST',integration())
|
||||
self.assertEqual(code,200)
|
||||
self.assertEqual(body['task']['merge_sources'],['source-a','source-b'])
|
||||
self.assertTrue(self.request(path,'POST',integration())[1]['duplicate'])
|
||||
self.assertEqual(self.request('/api/tasks/source-a/integrate','POST',
|
||||
integration(source_ids=['target']))[0],409)
|
||||
@@ -0,0 +1,222 @@
|
||||
"""Atomic worker receipts and execution versus acceptance state."""
|
||||
import concurrent.futures
|
||||
import http.client
|
||||
import json
|
||||
from pathlib import Path
|
||||
import tempfile
|
||||
import threading
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
from agent_control.store import Store, Invalid, Conflict
|
||||
from agent_control.server import Server
|
||||
|
||||
|
||||
def report(**changes):
|
||||
data=dict(report_id='report-1',task_id='task',session_id='worker',stage='execution_completed',
|
||||
current_progress='Implementation tested',summary='Execution finished',
|
||||
evidence=['Synthetic tests passed'],version='v1')
|
||||
data.update(changes)
|
||||
return data
|
||||
|
||||
|
||||
class ReportTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp=tempfile.TemporaryDirectory()
|
||||
self.path=Path(self.tmp.name)/'reports.sqlite3'
|
||||
self.store=Store(self.path)
|
||||
for sid in ['worker','other']:
|
||||
self.store.session(dict(id=sid,name=sid))
|
||||
self.store.task(dict(id='task',name='Work',session_id='worker',status='running'))
|
||||
self.store.task(dict(id='other-task',name='Other',session_id='other'))
|
||||
|
||||
def tearDown(self):
|
||||
self.tmp.cleanup()
|
||||
|
||||
def task(self):
|
||||
return next(t for t in self.store.state()['tasks'] if t['id']=='task')
|
||||
|
||||
def test_execution_completion_is_waiting_not_business_completion(self):
|
||||
result=self.store.task_report(report())
|
||||
self.assertFalse(result['duplicate'])
|
||||
task=result['task']
|
||||
self.assertEqual(task['execution_status'],'completed')
|
||||
self.assertEqual(task['status'],'waiting')
|
||||
self.assertEqual(task['phase'],'integration')
|
||||
self.assertEqual(task['current_progress'],'Implementation tested')
|
||||
self.assertEqual(task['report_evidence'],['Synthetic tests passed'])
|
||||
self.assertEqual(task['last_report_id'],'report-1')
|
||||
self.assertEqual(self.store.task_reports('task')['reports'],[result['report']])
|
||||
self.assertEqual(self.store.state()['attention']['items'],[])
|
||||
|
||||
def test_duplicate_persistent_receipt_returns_current_task_without_reapplying(self):
|
||||
first=self.store.task_report(report())
|
||||
self.store.task(dict(status='completed',phase='done',current_progress='Accepted'),'task')
|
||||
second=Store(self.path).task_report(report(needs_attention=False))
|
||||
self.assertTrue(second['duplicate'])
|
||||
self.assertEqual(second['report'],first['report'])
|
||||
self.assertEqual(second['task']['status'],'completed')
|
||||
self.assertEqual(second['task']['current_progress'],'Accepted')
|
||||
self.assertEqual(len(self.store.task_reports('task')['reports']),1)
|
||||
with self.assertRaises(Conflict): self.store.task_report(report(summary='Different'))
|
||||
with self.assertRaises(Conflict): self.store.task_report(report(task_id='other-task',session_id='other'))
|
||||
|
||||
def test_new_late_report_does_not_roll_back_integrated_or_accepted_task(self):
|
||||
self.store.task_report(report())
|
||||
for status in ['integrated','verified']:
|
||||
self.store.task(dict(integration_status=status,current_progress='Validated',version='accepted'),'task')
|
||||
before=self.task()
|
||||
result=self.store.task_report(report(report_id='late-'+status,stage='progress',
|
||||
current_progress='Old work',version='old'))
|
||||
self.assertEqual(result['task']['integration_status'],status)
|
||||
self.assertEqual(result['task']['current_progress'],'Validated')
|
||||
self.assertEqual(result['task']['execution_status'],'completed')
|
||||
self.assertEqual(result['task']['version'],'accepted')
|
||||
self.assertEqual(result['task']['progress_updated'],before['progress_updated'])
|
||||
self.store.task(dict(status='completed',phase='done'),'task')
|
||||
result=self.store.task_report(report(report_id='late-failure',stage='failed'))
|
||||
self.assertEqual(result['task']['status'],'completed')
|
||||
self.assertEqual(result['task']['phase'],'done')
|
||||
self.assertEqual(len(self.store.task_reports('task')['reports']),4)
|
||||
|
||||
def test_concurrent_completion_and_late_progress_preserve_waiting_but_new_blocker_applies(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
bodies=[report()]+[report(report_id='progress-'+str(i),stage='progress',
|
||||
current_progress='Older progress') for i in range(12)]
|
||||
list(pool.map(self.store.task_report,bodies))
|
||||
task=self.task()
|
||||
self.assertEqual(task['execution_status'],'completed')
|
||||
self.assertEqual(task['status'],'waiting')
|
||||
self.assertEqual(task['phase'],'integration')
|
||||
self.assertEqual(task['current_progress'],'Implementation tested')
|
||||
progress_updated=task['progress_updated']
|
||||
self.store.task_report(report(report_id='latest-arrival',stage='progress',
|
||||
current_progress='Still old progress'))
|
||||
task=self.task()
|
||||
self.assertEqual(task['current_progress'],'Implementation tested')
|
||||
self.assertEqual(task['progress_updated'],progress_updated)
|
||||
self.assertEqual(len(self.store.task_reports('task')['reports']),14)
|
||||
for stage in ['blocked','failed']:
|
||||
task=self.store.task_report(report(report_id='new-'+stage,stage=stage,
|
||||
current_progress='New issue discovered'))['task']
|
||||
self.assertEqual(task['status'],stage)
|
||||
self.assertEqual(task['execution_status'],stage)
|
||||
self.assertEqual(task['current_progress'],'New issue discovered')
|
||||
|
||||
def test_blocked_failed_progress_and_server_managed_defaults(self):
|
||||
self.assertEqual(self.task()['execution_status'],'pending')
|
||||
self.assertIsNone(self.task()['last_report_id'])
|
||||
for stage,expected in [('blocked','blocked'),('failed','failed'),('progress','running')]:
|
||||
task=self.store.task_report(report(report_id=stage,stage=stage))['task']
|
||||
self.assertEqual(task['execution_status'],expected)
|
||||
self.assertEqual(task['status'],expected)
|
||||
for field,value in [('execution_status','completed'),('last_report_id','fake'),
|
||||
('last_report_at',123),('report_evidence',['fake'])]:
|
||||
with self.assertRaises(Invalid): self.store.task({field:value},'task')
|
||||
self.store.task(dict(id='manual',name='Manual',status='completed',phase='done'))
|
||||
task=next(t for t in self.store.state()['tasks'] if t['id']=='manual')
|
||||
self.assertEqual(task['execution_status'],'pending')
|
||||
|
||||
def test_identity_and_schema_reject_without_mutation(self):
|
||||
before=self.task()
|
||||
cases=[dict(session_id='other'),dict(session_id='missing'),dict(task_id='other-task'),
|
||||
dict(stage='completed'),dict(current_progress=None),dict(current_progress='x'*1001),
|
||||
dict(summary='x'*2001),dict(evidence=['x']*21),dict(evidence=['x'*1001]),
|
||||
dict(evidence=[{}]),dict(version=None),dict(needs_attention=1),
|
||||
dict(needs_attention=True,summary=''),dict(source='admin')]
|
||||
for change in cases:
|
||||
with self.subTest(change=change):
|
||||
with self.assertRaises(Invalid): self.store.task_report(report(**change))
|
||||
self.assertEqual(self.task(),before)
|
||||
self.assertEqual(self.store.task_reports('task')['reports'],[])
|
||||
|
||||
def test_attention_only_on_explicit_true_and_same_task_item_updated(self):
|
||||
first=self.store.task_report(report(needs_attention=True))
|
||||
items=self.store.state()['attention']['items']
|
||||
self.assertEqual(len(items),1)
|
||||
aid=items[0]['id']
|
||||
self.store.task_report(report(report_id='r2',stage='blocked',needs_attention=True,summary='Need review'))
|
||||
items=self.store.state()['attention']['items']
|
||||
self.assertEqual(len(items),1)
|
||||
self.assertEqual(items[0]['id'],aid)
|
||||
self.assertEqual(items[0]['reason'],'Need review')
|
||||
self.store.attention(dict(id=aid,page='tasks',required=False,reason=''))
|
||||
self.store.task_report(report(needs_attention=True))
|
||||
self.assertEqual(self.store.state()['attention']['items'],[])
|
||||
self.assertEqual(self.store.hook_notifications()['pending_count'],0)
|
||||
self.assertEqual(self.store.decision_notifications()['pending_count'],0)
|
||||
|
||||
def test_failure_rolls_back_task_and_report_and_attention(self):
|
||||
before=self.task()
|
||||
with self.store.db() as db:
|
||||
db.execute("CREATE TRIGGER fail_report_attention BEFORE INSERT ON attention_history BEGIN SELECT RAISE(ABORT,'synthetic'); END")
|
||||
with self.assertRaises(Exception): self.store.task_report(report(needs_attention=True))
|
||||
self.assertEqual(self.task(),before)
|
||||
self.assertEqual(self.store.task_reports('task')['reports'],[])
|
||||
self.assertEqual(self.store.state()['attention']['items'],[])
|
||||
|
||||
def test_concurrent_receipt_only_one_record(self):
|
||||
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
|
||||
results=list(pool.map(lambda _:self.store.task_report(report()),range(12)))
|
||||
self.assertEqual(sum(not r['duplicate'] for r in results),1)
|
||||
self.assertEqual(len(self.store.task_reports('task')['reports']),1)
|
||||
|
||||
def test_recent_reports_are_bounded_and_task_scoped(self):
|
||||
for i in range(53):
|
||||
self.store.task_report(report(report_id='r-'+str(i),stage='progress'))
|
||||
self.store.task_report(report(report_id='other',task_id='other-task',session_id='other'))
|
||||
rows=self.store.task_reports('task')['reports']
|
||||
self.assertEqual(len(rows),50)
|
||||
self.assertEqual(rows[0]['report_id'],'r-52')
|
||||
self.assertTrue(all(row['task_id']=='task' for row in rows))
|
||||
self.assertEqual(len(self.store.task_reports('other-task')['reports']),1)
|
||||
with self.assertRaises(Invalid): self.store.task_reports('missing')
|
||||
|
||||
|
||||
class ReportHTTPTests(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.tmp = tempfile.TemporaryDirectory()
|
||||
root = Path(self.tmp.name)
|
||||
root.chmod(0o700)
|
||||
token = root / 'token'
|
||||
self.secret = 'h'*48
|
||||
token.write_text(self.secret)
|
||||
token.chmod(0o600)
|
||||
self.metrics = patch('agent_control.server.MetricsCache')
|
||||
mock = self.metrics.start()
|
||||
mock.return_value.snapshot.return_value = {'status':'synthetic'}
|
||||
self.server = Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(token)))
|
||||
self.thread = threading.Thread(target=self.server.serve_forever, daemon=True)
|
||||
self.thread.start()
|
||||
|
||||
def tearDown(self):
|
||||
self.server.shutdown()
|
||||
self.server.server_close()
|
||||
self.thread.join()
|
||||
self.metrics.stop()
|
||||
self.tmp.cleanup()
|
||||
|
||||
def request(self,path,method='GET',payload=None,headers=None):
|
||||
conn = http.client.HTTPConnection(*self.server.server_address,timeout=3)
|
||||
merged = {'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
|
||||
merged.update(headers or {})
|
||||
conn.request(method,path,json.dumps(payload) if payload is not None else None,merged)
|
||||
response = conn.getresponse()
|
||||
status, body = response.status,json.loads(response.read())
|
||||
conn.close()
|
||||
return status, body
|
||||
|
||||
|
||||
def test_security_query_and_roundtrip(self):
|
||||
self.server.store.session(dict(id='worker',name='Worker'))
|
||||
self.server.store.task(dict(id='task',name='Task',session_id='worker'))
|
||||
path='/api/task-reports'
|
||||
for headers,code in [({'Authorization':''},401),({'X-CSRF-Token':''},403),
|
||||
({'Origin':'https://untrusted.example'},403)]:
|
||||
self.assertEqual(self.request(path,'POST',report(),headers)[0],code)
|
||||
self.assertEqual(self.request(path+'?task_id=task',headers={'Authorization':''})[0],401)
|
||||
for query in ['', '?task_id=', '?task_id=task&task_id=other', '?session_id=worker']:
|
||||
self.assertEqual(self.request(path+query)[0],400)
|
||||
self.assertEqual(self.request(path,'POST',report())[0],200)
|
||||
self.assertTrue(self.request(path,'POST',report())[1]['duplicate'])
|
||||
self.assertEqual(self.request(path,'POST',report(summary='Changed'))[0],409)
|
||||
self.assertEqual(len(self.request(path+'?task_id=task')[1]['reports']),1)
|
||||
@@ -1,3 +1,4 @@
|
||||
import itertools
|
||||
import importlib.util
|
||||
import io
|
||||
import json
|
||||
@@ -27,7 +28,7 @@ class WorkerTests(unittest.TestCase):
|
||||
def test_desktop_queue_records_receipt_and_waits_for_final_hydration(self):
|
||||
import appserver
|
||||
with tempfile.TemporaryDirectory() as root:
|
||||
cfg={'codex':'codex','cwd':root,'max_turn_seconds':10}
|
||||
cfg={'codex':'codex','cwd':root,'max_turn_seconds':1000}
|
||||
message={'id':'a'*24,'text':'hello','source':'human'}
|
||||
prompt=worker.message_prompt(cfg,message)
|
||||
thread='11111111-1111-4111-8111-111111111111'
|
||||
@@ -44,7 +45,7 @@ class WorkerTests(unittest.TestCase):
|
||||
if self.calls>2:items.append({'type':'agentMessage','text':'real reply','phase':'final_answer'})
|
||||
return {'id':tid},[{'id':'observed-turn','status':'interrupted' if self.calls==2 else 'completed','items':items}]
|
||||
receipts=[]
|
||||
with patch.object(appserver,'AppServer',Reader),patch.object(worker.subprocess,'run',return_value=SimpleNamespace(returncode=0,stdout='Queued message '+receipt+' for thread '+thread+'.')),patch.object(worker.time,'sleep'):
|
||||
with patch.object(appserver,'AppServer',Reader),patch.object(worker.subprocess,'run',return_value=SimpleNamespace(returncode=0,stdout='Queued message '+receipt+' for thread '+thread+'.')),patch.object(worker.time,'sleep'),patch.object(worker.time,'monotonic',side_effect=itertools.count(0,15)):
|
||||
result=worker.run_queued(cfg,message,thread,lambda:None,lambda value:None,receipts.append)
|
||||
self.assertEqual(result['outcome'],'completed')
|
||||
self.assertEqual(result['text'],'real reply')
|
||||
|
||||
Reference in New Issue
Block a user