Add reusable multi-session skill and five-page project workbench

This commit is contained in:
aiagent
2026-10-19 03:03:29 -07:00
parent 939c14de77
commit 579dece104
75 changed files with 13096 additions and 0 deletions
+62
View File
@@ -0,0 +1,62 @@
import json
from http.server import BaseHTTPRequestHandler,ThreadingHTTPServer
from pathlib import Path
import sys
import tempfile
import threading
import unittest
from unittest.mock import patch
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'scripts'))
from control import Client,APIError
class ClientTests(unittest.TestCase):
def test_redirect_never_forwards_bearer_to_another_origin(self):
received=[]
class Target(BaseHTTPRequestHandler):
def log_message(self,*args):pass
def do_GET(self):
received.append(self.headers.get('Authorization'))
self.send_response(200);self.end_headers();self.wfile.write(b'{}')
target=ThreadingHTTPServer(('127.0.0.1',0),Target)
class Redirect(BaseHTTPRequestHandler):
def log_message(self,*args):pass
def do_GET(self):
self.send_response(302)
self.send_header('Location','http://127.0.0.1:'+str(target.server_port)+'/capture')
self.end_headers()
redirect=ThreadingHTTPServer(('127.0.0.1',0),Redirect)
threads=[threading.Thread(target=s.serve_forever,daemon=True) for s in (target,redirect)]
for t in threads:t.start()
try:
with tempfile.TemporaryDirectory() as root:
token=Path(root)/'token';token.write_text('test-only-token');token.chmod(0o600)
client=Client({'url':'http://127.0.0.1:'+str(redirect.server_port),'token_file':str(token)})
with self.assertRaises(APIError) as result:client.call('/api/state')
self.assertEqual(result.exception.status,302)
self.assertEqual(received,[])
finally:
for s in (redirect,target):s.shutdown();s.server_close()
for t in threads:t.join()
def test_ssh_json_is_explicit_utf8_and_preserves_chinese(self):
from types import SimpleNamespace
client=Client({'transport':'ssh','identity_file':'example-key','ssh_host':'observer@entry.example.test',
'remote_client':'/home/observer/agent-workbench/scripts/control.py','remote_config':'/home/observer/private/config.json'})
with patch('control.subprocess.run',return_value=SimpleNamespace(returncode=0,stdout=json.dumps({'text':'中文回复'}))) as run:
self.assertEqual(client.call('/api/state')['text'],'中文回复')
self.assertEqual(run.call_args.kwargs['encoding'],'utf-8')
self.assertIn('python3 -X utf8',run.call_args.args[0][-1])
def test_ssh_preserves_conflict_status_for_reconciliation(self):
from types import SimpleNamespace
client=Client({'transport':'ssh','identity_file':'example-key','ssh_host':'observer@entry.example.test',
'remote_client':'/home/observer/agent-workbench/scripts/control.py','remote_config':'/home/observer/private/config.json'})
with patch('control.subprocess.run',return_value=SimpleNamespace(returncode=2,stdout='{"api_error":409}')):
with self.assertRaises(APIError) as result:client.call('/api/state')
self.assertEqual(result.exception.status,409)
if __name__=='__main__':unittest.main()
+379
View File
@@ -0,0 +1,379 @@
"""Behavioral regression for the independent agent coordination service."""
import concurrent.futures
import http.client
import hashlib
import json
import os
from pathlib import Path
import sys
import tempfile
import threading
import unittest
from unittest.mock import patch
ROOT = Path(__file__).resolve().parents[1]
sys.path.insert(0, str(ROOT))
from agent_control.store import Store, Invalid, Conflict
from agent_control.server import Server, restricted_token
class StoreTests(unittest.TestCase):
def setUp(self):
self.tmp = tempfile.TemporaryDirectory()
self.path = Path(self.tmp.name) / 'db.sqlite3'
self.store = Store(self.path)
for sid in ('a','b'):
self.store.session(dict(id=sid,name=sid,wp='WP1'))
def tearDown(self):
self.tmp.cleanup()
def send(self, sid='a', key='one'):
return self.store.send(dict(session_id=sid,text='question',idempotency_key=key))
def claim(self, sid='a', worker='w'):
return self.store.claim(dict(session_id=sid,worker_id=worker))['message']
def completion(self, message, **overrides):
data=dict(session_id=message['session_id'],worker_id='w',message_id=message['id'],lease_id=message['lease_id'],text='real output',thread_id='thread-a')
data.update(overrides)
return data
def test_persistence_and_idempotency(self):
one = self.send()
self.assertEqual(one['id'], self.send()['id'])
reopened = Store(self.path)
self.assertEqual(reopened.messages('a')[0]['id'], one['id'])
with self.assertRaises(Conflict):
reopened.send(dict(session_id='a',text='different',idempotency_key='one'))
self.assertEqual(len(reopened.messages('a')),1)
def test_late_history_final_reconciles_without_duplicate_reply(self):
self.store.session(dict(id='a',name='a',thread_id='thread-a'))
original=self.send()
lease=self.claim()
owner=dict(session_id='a',worker_id='w',message_id=original['id'],lease_id=lease['lease_id'])
self.store.start(owner)
proof='exact submitted input'
self.store.submission(dict(owner,state='submitting',thread_id='thread-a',input_sha256=hashlib.sha256(proof.encode()).hexdigest()))
self.store.uncertain(owner)
batch=dict(session_id='a',thread_id='thread-a',items=[
dict(id='external-user',source='human',text='question',input_text=proof,related_message_id=original['id'],phase='input',turn_id='turn-a',turn_status='completed'),
dict(id='external-final',source='agent',text='real output',input_text=proof,related_message_id=original['id'],phase='final_answer',turn_id='turn-a',turn_status='completed')])
self.store.mirror(batch)
self.store.mirror(batch)
self.assertEqual(len(self.store.messages('a')),2)
self.assertEqual(self.store.messages('a')[0]['status'],'completed')
self.store.complete(self.completion(lease))
self.assertEqual(len(self.store.messages('a')),2)
with self.store.db() as db:
link=json.loads(db.execute('SELECT data FROM history_links WHERE external_id=?',('external-final',)).fetchone()['data'])
self.assertEqual(link['turn_id'],'turn-a')
def test_submission_is_bound_persistent_and_fenced(self):
self.store.session(dict(id='a',name='a',thread_id='thread-a'))
self.send()
lease=self.claim()
owner=self.completion(lease)
digest=hashlib.sha256(b'exact input').hexdigest()
data=dict(owner,state='submitting',input_sha256=digest)
with self.assertRaises(Conflict): self.store.submission(data)
self.store.start(owner)
self.store.submission(data)
self.store.submission(dict(data,state='accepted',receipt_id='receipt-a'))
self.store.submission(dict(data,state='observed',turn_id='turn-a'))
value=Store(self.path).messages('a')[0]['submission']
self.assertEqual(value['receipt_id'],'receipt-a')
self.assertEqual(value['turn_id'],'turn-a')
self.assertEqual(self.store.submission(data)['submission']['state'],'observed')
for change in ({'worker_id':'other'},{'thread_id':'wrong'},{'input_sha256':'b'*64},
{'state':'accepted','receipt_id':'other'},{'state':'observed','turn_id':'other'}):
with self.assertRaises(Conflict): self.store.submission(dict(data,**change))
with patch('agent_control.store.time.time',return_value=lease['lease_until']+1):
with self.assertRaises(Conflict): self.store.submission(data)
def test_mirror_requires_registered_full_input_and_turn(self):
self.store.session(dict(id='a',name='a',thread_id='thread-a'))
original=self.send(); lease=self.claim(); owner=self.completion(lease)
self.store.start(owner)
base=dict(session_id='a',thread_id='thread-a')
item=dict(id='f',source='agent',text='result',related_message_id=original['id'],
input_text='exact input',phase='final_answer',turn_id='turn-a',turn_status='completed')
with self.assertRaises(Conflict): self.store.mirror(dict(base,items=[item]))
self.store.submission(dict(owner,state='observed',turn_id='turn-a',input_sha256=hashlib.sha256(b'exact input').hexdigest()))
for change in ({'input_text':'forged exact input'},{'turn_id':'other-turn'}):
with self.assertRaises(Conflict): self.store.mirror(dict(base,items=[dict(item,**change)]))
self.assertIsNone(self.store.messages('a')[0]['response_id'])
def test_long_input_final_deferred_then_completed_without_duplicates(self):
self.store.session(dict(id='a',name='a',thread_id='thread-a'))
original=self.store.send(dict(session_id='a',text='q'*8001,idempotency_key='long'))
lease=self.claim(); owner=self.completion(lease); self.store.start(owner)
proof='envelope\n'+original['text']; answer='r'*8001
self.store.submission(dict(owner,state='accepted',receipt_id='r',input_sha256=hashlib.sha256(proof.encode()).hexdigest()))
items=[dict(id='u',source='human',text=original['text'],phase='input'),
dict(id='f',source='agent',text=answer,phase='final_answer')]
for i in items: i.update(related_message_id=original['id'],input_text=proof,turn_id='t',turn_status='completed')
batch=dict(session_id='a',thread_id='thread-a',items=items)
self.assertEqual(self.store.mirror(batch)['deferred'],['f'])
self.assertEqual(len(self.store.messages('a')),1)
self.store.complete(dict(owner,text=answer))
self.assertEqual(self.store.mirror(batch)['deferred'],[])
self.assertEqual(len(self.store.messages('a')),2)
self.assertEqual(self.store.messages('a')[1]['text'],answer)
def test_long_late_final_recovers_uncertain_and_rejects_oversize(self):
self.store.session(dict(id='a',name='a',thread_id='thread-a'))
original=self.send(); lease=self.claim(); owner=self.completion(lease); self.store.start(owner)
self.store.submission(dict(owner,state='submitting',input_sha256=hashlib.sha256(b'proof').hexdigest()))
self.store.uncertain(owner)
item=dict(id='f',source='agent',text='r'*8001,phase='final_answer',related_message_id=original['id'],
input_text='proof',turn_id='late',turn_status='completed')
batch=dict(session_id='a',thread_id='thread-a',items=[item])
self.store.mirror(batch); self.store.mirror(batch)
self.assertEqual(self.store.messages('a')[0]['status'],'completed')
self.assertEqual(len(self.store.messages('a')),2)
self.store.complete(dict(owner,text='r'*8001))
with self.assertRaises(Invalid): self.store.mirror(dict(batch,items=[dict(item,text='x'*262145)]))
with self.assertRaises(Invalid): self.store.complete(dict(owner,text='x'*262145))
self.assertEqual(len(self.store.messages('a')),2)
def test_live_visible_commentary_updates_same_item(self):
self.store.session(dict(id='a',name='a',thread_id='thread-a'))
payload=dict(session_id='a',thread_id='thread-a',items=[dict(id='comment',source='agent',text='first',phase='commentary',allow_update=True)])
self.store.mirror(payload)
payload['items'][0]['text']='updated'
self.store.mirror(payload)
self.assertEqual(len(self.store.messages('a')),1)
self.assertEqual(self.store.messages('a')[0]['text'],'updated')
def test_claim_is_atomic_and_serial_per_session(self):
self.send()
self.send(key='two')
with concurrent.futures.ThreadPoolExecutor(max_workers=8) as pool:
results=list(pool.map(lambda n:self.claim(worker='w'+str(n)),range(8)))
self.assertEqual(sum(value is not None for value in results),1)
self.assertIsNone(self.claim('b'))
def test_cross_session_replies_and_completion_rejected(self):
one=self.send()
with self.assertRaises(Invalid):
self.store.send(dict(session_id='b',text='bad',source='agent',reply_to=one['id'],idempotency_key='reply'))
claimed=self.claim()
with self.assertRaises(Conflict):
self.store.complete(self.completion(claimed,session_id='b'))
self.assertEqual(self.store.messages('b'),[])
self.assertEqual(self.store.messages('a')[0]['status'],'claimed')
def test_expired_lease_fencing_and_renew(self):
self.send()
with patch('agent_control.store.time.time', return_value=100):
first=self.claim()
with patch('agent_control.store.time.time', return_value=200):
result=self.store.renew(dict(self.completion(first),lease_seconds=300))
self.assertEqual(result['lease_until'],500)
with patch('agent_control.store.time.time', return_value=501):
with self.assertRaises(Conflict):
self.store.renew(self.completion(first))
second=self.claim()
self.assertNotEqual(first['lease_id'],second['lease_id'])
with self.assertRaises(Conflict):
self.store.complete(self.completion(first))
self.store.complete(self.completion(second))
self.assertEqual(len(self.store.messages('a')),2)
def test_completion_is_atomic_idempotent_and_failure_terminal(self):
self.send()
claim=self.claim()
response=self.store.complete(self.completion(claim))
self.assertEqual(response['id'],self.store.complete(self.completion(claim))['id'])
self.assertEqual(len(Store(self.path).messages('a')),2)
self.assertIsNone(self.claim())
self.send(key='two')
next_claim=self.claim()
self.store.complete(self.completion(next_claim,outcome='failed',thread_id=None,text='authentication unavailable'))
self.assertEqual(self.store.messages('a')[-2]['status'],'failed')
self.assertIsNone(self.claim())
def test_task_validation_is_atomic(self):
self.store.task(dict(id='WP0',name='总控',status='running',phase='planning',eta_min_minutes=2,eta_max_minutes=5,eta_basis='remaining checks'))
before=self.store.state()['tasks']
for update in ({'phase':'imaginary'},{'eta_max_minutes':1},{'eta_min_minutes':float('nan')},{'eta_basis':''},{'status':'completed'}):
with self.assertRaises(Invalid):
self.store.task(update,'WP0')
self.assertEqual(before,self.store.state()['tasks'])
self.store.task(dict(status='completed',phase='done'),'WP0')
def test_started_expiry_blocks_session_until_reconciliation(self):
self.send()
self.send(key='next')
with patch('agent_control.store.time.time', return_value=100):
first=self.claim()
self.store.start(self.completion(first))
with patch('agent_control.store.time.time', return_value=401):
result=self.store.claim(dict(session_id='a',worker_id='other'))
self.assertIsNone(result['message'])
self.assertTrue(result['needs_reconciliation'])
with self.assertRaises(Conflict):
self.store.complete(self.completion(first))
self.assertEqual(self.store.messages('a')[0]['outcome'],'uncertain')
self.assertNotIn('lease_id',self.store.messages('a')[0])
self.store.reconcile(dict(session_id='a',message_id=first['id'],action='failed',resolution='人工确认已产生副作用,禁止重跑'))
second=self.claim()
self.assertNotEqual(first['id'],second['id'])
def test_uncertain_signal_fences_worker_and_requires_explicit_retry(self):
self.send()
first=self.claim()
self.store.start(self.completion(first))
with self.assertRaises(Conflict):
self.store.uncertain(self.completion(first,worker_id='other'))
self.store.uncertain(self.completion(first))
self.assertIsNone(self.claim())
with self.assertRaises(Invalid):
self.store.reconcile(dict(session_id='a',message_id=first['id'],action='retry',resolution=''))
self.store.reconcile(dict(session_id='a',message_id=first['id'],action='retry',resolution='人工确认未产生副作用,允许重试'))
second=self.claim()
self.assertEqual(first['id'],second['id'])
self.assertNotEqual(first['lease_id'],second['lease_id'])
with self.assertRaises(Conflict):
self.store.complete(self.completion(first))
def test_mirror_is_bound_idempotent_and_never_executable(self):
self.store.session(dict(id='a',name='bound',thread_id='actual-thread'))
batch=dict(session_id='a',thread_id='actual-thread',items=[
dict(id='external-1',source='human',text='historical question',created=10),
dict(id='external-1',source='agent',text='historical reply',created=11)])
self.assertEqual(self.store.mirror(batch)['imported'],2)
self.assertEqual(self.store.mirror(batch)['imported'],0)
self.assertEqual(self.store.state()['sessions'][0]['connection'],'waiting')
self.assertIsNone(self.claim())
rows=self.store.messages('a')
self.assertTrue(all(r['status']=='delivered' and r['origin']=='session_history' for r in rows))
self.assertEqual(rows[0]['created'],10)
self.assertEqual(self.store.messages('b'),[])
with self.assertRaises(Conflict):
self.store.mirror(dict(batch,thread_id='other-thread'))
with self.assertRaises(Conflict):
self.store.mirror(dict(batch,session_id='b'))
def test_mirror_batch_failure_rolls_back_and_rejects_changed_items(self):
self.store.session(dict(id='a',name='bound',thread_id='actual-thread'))
base=dict(session_id='a',thread_id='actual-thread')
one=dict(id='one',source='human',text='first')
self.store.mirror(dict(base,items=[one]))
with self.assertRaises(Conflict):
self.store.mirror(dict(base,items=[dict(one,id='new'),dict(one,text='changed')]))
self.assertEqual(len(self.store.messages('a')),1)
for items in ([dict(one,created=float('nan'))],[one]*101,[dict(one,source='tool')]):
with self.assertRaises(Invalid):
self.store.mirror(dict(base,items=items))
self.assertEqual(len(self.store.messages('a')),1)
def test_requested_is_not_real_session(self):
self.store.request(dict(name='new',prompt='work'))
state=self.store.state()
self.assertEqual(len(state['sessions']),2)
self.assertEqual(state['requests'][0]['status'],'requested')
self.assertTrue(all(s['connection']=='waiting' for s in state['sessions']))
class HTTPTests(unittest.TestCase):
def setUp(self):
self.tmp=tempfile.TemporaryDirectory()
root=Path(self.tmp.name)
root.chmod(0o700)
self.secret='a'*48
self.token=root/'token'
self.token.write_text(self.secret)
self.token.chmod(0o600)
web=root/'web'
web.mkdir()
(web/'index.html').write_text('<title>Agent Workbench</title>')
(web/'app.js').write_text('console.log("test fixture")')
self.server=Server(dict(bind='127.0.0.1',port=0,runtime_root=str(root),token_file=str(self.token),web_root=str(web)))
self.thread=threading.Thread(target=self.server.serve_forever,daemon=True)
self.thread.start()
def tearDown(self):
self.server.shutdown()
self.server.server_close()
self.thread.join()
self.tmp.cleanup()
def request(self,path='/api/state',method='GET',data=None,headers=None):
conn=http.client.HTTPConnection(*self.server.server_address,timeout=3)
base={'Authorization':'Bearer '+self.secret,'X-CSRF-Token':'1','Content-Type':'application/json'}
base.update(headers or {})
conn.request(method,path,body=json.dumps(data) if data is not None else None,headers=base)
response=conn.getresponse()
body=response.read()
result=(response.status,body,response.getheaders())
conn.close()
return result
def test_auth_host_origin_csrf_and_static_boundary(self):
self.assertEqual(self.request()[0],200)
self.assertEqual(self.request(headers={'Authorization':''})[0],401)
self.assertEqual(self.request(headers={'Host':'evil.example'})[0],403)
self.assertEqual(self.request(headers={'Origin':'https://evil.example'})[0],403)
self.assertEqual(self.request('/api/sessions','POST',{'name':'test'}, {'X-CSRF-Token':''})[0],403)
self.assertEqual(self.request('/../store.py')[0],404)
self.assertEqual(self.request('/%2e%2e/server.py')[0],404)
self.assertEqual(self.request('/?token=secret')[0],404)
code,body,headers=self.request('/',headers={'Authorization':''})
self.assertEqual(code,200)
self.assertIn(b'Agent Workbench',body)
self.assertIn('Content-Security-Policy',dict(headers))
def test_http_roundtrip_and_errors_do_not_leak(self):
code,body,_=self.request('/api/sessions','POST',{'id':'a','name':'session'})
self.assertEqual(code,200)
code,body,_=self.request('/api/messages','POST',{'session_id':'a','text':'hello','idempotency_key':'k'})
self.assertEqual(code,200)
code,body,_=self.request('/api/worker/claim','POST',{'session_id':'a','worker_id':'w'})
message=json.loads(body)['message']
self.assertEqual(code,200)
code,body,_=self.request('/api/worker/complete','POST',dict(session_id='a',worker_id='w',message_id=message['id'],lease_id=message['lease_id'],thread_id='real-thread',text='output'))
self.assertEqual(code,200)
code,body,_=self.request('/api/messages?session_id=a')
self.assertEqual(len(json.loads(body)['messages']),2)
with patch.object(self.server.store,'state',side_effect=RuntimeError(self.secret)):
code,body,_=self.request()
self.assertEqual(code,500)
self.assertNotIn(self.secret.encode(),body)
def test_http_mirror_accepts_bounded_long_history(self):
self.request('/api/sessions','POST',dict(id='history',name='history',thread_id='bound-thread'))
batch=dict(session_id='history',thread_id='bound-thread',items=[dict(id='long',source='agent',text='历史正文'*6000)])
self.assertGreater(len(json.dumps(batch).encode()),65536)
code,body,_=self.request('/api/mirror','POST',batch)
self.assertEqual(code,200)
self.assertEqual(json.loads(body)['imported'],1)
code,body,_=self.request('/api/worker/claim','POST',dict(session_id='history',worker_id='w'))
self.assertIsNone(json.loads(body)['message'])
def test_http_submission_and_large_completion(self):
self.request('/api/sessions','POST',dict(id='large',name='large',thread_id='thread-large'))
self.request('/api/messages','POST',dict(session_id='large',text='q',idempotency_key='one'))
_,body,_=self.request('/api/worker/claim','POST',dict(session_id='large',worker_id='w'))
claimed=json.loads(body)['message']
owner=dict(session_id='large',worker_id='w',message_id=claimed['id'],lease_id=claimed['lease_id'])
self.assertEqual(self.request('/api/worker/start','POST',owner)[0],200)
payload=dict(owner,state='submitting',thread_id='thread-large',input_sha256=hashlib.sha256(b'input').hexdigest())
self.assertEqual(self.request('/api/worker/submission','POST',payload)[0],200)
text='result'*20000
result=dict(owner,thread_id='thread-large',text=text)
self.assertGreater(len(json.dumps(result)),65536)
self.assertEqual(self.request('/api/worker/complete','POST',result)[0],200)
self.assertEqual(self.request('/api/worker/complete','POST',dict(result,text='x'*262145))[0],400)
@unittest.skipUnless(os.name=='posix','POSIX permissions')
def test_permissive_secret_file_rejected(self):
self.token.chmod(0o644)
with self.assertRaises(ValueError):
restricted_token(self.token)
if __name__=='__main__':
unittest.main()
+150
View File
@@ -0,0 +1,150 @@
"""Synthetic public fixtures: no site inventory or production measurements."""
import json
from pathlib import Path
import subprocess
import sys
import time
import unittest
from unittest.mock import Mock, patch
sys.path.insert(0, str(Path(__file__).resolve().parents[1]))
from agent_control import metrics as m
NODE = 'MemTotal: 1000 kB\nMemAvailable: 400 kB\n__LOAD__\n1.20 2.30 3.40 1/20 123\n'
NFS = 'Filesystem Type 1B-blocks Used Avail Use% Mounted on\nstorage.example.test:/export nfs4 1000 300 700 30% /mnt/shared\n'
HOSTS = 'HOST_NAME STATUS JL/U MAX NJOBS RUN SSUSP USUSP RSV\nnode-a.example.test ok - 8 6 6 0 0 0\nnode-b.example.test unavail - 1 0 0 0 0 0\n'
QUEUES = 'QUEUE_NAME PRIO STATUS MAX JL/U JL/P JL/H NJOBS PEND RUN SUSP RSV\nbatch 30 Open:Active - - - - 6 0 6 0 0\n'
CONFIG = {'entry_hostname': 'entry.example.test', 'hosts': ['entry.example.test', 'node-a.example.test'],
'authorized_hosts': ['node-a.example.test'], 'ssh_user': 'observer',
'ssh_identity_file': '/home/observer/.ssh/metrics',
'storage_hosts': ['entry.example.test'],
'wrapper_paths': {'slurm': '/opt/site/bin/scheduler-exec', 'lsf': '/opt/site/bin/scheduler-exec'}}
class MetricsTests(unittest.TestCase):
def test_memory_load_units(self):
row = m.parse_node(NODE)
self.assertEqual(row['memory_total_bytes'], 1024000)
self.assertEqual(row['memory_available_bytes'], 409600)
self.assertEqual(row['memory_used_percent'], 60)
self.assertEqual(row['load_15m'], 3.4)
def test_missing_memory_and_nan_are_not_zero(self):
for data in (NODE.replace('MemAvailable:', 'Other:'), NODE.replace('1.20', 'nan')):
with self.assertRaises((ValueError, KeyError)):
m.parse_node(data)
def test_nfs_actual_mount(self):
row = m.parse_nfs(NFS)[0]
self.assertEqual(row['mount_source'], 'storage.example.test:/export')
self.assertEqual(row['available_bytes'], 700)
self.assertEqual(row['used_percent'], 30)
with self.assertRaises(ValueError):
m.parse_nfs('Permission denied')
def test_slurm_dedup_and_empty_jobs(self):
line = 'node-a.example.test|idle|8|16000\n'
self.assertEqual(m.parse_slurm_nodes(line * 2)['count'], 1)
self.assertEqual(m.parse_jobs('')['count'], 0)
self.assertEqual(m.parse_jobs('RUNNING\nPENDING\nRUNNING')['by_state'], {'RUNNING': 2, 'PENDING': 1})
with self.assertRaises(ValueError):
m.parse_jobs('Permission denied')
def test_lsf_slots_are_not_job_records(self):
hosts = m.parse_lsf_table(HOSTS, 'hosts')
self.assertEqual(hosts['hosts'][1]['state'], 'unavail')
self.assertEqual(hosts['hosts'][0]['run'], 6)
self.assertEqual(m.parse_lsf_table(QUEUES, 'queues')['queues'][0]['run'], 6)
self.assertEqual(m.parse_jobs('RUN\nRUN\n')['count'], 2)
with self.assertRaises(ValueError):
m.parse_lsf_table(HOSTS.replace('6', '-'), 'hosts')
def test_default_only_executes_local_node(self):
with patch.object(m, '_run', return_value=NODE) as run:
result = m.collect({})
self.assertEqual(run.call_count, 1)
self.assertEqual(run.call_args.args[:2], ('localhost', 'node'))
self.assertEqual(result['storage'], [])
self.assertEqual(result['schedulers']['slurm']['jobs']['status'], 'not_configured')
self.assertIsNone(result['schedulers']['lsf']['jobs']['data'])
self.assertEqual(result['status'], 'partial')
def test_collect_partial_no_fabricated_data(self):
def run(host, collector, timeout, settings):
if host != settings['entry']:
raise RuntimeError('permission denied (exit 255)')
return {'node': NODE, 'nfs': NFS, 'slurm_nodes': 'node-a.example.test|idle|8|16000',
'slurm_jobs': '', 'lsf_hosts': HOSTS, 'lsf_queues': QUEUES, 'lsf_jobs': 'RUN\nRUN'}[collector]
with patch.object(m, '_run', side_effect=run):
result = m.collect(CONFIG)
self.assertEqual(result['status'], 'partial')
self.assertIsNone(result['nodes'][1]['data'])
self.assertEqual(len(result['errors']), 1)
self.assertEqual(result['schedulers']['lsf']['jobs']['data']['count'], 2)
self.assertTrue(result['collected_at'].endswith('Z'))
json.dumps(result, allow_nan=False)
def test_config_rejects_shell_and_unapproved_targets(self):
invalid = [None, {'command': 'id'}, {'hosts': ['node-a.example.test']},
{'hosts': ['localhost', 'localhost']}, {'hosts': ['localhost'], 'nodes': []},
{'schedulers': ['lsf; id']}, {'total_timeout_seconds': float('nan')},
{'hosts': 'all'}, {'wrapper_paths': {'slurm': '/bin/sh -c id'}},
{'wrapper_paths': {'slurm': '/opt/../bin/sh'}}, {'ssh_user': '-oProxyCommand=id'},
{'entry_hostname': 'localhost;id'}, {'authorized_hosts': [['bad']]}]
with patch.object(m, '_run') as run:
for config in invalid:
with self.subTest(config=config):
self.assertEqual(m.collect(config)['status'], 'error')
run.assert_not_called()
def test_timeout_no_zero(self):
with patch.object(m, '_run', side_effect=TimeoutError('collector timed out')):
result = m.collect({'schedulers': []})
self.assertEqual(result['status'], 'error')
self.assertIsNone(result['nodes'][0]['data'])
def test_real_subprocess_timeout_kills_group(self):
with patch.dict(m.COMMANDS, {'node': [sys.executable, '-c', 'import time; time.sleep(10)']}):
started = time.monotonic()
with self.assertRaises(TimeoutError):
m._run('localhost', 'node', 0.15, m._settings({}))
self.assertLess(time.monotonic() - started, 2)
def test_remote_ssh_strict_fixed_and_identity_not_source(self):
process = Mock(returncode=0)
process.communicate.return_value = (NODE, '')
settings = m._settings(CONFIG)
with patch.object(subprocess, 'Popen', return_value=process) as popen:
record = m._sample('node-a.example.test', 'node', settings, time.monotonic()+5)
command = popen.call_args.args[0]
self.assertIn('StrictHostKeyChecking=yes', command)
self.assertIn('BatchMode=yes', command)
self.assertIn('observer@node-a.example.test', command)
self.assertIn(CONFIG['ssh_identity_file'], command)
self.assertNotIn(CONFIG['ssh_identity_file'], record['source'])
self.assertNotIn('shell', popen.call_args.kwargs)
def test_scheduler_argv_and_empty_lsf(self):
command = m._command('slurm_jobs', m._settings(CONFIG))
self.assertEqual(command, ['/opt/site/bin/scheduler-exec', 'slurm', 'squeue', '-h', '-o', '%T'])
process = Mock(returncode=255)
process.communicate.return_value = ('', 'No unfinished job found.\n')
with patch.object(subprocess, 'Popen', return_value=process):
self.assertEqual(m._run('entry.example.test', 'lsf_jobs', 1, m._settings(CONFIG)), '')
def test_deadline_does_not_start_command(self):
with patch.object(m, '_run') as run:
result = m._sample('localhost', 'node', m._settings({}), time.monotonic()-1)
run.assert_not_called()
self.assertEqual(result['status'], 'error')
self.assertIsNone(result['data'])
def test_missing_wrapper_is_error_not_empty_jobs(self):
with patch.object(subprocess, 'Popen', side_effect=FileNotFoundError):
result = m.collect(dict(CONFIG, hosts=[], storage_hosts=[]))
self.assertEqual(result['status'], 'error')
self.assertIsNone(result['schedulers']['lsf']['jobs']['data'])
if __name__ == '__main__':
unittest.main()
+109
View File
@@ -0,0 +1,109 @@
import sys
import hashlib
import json
from pathlib import Path
import tempfile
import unittest
ROOT=Path(__file__).resolve().parents[1]
sys.path.insert(0,str(ROOT/'scripts'))
from mirror import visible_items,sync
from worker import message_prompt
class MirrorTests(unittest.TestCase):
def test_quoted_marker_does_not_hide_human_or_reply(self):
turns=[{'id':'t','status':'completed','items':[
{'type':'userMessage','id':'u','content':[{'type':'text','text':'Please explain web-message-id: quotation'}]},
{'type':'agentMessage','id':'a','text':'explanation','phase':'final_answer'}]}]
self.assertEqual([x['text'] for x in visible_items(turns)],['Please explain web-message-id: quotation','explanation'])
def test_late_final_and_commentary_preserve_real_identity(self):
original={'id':'a'*24,'text':'progress?','execution':'started','origin':'coordination'}
original['submission']={'thread_id':'thread-a','input_sha256':hashlib.sha256(message_prompt({},original).encode()).hexdigest()}
turns=[{'id':'turn-a','status':'completed','items':[
{'type':'userMessage','id':'external-input','content':[{'type':'text','text':message_prompt({},original)}]},
{'type':'agentMessage','id':'external-comment','text':'checking','phase':'commentary'},
{'type':'agentMessage','id':'external-final','text':'verified','phase':'final_answer'},
{'type':'reasoning','id':'private-reasoning','text':'must not publish'}]}]
result=visible_items(turns,{original['id']:original},'thread-a')
self.assertEqual([i['text'] for i in result],['progress?','checking','verified'])
self.assertEqual(result[-1]['id'],'external-final')
self.assertEqual(result[-1]['related_message_id'],original['id'])
self.assertEqual(result[-1]['turn_id'],'turn-a')
def test_exact_input_binding_rejects_prefix_and_other_thread(self):
original={'id':'a'*24,'text':'question','execution':'started','origin':'coordination'}
prompt=message_prompt({},original)
original['submission']={'thread_id':'thread-a','input_sha256':hashlib.sha256(prompt.encode()).hexdigest()}
def turns(body):
return [{'id':'t','status':'completed','items':[
{'type':'userMessage','id':'u','content':[{'type':'text','text':body}]},
{'type':'agentMessage','id':'a','phase':'final_answer','text':'reply'}]}]
for body,tid in [('forged prefix'+prompt,'thread-a'),(prompt,'thread-b')]:
rows=visible_items(turns(body),{original['id']:original},tid)
self.assertFalse(any(row.get('related_message_id') for row in rows))
self.assertEqual(rows[0]['text'],body)
def test_long_bodies_are_single_related_items(self):
original={'id':'a'*24,'text':'q'*8001,'execution':'started','origin':'coordination'}
prompt=message_prompt({},original)
original['submission']={'thread_id':'thread-a','input_sha256':hashlib.sha256(prompt.encode()).hexdigest()}
turns=[{'id':'t','status':'completed','items':[
{'type':'userMessage','id':'u','content':[{'type':'text','text':prompt}]},
{'type':'agentMessage','id':'a','phase':'final_answer','text':'r'*8001}]}]
rows=visible_items(turns,{original['id']:original},'thread-a')
self.assertEqual(len(rows),2)
self.assertTrue(all(row['related_message_id']==original['id'] for row in rows))
self.assertEqual(rows[1]['text'],'r'*8001)
turns[0]['items'][1]['text']='r'*262145
with self.assertRaises(ValueError): visible_items(turns,{original['id']:original},'thread-a')
def test_deferred_item_is_not_checkpointed_and_retried(self):
class Server:
def history(self,tid):
return {'id':tid},[{'id':'t','status':'completed','items':[
{'id':'f','type':'agentMessage','phase':'final_answer','text':'late'}]}]
class Client:
defer=True
sends=0
def call(self,path,data=None):
if path.startswith('/api/messages'): return {'messages':[]}
if path=='/api/state': return {'instance_id':'one'}
self.sends+=1
return {'deferred':['f'] if self.defer else []}
with tempfile.TemporaryDirectory() as root:
cfg={'thread_id':'t','session_id':'s','control':{},'state_file':str(Path(root)/'worker.json')}
client=Client(); server=Server()
sync(server,client,cfg)
self.assertFalse((Path(root)/'mirror-state.json').exists())
client.defer=False
sync(server,client,cfg); sync(server,client,cfg)
self.assertEqual(client.sends,2)
self.assertIn('f',json.loads((Path(root)/'mirror-state.json').read_text())['seen'])
def test_active_commentary_visible_but_unfinished_final_not_published(self):
turn={'id':'t','status':'inProgress','items':[
{'type':'agentMessage','id':'c','text':'running','phase':'commentary'},
{'type':'agentMessage','id':'f','text':'partial','phase':'final_answer'}]}
self.assertEqual([x['text'] for x in visible_items([turn])],['running'])
def test_checkpoint_rejects_new_service_instance(self):
class Server:
def history(self,tid): return {'id':tid},[]
class Client:
instance='one'
def call(self,path,data=None):
return {'messages':[]} if path.startswith('/api/messages') else {'instance_id':self.instance}
with tempfile.TemporaryDirectory() as root:
cfg={'thread_id':'t','session_id':'s','control':{'url':'http://127.0.0.1:8796'},'state_file':str(Path(root)/'worker.json')}
client=Client()
# An empty initial mirror still establishes its target binding.
from worker import atomic_json
atomic_json(Path(root)/'mirror-state.json',{'binding':{'schema_version':2,'thread_id':'t','session_id':'s','control':cfg['control'],'instance_id':'one'},'seen':{}})
sync(Server(),client,cfg)
client.instance='two'
with self.assertRaisesRegex(RuntimeError,'another control'): sync(Server(),client,cfg)
if __name__=='__main__':unittest.main()
+104
View File
@@ -0,0 +1,104 @@
import importlib.util
import io
import json
from pathlib import Path
import sys
import tempfile
import unittest
from unittest.mock import patch
from types import SimpleNamespace
ROOT=Path(__file__).resolve().parents[1]
SCRIPT=ROOT/'scripts'
sys.path.insert(0,str(SCRIPT))
import worker
class Process:
pid=12345
def __init__(self,events):
self.stdin=io.StringIO()
self.stdout=io.StringIO(''.join(json.dumps(e)+'\n' for e in events))
def wait(self,timeout=None): return 0
def poll(self): return 0
class WorkerTests(unittest.TestCase):
def test_desktop_queue_records_receipt_and_waits_for_final_hydration(self):
import appserver
with tempfile.TemporaryDirectory() as root:
cfg={'codex':'codex','cwd':root,'max_turn_seconds':10}
message={'id':'a'*24,'text':'hello','source':'human'}
prompt=worker.message_prompt(cfg,message)
thread='11111111-1111-4111-8111-111111111111'
receipt='22222222-2222-4222-8222-222222222222'
class Reader:
calls=0
def __init__(self,*args):pass
def __enter__(self):return self
def __exit__(self,*args):pass
def history(self,tid):
self.calls+=1
if self.calls==1:return {'id':tid},[]
items=[{'type':'userMessage','content':[{'type':'text','text':prompt}]}]
if self.calls>2:items.append({'type':'agentMessage','text':'real reply','phase':'final_answer'})
return {'id':tid},[{'id':'observed-turn','status':'interrupted' if self.calls==2 else 'completed','items':items}]
receipts=[]
with patch.object(appserver,'AppServer',Reader),patch.object(worker.subprocess,'run',return_value=SimpleNamespace(returncode=0,stdout='Queued message '+receipt+' for thread '+thread+'.')),patch.object(worker.time,'sleep'):
result=worker.run_queued(cfg,message,thread,lambda:None,lambda value:None,receipts.append)
self.assertEqual(result['outcome'],'completed')
self.assertEqual(result['text'],'real reply')
self.assertEqual(next(r for r in receipts if r['state']=='accepted')['receipt_id'],receipt)
def test_progress_without_terminal_event_is_not_success(self):
with tempfile.TemporaryDirectory() as root:
proc=Process([{'type':'item.completed','item':{'type':'agent_message','text':'working'}}])
with patch.object(worker.subprocess,'Popen',return_value=proc):
result=worker.run_codex({'codex':'codex','cwd':root},{'id':'m1','text':'hello','source':'agent'},None,lambda:None)
self.assertEqual(result['outcome'],'failed')
def test_real_terminal_event_and_thread_saved_immediately(self):
with tempfile.TemporaryDirectory() as root:
saved=[]
proc=Process([{'type':'thread.started','thread_id':'thread-1'},
{'type':'item.completed','item':{'type':'agent_message','text':'ready'}},
{'type':'turn.completed'}])
with patch.object(worker.subprocess,'Popen',return_value=proc):
result=worker.run_codex({'codex':'codex','cwd':root},{'id':'m1','text':'hello','source':'agent'},None,lambda:None,on_thread=saved.append)
self.assertEqual(saved,['thread-1'])
self.assertEqual(result,{'text':'ready','thread_id':'thread-1','outcome':'completed'})
def test_once_completes_without_claiming_second_message(self):
with tempfile.TemporaryDirectory() as root:
calls=[]
class Client:
def __init__(self,cfg): pass
def call(self,path,data):
calls.append(path)
if path=='/api/worker/claim': return {'message':{'id':'m1','lease_id':'l1','text':'hello','source':'human'}}
return {}
cfg={'control':{},'session_id':'s1','cwd':root,'state_file':str(Path(root)/'state.json')}
with patch.object(worker,'Client',Client),patch.object(worker,'run_codex',return_value={'text':'reply','thread_id':'t1','outcome':'completed'}) as run:
worker.work(cfg,once=True)
self.assertEqual(calls,['/api/worker/claim','/api/worker/start','/api/worker/complete'])
self.assertEqual(run.call_count,1)
state=json.loads(Path(cfg['state_file']).read_text())
self.assertEqual(state['thread_id'],'t1')
self.assertNotIn('inflight',state)
def test_interrupted_execution_is_never_rerun(self):
with tempfile.TemporaryDirectory() as root:
cfg={'control':{},'session_id':'s1','cwd':root,'state_file':str(Path(root)/'state.json')}
worker.atomic_json(Path(cfg['state_file']),{'binding':{'control':{},'session_id':'s1','cwd':str(Path(root).resolve())},'inflight':{'message_id':'m1'}})
with patch.object(worker,'run_codex') as run:
with self.assertRaisesRegex(RuntimeError,'interrupted'): worker.work(cfg,once=True)
run.assert_not_called()
def test_changed_session_refuses_saved_thread(self):
with tempfile.TemporaryDirectory() as root:
cfg={'control':{},'session_id':'s2','cwd':root,'state_file':str(Path(root)/'state.json')}
worker.atomic_json(Path(cfg['state_file']),{'binding':{'control':{},'session_id':'s1','cwd':root},'thread_id':'t1'})
with self.assertRaisesRegex(RuntimeError,'another session'): worker.work(cfg,once=True)
if __name__=='__main__': unittest.main()