mayge/agent-workbench
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Schedule consultation bridges on demand and preserve delivery evidence

Browse Source
This commit is contained in:
aiagent
2026-10-20 00:39:12 -07:00
parent 557f1ee03c
commit 2f633869fe
18 changed files with 483 additions and 56 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/decision_fleet.py
+91 -43
View File
@@ -19,6 +19,28 @@ def busy(path):
return bool(state.get('inflight') or state.get('pending_completion') or state.get('needs_reconciliation'))
def reconcile_completed(path,client,session_id,thread_id):
"""Clear only a proven terminal original execution, never submit another input."""
state=json.loads(path.read_text(encoding='utf-8'))
record=state.get('inflight') or {}
proof=record.get('submission') or {}
if not record.get('message_id') or not proof.get('input_sha256'):return False
from urllib.parse import quote
rows=client.call('/api/messages?session_id='+quote(session_id,safe=''))['messages']
original=next((m for m in rows if m['id']==record['message_id']),None)
if not original:return False
reply=next((m for m in rows if m['id']==original.get('response_id')),None)
remote=original.get('submission') or {}
if (not reply or original.get('status') not in ('completed','failed')
or remote.get('thread_id')!=thread_id
or remote.get('input_sha256')!=proof['input_sha256']
or (proof.get('turn_id') and remote.get('turn_id')!=proof['turn_id'])):return False
state.setdefault('delivery_receipts',{})[record['message_id']]=remote
for key in ('inflight','pending_completion','needs_reconciliation'):state.pop(key,None)
atomic_json(path,state)
return True
def child_config(base,binding,root):
did=binding['decision_id'];sid=binding['session_id'];thread=binding['thread_id']
if any(not isinstance(v,str) or not re.fullmatch(r'[A-Za-z0-9_.:-]{1,128}',v) for v in (did,sid)):
@@ -54,20 +76,20 @@ class Fleet:
def stop(self,did,force=False):
child=self.children[did]
if not force and busy(Path(child['cfg']['state_file'])):return False
for process in child['processes'].values():
if process.poll() is None:process.terminate()
for process in child['processes'].values():
try:process.wait(timeout=10)
except subprocess.TimeoutExpired:process.kill();process.wait(timeout=10)
# Single-cycle children exit naturally; never interrupt a native writer.
if any(p.poll() is None for p in child['processes'].values()):return False
for stream in child['logs'].values():stream.close()
del self.children[did]
return True
def tick(self,bindings):
live={}
waiting=0;problems=[]
threads=set();sessions=set()
now=time.time();problems=[];live={};threads=set();sessions=set()
journal_path=self.root/'schedule.json'
journal=json.loads(journal_path.read_text(encoding='utf-8')) if journal_path.exists() else {}
notifications={}
if self.cfg.get('notification_state_file'):
notification_path=Path(self.cfg['notification_state_file'])
if notification_path.exists():notifications=json.loads(notification_path.read_text(encoding='utf-8')).get('deliveries',{})
for binding in bindings:
did=binding['decision_id']
if did in live or binding['thread_id'] in threads or binding['session_id'] in sessions:
@@ -75,43 +97,66 @@ class Fleet:
live[did]=binding;threads.add(binding['thread_id']);sessions.add(binding['session_id'])
for did in list(self.children):
binding=live.get(did)
if not binding or binding.get('archive_status')=='archived':
if not self.stop(did):problems.append('archive_requires_reconciliation:'+did)
for did,binding in sorted(live.items()):
child=self.children[did]
mirror=child['processes'].get('mirror')
if mirror and mirror.poll()==0:
journal.setdefault(did,{}).setdefault('fact_synced',{}).update(child.get('terminal_facts',{}))
if binding and child_config(self.cfg,binding,self.root)[0]!=self.children[did]['cfg']:
problems.append('binding_changed:'+did);continue
if not self.stop(did) and (not binding or binding.get('archive_status')=='archived'):
problems.append('archive_requires_reconciliation:'+did)
candidates=[]
for did,binding in live.items():
if binding.get('archive_status')=='archived':continue
candidate,runtime=child_config(self.cfg,binding,self.root)
child=self.children.get(did)
if child and child['cfg']!=candidate:
problems.append('binding_changed:'+did);continue
if not child:
if len(self.children)>=self.limit:waiting+=1;continue
config_path=runtime/'bridge.json'
if config_path.exists() and json.loads(config_path.read_text())!=candidate:
problems.append('saved_binding_changed:'+did);continue
atomic_json(config_path,candidate)
child={'cfg':candidate,'runtime':runtime,'path':config_path,'processes':{},'logs':{},'retry_at':{},'started':{}}
self.children[did]=child
demand=binding.get('demand')
state_path=Path(candidate['state_file'])
for role in ('worker','mirror'):
process=child['processes'].get(role)
if process and process.poll() is None:
try:
health=json.loads((runtime/(role+'-health.json')).read_text())
healthy=health.get('status')=='ok' and time.time()-health.get('updated',0)<60
except (OSError,ValueError):healthy=False
if not healthy and time.time()-child['started'].get(role,0)>60:
problems.append('bridge_health_stale:'+did+':'+role)
continue
if role=='worker' and busy(state_path):
problems.append('execution_requires_reconciliation:'+did);continue
if time.monotonic()<child['retry_at'].get(role,0):continue
if role not in child['logs']:child['logs'][role]=open(runtime/(role+'.log'),'a',encoding='utf-8')
command=[sys.executable,'-X','utf8',str(Path(__file__).with_name(role+'.py')),'--config',str(child['path'])]
child['processes'][role]=subprocess.Popen(command,stdout=child['logs'][role],stderr=child['logs'][role],
local_busy=busy(state_path)
worker=self.children.get(did,{}).get('processes',{}).get('worker')
worker_alive=worker is not None and worker.poll() is None
if local_busy and not worker_alive and demand is not None and not demand.get('uncertain',0) and not demand.get('inflight',0):
try:
if reconcile_completed(state_path,Client(self.cfg['control']),binding['session_id'],binding['thread_id']):local_busy=False
except Exception:pass
if demand is None:
problems.append('queue_summary_unavailable:'+did);demand={}
uncertain=bool(demand.get('uncertain',0) or local_busy)
facts={key:r for key,r in notifications.items() if r.get('decision_id')==did and r.get('event_kind')=='fact_answered'}
terminal_facts={key:str(r.get('completed_at') or r.get('turn_id') or 'done') for key,r in facts.items() if r.get('processing_done')}
synced=journal.get(did,{}).get('fact_synced',{})
fact_demand=any(not r.get('processing_done') for r in facts.values()) or any(synced.get(key)!=value for key,value in terminal_facts.items())
needed=bool(demand.get('queued',0) or demand.get('inflight',0) or uncertain or fact_demand)
if not needed:
if did in journal:journal[did].pop('waiting_since',None)
continue
record=journal.setdefault(did,{})
record.setdefault('waiting_since',min(now,demand.get('oldest_queued_at') or now))
if did in self.children:
if uncertain and not worker_alive:problems.append('execution_requires_reconciliation:'+did)
continue
# Finite human boost: old ordinary work eventually precedes new human work.
score=now-record['waiting_since']+(60 if demand.get('human_queued',0) else 0)
candidates.append((-score,record['waiting_since'],record.get('last_served',0),did,binding,candidate,runtime,uncertain,terminal_facts))
candidates.sort(key=lambda row:row[:4]);waiting=0
for _,_,_,did,binding,candidate,runtime,uncertain,terminal_facts in candidates:
if len(self.children)>=self.limit:waiting+=1;continue
config_path=runtime/'bridge.json'
if config_path.exists() and json.loads(config_path.read_text(encoding='utf-8'))!=candidate:
problems.append('saved_binding_changed:'+did);continue
atomic_json(config_path,candidate)
child={'cfg':candidate,'runtime':runtime,'path':config_path,'processes':{},'logs':{},'terminal_facts':terminal_facts}
self.children[did]=child
journal[did].update(waiting_since=now,last_served=now)
atomic_json(journal_path,journal)
roles=('mirror',) if uncertain or binding.get('demand',{}).get('inflight',0) or not binding.get('demand',{}).get('queued',0) else ('worker','mirror')
if uncertain:problems.append('execution_requires_reconciliation:'+did)
for role in roles:
stream=open(runtime/(role+'.log'),'a',encoding='utf-8');child['logs'][role]=stream
command=[sys.executable,'-X','utf8',str(Path(__file__).with_name(role+'.py')),'--config',str(config_path),'--once']
child['processes'][role]=subprocess.Popen(command,stdout=stream,stderr=stream,
**({'creationflags':subprocess.CREATE_NO_WINDOW} if os.name=='nt' else {}))
child['started'][role]=time.time()
child['retry_at'][role]=time.monotonic()+15
snapshot={'status':'failed' if problems else 'ok','updated':time.time(),'active':len(self.children),
atomic_json(journal_path,journal)
snapshot={'status':'failed' if problems else 'ok','updated':now,'active':len(self.children),
'waiting':waiting,'problems':problems,'bindings':{did:{'session_id':c['cfg']['session_id'],
'thread_id':c['cfg']['thread_id'],'processes':{r:{'pid':p.pid,'running':p.poll() is None}
for r,p in c['processes'].items()}} for did,c in self.children.items()}}
@@ -119,7 +164,10 @@ class Fleet:
return snapshot
def close(self):
for did in list(self.children):self.stop(did,force=True)
# Active cycles retain their durable state and host writer locks.
for did in list(self.children):
for stream in self.children[did]['logs'].values():stream.close()
self.stop(did)
def main():
scripts/notifier.py
+35 -3
View File
@@ -59,8 +59,26 @@ def notification_text(n):
+json.dumps(data,ensure_ascii=False)+'\n请核实事件与影响，按既定权限处理。面向用户仅汇报持续重大影响、需行动或待决策；短暂自动恢复只进日志，不向人类入口转发。若有log_id，给一句摘要并链接日志页#/logs?incident=<log_id>，不复制原载荷；重复状态不打扰。消息已投递不代表故障已修复。')
def target_ready(cfg,records,thread,cache):
if not cfg.get('enforce_backpressure'):
def schedule_notifications(notifications,state,now):
"""Prefer decisions while aging waiting work into FIFO; persist across restarts.
This changes dispatch order only, never authorization, receipt or event identity.
A busy/uncertain target remains blocked regardless of waiting time.
"""
waiting=state.setdefault('notification_waiting',{})
def identity(n):
return ('decision:' if 'decision' in n else 'eta:' if 'checkpoint' in n else 'hook:')+n['notification_id']
for n in notifications:
waiting.setdefault(identity(n),now)
def rank(n):
first=waiting[identity(n)]
priority=0 if 'decision' in n else 1 if n.get('hook',{}).get('severity')=='critical' else 2 if 'hook' in n else 3
return (0 if now-first>=120 else priority+1,first,identity(n))
return sorted(notifications,key=rank)
def target_ready(cfg,records,thread,cache,force_reconcile=False):
if not cfg.get('enforce_backpressure') and not force_reconcile:
return True
limit=cfg.get('max_inflight_per_target',1)
if not isinstance(limit,int) or isinstance(limit,bool) or not 1<=limit<=3:
@@ -96,6 +114,15 @@ def deliver(client,cfg,state,path):
records=state.setdefault('deliveries',{})
errors=[]
cache={}
# Fact answers enter the native consultation thread directly. Reconcile their
# completion even with an empty outbox, so a demand-driven mirror can retire.
for thread in {r['thread_id'] for r in records.values()
if r.get('event_kind')=='fact_answered' and not r.get('processing_done')}:
try:
target_ready(cfg,records,thread,cache,force_reconcile=True)
except Exception:
errors.append('fact_answer_reconciliation_unavailable')
atomic_json(path,state)
if cfg.get('coalesce_stale_hooks'):
# Upgrade old accepted-but-unacked receipts before pruning pending data.
# Unknown submissions are protected, never silently cancelled or replayed.
@@ -128,7 +155,9 @@ def deliver(client,cfg,state,path):
notifications += batch['notifications']
queue_counts[endpoint]=batch.get('pending_count',len(batch['notifications']))
except Exception:errors.append('outbox_unavailable')
notifications.sort(key=lambda n:0 if 'decision' in n else 1 if n.get('hook',{}).get('severity')=='critical' else 2 if 'hook' in n else 3)
notifications=schedule_notifications(notifications,state,time.time())
# Waiting age must survive offline delivery/ACK failures, not just successful sends.
atomic_json(path,state)
task_owners={}
if cfg.get('execution_notification_targets'):
snapshot=client.call('/api/state')
@@ -161,6 +190,8 @@ def deliver(client,cfg,state,path):
if 'decision' in n:
client.call('/api/decision-notifications/submitting',{'notification_id':n['notification_id']})
old={'kind':'hook' if 'hook' in n else 'decision' if 'decision' in n else 'eta','notification_ids':([item['notification_id'] for item in notifications if item.get('hook')==n.get('hook')] if internal_event else [n['notification_id']]),'created_at':time.time(),'thread_id':thread,'input_sha256':digest,'state':'submitting'}
if 'decision' in n:
old.update(decision_id=n['decision']['id'],event_kind=n.get('event',{}).get('kind'))
records[key]=old
atomic_json(path,state)
if 'hook' in n and cfg.get('coalesce_stale_hooks'):
@@ -177,6 +208,7 @@ def deliver(client,cfg,state,path):
# Crash after queue acceptance: reconcile exact actual input, never requeue.
with AppServer(cfg['codex']) as reader:
meta,turns=reader.history(thread)
if meta.get('id')!=thread:raise ValueError('Unexpected reconciliation thread identity')
observed=next((t['id'] for t in turns if any(
i.get('type')=='userMessage' and '\n'.join(p.get('text','') for p in i.get('content',[]) if p.get('type')=='text')==prompt
for i in t.get('items',[]))),None)