Add event-driven reporting and durable failure recovery hooks

scripts/hooks.py

@@ -0,0 +1,65 @@
+#!/usr/bin/env python3
+"""Post bounded failure/recovery events, or wrap an explicitly authorized local check."""
+import argparse
+import hashlib
+import json
+from pathlib import Path
+import subprocess
+import sys
+from control import Client
+from worker import atomic_json
+
+
+def main():
+    p=argparse.ArgumentParser(description=__doc__)
+    p.add_argument('--config',required=True)
+    p.add_argument('--source',required=True)
+    p.add_argument('--event-id',required=True)
+    p.add_argument('--severity',choices=['info','warning','critical'],default='warning')
+    p.add_argument('--summary',required=True)
+    p.add_argument('--evidence',default='')
+    p.add_argument('--task-id',default='')
+    p.add_argument('--status',choices=['firing','resolved'],default='firing')
+    p.add_argument('--timeout',type=int,default=300)
+    p.add_argument('command',nargs=argparse.REMAINDER)
+    a=p.parse_args()
+    cfg=json.loads(Path(a.config).read_text(encoding='utf-8-sig'))
+    client=Client(cfg.get('control',cfg))
+    command=a.command[1:] if a.command[:1]==['--'] else a.command
+    status=a.status
+    code=0
+    if command:
+        if not 1<=a.timeout<=86400:raise ValueError('Timeout must be bounded')
+        # Inherit output for the operator, never send raw logs or command arguments.
+        # No shell and no command is ever accepted by the HTTP service.
+        try:code=subprocess.run(command,timeout=a.timeout).returncode
+        except subprocess.TimeoutExpired:code=124
+        except OSError:code=127
+        status='resolved' if code==0 else 'firing'
+    payload={'source':a.source,'event_id':a.event_id,'severity':a.severity,
+             'status':status,'summary':a.summary,'evidence':[a.evidence] if a.evidence else []}
+    if a.task_id:payload['task_id']=a.task_id
+    state_path=Path(a.config).resolve().with_suffix('.hook-check-state.json')
+    state=json.loads(state_path.read_text(encoding='utf-8')) if state_path.exists() else {}
+    key=a.source+':'+a.event_id
+    previous=state.get(key)
+    if command and code==0 and not previous:
+        print(json.dumps({'status':'ok','recovery_required':False}))
+        return 0
+    if status=='firing':
+        state[key]={'payload':payload,'accepted':False}
+        atomic_json(state_path,state)
+    try:
+        if status=='resolved' and previous and not previous.get('accepted'):
+            client.call('/api/hooks',previous['payload'])
+        result=client.call('/api/hooks',payload)
+    except Exception:
+        print('Hook delivery failed; original command exit='+str(code)+'. Inspect event delivery before retry.',file=sys.stderr)
+        return code or 75
+    state[key]={'payload':payload,'accepted':True}
+    atomic_json(state_path,state)
+    print(json.dumps(result,ensure_ascii=True))
+    return code
+
+
+if __name__=='__main__':sys.exit(main())