|
| 1 | +#!/usr/bin/env python3 |
| 2 | +"""Compare actual local gateway responses for supplied scoped-token request cases.""" |
| 3 | +import argparse |
| 4 | +import asyncio |
| 5 | +import hashlib |
| 6 | +import json |
| 7 | +import os |
| 8 | +from pathlib import Path |
| 9 | +import re |
| 10 | +import sys |
| 11 | +import socket |
| 12 | +import subprocess |
| 13 | +import inspect |
| 14 | +import tempfile |
| 15 | +from unittest.mock import patch |
| 16 | + |
| 17 | +BOUNDARY = ('Supplied-case comparison on native mock-provider gateway requests with fresh local bearer tokens. ' |
| 18 | + 'Not a proof of production access, all possible requests, upstream health, distributed revocation, ' |
| 19 | + 'proof of possession or deployment safety. No configuration is installed.') |
| 20 | +IDENT = re.compile(r'[A-Za-z0-9][A-Za-z0-9._~:/@+-]{0,127}\Z') |
| 21 | + |
| 22 | + |
| 23 | +def read_json(path): |
| 24 | + raw = Path(path).read_bytes() |
| 25 | + if len(raw) > 65536: |
| 26 | + raise ValueError('input exceeds 64 KiB') |
| 27 | + def unique(pairs): |
| 28 | + d = {} |
| 29 | + for k, v in pairs: |
| 30 | + if k in d: |
| 31 | + raise ValueError('duplicate JSON field') |
| 32 | + d[k] = v |
| 33 | + return d |
| 34 | + return json.loads(raw, object_pairs_hook=unique), hashlib.sha256(raw).hexdigest() |
| 35 | + |
| 36 | + |
| 37 | +def text(x): |
| 38 | + return isinstance(x, str) and bool(IDENT.fullmatch(x)) |
| 39 | + |
| 40 | + |
| 41 | +def validate_config(cfg): |
| 42 | + if not isinstance(cfg, dict) or not isinstance(cfg.get('providers'), list) or not 1 <= len(cfg['providers']) <= 8: |
| 43 | + raise ValueError('expected 1-8 mock providers') |
| 44 | + # Never instantiate an HTTPProvider or silently remove a declared real route. |
| 45 | + for p in cfg['providers']: |
| 46 | + if not isinstance(p, dict) or p.get('type') != 'mock': |
| 47 | + raise ValueError('only all-mock configurations are supported; real/mixed routes are not projected or activated') |
| 48 | + if not text(p.get('name')) or not isinstance(p.get('models'), list) or not 1 <= len(p['models']) <= 32 or not all(text(m) for m in p['models']): |
| 49 | + raise ValueError('invalid mock provider identifiers/models') |
| 50 | + if p.get('latency', .02) != 0 or p.get('fail_rate', 0) != 0: |
| 51 | + raise ValueError('set mock latency and fail_rate to zero for deterministic permission checks') |
| 52 | + ta = cfg.get('token_auth') |
| 53 | + if not isinstance(ta, dict) or ta.get('enabled') is not True or not text(ta.get('issuer')) or not text(ta.get('audience')): |
| 54 | + raise ValueError('enabled native token_auth with bounded issuer/audience required') |
| 55 | + if type(ta.get('demo', {}).get('enabled', False)) is not bool or ta.get('demo', {}).get('enabled', False): |
| 56 | + raise ValueError('delegation demo must be disabled for this local preflight') |
| 57 | + if not isinstance(ta.get('scope_models'), dict) or len(ta['scope_models']) > 32: |
| 58 | + raise ValueError('invalid scope mapping') |
| 59 | + for scope, models in ta['scope_models'].items(): |
| 60 | + if not text(scope) or not isinstance(models, list) or len(models) > 32 or not all(text(m) for m in models): |
| 61 | + raise ValueError('invalid scope mapping values') |
| 62 | + if not text(ta.get('required_scope', 'llm:complete')): |
| 63 | + raise ValueError('invalid required scope') |
| 64 | + |
| 65 | + |
| 66 | +def validate_cases(cases): |
| 67 | + if not isinstance(cases, list) or not 1 <= len(cases) <= 32: |
| 68 | + raise ValueError('expected 1-32 request cases') |
| 69 | + names = set() |
| 70 | + for c in cases: |
| 71 | + if not isinstance(c, dict) or set(c) != {'name', 'scope', 'model', 'expect_status'}: |
| 72 | + raise ValueError('each case needs exactly name, scope, model and expect_status') |
| 73 | + if not text(c['name']) or c['name'] in names or not text(c['model']): |
| 74 | + raise ValueError('invalid or duplicate case identity') |
| 75 | + names.add(c['name']) |
| 76 | + if not isinstance(c['scope'], list) or len(c['scope']) > 32 or not all(text(s) for s in c['scope']) or len(set(c['scope'])) != len(c['scope']): |
| 77 | + raise ValueError('invalid case scopes') |
| 78 | + if type(c['expect_status']) is not int or c['expect_status'] not in (200, 401, 403, 404): |
| 79 | + raise ValueError('unsupported expected status') |
| 80 | + |
| 81 | + |
| 82 | +async def one_request(cfg, case): |
| 83 | + import httpx |
| 84 | + import llmgate.config as native_config |
| 85 | + from actas.tokens import issue_token |
| 86 | + original_create = native_config.create_app |
| 87 | + captured = {} |
| 88 | + def create(*args, **kw): |
| 89 | + captured['token_auth'] = kw['token_auth'] |
| 90 | + return original_create(*args, **kw) |
| 91 | + with tempfile.TemporaryDirectory(prefix='gateway-policy-') as tmp: |
| 92 | + path = Path(tmp, 'config.json') |
| 93 | + path.write_text(json.dumps(cfg)) |
| 94 | + with patch.dict(os.environ, {cfg.get('keys_env', 'LLMGATE_KEYS'): '{}'}, clear=False), patch.object(native_config, 'create_app', create): |
| 95 | + app = native_config.build_app(str(path)) |
| 96 | + ta = captured['token_auth'] |
| 97 | + token, _ = issue_token(ta.verifier.keystore, issuer=cfg['token_auth']['issuer'], |
| 98 | + subject='local-preflight-worker', scope=' '.join(case['scope']), |
| 99 | + audience=[cfg['token_auth']['audience']], expires_in=60) |
| 100 | + # Native app lifecycle, auth verification, required-scope check, allowlist, |
| 101 | + # known-route check and actual MockProvider completion all execute. |
| 102 | + async with app.router.lifespan_context(app): |
| 103 | + async with httpx.AsyncClient(transport=httpx.ASGITransport(app=app), base_url='http://local-preflight') as client: |
| 104 | + response = await client.post('/v1/chat/completions', headers={'Authorization': 'Bearer ' + token}, |
| 105 | + json={'model': case['model'], 'messages': [{'role': 'user', 'content': 'local permission check'}], 'max_tokens': 1}) |
| 106 | + calls = sum(route.provider.calls for route in app.state.router.routes) |
| 107 | + if response.status_code not in (200, 401, 403, 404): |
| 108 | + raise ValueError('native request produced an unsupported operational status') |
| 109 | + if (response.status_code == 200) != (calls > 0): |
| 110 | + raise ValueError('native response/provider execution evidence disagrees') |
| 111 | + return {'status': response.status_code, 'provider_executed': calls > 0} |
| 112 | + |
| 113 | + |
| 114 | +async def compare(baseline, candidate, cases): |
| 115 | + validate_config(baseline); validate_config(candidate); validate_cases(cases) |
| 116 | + changes = [] |
| 117 | + # No caller credentials/store/env-based provider activation inherited. |
| 118 | + def no_network(*args, **kwargs): |
| 119 | + raise ValueError('network access is disabled during this local comparison') |
| 120 | + with patch.dict(os.environ, {}, clear=True), patch.object(socket.socket, 'connect', no_network), patch.object(socket.socket, 'connect_ex', no_network), patch.object(socket, 'create_connection', no_network): |
| 121 | + for case in cases: |
| 122 | + before = await one_request(baseline, case) |
| 123 | + after = await one_request(candidate, case) |
| 124 | + transition = 'newly_reachable' if before['status'] != 200 and after['status'] == 200 else 'newly_refused' if before['status'] == 200 and after['status'] != 200 else 'unchanged' if before['status'] == after['status'] else 'refusal_changed' |
| 125 | + changes.append({'name': case['name'], 'before': before, 'after': after, 'transition': transition, |
| 126 | + 'candidate_expectation_matched': after['status'] == case['expect_status']}) |
| 127 | + changed = any(x['before']['status'] != x['after']['status'] for x in changes) |
| 128 | + matched = all(x['candidate_expectation_matched'] for x in changes) |
| 129 | + return {'schema_version': 1, 'state': 'review_required' if changed or not matched else 'pass', |
| 130 | + 'total': len(changes), 'newly_reachable': sum(x['transition'] == 'newly_reachable' for x in changes), |
| 131 | + 'newly_refused': sum(x['transition'] == 'newly_refused' for x in changes), |
| 132 | + 'refusal_changed': sum(x['transition'] == 'refusal_changed' for x in changes), |
| 133 | + 'candidate_matched': sum(x['candidate_expectation_matched'] for x in changes), |
| 134 | + 'cases': changes, 'boundary': BOUNDARY} |
| 135 | + |
| 136 | + |
| 137 | +def source_identity(): |
| 138 | + import llmgate.config |
| 139 | + import llmgate.tokenauth |
| 140 | + import actas.tokens |
| 141 | + root = Path(inspect.getfile(llmgate.config)).resolve().parents[1] |
| 142 | + git = subprocess.run(['git', '-C', str(root), 'rev-parse', 'HEAD'], capture_output=True, text=True, timeout=10) |
| 143 | + status = subprocess.run(['git', '-C', str(root), 'status', '--porcelain'], capture_output=True, text=True, timeout=10) |
| 144 | + if git.returncode or status.returncode or status.stdout: |
| 145 | + raise ValueError('clean gateway Git source required') |
| 146 | + modules = {} |
| 147 | + for module in (llmgate.config, llmgate.tokenauth, actas.tokens): |
| 148 | + path = Path(inspect.getfile(module)).resolve() |
| 149 | + modules[module.__name__] = {'relative_path': str(path.relative_to(root)), 'sha256': hashlib.sha256(path.read_bytes()).hexdigest()} |
| 150 | + return {'gateway_git_head': git.stdout.strip(), 'native_imports': modules} |
| 151 | + |
| 152 | + |
| 153 | +def main(): |
| 154 | + p = argparse.ArgumentParser(description=__doc__) |
| 155 | + for arg in ('baseline', 'candidate', 'cases'): |
| 156 | + p.add_argument('--' + arg, required=True) |
| 157 | + p.add_argument('--gateway-source', required=True, help='Clean gateway checkout containing its embedded Actas runtime') |
| 158 | + args = p.parse_args() |
| 159 | + try: |
| 160 | + expected_source = Path(args.gateway_source).resolve(strict=True) |
| 161 | + sys.path.insert(0, str(expected_source)) |
| 162 | + baseline, bh = read_json(args.baseline); candidate, ch = read_json(args.candidate); cases, qh = read_json(args.cases) |
| 163 | + identity = source_identity() |
| 164 | + import llmgate.config |
| 165 | + if Path(inspect.getfile(llmgate.config)).resolve().parents[1] != expected_source: |
| 166 | + raise ValueError('native gateway import mismatch') |
| 167 | + report = asyncio.run(compare(baseline, candidate, cases)) |
| 168 | + if source_identity() != identity: |
| 169 | + raise ValueError('native source changed during comparison') |
| 170 | + report['source'] = identity |
| 171 | + report['input_sha256'] = {'baseline': bh, 'candidate': ch, 'cases': qh} |
| 172 | + except Exception as exc: |
| 173 | + # Ordinary dependency/native execution failures are not policy regressions. |
| 174 | + # BaseException (KeyboardInterrupt/SystemExit) is deliberately not caught. |
| 175 | + report = {'schema_version': 1, 'state': 'error', 'error_type': type(exc).__name__, 'boundary': BOUNDARY} |
| 176 | + print(json.dumps(report, indent=2)) |
| 177 | + return {'pass': 0, 'review_required': 1, 'error': 2}[report['state']] |
| 178 | + |
| 179 | + |
| 180 | +if __name__ == '__main__': |
| 181 | + sys.exit(main()) |
0 commit comments