Skip to content

Commit 25261e5

Browse files
committed
Validate actual UCI option contracts before starting strength campaigns
1 parent c321b46 commit 25261e5

4 files changed

Lines changed: 481 additions & 127 deletions

File tree

‎.github/workflows/ci.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@ jobs:
3434
./build/network_test models/diagnostic-v2.cai 10000
3535
python tests/format_controls.py --network models/diagnostic-v2.cai
3636
python -m tests.statistics_test
37+
python -m tests.arena_options
3738
python -m tests.features_test
3839
python -m training.corpus --engine build/caissa --pairs 50 --nodes 500 --workers 2 --output data/ci-original
3940
python -m training.prepare data/ci-original data/ci-prepared

‎arena/client.py‎

Lines changed: 188 additions & 49 deletions
Original file line numberDiff line numberDiff line change
@@ -1,60 +1,199 @@
11
"""A bounded UCI process owner. The arena, not this client, judges chess legality."""
2-
import queue,subprocess,threading,time
3-
class EngineFailure(RuntimeError):pass
2+
3+
import queue, subprocess, threading, time
4+
5+
6+
class EngineFailure(RuntimeError):
7+
pass
8+
9+
10+
def parse_options(lines):
11+
options = {}
12+
for line in lines:
13+
if not line.startswith("option name "):
14+
continue
15+
fields = line.split()
16+
try:
17+
marker = fields.index("type", 2)
18+
except ValueError:
19+
raise EngineFailure("malformed engine option declaration")
20+
name = " ".join(fields[2:marker])
21+
kind = fields[marker + 1] if marker + 1 < len(fields) else ""
22+
if (
23+
not name
24+
or kind not in ["check", "spin", "combo", "button", "string"]
25+
or name in options
26+
):
27+
raise EngineFailure("invalid or duplicate engine option declaration")
28+
option = {"type": kind}
29+
position = marker + 2
30+
while position < len(fields):
31+
key = fields[position]
32+
position += 1
33+
if key not in ["default", "min", "max", "var"]:
34+
raise EngineFailure("malformed option metadata")
35+
end = position
36+
while end < len(fields) and fields[end] not in [
37+
"default",
38+
"min",
39+
"max",
40+
"var",
41+
]:
42+
end += 1
43+
value = " ".join(fields[position:end])
44+
position = end
45+
if key == "var":
46+
option.setdefault("choices", []).append(value)
47+
else:
48+
option[key] = value
49+
options[name] = option
50+
return options
51+
52+
53+
def validate_option(name, value, available):
54+
if name not in available:
55+
raise EngineFailure("requested engine option is undeclared: " + str(name))
56+
contract = available[name]
57+
kind = contract["type"]
58+
if kind == "spin":
59+
if not isinstance(value, int) or isinstance(value, bool):
60+
raise EngineFailure("integer engine option required: " + name)
61+
try:
62+
low = int(contract["min"])
63+
high = int(contract["max"])
64+
except (KeyError, ValueError):
65+
raise EngineFailure("engine spin bounds are malformed: " + name)
66+
if not low <= value <= high:
67+
raise EngineFailure("engine option outside declared bounds: " + name)
68+
elif kind == "check" and not isinstance(value, bool):
69+
raise EngineFailure("boolean engine option required: " + name)
70+
elif kind == "combo" and str(value) not in contract.get("choices", []):
71+
raise EngineFailure("engine option choice undeclared: " + name)
72+
elif kind == "string" and not isinstance(value, str):
73+
raise EngineFailure("string engine option required: " + name)
74+
75+
476
class UciClient:
5-
def __init__(self,command,options,timeout=10,warmup_nodes=0):
6-
try:self.process=subprocess.Popen(command,stdin=subprocess.PIPE,stdout=subprocess.PIPE,stderr=subprocess.DEVNULL,text=True,bufsize=1)
7-
except OSError as e:raise EngineFailure('engine startup failed: '+str(e)) from e
8-
self.lines=queue.Queue(maxsize=10000);self.timeout=timeout
77+
def __init__(self, command, options, timeout=10, warmup_nodes=0):
78+
try:
79+
self.process = subprocess.Popen(
80+
command,
81+
stdin=subprocess.PIPE,
82+
stdout=subprocess.PIPE,
83+
stderr=subprocess.DEVNULL,
84+
text=True,
85+
bufsize=1,
86+
)
87+
except OSError as e:
88+
raise EngineFailure("engine startup failed: " + str(e)) from e
89+
self.lines = queue.Queue(maxsize=10000)
90+
self.timeout = timeout
91+
992
def reader():
1093
try:
11-
for line in self.process.stdout:
12-
if len(line)>16384:break
13-
try:self.lines.put(line.strip(),timeout=1)
14-
except queue.Full:break
94+
while True:
95+
line = self.process.stdout.readline(16385)
96+
if not line or len(line) > 16384:
97+
break
98+
try:
99+
self.lines.put(line.strip(), timeout=1)
100+
except queue.Full:
101+
break
15102
finally:
16-
try:self.lines.put(None,timeout=.1)
17-
except queue.Full:pass
18-
self.reader=threading.Thread(target=reader,daemon=True);self.reader.start()
103+
try:
104+
self.lines.put(None, timeout=0.1)
105+
except queue.Full:
106+
pass
107+
108+
self.reader = threading.Thread(target=reader, daemon=True)
109+
self.reader.start()
19110
try:
20-
self.send('uci');self.until('uciok',timeout)
21-
for name,value in options.items():
22-
if any(c in str(name)+str(value) for c in '\r\n\0'):raise ValueError('invalid option delimiter')
23-
self.send(f'setoption name {name} value {str(value).lower() if isinstance(value,bool) else value}')
24-
self.send('isready');self.until('readyok',timeout)
25-
self.warmup_ms=0.
111+
self.send("uci")
112+
self.handshake = self.until("uciok", timeout)
113+
self.available_options = parse_options(self.handshake)
114+
for name, value in options.items():
115+
if any(c in str(name) + str(value) for c in "\r\n\0"):
116+
raise ValueError("invalid option delimiter")
117+
validate_option(name, value, self.available_options)
118+
self.send(
119+
f"setoption name {name}"
120+
if self.available_options[name]["type"] == "button"
121+
else f"setoption name {name} value {str(value).lower() if isinstance(value,bool) else value}"
122+
)
123+
self.send("isready")
124+
self.until("readyok", timeout)
125+
self.warmup_ms = 0.0
26126
if warmup_nodes:
27-
start=time.monotonic_ns();self.send('position startpos');self.send(f'go nodes {warmup_nodes}');self.until('bestmove ',timeout)
28-
self.warmup_ms=(time.monotonic_ns()-start)/1e6
29-
self.send('ucinewgame');self.send('isready');self.until('readyok',timeout)
30-
except BaseException:self.close();raise
31-
def send(self,line):
32-
if len(line)>8192:raise EngineFailure('command too long')
33-
try:self.process.stdin.write(line+'\n');self.process.stdin.flush()
34-
except (BrokenPipeError,OSError) as e:raise EngineFailure('engine pipe closed') from e
35-
def until(self,prefix,timeout):
36-
deadline=time.monotonic()+timeout;lines=[]
127+
start = time.monotonic_ns()
128+
self.send("position startpos")
129+
self.send(f"go nodes {warmup_nodes}")
130+
self.until("bestmove ", timeout)
131+
self.warmup_ms = (time.monotonic_ns() - start) / 1e6
132+
self.send("ucinewgame")
133+
self.send("isready")
134+
self.until("readyok", timeout)
135+
except BaseException:
136+
self.close()
137+
raise
138+
139+
def send(self, line):
140+
if len(line) > 8192:
141+
raise EngineFailure("command too long")
142+
try:
143+
self.process.stdin.write(line + "\n")
144+
self.process.stdin.flush()
145+
except (BrokenPipeError, OSError) as e:
146+
raise EngineFailure("engine pipe closed") from e
147+
148+
def until(self, prefix, timeout):
149+
deadline = time.monotonic() + timeout
150+
lines = []
37151
while True:
38-
try:line=self.lines.get(timeout=max(.001,deadline-time.monotonic()))
39-
except queue.Empty as e:raise EngineFailure('engine response timeout') from e
40-
if line is None:raise EngineFailure('engine exited')
152+
try:
153+
line = self.lines.get(timeout=max(0.001, deadline - time.monotonic()))
154+
except queue.Empty as e:
155+
raise EngineFailure("engine response timeout") from e
156+
if line is None:
157+
raise EngineFailure("engine exited")
41158
lines.append(line)
42-
if len(lines)>5000:raise EngineFailure('engine output flood')
43-
if line.startswith(prefix):return lines
44-
if time.monotonic()>deadline:raise EngineFailure('engine response timeout')
45-
def play(self,initial,moves,nodes=None,milliseconds=None):
46-
self.send('position fen '+initial+(' moves '+' '.join(moves) if moves else ''))
47-
command=f'go nodes {nodes}' if nodes is not None else f'go movetime {milliseconds}'
48-
start=time.monotonic_ns();self.send(command)
49-
lines=self.until('bestmove ',max(2.,(milliseconds or 0)/1000+1))
50-
elapsed=(time.monotonic_ns()-start)/1e6
51-
return lines[-1].split()[1],elapsed,lines
159+
if line.startswith("info string error "):
160+
raise EngineFailure("engine rejected command: " + line[18:])
161+
if len(lines) > 5000:
162+
raise EngineFailure("engine output flood")
163+
if line.startswith(prefix):
164+
return lines
165+
if time.monotonic() > deadline:
166+
raise EngineFailure("engine response timeout")
167+
168+
def play(self, initial, moves, nodes=None, milliseconds=None):
169+
self.send(
170+
"position fen " + initial + (" moves " + " ".join(moves) if moves else "")
171+
)
172+
command = (
173+
f"go nodes {nodes}" if nodes is not None else f"go movetime {milliseconds}"
174+
)
175+
start = time.monotonic_ns()
176+
self.send(command)
177+
lines = self.until("bestmove ", max(2.0, (milliseconds or 0) / 1000 + 1))
178+
elapsed = (time.monotonic_ns() - start) / 1e6
179+
return lines[-1].split()[1], elapsed, lines
180+
52181
def close(self):
53-
p=self.process
182+
p = self.process
54183
if p.poll() is None:
55-
try:self.send('stop');self.send('quit');p.wait(timeout=.5)
56-
except (EngineFailure,subprocess.TimeoutExpired):p.kill();p.wait(timeout=2)
57-
for stream in [p.stdin,p.stdout]:
58-
if stream:stream.close()
59-
def __enter__(self):return self
60-
def __exit__(self,*_):self.close()
184+
try:
185+
self.send("stop")
186+
self.send("quit")
187+
p.wait(timeout=0.5)
188+
except (EngineFailure, subprocess.TimeoutExpired):
189+
p.kill()
190+
p.wait(timeout=2)
191+
for stream in [p.stdin, p.stdout]:
192+
if stream:
193+
stream.close()
194+
195+
def __enter__(self):
196+
return self
197+
198+
def __exit__(self, *_):
199+
self.close()

0 commit comments

Comments
 (0)