Repository navigation
Expand file tree
/
Copy pathtest_core.py
More file actions
904 lines (785 loc) · 40 KB
/
Copy pathtest_core.py
File metadata and controls
904 lines (785 loc) · 40 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
517
518
519
520
521
522
523
524
525
526
527
528
529
530
531
532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
563
564
565
566
567
568
569
570
571
572
573
574
575
576
577
578
579
580
581
582
583
584
585
586
587
588
589
590
591
592
593
594
595
596
597
598
599
600
601
602
603
604
605
606
607
608
609
610
611
612
613
614
615
616
617
618
619
620
621
622
623
624
625
626
627
628
629
630
631
632
633
634
635
636
637
638
639
640
641
642
643
644
645
646
647
648
649
650
651
652
653
654
655
656
657
658
659
660
661
662
663
664
665
666
667
668
669
670
671
672
673
674
675
676
677
678
679
680
681
682
683
684
685
686
687
688
689
690
691
692
693
694
695
696
697
698
699
700
701
702
703
704
705
706
707
708
709
710
711
712
713
714
715
716
717
718
719
720
721
722
723
724
725
726
727
728
729
730
731
732
733
734
735
736
737
738
739
740
741
742
743
744
745
746
747
748
749
750
751
752
753
754
755
756
757
758
759
760
761
762
763
764
765
766
767
768
769
770
771
772
773
774
775
776
777
778
779
780
781
782
783
784
785
786
787
788
789
790
791
792
793
794
795
796
797
798
799
800
801
802
803
804
805
806
807
808
809
810
811
812
813
814
815
816
817
818
819
820
821
822
823
824
825
826
827
828
829
830
831
832
833
834
835
836
837
838
839
840
841
842
843
844
845
846
847
848
849
850
851
852
853
854
855
856
857
858
859
860
861
862
863
864
865
866
867
868
869
870
871
872
873
874
875
876
877
878
879
880
881
882
883
884
885
886
887
888
889
890
891
892
893
894
895
896
897
898
899
900
901
902
903
904
"""Smoke tests for ephemeral_core parser and models."""
import os
from ephemeral_core.parser import (
parse_codeblocks,
resolve_runtime_config,
infer_python_dependencies,
extract_declared_dependencies,
inject_python_dependency_metadata,
prepare_python_block,
)
from ephemeral_core.models import ExecutionResult, GroupResult, BlockResult
# --- Test 1: Basic fenced codeblock ---
md = "```python\nprint(42)\n```"
blocks = parse_codeblocks(md)
assert len(blocks) == 1, f"Expected 1 block, got {len(blocks)}"
assert blocks[0]['type'] == 'code'
assert blocks[0]['config']['image'] == 'docker.io/tymills620/ephemeral-python-uv:latest'
assert blocks[0]['config']['cmd'] == ['uv', 'run', '-']
assert blocks[0]['config']['allow_network'] == False
print("PASS: Basic fenced codeblock")
# --- Test 2: Network flag (unsafe) ---
md2 = "```node unsafe\nconsole.log(1)\n```"
blocks2 = parse_codeblocks(md2)
assert blocks2[0]['config']['allow_network'] == True
assert blocks2[0]['config']['image'] == 'docker.io/library/node:18-alpine'
print("PASS: Network flag (unsafe)")
# --- Test 3: No-chain flag ---
md3 = "```python nopipe\nprint(1)\n```"
blocks3 = parse_codeblocks(md3)
assert blocks3[0]['config']['allow_chain'] == False
print("PASS: No-chain flag (nopipe)")
# --- Test 4: Version override ---
md4 = "```python:3.11\nprint(1)\n```"
blocks4 = parse_codeblocks(md4)
assert blocks4[0]['config']['image'] == 'docker.io/tymills620/ephemeral-python-uv:3.11'
print("PASS: Version override (python:3.11)")
# --- Test 5: Alias resolution ---
md5 = "```js\nconsole.log(1)\n```"
blocks5 = parse_codeblocks(md5)
assert blocks5[0]['config']['image'] == 'docker.io/library/node:18-alpine'
print("PASS: Alias resolution (js -> node)")
# --- Test 6: Shebang inside fenced block overwrites header ---
md6 = "```bash\n#! python\nprint(1)\n```"
blocks6 = parse_codeblocks(md6)
assert blocks6[0]['config']['image'] == 'docker.io/tymills620/ephemeral-python-uv:latest'
print("PASS: Shebang overwrites fenced header")
# --- Test 7: Seed file detection ---
md7 = "```data.csv\na,b,c\n1,2,3\n```"
blocks7 = parse_codeblocks(md7)
assert blocks7[0]['type'] == 'seed'
assert blocks7[0]['name'] == 'data.csv'
print("PASS: Seed file detection")
# --- Test 8: Seed file with b64 flag ---
md8 = "```image.png b64\naGVsbG8=\n```"
blocks8 = parse_codeblocks(md8)
assert blocks8[0]['type'] == 'seed'
assert blocks8[0]['is_b64'] == True
print("PASS: Seed file with b64 flag")
# --- Test 9: Multiple blocks with different languages ---
md9 = "```python\nprint(1)\n```\n\n```node\nconsole.log(2)\n```"
blocks9 = parse_codeblocks(md9)
assert len(blocks9) == 2
assert blocks9[0]['config']['image'] == 'docker.io/tymills620/ephemeral-python-uv:latest'
assert blocks9[1]['config']['image'] == 'docker.io/library/node:18-alpine'
print("PASS: Multiple blocks with different languages")
# --- Test 10: Shebang-only (no fences) ---
md10 = "#! python\nprint('hello')\n"
blocks10 = parse_codeblocks(md10)
assert len(blocks10) == 1
assert blocks10[0]['config']['image'] == 'docker.io/tymills620/ephemeral-python-uv:latest'
print("PASS: Shebang-only (no fences)")
# --- Test 11: Empty input ---
blocks_empty = parse_codeblocks("")
assert blocks_empty == []
print("PASS: Empty input")
# --- Test 12: Quad-backtick stripping ---
md12 = "````markdown\n```python\nprint(1)\n```\n````"
blocks12 = parse_codeblocks(md12)
assert len(blocks12) == 1
assert blocks12[0]['config']['image'] == 'docker.io/tymills620/ephemeral-python-uv:latest'
print("PASS: Quad-backtick stripping")
# --- Test 13: Dataclass construction ---
result = ExecutionResult(
stdout="hello", stderr="", exit_code=0,
artifact_paths=["/tmp/out.txt"], artifact_dir="/tmp"
)
assert result.exit_code == 0
assert result.artifact_paths == ["/tmp/out.txt"]
print("PASS: ExecutionResult dataclass")
# --- Test 14: resolve_runtime_config with custom image override ---
cfg = resolve_runtime_config("python image=myrepo/mypython:latest")
assert cfg['image'] == 'myrepo/mypython:latest'
assert cfg['cmd'] == ['uv', 'run', '-']
print("PASS: Custom image override")
# --- Test 15: Entrypoint override ---
cfg2 = resolve_runtime_config("tiddlywiki")
assert cfg2['entrypoint'] == ''
print("PASS: Entrypoint from LANG_MAP")
# --- Test 16: Infer third-party deps from imports ---
code16 = """import requests
from numpy import array
import pandas as pd
import os, sys, json
from collections import OrderedDict
import matplotlib.pyplot as plt
"""
deps16 = infer_python_dependencies(code16)
assert deps16 == ['matplotlib', 'numpy', 'pandas', 'requests'], f"Unexpected deps: {deps16}"
assert infer_python_dependencies("print(1)\nimport math\n") == []
print("PASS: Import-based dependency inference")
# --- Test 17: Dotted and multi-line imports ---
code17 = """from sklearn.ensemble import (
RandomForestClassifier,
GradientBoosting,
)
import urllib.request
"""
assert infer_python_dependencies(code17) == ['sklearn']
assert infer_python_dependencies("from . import helper\nfrom ..pkg import thing\n") == []
print("PASS: Dotted/multi-line import handling")
# --- Test 18: PEP 723 metadata respected (no double injection) ---
code18 = """# /// script
# dependencies = ["requests<3", "rich"]
# ///
import requests
"""
assert extract_declared_dependencies(code18) == ['requests<3', 'rich']
assert inject_python_dependency_metadata(code18, ['numpy']) == code18
print("PASS: Existing PEP 723 metadata respected")
# --- Test 19: Multi-line declared dependencies preserved ---
code19 = """# /// script
# dependencies = [
# "aiohttp",
# "pydantic>=2",
# ]
# ///
import aiohttp
"""
assert extract_declared_dependencies(code19) == ['aiohttp', 'pydantic>=2']
print("PASS: Multi-line declared dependencies")
# --- Test 20: Header injection preserves shebang ---
code20 = "#! python\nimport requests\nprint(1)\n"
injected20 = inject_python_dependency_metadata(code20, ['requests'])
assert injected20.startswith("#! python\n# /// script\n"), injected20
print("PASS: Header injection after shebang")
# --- Test 21: prepare_python_block returns deps + injected content ---
block21 = {'type': 'code', 'header': 'python', 'content': 'import numpy\nprint(1)\n', 'config': {}}
prepared21, deps21 = prepare_python_block(block21)
assert deps21 == ['numpy']
assert prepared21['content'].startswith("# /// script\n")
assert block21['content'] == 'import numpy\nprint(1)\n' # original untouched
print("PASS: prepare_python_block")
# --- Test 22: language-map image set (super-seed hydration) ---
# The hydrate set must equal the receiver-side allowlist exactly, or a
# "super-seed" node could be missing an image remote jobs may request.
from ephemeral_core.config import LANG_MAP, mapped_images
imgs = mapped_images()
assert len(imgs) > 30, f"expected a rich language map, got {len(imgs)} images"
assert len(imgs) == len(set(imgs)), "mapped_images must dedupe (gcc backs c/cpp/fortran)"
assert "docker.io/tymills620/ephemeral-python-uv:latest" in imgs
assert "docker.io/library/node:18-alpine" in imgs
assert "docker.io/library/gcc:latest" in imgs
# SQL: the image must have a shell (executor appends `sh`) and the aliases
# must point at the real entry, or ```sql/sqlite3 silently resolves to nothing.
assert "docker.io/keinos/sqlite3:latest" in imgs
assert LANG_MAP["sqlite"] == "sql" and LANG_MAP["sqlite3"] == "sql"
assert LANG_MAP["sql"]["cmd"][0] == "sh"
# Mermaid definitions render as PNG artifacts; `mm` is the short alias.
mermaid_cfg = resolve_runtime_config("mermaid")
assert mermaid_cfg["image"] == "ghcr.io/mermaid-js/mermaid-cli/mermaid-cli:latest"
assert mermaid_cfg["entrypoint"] == ""
assert mermaid_cfg["user"] == "0"
assert mermaid_cfg["cmd"] == ["sh", "-c", "cat > /tmp/diagram.mmd && mmdc -p /puppeteer-config.json -i /tmp/diagram.mmd -o /output/diagram.png"]
assert LANG_MAP["mm"] == "mermaid"
assert resolve_runtime_config("mm") == mermaid_cfg
assert mermaid_cfg["image"] in imgs
from unittest.mock import patch
from ephemeral_core.executor import _build_podman_cmd
with patch("ephemeral_core.executor.podman_supports_cgroup_limits", return_value=False):
mermaid_podman_cmd = _build_podman_cmd(mermaid_cfg, "/tmp/output")
assert mermaid_podman_cmd[mermaid_podman_cmd.index("--user") + 1] == "0"
assert mermaid_podman_cmd[-2:] == [mermaid_cfg["image"], "sh"]
from ephemeral_net.sandbox import default_image_allowlist
assert set(imgs) == set(default_image_allowlist()), \
"hydrate set must match the receiver-side allowlist"
print(f"PASS: language-map image set ({len(imgs)} unique images == allowlist)")
# --- Test 22b: local-LLM monolith wiring ---
# The image ENTRYPOINT boots llama.cpp before user code, so the language entry
# must NOT override the entrypoint (tiddlywiki-style `entrypoint: ''` would
# replace the boot script with plain sh) and must not use `uv run` (the
# two-stage dependency path re-invokes the payload as /deps/venv/bin/python -,
# skipping the boot step). It also must stay usable under --network none.
llm_cfg = resolve_runtime_config("llm")
assert llm_cfg["image"] == "docker.io/tymills620/ephemeral-qwen-monolith:latest"
assert llm_cfg.get("entrypoint") is None, "llm must inherit the image ENTRYPOINT"
assert llm_cfg["cmd"] == ["python", "-"]
assert llm_cfg["allow_network"] is False
assert llm_cfg["allow_chain"] is False
assert llm_cfg["image"] in imgs
assert LANG_MAP["qwen"] == "llm" and LANG_MAP["local-llm"] == "llm"
assert resolve_runtime_config("qwen")["image"] == llm_cfg["image"]
print("PASS: local-LLM monolith wiring")
# --- Test 22c: bare file extensions resolve as language declarations ---
# An editor that knows only a filename's suffix can serialize `report.tex` as a
# `tex` fence without probing Ephemeral's language list. Extensions are declared
# without a dot, so they can never be read as a seed filename, whose header
# always carries one (`data.csv`).
from ephemeral_core.config import FILE_EXTENSIONS
assert FILE_EXTENSIONS["py"] == "python" and FILE_EXTENSIONS["tex"] == "latex"
assert FILE_EXTENSIONS["rb"] == "ruby" and FILE_EXTENSIONS["rs"] == "rust"
# `.pl` is Perl to editors, so Prolog is addressed by name or `swipl` only.
assert FILE_EXTENSIONS["pl"] == "perl" and LANG_MAP["swipl"] == "prolog"
assert "pl" not in {k for k, v in LANG_MAP.items() if isinstance(v, str) and v == "prolog"}
# `.cc` is C++ by suffix convention; the C++ image also runs C-style code.
assert FILE_EXTENSIONS["cc"] == "cpp" and LANG_MAP["c"]
for ext, lang in FILE_EXTENSIONS.items():
assert isinstance(LANG_MAP.get(lang), dict), f"{ext} -> {lang} is not a canonical language"
assert LANG_MAP[ext] == lang
ext_cfg = resolve_runtime_config(ext)
assert ext_cfg["image"] == LANG_MAP[lang]["image"], f"{ext} must resolve to {lang}"
assert ext_cfg["cmd"] == LANG_MAP[lang]["cmd"], f"{ext} must reuse {lang}'s command"
# The dotted form stays a seed file; only the bare form is a declaration.
seed_blocks = parse_codeblocks("```data.rb\nputs 1\n```")
assert seed_blocks[0]["type"] == "seed" and seed_blocks[0]["name"] == "data.rb"
assert parse_codeblocks("```rb\nputs 1\n```")[0]["type"] == "code"
assert parse_codeblocks("```tex\nx\n```")[0]["config"]["image"] == "docker.io/pandoc/extra"
print(f"PASS: file extensions resolve as languages ({len(FILE_EXTENSIONS)} aliases)")
# --- Test 23: Chaining is OFF by default ---
md23 = "```python\nprint(1)\n```"
blocks23 = parse_codeblocks(md23)
assert blocks23[0]['config']['allow_chain'] == False
print("PASS: Chaining off by default (no flag)")
# --- Test 24: Chaining opt-in flags (chain / piping / pipe) ---
for flag in ('chain', 'piping', 'pipe'):
cfg = resolve_runtime_config(f"python {flag}")
assert cfg['allow_chain'] == True, f"{flag} should enable chaining"
print("PASS: chain / piping / pipe opt in to chaining")
# --- Test 25: nopipe still overrides chain ---
cfg = resolve_runtime_config("python chain nopipe")
assert cfg['allow_chain'] == False
print("PASS: nopipe overrides chain")
# --- Test 26: Run grouping + chained detection ---
from ephemeral_core.executor import group_into_runs, request_is_chained, MAX_PARALLEL_RUNS
md26 = "```python\nprint(1)\n```\n\n```node\nconsole.log(2)\n```\n\n```python\nprint(3)\n```"
runs26 = group_into_runs(parse_codeblocks(md26))
assert len(runs26) == 3, f"3 different runs expected, got {len(runs26)}"
assert not request_is_chained(runs26)
# Same-config blocks merge into one run; chaining declared anywhere -> True
md26b = "```python\nprint(1)\n```\n\n```python\nprint(2)\n```"
assert len(group_into_runs(parse_codeblocks(md26b))) == 1
md26c = "```python chain\nprint(1)\n```\n\n```node\nconsole.log(2)\n```"
assert request_is_chained(group_into_runs(parse_codeblocks(md26c)))
assert MAX_PARALLEL_RUNS == 4, "local parallel guardrail must be 4"
print("PASS: Run grouping + chained detection")
# --- Test 27: Container resource limits scale to small hosts ---
from ephemeral_core import executor as executor_mod
_saved_host_mib = executor_mod._host_memory_mib
_saved_limits_env = {k: os.environ.pop(k, None) for k in (
"EPHEMERAL_MEMORY_LIMIT", "EPHEMERAL_CPU_LIMIT", "EPHEMERAL_PIDS_LIMIT")}
try:
# 1 GiB host -> ~half of RAM (min 256 MiB, 64 MiB steps), 1 cpu, 256 pids
executor_mod._host_memory_mib = lambda: 1024
assert executor_mod._container_resource_limits() == \
['--memory', '512m', '--cpus', '1', '--pids-limit', '256'], \
executor_mod._container_resource_limits()
# 2 GiB host -> 1024m ceiling
executor_mod._host_memory_mib = lambda: 2048
assert executor_mod._container_resource_limits()[1] == '1024m'
# Tiny 512 MiB host -> clamped to the 256 MiB floor
executor_mod._host_memory_mib = lambda: 512
assert executor_mod._container_resource_limits() == \
['--memory', '256m', '--cpus', '1', '--pids-limit', '256']
# Unknown memory (macOS/Windows) and large hosts keep the historical defaults
executor_mod._host_memory_mib = lambda: None
assert executor_mod._container_resource_limits() == \
['--memory', '2g', '--cpus', '2', '--pids-limit', '512']
executor_mod._host_memory_mib = lambda: 8192
assert executor_mod._container_resource_limits() == \
['--memory', '2g', '--cpus', '2', '--pids-limit', '512']
# Explicit env overrides win on any host
os.environ["EPHEMERAL_MEMORY_LIMIT"] = "700m"
os.environ["EPHEMERAL_CPU_LIMIT"] = "1.0"
os.environ["EPHEMERAL_PIDS_LIMIT"] = "128"
executor_mod._host_memory_mib = lambda: 1024
assert executor_mod._container_resource_limits() == \
['--memory', '700m', '--cpus', '1.0', '--pids-limit', '128']
finally:
executor_mod._host_memory_mib = _saved_host_mib
# Clear the overrides this test set, then restore the caller's values.
for k, v in _saved_limits_env.items():
os.environ.pop(k, None)
if v is not None:
os.environ[k] = v
print("PASS: Container resource limits scale to host RAM (small-VPS safe)")
# RAM is read from the Podman machine (WSL2 on Windows), not the tray host.
_saved_check_output = executor_mod.subprocess.check_output
try:
executor_mod.subprocess.check_output = lambda *a, **k: b"2147483648\n" # 2 GiB
assert executor_mod._host_memory_mib() == 2048
assert executor_mod._container_resource_limits()[1] == '1024m'
executor_mod.subprocess.check_output = lambda *a, **k: b"17179869184\n" # 16 GiB
assert executor_mod._container_resource_limits()[1] == '2g'
finally:
executor_mod.subprocess.check_output = _saved_check_output
print("PASS: Host RAM read from podman info; memory = half of RAM, capped at 2g")
# --- Test 28: native image architecture checks --------------------------
# A foreign-architecture image must not be considered warm. Multi-platform
# tags are resolved by the registry/Podman pull for the local architecture;
# the local cache check only accepts the architecture actually installed.
_saved_machine = executor_mod.platform.machine
_saved_check_call = executor_mod.subprocess.check_call
_saved_check_output = executor_mod.subprocess.check_output
try:
executor_mod.platform.machine = lambda: "aarch64"
assert executor_mod.host_arch() == "arm64"
executor_mod.subprocess.check_output = lambda *args, **kwargs: b"amd64\n"
assert not executor_mod.image_is_compatible("octave")
executor_mod.subprocess.check_output = lambda *args, **kwargs: b"arm64\n"
assert executor_mod.image_is_compatible("octave")
executor_mod.subprocess.check_call = lambda *args, **kwargs: 0
# octave is amd64-only upstream: even a cached arm64 copy is never warm
# on an arm64 host because the host must not pull or advertise it.
octave_image = executor_mod.LANG_MAP["octave"]["image"]
assert octave_image in executor_mod.images_blocked_on_host()
assert not executor_mod.check_image_exists(octave_image)
assert executor_mod.check_image_exists("amd-image")
executor_mod.platform.machine = lambda: "x86_64"
assert octave_image not in executor_mod.images_blocked_on_host()
executor_mod.platform.machine = lambda: "aarch64"
inventory = [
{"Architecture": "amd64", "Names": ["amd-image"]},
{"Architecture": "arm64", "Names": ["arm-image"]},
]
executor_mod.subprocess.check_output = lambda *args, **kwargs: __import__("json").dumps(inventory).encode()
assert executor_mod.list_local_images() == ["arm-image"]
finally:
executor_mod.platform.machine = _saved_machine
executor_mod.subprocess.check_call = _saved_check_call
executor_mod.subprocess.check_output = _saved_check_output
print("PASS: foreign cached images are excluded from native warm-image routing")
# --- Test 28b: amd64-only images are blocked on arm64 hosts only ----------
# Every entry must name a real LANG_MAP image (so the table cannot drift from
# config.py), be blocked on arm64, be allowed on amd64, and never be listed
# as warm on arm64 even when Podman names it with an explicit :latest tag.
configured = {cfg["image"] for cfg in executor_mod.LANG_MAP.values() if isinstance(cfg, dict) and cfg.get("image")}
assert set(executor_mod._NATIVE_ARCH_ONLY_IMAGES) <= configured, "native-arch table names an image missing from LANG_MAP"
_saved_machine = executor_mod.platform.machine
_saved_check_output = executor_mod.subprocess.check_output
try:
for image, arch in executor_mod._NATIVE_ARCH_ONLY_IMAGES.items():
executor_mod.platform.machine = lambda: "aarch64"
blocked_arm = executor_mod.images_blocked_on_host()
assert image in blocked_arm or image + ":latest" in blocked_arm, image
executor_mod.platform.machine = lambda: "x86_64"
blocked_amd = executor_mod.images_blocked_on_host()
assert image not in blocked_amd and image + ":latest" not in blocked_amd, image
executor_mod.platform.machine = lambda: "aarch64"
# No Architecture key (older Podman): these names must be stopped by the
# blocklist itself, not by the architecture comparison.
inventory = [{"Names": ["docker.io/esolang/piet:latest", "docker.io/pandoc/extra:latest"]}]
executor_mod.subprocess.check_output = lambda *args, **kwargs: __import__("json").dumps(inventory).encode()
assert executor_mod.list_local_images() == []
finally:
executor_mod.platform.machine = _saved_machine
executor_mod.subprocess.check_output = _saved_check_output
print("PASS: amd64-only images are blocked on arm64 hosts and match LANG_MAP")
# --- Test 28c: Octave image ships the MATLAB parity it advertises ---------
# These live in the image and in the octave command, so nothing else in the
# suite can notice their removal: the control package has no pzplot at all,
# and Octave throws figures away before atexit runs, so a plot only exists
# while the block itself is running. Both were measured on Octave 11.3.0.
from pathlib import Path as _Path
_OCT_ROOT = _Path(__file__).resolve().parent
_OCT_DF = (_OCT_ROOT / "dockerfiles" / "ephemeral-octave-forge.dockerfile").read_text(encoding="utf-8")
_OCT_SHIM = (_OCT_ROOT / "dockerfiles" / "octave" / "pzplot.m").read_text(encoding="utf-8")
_OCT_HOOK = (_OCT_ROOT / "dockerfiles" / "octave" / "__ephemeral_save_figures.m").read_text(encoding="utf-8")
_OCT_CMD = executor_mod.LANG_MAP["octave"]["cmd"]
assert "pkg install -forge control" in _OCT_DF
# Site-wide startup, not ~/.octaverc: loading must not depend on HOME.
assert "/usr/share/octave/site/m/startup/octaverc" in _OCT_DF
assert ">> /root/.octaverc" not in _OCT_DF # the load lives in the site startup file
assert "pkg load control" in _OCT_DF and "pkg load signal" in _OCT_DF
assert '"defaultfigurevisible", "off"' in _OCT_DF
# No atexit call and no pinned toolkit: the former never sees a figure, and
# the latter would put gnuplot's warning in blocks that never plot.
assert 'atexit ("' not in _OCT_DF and 'graphics_toolkit ("' not in _OCT_DF
# Both .m files must land on the site path, or the shim and hook never
# resolve by name.
assert "site/m/pzplot.m" in _OCT_DF and "site/m/__ephemeral_save_figures.m" in _OCT_DF
assert "pzmap (varargin{:})" in _OCT_SHIM and "varargout{1:nargout}" in _OCT_SHIM
assert '"/output"' in _OCT_HOOK and "figure-%d.png" in _OCT_HOOK
# An explicit print () wins: the hook may not overwrite a file it wrote.
assert 'exist (target, "file")' in _OCT_HOOK
# The command must source the payload (the figure still exists then) and
# save before exiting, keeping the block's own error and exit status.
assert _OCT_CMD[:2] == ["sh", "-c"], _OCT_CMD
assert 'source ("/dev/stdin")' in _OCT_CMD[2], _OCT_CMD
assert "__ephemeral_save_figures ()" in _OCT_CMD[2], _OCT_CMD
assert "rethrow (err)" in _OCT_CMD[2], _OCT_CMD
print("PASS: Octave image provides pzplot and saves figures to /output")
# --- Test 29: result fences outgrow the payload's own backticks -----------
# Block output is arbitrary text, and a model reply routinely contains its own
# ``` fences. A fixed three-backtick wrapper ends at the first of those, so the
# rest of the result spills out of its panel and renders as stray markdown.
# CommonMark closes a fence only with a run at least as long, so the wrapper
# must be one backtick longer than the longest run inside the payload.
import re
from ephemeral_core.executor import _md_fence
assert _md_fence("42") == "```text\n42\n```", _md_fence("42")
assert _md_fence("a ``` b") == "````text\na ``` b\n````", _md_fence("a ``` b")
assert _md_fence("`````") == "``````text\n`````\n``````", _md_fence("`````")
def _scan_one(text):
"""CommonMark-style scan: the body of the single outermost fence."""
lines = text.split("\n")
opened = re.match(r"^(`{3,})", lines[0])
assert opened, lines[0]
width = len(opened.group(1))
closers = [i for i, line in enumerate(lines[1:], 1) if re.fullmatch(rf"`{{{width},}}", line)]
assert len(closers) == 1, closers
return "\n".join(lines[1:closers[0]])
for payload in ["plain", "```python\nprint(1)\n```", "````css\nx\n````", "inline `code` only"]:
assert _scan_one(_md_fence(payload)) == payload, payload
# The envelope the executor actually returns for a reply that contains a
# fenced block: four backticks around the whole payload, so the inner ```python
# cannot close it.
import tempfile
from ephemeral_core.executor import _format_success_result
_out_dir = tempfile.mkdtemp()
_envelope = _format_success_result(
"MARK\n```python\nprint(1)\n```\n", "", [(0, "MARK", "llm")],
"llm", 1, 1, True, _out_dir, False,
).stdout_formatted
assert _envelope.startswith("## Llm Result\n\n````text\n"), repr(_envelope)
assert _envelope.rstrip("\n").endswith("\n````"), repr(_envelope)
assert _scan_one(_envelope.split("\n", 2)[2].rstrip("\n")) == "```python\nprint(1)\n```", repr(_envelope)
# The "this is already a result" safety guard must still recognise a widened
# fence, or a pasted result would be executed a second time.
assert re.search(r"^## (Run|Result) .*[\r\n]+`{3,}text", "## Run 1\n" + _md_fence("```"), re.MULTILINE)
print("PASS: result fences outgrow the payload's own backticks")
# --- Test 30: llm blocks are chat boxes that stay Python -------------------
# Bare prose becomes print(chat("...")) and `chat`/`client` are injected, so
# an llm block needs no boilerplate. Real Python still runs as Python, blocks
# that import the helper themselves are untouched, and nothing here touches a
# non-monolith image (plain `python` blocks keep their own behaviour).
import json
from ephemeral_core.parser import (
is_local_llm_config,
prepare_llm_block,
reads_as_python,
)
def _llm(content):
return prepare_llm_block({"type": "code", "header": "llm", "content": content, "config": {}})["content"]
prompt = "Can you write a small python script to calculate the first 10 primes?"
assert _llm(prompt) == (
"from ephemeral_llm import chat, client\n"
'print(chat("Can you write a small python script to calculate the first 10 primes?", prose=True))\n'
), _llm(prompt)
# Multi-line prose, quotes and newlines arrive intact in one valid literal.
prose = 'Explain "groupby" in\none sentence.'
wrapped = _llm(prose)
assert wrapped.startswith("from ephemeral_llm import chat, client\nprint(chat("), wrapped
assert json.loads(wrapped.split("print(chat(", 1)[1].rsplit(", prose=True))\n", 1)[0]) == prose
assert wrapped.endswith(", prose=True))\n"), wrapped
# Code keeps running as code, with the names injected.
code30 = 'df = pd.DataFrame({"n": [1]})\nprint(chat("sum?", max_tokens=8))'
assert _llm(code30) == "from ephemeral_llm import chat, client\n" + code30
# Explicit imports are respected (no duplicate preamble), and blank is a no-op.
own = 'from ephemeral_llm import chat\nprint(chat("hi"))'
assert _llm(own) == own
assert _llm(" \n") == " \n"
# Prose vs program: the heuristic that decides which branch a block takes.
for source in (prompt, "hello", "summarise this please:"):
assert not reads_as_python(source), source
for source in ('print("hi")', "1 + 1", "x = 1", "import pandas as pd", "def f():\n return 1"):
assert reads_as_python(source), source
# Image gating: only the monolith gets the rewrite.
assert is_local_llm_config({"image": "docker.io/tymills620/ephemeral-qwen-monolith:latest"})
assert not is_local_llm_config({"image": "docker.io/tymills620/ephemeral-python-uv:latest"})
assert not is_local_llm_config({})
# Idempotent: a re-prepared (already rewritten) block is left alone.
once = _llm(prompt)
assert prepare_llm_block({"type": "code", "content": once})["content"] == once
print("PASS: llm bare prompts and implicit chat()/client()")
# --- Test 31: generated code is saved for the next container --------------
# The image's helper strips the prose off a model reply and writes the code to
# /output, where `chain` stages it into the following container. The suffix
# follows whatever language the model fenced (not Python specifically), so the
# next block can import/require/run it in its own language.
import importlib.util
import io
import pathlib
import sys
import tempfile
import types
from contextlib import redirect_stdout
ROOT = pathlib.Path(__file__).resolve().parent
LLM_MODULE = ROOT / "dockerfiles" / "ephemeral-qwen-monolith" / "ephemeral_llm.py"
try: # the image's module imports the OpenAI SDK; stub it when absent
import openai # noqa: F401
_openai_stub = False
except ImportError:
_stub = types.ModuleType("openai")
class _OpenAI:
def __init__(self, **kwargs):
self.kwargs = kwargs
_stub.OpenAI = _OpenAI
sys.modules["openai"] = _stub
_openai_stub = True
_spec = importlib.util.spec_from_file_location("ephemeral_llm_under_test", LLM_MODULE)
llm = importlib.util.module_from_spec(_spec)
_spec.loader.exec_module(llm)
reply = (
"Certainly! Here's a small Python script:\n\n"
"```python\n"
"def is_prime(n):\n"
" return n > 1 and all(n % i for i in range(2, int(n ** 0.5) + 1))\n"
"\n"
"print([n for n in range(2, 30) if is_prime(n)][:10])\n"
"```\n\n"
"Run it anywhere.\n"
)
code31 = llm.extract_code(reply)
assert code31.startswith("def is_prime"), code31
assert "Certainly" not in code31 and "Run it anywhere" not in code31
assert code31.endswith("\n") and not code31.endswith("\n\n")
# Code-agnostic: the fence label, not Python, decides the file's suffix.
assert llm.extract_code("Here you go:\n\n```js\nconsole.log('hi');\n```\n") == "console.log('hi');\n"
assert llm._filename("generated", "javascript") == "generated.js"
assert llm._filename("generated", "bash") == "generated.sh"
assert llm._filename("generated", "") == "generated"
assert llm._filename("generated", "piet") == "generated" # unknown label, no suffix
assert llm._filename("primes.py", "python") == "primes.py"
assert llm._filename("../../etc/passwd", "python") == "passwd.py"
# Fence selection: first labelled block by default, the requested one on ask.
mixed = (
"First the shell step:\n\n```bash\necho hi\n```\n\n"
"Then the Python:\n\n```python\nprint(1)\n```\n"
)
assert llm.extract_code(mixed) == "echo hi\n"
assert llm.extract_code(mixed, lang="python") == "print(1)\n"
assert llm.extract_code(mixed, lang="py") == "print(1)\n" # alias
try:
llm.extract_code(mixed, lang="ruby")
raise AssertionError("a missing language must not silently save another")
except ValueError as exc31:
assert "bash" in str(exc31) and "python" in str(exc31), str(exc31)
# A wider fence keeps an inner ```python``` intact as code.
assert llm.extract_code("Look:\n\n````markdown\n```python\nprint(1)\n```\n````\n") == (
"```python\nprint(1)\n```\n"
)
assert llm.extract_code("~~~{.js}\nlet x = 1;\n~~~") == "let x = 1;\n"
# Unlabelled fences follow the caller's language — but only if they're code.
assert llm.extract_code("```\nprint(2)\n```", lang="python") == "print(2)\n"
try:
llm.extract_code("```\nnot python at all here.\n```", lang="python")
raise AssertionError("an unlabelled prose block must not become Python")
except ValueError:
pass
# Unfenced replies: bare code is accepted, prose is not.
assert llm.extract_code("print(3)\n") == "print(3)\n"
try:
llm.extract_code("I can't help with that.")
raise AssertionError("a prose-only reply must not be written as code")
except ValueError as exc31:
assert "can't help" in str(exc31), str(exc31)
with tempfile.TemporaryDirectory() as tmp31:
path31 = llm.save_code(reply, directory=tmp31)
assert os.path.basename(path31) == "generated.py", path31
with open(path31, encoding="utf-8") as fh31:
assert fh31.read() == code31 # written verbatim
# By default the code itself is echoed, fenced in its own language, so one
# block shows the program and the next block shows what it printed.
printed31 = io.StringIO()
with redirect_stdout(printed31):
llm.save_code("```bash\necho hi\n```", directory=tmp31)
assert "generated.sh" in printed31.getvalue(), printed31.getvalue()
assert "```bash\necho hi\n```" in printed31.getvalue(), printed31.getvalue()
only_api31 = io.StringIO()
with redirect_stdout(only_api31):
llm.save_code(reply, directory=tmp31, echo=False)
assert "saved" in only_api31.getvalue(), only_api31.getvalue()
assert "def is_prime" not in only_api31.getvalue(), only_api31.getvalue()
silent31 = io.StringIO()
with redirect_stdout(silent31):
llm.save_code(reply, directory=tmp31, quiet=True)
assert silent31.getvalue() == "", silent31.getvalue()
# Created on demand, and an unknown language still lands (no suffix).
nested31 = os.path.join(tmp31, "nested")
assert llm.save_code("```piet\nnope\n```", directory=nested31, quiet=True).endswith("generated")
# A __main__ guard doesn't fire on import — point at runpy instead.
printed31 = io.StringIO()
with redirect_stdout(printed31):
llm.save_code("```python\nif __name__ == '__main__':\n print(4)\n```", directory=tmp31)
# runpy without run_name="__main__" would skip the guard as well.
assert "runpy.run_path" in printed31.getvalue(), printed31.getvalue()
assert 'run_name="__main__"' in printed31.getvalue(), printed31.getvalue()
# write_code = chat + save in one call (chat is stubbed, then restored so
# the tests below exercise the real one). `ask_code` is the old name and
# must keep working, or every document written before the rename breaks.
_real_chat31 = llm.chat
llm.chat = lambda *a, **k: "```python\nprint(5)\n```"
try:
asked31 = llm.write_code("write a script", directory=tmp31, quiet=True)
finally:
llm.chat = _real_chat31
with open(asked31, encoding="utf-8") as fh31:
assert fh31.read() == "print(5)\n"
assert llm.ask_code is llm.write_code, "ask_code must stay an alias of write_code"
if _openai_stub:
del sys.modules["openai"]
print("PASS: model replies become runnable, code-agnostic files")
# --- Test 32: locally staged modules are not PyPI requirements ------------
# Seeds and chained /output files are written into the workdir before the code
# runs, so `import generated` (or `import helpers` from a seeded file) must
# resolve locally instead of sending uv to the package registry.
from ephemeral_core.executor import _local_module_names
assert _local_module_names([]) == set()
seeds32 = [
{"type": "seed", "name": "generated.py"},
{"type": "seed", "name": "pkg/__init__.py"},
{"type": "seed", "name": "data.json"},
{"type": "seed", "name": "lib\\util.py"}, # windows-style separator
{"type": "code", "name": "ignored.py"},
]
local32 = _local_module_names(seeds32)
assert local32 == {"generated", "pkg", "util"}, local32
assert infer_python_dependencies("import generated\nimport pandas as pd") == ["generated", "pandas"]
assert infer_python_dependencies("import generated\nimport pandas as pd", local32) == ["pandas"]
assert infer_python_dependencies("from generated import first_n_primes", {"generated"}) == []
# No PEP 723 header is injected for a locally staged module.
staged32, deps32 = prepare_python_block(
{"type": "code", "content": "import generated\nprint(generated.__name__)"}, local_modules=local32
)
assert deps32 == [], deps32
assert staged32["content"] == "import generated\nprint(generated.__name__)"
print("PASS: staged modules (seeds, chained generated code) aren't PyPI deps")
# --- Test 33: every run gets a time cap, `notimeout` lifts it locally -----
# A caller that passes no timeout (the tray's local path) used to get an
# unbounded run, so one `while True` could hold the host forever. The core now
# applies DEFAULT_TIMEOUT; only a local header can opt out, via a keyword the
# swarm strips (see test_net's sanitizer test).
from ephemeral_core.config import DEFAULT_TIMEOUT, NO_TIMEOUT_FLAGS
from ephemeral_core.executor import effective_timeout
assert DEFAULT_TIMEOUT == 300, DEFAULT_TIMEOUT
assert resolve_runtime_config("python")["ignore_timeout"] is False
for flag33 in sorted(NO_TIMEOUT_FLAGS):
cfg33 = resolve_runtime_config(f"python {flag33}")
assert cfg33["ignore_timeout"] is True, flag33
# The keyword is consumed, not passed through as a language or an override.
assert cfg33["image"].endswith("ephemeral-python-uv:latest"), cfg33
assert cfg33["cmd"] == ["uv", "run", "-"], cfg33
# The cap fills in only where the caller left it open; explicit wins.
assert effective_timeout({}, None) == DEFAULT_TIMEOUT
assert effective_timeout({"ignore_timeout": False}, None) == DEFAULT_TIMEOUT
assert effective_timeout({"ignore_timeout": True}, None) is None
assert effective_timeout({"ignore_timeout": True}, 5) == 5
assert effective_timeout({"ignore_timeout": False}, 60) == 60
print("PASS: default time cap, with a local-only notimeout opt-out")
# --- Test 34: every reply reports the seed that produced it ---------------
# llama.cpp honours `seed` but never reports the one it used, so the helper
# picks one itself and prints it — that is what makes a reply reproducible.
class _Msg34:
content = "Five."
class _Choice34:
message = _Msg34()
class _Resp34:
choices = [_Choice34()]
system_fingerprint = "b11706-f74e2fa6a"
class _Completions34:
def __init__(self, seen, reply=None):
self.seen = seen
self.reply = _Resp34() if reply is None else reply
def create(self, **kwargs):
self.seen.append(kwargs)
return self.reply
class _Chat34:
def __init__(self, seen, reply=None):
self.completions = _Completions34(seen, reply)
class _Client34:
def __init__(self, seen, reply=None):
self.chat = _Chat34(seen, reply)
seen34 = []
llm.client = lambda **kw: _Client34(seen34)
llm._server_params = {"temperature": 0.800000011920929} # as /props reports it
llm._last_call = None
llm._last_fingerprint = None
buf34 = io.StringIO()
with redirect_stdout(buf34):
reply34 = llm.chat("hi")
request34 = seen34[-1]
assert reply34 == "Five.", reply34
assert isinstance(request34["seed"], int) and 0 <= request34["seed"] < 2 ** 31
# The line is shaped as the call to replay, so it can be pasted straight back.
assert (
f"[ephemeral-llm] chat(prompt, seed={request34['seed']}, temperature=0.8)"
in buf34.getvalue()
), buf34.getvalue()
# The temperature reported is the server's own default, not an invention.
assert llm.last_call()["temperature"] == 0.8
assert llm.last_call()["seed"] == request34["seed"]
assert llm.last_call()["temperature"] == 0.8
assert llm.last_call()["system_fingerprint"] == "b11706-f74e2fa6a"
# An explicit seed and temperature go through unchanged.
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi", seed=12345, temperature=0.25)
assert seen34[-1]["seed"] == 12345 and seen34[-1]["temperature"] == 0.25
assert (
"[ephemeral-llm] chat(prompt, seed=12345, temperature=0.25)"
in buf34.getvalue()
), buf34.getvalue()
# A prose block is labelled as such, so the reader knows it was not run as code.
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi", seed=99, temperature=0.5, prose=True)
assert (
"[ephemeral-llm] prose: chat(prompt, seed=99, temperature=0.5)"
in buf34.getvalue()
), buf34.getvalue()
# An ordinary call keeps the plain label.
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi", seed=99, temperature=0.5)
assert "prose:" not in buf34.getvalue(), buf34.getvalue()
# report=False (or EPHEMERAL_LLM_REPORT=0) leaves the output to the reply.
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi", seed=1, report=False)
assert buf34.getvalue() == "", buf34.getvalue()
os.environ[llm.SEED_ENV] = "777"
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi")
assert seen34[-1]["seed"] == 777, seen34[-1]
assert llm.new_seed() == 777
del os.environ[llm.SEED_ENV]
os.environ["EPHEMERAL_LLM_REPORT"] = "0"
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi")
assert buf34.getvalue() == "", buf34.getvalue()
del os.environ["EPHEMERAL_LLM_REPORT"]
# A new server build is called out: the seed only reproduces within one build.
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.chat("hi", seed=1)
_Resp34.system_fingerprint = "other-build-123"
llm.chat("hi", seed=1)
assert "server build changed" in buf34.getvalue(), buf34.getvalue()
# write_code forwards the seed, echoes the code, and names itself — not the
# chat() it used internally, which would print a second, wrong line.
class _CodeMsg34:
content = "```python\nprint(6)\n```"
class _CodeResp34(_Resp34):
choices = [type("_C34", (), {"message": _CodeMsg34()})()]
llm.client = lambda **kw: _Client34(seen34, reply=_CodeResp34())
with tempfile.TemporaryDirectory() as tmp34:
buf34 = io.StringIO()
with redirect_stdout(buf34):
path34 = llm.write_code("write a script", seed=99, directory=tmp34)
out34 = buf34.getvalue()
with open(path34, encoding="utf-8") as fh34:
assert fh34.read() == "print(6)\n"
assert path34.endswith("generated.py")
assert out34.startswith(
"[ephemeral-llm] write_code(prompt, seed=99, temperature=0.8)\n"
), out34
assert "[ephemeral-llm] chat(" not in out34, out34 # one line, not two
assert "saved " not in out34, out34 # off by default
assert "```python\nprint(6)\n```" in out34, out34 # code still echoed
# ... and can be brought back, or silenced entirely.
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.write_code(
"write a script", seed=99, directory=tmp34, echo=False,
save_confirmation=True,
)
assert "saved " in buf34.getvalue(), buf34.getvalue()
assert "print(6)" not in buf34.getvalue(), buf34.getvalue()
buf34 = io.StringIO()
with redirect_stdout(buf34):
llm.write_code("write a script", seed=99, directory=tmp34, quiet=True)
assert buf34.getvalue() == "", buf34.getvalue()
print("PASS: reported seed/temperature make a reply reproducible")
print("\n=== ALL 35 TESTS PASSED ===")