Skip to content

Commit bb250fd

Browse files
committed
test: require snapshot copy mode to stay inside the workspace
Copy mode currently materializes any artifact URI stored in the catalog. These tests refuse local paths and object-store URLs that resolve outside the catalog's parent workspace, including symlink and lookup swaps, and they fail until that containment exists.
1 parent 3aacb69 commit bb250fd

1 file changed

Lines changed: 359 additions & 0 deletions

File tree

Lines changed: 359 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,359 @@
1+
"""Copy-mode snapshot export must not read artifact URIs outside the workspace.
2+
3+
The workspace root is the parent of the catalog directory (the data root in the
4+
standard layout). Artifact files the pipeline publishes live under that parent,
5+
not inside the catalog directory, so copy mode still materializes those files.
6+
A URI whose resolved file is outside that root, or an object-store URL when the
7+
catalog itself is local, is attacker-controlled catalog data: copy mode must
8+
refuse it before those bytes are written anywhere the export creates.
9+
"""
10+
11+
from __future__ import annotations
12+
13+
import os
14+
import shutil
15+
import threading
16+
from collections.abc import Generator
17+
from contextlib import contextmanager
18+
from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
19+
from pathlib import Path
20+
21+
import duckdb
22+
import pytest
23+
from catalog_test_helpers import SNAPSHOT_EPISODE_STAMPS
24+
25+
import hflow
26+
import hflow.snapshot as snapshot_module
27+
import hflow.storage as storage_module
28+
from hflow.catalog import Catalog, CheckRunRow
29+
from hflow.cli import main as cli_main
30+
31+
OUTSIDE_MARKER = b"outside-workspace-artifact-bytes"
32+
INSIDE_MARKER = b"inside-workspace-artifact-bytes"
33+
34+
35+
def _workspace(tmp_path: Path) -> Path:
36+
workspace = tmp_path / "data"
37+
workspace.mkdir()
38+
return workspace
39+
40+
41+
def _append_artifact(catalog: Catalog, workspace: Path, artifact_uri: str) -> None:
42+
canonical_episode = workspace / "episode.canonical.mcap"
43+
canonical_episode.write_bytes(b"canonical bytes for artifact uri containment")
44+
catalog.append_episode(
45+
canonical_path=canonical_episode,
46+
stamps=SNAPSHOT_EPISODE_STAMPS,
47+
episode_metadata={"task": "containment"},
48+
check_rows=[
49+
CheckRunRow(
50+
check_name="media/contact_sheet",
51+
check_version="media-v1",
52+
critical=False,
53+
status=hflow.CheckStatus.MEASURED,
54+
duration_s=0.1,
55+
measurements={"artifact/preview.jpg": artifact_uri},
56+
)
57+
],
58+
)
59+
60+
61+
def _tree_contains(root: Path, marker: bytes) -> bool:
62+
if not root.exists():
63+
return False
64+
for path in root.rglob("*"):
65+
if path.is_file() and not path.is_symlink() and marker in path.read_bytes():
66+
return True
67+
return False
68+
69+
70+
def _record_marker_writes(monkeypatch: pytest.MonkeyPatch, marker: bytes) -> list[str]:
71+
"""Remember every copy or bucket download that persisted ``marker``."""
72+
hits: list[str] = []
73+
real_copyfile = shutil.copyfile
74+
75+
def copyfile(
76+
source: Path | str, destination: Path | str, *, follow_symlinks: bool = True
77+
) -> Path | str:
78+
result = real_copyfile(source, destination, follow_symlinks=follow_symlinks)
79+
written = Path(destination)
80+
if written.is_file() and marker in written.read_bytes():
81+
hits.append(str(written))
82+
return result
83+
84+
# snapshot and storage both bind the stdlib shutil module.
85+
monkeypatch.setattr(snapshot_module.shutil, "copyfile", copyfile)
86+
monkeypatch.setattr(storage_module.shutil, "copyfile", copyfile)
87+
real_download = storage_module._download_to_file_atomically
88+
89+
def download(get_result: object, destination: Path) -> None:
90+
real_download(get_result, destination)
91+
if destination.is_file() and marker in destination.read_bytes():
92+
hits.append(str(destination))
93+
94+
monkeypatch.setattr(storage_module, "_download_to_file_atomically", download)
95+
return hits
96+
97+
98+
def _assert_marker_absent(*roots: Path, hits: list[str]) -> None:
99+
assert hits == []
100+
for root in roots:
101+
assert not _tree_contains(root, OUTSIDE_MARKER)
102+
103+
104+
def test_copy_mode_still_materializes_artifact_inside_workspace(tmp_path: Path) -> None:
105+
workspace = _workspace(tmp_path)
106+
preview = workspace / "episodes" / "preview.jpg"
107+
preview.parent.mkdir()
108+
preview.write_bytes(INSIDE_MARKER)
109+
catalog = Catalog(workspace / "catalog")
110+
_append_artifact(catalog, workspace, str(preview.resolve()))
111+
output = tmp_path / "snapshot"
112+
113+
report = hflow.export_dataset_snapshot(
114+
catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY
115+
)
116+
117+
assert report.copied_media_count == 1
118+
assert _tree_contains(output, INSIDE_MARKER)
119+
assert not _tree_contains(output, OUTSIDE_MARKER)
120+
121+
122+
def test_copy_mode_refuses_artifact_uri_outside_workspace(
123+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
124+
) -> None:
125+
workspace = _workspace(tmp_path)
126+
secret = tmp_path / "outside.bin"
127+
secret.write_bytes(OUTSIDE_MARKER)
128+
catalog = Catalog(workspace / "catalog")
129+
_append_artifact(catalog, workspace, str(secret.resolve()))
130+
output = tmp_path / "snapshot"
131+
hits = _record_marker_writes(monkeypatch, OUTSIDE_MARKER)
132+
133+
with pytest.raises(ValueError, match="outside the workspace"):
134+
hflow.export_dataset_snapshot(catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY)
135+
136+
_assert_marker_absent(output, hits=hits)
137+
138+
139+
def test_cli_copy_mode_refuses_artifact_uri_outside_workspace(
140+
tmp_path: Path, capsys: pytest.CaptureFixture[str]
141+
) -> None:
142+
workspace = _workspace(tmp_path)
143+
secret = tmp_path / "outside.bin"
144+
secret.write_bytes(OUTSIDE_MARKER)
145+
catalog = Catalog(workspace / "catalog")
146+
_append_artifact(catalog, workspace, str(secret.resolve()))
147+
output = tmp_path / "snapshot"
148+
149+
exit_code = cli_main(
150+
[
151+
"export",
152+
"snapshot",
153+
"--catalog",
154+
str(catalog.root),
155+
"--output",
156+
str(output),
157+
"--media",
158+
"copy",
159+
]
160+
)
161+
162+
assert exit_code == 2
163+
assert "outside the workspace" in capsys.readouterr().err
164+
assert not _tree_contains(output, OUTSIDE_MARKER)
165+
166+
167+
def test_copy_mode_refuses_symlink_that_resolves_outside_workspace(
168+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
169+
) -> None:
170+
workspace = _workspace(tmp_path)
171+
secret = tmp_path / "outside.bin"
172+
secret.write_bytes(OUTSIDE_MARKER)
173+
link = workspace / "episodes" / "preview.jpg"
174+
link.parent.mkdir()
175+
link.symlink_to(secret)
176+
catalog = Catalog(workspace / "catalog")
177+
_append_artifact(catalog, workspace, str(link))
178+
output = tmp_path / "snapshot"
179+
hits = _record_marker_writes(monkeypatch, OUTSIDE_MARKER)
180+
181+
with pytest.raises(ValueError, match="outside the workspace"):
182+
hflow.export_dataset_snapshot(catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY)
183+
184+
_assert_marker_absent(output, hits=hits)
185+
186+
187+
def test_copy_mode_refuses_parent_segments_that_escape_workspace(
188+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
189+
) -> None:
190+
workspace = _workspace(tmp_path)
191+
secret = tmp_path / "outside.bin"
192+
secret.write_bytes(OUTSIDE_MARKER)
193+
(workspace / "episodes").mkdir()
194+
catalog = Catalog(workspace / "catalog")
195+
escaped = workspace / "episodes" / ".." / ".." / "outside.bin"
196+
_append_artifact(catalog, workspace, str(escaped))
197+
output = tmp_path / "snapshot"
198+
hits = _record_marker_writes(monkeypatch, OUTSIDE_MARKER)
199+
200+
with pytest.raises(ValueError, match="outside the workspace"):
201+
hflow.export_dataset_snapshot(catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY)
202+
203+
_assert_marker_absent(output, hits=hits)
204+
205+
206+
def test_copy_mode_refuses_relative_uri_that_escapes_workspace(
207+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
208+
) -> None:
209+
workspace = _workspace(tmp_path)
210+
secret = tmp_path / "outside.bin"
211+
secret.write_bytes(OUTSIDE_MARKER)
212+
catalog = Catalog(workspace / "catalog")
213+
_append_artifact(catalog, workspace, "../outside.bin")
214+
monkeypatch.chdir(workspace)
215+
output = tmp_path / "snapshot"
216+
hits = _record_marker_writes(monkeypatch, OUTSIDE_MARKER)
217+
218+
with pytest.raises(ValueError, match="outside the workspace"):
219+
hflow.export_dataset_snapshot(catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY)
220+
221+
_assert_marker_absent(output, hits=hits)
222+
223+
224+
def test_copy_mode_does_not_read_file_replaced_after_lookup(
225+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
226+
) -> None:
227+
"""A containment check that opens the path again must not observe a new file.
228+
229+
The catalog names a regular file inside the workspace. After the first
230+
status lookup of that path, the file is replaced with a symlink to a file
231+
outside the workspace. Copy mode must keep the bytes it already opened, or
232+
refuse, and must not write the outside file.
233+
"""
234+
workspace = _workspace(tmp_path)
235+
secret = tmp_path / "outside.bin"
236+
secret.write_bytes(OUTSIDE_MARKER)
237+
preview = workspace / "episodes" / "preview.jpg"
238+
preview.parent.mkdir()
239+
preview.write_bytes(INSIDE_MARKER)
240+
catalog = Catalog(workspace / "catalog")
241+
_append_artifact(catalog, workspace, str(preview))
242+
output = tmp_path / "snapshot"
243+
hits = _record_marker_writes(monkeypatch, OUTSIDE_MARKER)
244+
swapped = False
245+
real_stat = os.stat
246+
preview_path = os.fspath(preview)
247+
248+
def stat_and_swap(
249+
path: int | os.PathLike[str] | str | bytes,
250+
*,
251+
dir_fd: int | None = None,
252+
follow_symlinks: bool = True,
253+
) -> os.stat_result:
254+
nonlocal swapped
255+
result = real_stat(path, dir_fd=dir_fd, follow_symlinks=follow_symlinks)
256+
if not swapped and not isinstance(path, int) and os.fspath(path) == preview_path:
257+
swapped = True
258+
preview.unlink()
259+
preview.symlink_to(secret)
260+
return result
261+
262+
monkeypatch.setattr(os, "stat", stat_and_swap)
263+
264+
try:
265+
hflow.export_dataset_snapshot(catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY)
266+
except ValueError as error:
267+
assert "outside the workspace" in str(error)
268+
else:
269+
assert not _tree_contains(output, OUTSIDE_MARKER)
270+
assert _tree_contains(output, INSIDE_MARKER)
271+
272+
assert swapped
273+
_assert_marker_absent(output, hits=hits)
274+
275+
276+
class _ObjectStoreHandler(BaseHTTPRequestHandler):
277+
payload = OUTSIDE_MARKER
278+
279+
def do_HEAD(self) -> None:
280+
self._respond(include_body=False)
281+
282+
def do_GET(self) -> None:
283+
self._respond(include_body=True)
284+
285+
def _respond(self, *, include_body: bool) -> None:
286+
body = self.payload
287+
self.send_response(200)
288+
self.send_header("Content-Length", str(len(body)))
289+
self.send_header("ETag", '"outside"')
290+
self.end_headers()
291+
if include_body:
292+
self.wfile.write(body)
293+
294+
def log_message(self, format: str, *args: object) -> None:
295+
del format, args
296+
297+
298+
@contextmanager
299+
def _serve_object() -> Generator[str, None, None]:
300+
server = ThreadingHTTPServer(("127.0.0.1", 0), _ObjectStoreHandler)
301+
thread = threading.Thread(target=server.serve_forever, daemon=True)
302+
thread.start()
303+
host, port = server.server_address[:2]
304+
try:
305+
yield f"http://{host}:{port}"
306+
finally:
307+
server.shutdown()
308+
thread.join(timeout=5)
309+
310+
311+
def test_copy_mode_does_not_fetch_object_store_url_outside_workspace(
312+
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
313+
) -> None:
314+
"""An s3 URL in a local catalog is fetched with ambient client configuration.
315+
316+
``fetch_uri`` sends every ``s3``/``gs``/``az`` URL through the same bucket
317+
branch. This uses ``s3`` because that client reads ``AWS_ENDPOINT`` from
318+
the environment; the endpoint is a local stand-in for the object store.
319+
"""
320+
workspace = _workspace(tmp_path)
321+
catalog = Catalog(workspace / "catalog")
322+
_append_artifact(catalog, workspace, "s3://other-bucket/outside/secret.bin")
323+
output = tmp_path / "snapshot"
324+
hits = _record_marker_writes(monkeypatch, OUTSIDE_MARKER)
325+
mirror = tmp_path / "mirrors"
326+
327+
with _serve_object() as endpoint:
328+
monkeypatch.setenv("HFLOW_MIRROR_DIR", str(mirror))
329+
monkeypatch.setenv("AWS_ENDPOINT", endpoint)
330+
monkeypatch.setenv("AWS_ALLOW_HTTP", "true")
331+
monkeypatch.setenv("AWS_SKIP_SIGNATURE", "true")
332+
monkeypatch.setenv("AWS_DEFAULT_REGION", "us-east-1")
333+
with pytest.raises(ValueError, match="outside the workspace"):
334+
hflow.export_dataset_snapshot(
335+
catalog.root, output, media_mode=hflow.SnapshotMediaMode.COPY
336+
)
337+
338+
_assert_marker_absent(output, mirror, hits=hits)
339+
340+
341+
def test_references_mode_does_not_read_artifact_uris(tmp_path: Path) -> None:
342+
workspace = _workspace(tmp_path)
343+
secret = tmp_path / "outside.bin"
344+
secret.write_bytes(OUTSIDE_MARKER)
345+
catalog = Catalog(workspace / "catalog")
346+
outside_uri = str(secret.resolve())
347+
_append_artifact(catalog, workspace, outside_uri)
348+
output = tmp_path / "snapshot"
349+
350+
report = hflow.export_dataset_snapshot(
351+
catalog.root, output, media_mode=hflow.SnapshotMediaMode.REFERENCES
352+
)
353+
354+
assert report.copied_media_count == 0
355+
assert not _tree_contains(output, OUTSIDE_MARKER)
356+
media_uri = output / "media.parquet"
357+
assert duckdb.execute("SELECT uri FROM read_parquet(?)", [str(media_uri)]).fetchone() == (
358+
outside_uri,
359+
)

0 commit comments

Comments
 (0)