tests: Fix parallel runs

mkosi/__init__.py

@@ -2565,11 +2565,22 @@ def calculate_signature_gpg(context: Context) -> None:
     ]  # fmt: skip
 
     with complete_step("Signing SHA256SUMS…"):
-        run(
-            cmdline,
-            env=env,
-            sandbox=context.sandbox(options=options),
-        )
+        try:
+            run(
+                cmdline,
+                env=env,
+                sandbox=context.sandbox(options=options),
+            )
+        finally:
+            # gpg autostarts a gpg-agent to sign and, as the sandbox has no PID namespace, that agent is
+            # leaked when the sandbox goes away. Note this will also kill a "real" user agent, but
+            # gpg auto-starts a new one.
+            run(
+                ["gpgconf", "--kill", "gpg-agent"],
+                env=env,
+                sandbox=context.sandbox(options=options),
+                check=False,
+            )
 
 
 def calculate_signature_sop(context: Context) -> None:

mkosi/config.py

@@ -5445,7 +5445,15 @@ def want_default_initrd(config: Config) -> bool:
     return Path("default") in config.initrds
 
 
-def finalize_historydir(args: Args) -> Path:
+def finalize_historydir(args: Args, output_dir: Optional[Path] = None) -> Path:
+    # When an output dir is given on the CLI, store the build history there so that concurrent builds with
+    # different output dirs don't clobber a shared history. Don't check the finalized OutputDirectory=
+    # config, only the CLI value: the former isn't known yet here (config files and includes are
+    # parsed later) and vm/boot can't see it anyway since they recover the config from the history instead of
+    # parsing it. An output dir set only in config files keeps the history in the config dir.
+    if output_dir is not None:
+        return output_dir / ".mkosi-private/history"
+
     configdir = finalize_configdir(args.directory)
     return (configdir or Path.cwd()) / ".mkosi-private/history"
 
@@ -5502,7 +5510,7 @@ def parse_config(
         return args, None, ()
 
     configdir = finalize_configdir(args.directory)
-    historydir = finalize_historydir(args)
+    historydir = finalize_historydir(args, context.cli.get("output_dir"))
 
     if have_history(args, historydir):
         history = Config.from_partial_json((historydir / "latest.json").read_text())

mkosi/installer/__init__.py

@@ -1,15 +1,15 @@
 # SPDX-License-Identifier: LGPL-2.1-or-later
 
-from collections.abc import Sequence
-from contextlib import AbstractContextManager
+import contextlib
+from collections.abc import Iterator, Sequence
 from pathlib import Path
 
 from mkosi.config import Config, ConfigFeature, OutputFormat
 from mkosi.context import Context
 from mkosi.mounts import finalize_certificate_mounts
 from mkosi.run import apivfs_options, finalize_interpreter, finalize_passwd_symlinks, find_binary
 from mkosi.tree import rmtree
-from mkosi.util import PathString, flatten, startswith
+from mkosi.util import PathString, flatten, flock, startswith
 
 
 class PackageManager:
@@ -153,22 +153,32 @@ def apivfs_script_cmd(cls, context: Context) -> list[PathString]:
         ]  # fmt: skip
 
     @classmethod
+    @contextlib.contextmanager
     def sandbox(
         cls,
         context: Context,
         *,
         apivfs: bool,
         options: Sequence[PathString] = (),
-    ) -> AbstractContextManager[list[PathString]]:
-        return context.sandbox(
-            network=True,
-            options=[
-                *context.rootoptions(),
-                *cls.mounts(context),
-                *cls.options(root=context.root, apivfs=apivfs),
-                *options,
-            ],
-        )  # fmt: skip
+    ) -> Iterator[list[PathString]]:
+        # The package cache directory is shared between all mkosi builds of the same distribution (see
+        # Config.package_cache_dir_or_default()) and is bind mounted read-write into every package manager
+        # sandbox by mounts(). Avoid concurrent downloads, as those result in corruption/truncation and
+        # unnecessary duplicate fetches. Builds of different distributions use different cache directories
+        # and so don't contend.
+        with (
+            flock(context.config.package_cache_dir_or_default()),
+            context.sandbox(
+                network=True,
+                options=[
+                    *context.rootoptions(),
+                    *cls.mounts(context),
+                    *cls.options(root=context.root, apivfs=apivfs),
+                    *options,
+                ],
+            ) as sandbox,
+        ):
+            yield sandbox
 
     @classmethod
     def install(

tests/__init__.py

@@ -37,7 +37,10 @@ def __enter__(self) -> "Image":
         else:
             tmpdir = Path("/var/tmp")
 
-        self.output_dir = Path(os.getenv("TMPDIR", tmpdir)) / uuid.uuid4().hex[:16]
+        token = uuid.uuid4().hex[:16]
+        self.output_dir = Path(os.getenv("TMPDIR", tmpdir)) / token
+        # Unique VM name to support parallel runs; CID name is derived from machine name
+        self.machine = f"mkosi-{token}"
 
         return self
 
@@ -117,19 +120,27 @@ def boot(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> Complet
                 "--runtime-build-sources=no",
                 "--ephemeral=yes",
                 "--register=no",
+                "--machine",
+                self.machine,
+                "--output-directory", self.output_dir,
                 *options,
             ],
             args,
             stdin=sys.stdin if sys.stdin.isatty() else None,
             check=False,
-        )
+        )  # fmt: skip
 
         if result.returncode != 123:
             raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr)
 
         return result
 
-    def vm(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> CompletedProcess:
+    def vm(
+        self,
+        options: Sequence[str] = (),
+        args: Sequence[str] = (),
+        ram: str = "1536M",
+    ) -> CompletedProcess:
         need_hyperv_workaround = os.uname().machine == "x86_64"
 
         result = self.mkosi(
@@ -139,15 +150,18 @@ def vm(self, options: Sequence[str] = (), args: Sequence[str] = ()) -> Completed
                 "--vsock=yes",
                 # TODO: Drop once both Hyper-V bugs are fixed in Github Actions.
                 *(["--qemu-args=-cpu max,pcid=off"] if need_hyperv_workaround else []),
-                "--ram=2G",
+                f"--ram={ram}",
                 "--ephemeral=yes",
                 "--register=no",
+                "--machine",
+                self.machine,
+                "--output-directory", self.output_dir,
                 *options,
             ],
             args,
             stdin=sys.stdin if sys.stdin.isatty() else None,
             check=False,
-        )
+        )  # fmt: skip
 
         if result.returncode != 123:
             raise subprocess.CalledProcessError(result.returncode, result.args, result.stdout, result.stderr)

tests/test_initrd.py

@@ -150,7 +150,9 @@ def test_initrd_luks(config: ImageConfig, passphrase: Path) -> None:
 
         with Image(config) as image:
             image.build(["--repart-directory", repartd, "--passphrase", passphrase, "--format=disk"])
-            image.vm(["--credential=cryptsetup.passphrase=mkosi"])
+            # repart's default LUKS2 KDF (Argon2id) is memory-hard and needs ~1 GiB of RAM just to derive
+            # the key, so the default VM RAM isn't enough to unlock the root.
+            image.vm(["--credential=cryptsetup.passphrase=mkosi"], ram="2G")
 
 
 @pytest.mark.skipif(os.getuid() != 0, reason="mkosi-initrd LUKS+LVM test can only be executed as root")