diff --git a/README.md b/README.md index c7575e15..409c59ac 100644 --- a/README.md +++ b/README.md @@ -397,8 +397,10 @@ We provide the following list of available keys: - `modules` --- Declare submodules to be included inside source preparation (source archives creation and placeholders substitution) - `files` --- List of external files to be downloaded. It has to be provided - with the combination of a `url` and a verification method. A verification - method is either a checksum file or a signature file with public GPG keys. + with the combination of a `url`/`git-url` and a verification method. For + `url`, a verification method is either a checksum file or a signature file + with public GPG keys. For `git-url`, it's either GPG key to verity a tag, or + explicit commit id. - `url` --- URL of the external file to download. - `sha256` --- Path to `sha256` checksum file relative to source directory (in combination with `url`). @@ -409,6 +411,15 @@ We provide the following list of available keys: - `uncompress` --- Uncompress external file downloaded before verification. - `pubkeys` --- List of public GPG keys to use for verifying the downloaded signature file (in combination with `url` and `signature`). +- `git-url` --- URL of a repository to fetch and create a tarball from. +- `tag` --- Specific tag to fetch from `git-url` and verify with a key + specified in `pubkeys`. Can use `@VERSION@` placeholder. +- `commit-id` --- Specific pre-verified commit to fetch from `git-url`. No + extra verification is performed. +- `git-basename` --- Specify basename for archive created out of git + repository. This is also used for the top level directory inside the archive. + If not set, final component of the `git-url` (minus `.git` if applicable), + plus a commit id or tag will be used. Here is a non-exhaustive list of distribution-specific keys: - `host-fc32` --- Fedora 32 for the `host` package set content only diff --git a/qubesbuilder/common.py b/qubesbuilder/common.py index af725ef2..f6366c09 100644 --- a/qubesbuilder/common.py +++ b/qubesbuilder/common.py @@ -16,6 +16,7 @@ # with this program. If not, see . # # SPDX-License-Identifier: GPL-3.0-or-later +import os import re import shutil import tempfile @@ -76,6 +77,24 @@ def is_filename_valid( return True +def get_archive_name(file: dict): + if "url" in file: + fn = os.path.basename(file["url"]) + if file.get("uncompress", False): + return Path(fn).with_suffix("").name + return fn + if "git-basename" in file: + return f"{file['git-basename']}.tar.gz" + else: + archive_base = os.path.basename(file["git-url"]).partition(".git")[0] + if "tag" in file: + assert "/" not in file["tag"] + return f"{archive_base}-{file['tag']}.tar.gz" + if "commit-id" in file: + return f"{archive_base}-{file['commit-id']}.tar.gz" + return None + + # Originally from QubesOS/qubes-builder/rpc-services/qubesbuilder.BuildLog def sanitize_line(untrusted_line): line = bytearray(untrusted_line) diff --git a/qubesbuilder/plugins/fetch/__init__.py b/qubesbuilder/plugins/fetch/__init__.py index 22bbe122..8495f880 100644 --- a/qubesbuilder/plugins/fetch/__init__.py +++ b/qubesbuilder/plugins/fetch/__init__.py @@ -27,7 +27,7 @@ from shlex import quote from typing import Any, List, Union -from qubesbuilder.common import VerificationMode +from qubesbuilder.common import VerificationMode, get_archive_name from qubesbuilder.component import QubesComponent from qubesbuilder.config import Config from qubesbuilder.exc import NoQubesBuilderFileError @@ -188,156 +188,19 @@ def run(self, stage: str): super().run(stage) executor = self.get_executor(stage) - source_dir = executor.get_builder_dir() / self.component.name parameters = self.get_parameters(stage) distfiles_dir = self.get_component_distfiles_dir() distfiles_dir.mkdir(parents=True, exist_ok=True) # Download and verify files given in .qubesbuilder for file in parameters.get("files", []): - # Temporary dir for downloaded file - temp_dir = Path(tempfile.mkdtemp(dir=self.get_temp_dir())) - - # - # download - # - parsed_url = urllib.parse.urlparse(file["url"]) - fn = str(os.path.basename(parsed_url.geturl())) - - # If we request to uncompress the file we drop the archive suffix - if file.get("uncompress", False): - final_fn = Path(fn).with_suffix("").name + if "url" in file: + self.download_file(file, executor, distfiles_dir) + elif "git-url" in file: + self.download_git_archive(file, executor, distfiles_dir) else: - final_fn = fn - - untrusted_final_fn = "untrusted_" + final_fn - - if (distfiles_dir / final_fn).exists(): - if self.config.force_fetch: - os.remove(distfiles_dir / final_fn) - else: - self.log.info( - f"{self.component}: file {final_fn} already downloaded. Skipping." - ) - continue - copy_in = [ - ( - self.manager.entities["fetch"].directory, - executor.get_plugins_dir(), - ), - (self.component.source_dir, executor.get_builder_dir()), - ] - copy_out = [(source_dir / untrusted_final_fn, temp_dir)] - - # Construct command for "download-file". - download_cmd = [ - str(executor.get_plugins_dir() / "fetch/scripts/download-file"), - "--output-dir", - str(executor.get_builder_dir() / self.component.name), - "--file-name", - fn, - "--file-url", - file["url"], - ] - if file.get("signature", None): - download_cmd += ["--signature-url", file["signature"]] - signature_fn = os.path.basename(file["signature"]) - untrusted_signature_fn = "untrusted_" + signature_fn - copy_out += [ - ( - executor.get_builder_dir() - / self.component.name - / untrusted_signature_fn, - temp_dir, - ) - ] - if file.get("uncompress", False): - download_cmd += ["--uncompress"] - cmd = [" ".join(map(shlex.quote, download_cmd))] - try: - executor.run( - cmd, copy_in, copy_out, environment=self.environment - ) - except ExecutorError as e: - shutil.rmtree(temp_dir) - raise FetchError(f"Failed to download file '{file}': {str(e)}.") - - # - # verify - # - - # Keep executor workflow if we move verification of files in another - # cage type (copy-in, copy-out and cmd would need adjustments). - - if isinstance(executor, LocalExecutor): - # If executor is a LocalExecutor, use the same base - # directory for temporary directory - local_executor = LocalExecutor( - directory=executor.get_directory() - ) - else: - local_executor = LocalExecutor() - - copy_in = [] - copy_out = [(temp_dir / final_fn, distfiles_dir)] - - # Construct command for "verify-file". - verify_cmd = [ - str( - self.manager.entities["fetch"].directory - / "scripts/verify-file" - ), - "--output-dir", - str(temp_dir), - "--untrusted-file", - str(temp_dir / untrusted_final_fn), - ] - if file.get("sha256", None): - verify_cmd += [ - "--checksum-cmd", - "sha256sum", - "--checksum-file", - str(self.component.source_dir / file["sha256"]), - ] - elif file.get("sha512", None): - verify_cmd += [ - "--checksum-cmd", - "sha512sum", - "--checksum-file", - str(self.component.source_dir / file["sha512"]), - ] - elif file.get("signature", None): - signature_fn = os.path.basename(file["signature"]) - untrusted_signature_fn = "untrusted_" + signature_fn - verify_cmd += [ - "--untrusted-signature-file", - str(temp_dir / untrusted_signature_fn), - ] - copy_out += [ - ( - temp_dir / signature_fn, - distfiles_dir, - ) - ] - else: - raise FetchError(f"No verification method for {final_fn}") - - if file.get("pubkeys", None): - for pubkey in file["pubkeys"]: - verify_cmd += [ - "--pubkey-file", - str(self.component.source_dir / pubkey), - ] - - cmd = [" ".join(map(shlex.quote, verify_cmd))] - try: - local_executor.run( - cmd, copy_in, copy_out, environment=self.environment - ) - except ExecutorError as e: - raise FetchError(f"Failed to verify file '{file}': {str(e)}.") - finally: - shutil.rmtree(temp_dir) + msg = "'files' entries must have either url or git-url entry" + raise FetchError(msg) # # source hash and version tags determination @@ -533,5 +396,210 @@ def run(self, stage: str): msg = f"{self.component}: Failed to clean artifacts: {str(e)}." raise FetchError(msg) from e + def download_git_archive(self, file, executor, distfiles_dir): + repo_bn = os.path.basename(file["git-url"]).partition(".git")[0] + if "git-basename" in file: + archive_base = file["git-basename"] + else: + archive_base = repo_bn + if "tag" in file: + if "/" in file["tag"]: + msg = "Tags with '/' are not supported" + raise FetchError(msg) + elif "commit-id" in file: + if not re.match(r"\A[a-z0-9]*\Z", file["commit-id"]) or len( + file["commit-id"] + ) not in (40, 64): + msg = "Full commit id is needed for 'commit-id'" + raise FetchError(msg) + else: + msg = "Fetching git archive requires either 'tag' or 'commit-id'" + raise FetchError(msg) + archive_name = get_archive_name(file) + + if (distfiles_dir / archive_name).exists(): + if self.config.force_fetch: + os.remove(distfiles_dir / archive_name) + else: + self.log.info( + f"{self.component}: file {archive_name} already downloaded. Skipping." + ) + return + copy_in = [ + ( + self.manager.entities["fetch"].directory, + executor.get_plugins_dir(), + ), + ] + local_source_dir = self.get_sources_dir() / self.component.name + for key_file in file.get("pubkeys", []): + copy_in += [ + ( + local_source_dir / key_file, + executor.get_builder_dir() / "keys", + ) + ] + + source_dir = executor.get_builder_dir() / repo_bn + + get_sources_cmd = [ + str( + executor.get_plugins_dir() + / "fetch/scripts/get-and-verify-source.py" + ), + "--shallow-clone", + "--trust-all-keys", + file["git-url"], # clone from + str(source_dir), # clone into + str(executor.get_builder_dir() / "keyring"), # git keyring dir + str(executor.get_builder_dir() / "keys"), # keys to import + ] + if "tag" in file: + get_sources_cmd += ["--git-branch", file["tag"]] + elif "commit-id" in file: + get_sources_cmd += ["--git-commit", file["commit-id"]] + + cmd = [ + f"cd {str(executor.get_builder_dir())}", + " ".join(get_sources_cmd), + f"{executor.get_plugins_dir()}/fetch/scripts/create-archive {source_dir} {archive_name} {archive_base}/", + ] + + copy_out = [(source_dir / archive_name, distfiles_dir)] + + try: + executor.run(cmd, copy_in, copy_out, environment=self.environment) + except ExecutorError as e: + raise FetchError(f"Failed to download file '{file}': {str(e)}.") + + def download_file(self, file, executor, distfiles_dir): + # Temporary dir for downloaded file + temp_dir = Path(tempfile.mkdtemp(dir=self.get_temp_dir())) + # + # download + # + parsed_url = urllib.parse.urlparse(file["url"]) + fn = str(os.path.basename(parsed_url.geturl())) + # If we request to uncompress the file we drop the archive suffix + if file.get("uncompress", False): + final_fn = Path(fn).with_suffix("").name + else: + final_fn = fn + untrusted_final_fn = "untrusted_" + final_fn + if (distfiles_dir / final_fn).exists(): + if self.config.force_fetch: + os.remove(distfiles_dir / final_fn) + else: + self.log.info( + f"{self.component}: file {final_fn} already downloaded. Skipping." + ) + return + copy_in = [ + ( + self.manager.entities["fetch"].directory, + executor.get_plugins_dir(), + ), + (self.component.source_dir, executor.get_builder_dir()), + ] + source_dir = executor.get_builder_dir() / self.component.name + copy_out = [(source_dir / untrusted_final_fn, temp_dir)] + # Construct command for "download-file". + download_cmd = [ + str(executor.get_plugins_dir() / "fetch/scripts/download-file"), + "--output-dir", + str(executor.get_builder_dir() / self.component.name), + "--file-name", + fn, + "--file-url", + file["url"], + ] + if file.get("signature", None): + download_cmd += ["--signature-url", file["signature"]] + signature_fn = os.path.basename(file["signature"]) + untrusted_signature_fn = "untrusted_" + signature_fn + copy_out += [ + ( + executor.get_builder_dir() + / self.component.name + / untrusted_signature_fn, + temp_dir, + ) + ] + if file.get("uncompress", False): + download_cmd += ["--uncompress"] + cmd = [" ".join(map(shlex.quote, download_cmd))] + try: + executor.run(cmd, copy_in, copy_out, environment=self.environment) + except ExecutorError as e: + shutil.rmtree(temp_dir) + raise FetchError(f"Failed to download file '{file}': {str(e)}.") + # + # verify + # + # Keep executor workflow if we move verification of files in another + # cage type (copy-in, copy-out and cmd would need adjustments). + if isinstance(executor, LocalExecutor): + # If executor is a LocalExecutor, use the same base + # directory for temporary directory + local_executor = LocalExecutor(directory=executor.get_directory()) + else: + local_executor = LocalExecutor() + copy_in = [] + copy_out = [(temp_dir / final_fn, distfiles_dir)] + # Construct command for "verify-file". + verify_cmd = [ + str( + self.manager.entities["fetch"].directory / "scripts/verify-file" + ), + "--output-dir", + str(temp_dir), + "--untrusted-file", + str(temp_dir / untrusted_final_fn), + ] + if file.get("sha256", None): + verify_cmd += [ + "--checksum-cmd", + "sha256sum", + "--checksum-file", + str(self.component.source_dir / file["sha256"]), + ] + elif file.get("sha512", None): + verify_cmd += [ + "--checksum-cmd", + "sha512sum", + "--checksum-file", + str(self.component.source_dir / file["sha512"]), + ] + elif file.get("signature", None): + signature_fn = os.path.basename(file["signature"]) + untrusted_signature_fn = "untrusted_" + signature_fn + verify_cmd += [ + "--untrusted-signature-file", + str(temp_dir / untrusted_signature_fn), + ] + copy_out += [ + ( + temp_dir / signature_fn, + distfiles_dir, + ) + ] + else: + raise FetchError(f"No verification method for {final_fn}") + if file.get("pubkeys", None): + for pubkey in file["pubkeys"]: + verify_cmd += [ + "--pubkey-file", + str(self.component.source_dir / pubkey), + ] + cmd = [" ".join(map(shlex.quote, verify_cmd))] + try: + local_executor.run( + cmd, copy_in, copy_out, environment=self.environment + ) + except ExecutorError as e: + raise FetchError(f"Failed to verify file '{file}': {str(e)}.") + finally: + shutil.rmtree(temp_dir) + PLUGINS = [FetchPlugin] diff --git a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py index a60d2003..af8f2a9d 100755 --- a/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py +++ b/qubesbuilder/plugins/fetch/scripts/get-and-verify-source.py @@ -79,7 +79,7 @@ def verify_git_obj(gpg_client, keyring_dir, repository_dir, obj_type, obj_path): def main(args): # Sanity check on branch and repo - if not re.match(r"^[A-Za-z][A-Za-z0-9/._-]+$", args.git_branch): + if not re.match(r"^[A-Za-z0-9][A-Za-z0-9/._-]+$", args.git_branch): raise ValueError(f"Invalid branch {args.git_branch}") elif not re.match(r"^/[A-Za-z][A-Za-z0-9-/_]*$", args.component_directory): raise ValueError( @@ -91,7 +91,7 @@ def main(args): keys_dir = Path(args.keys_dir).expanduser().resolve() git_keyring_dir = Path(args.git_keyring_dir).expanduser().resolve() - git_branch = args.git_branch + git_branch = args.git_commit or args.git_branch clean = args.clean fetch_only = args.fetch_only ignore_missing = args.ignore_missing @@ -120,10 +120,18 @@ def main(args): if not re.match(r"^[a-fA-F0-9]{40}$", maintainer): raise ValueError(f"Invalid maintainer provided: {maintainer}") + if args.trust_all_keys and maintainers: + raise ValueError( + "--maintainer cannot be used together with --trust-all-keys" + ) + # Define common git options git_options: List[str] = [] git_merge_opts = ["--ff-only"] + if args.shallow_clone: + git_options += ["--depth=1"] + fresh_clone = False if clean: shutil.rmtree(repo) @@ -173,20 +181,38 @@ def main(args): if repo.exists(): shutil.rmtree(repo) try: - subprocess.run( - [ - "git", - "clone", - "-n", - "-q", - "-b", - git_branch, - git_url, - str(repo), - ], - capture_output=True, - check=True, - ) + if args.git_commit: + # git clone can't handle commit reference, use fetch instead + repo.mkdir() + subprocess.run( + ["git", "init"], + capture_output=True, + cwd=repo, + check=True, + ) + subprocess.run( + ["git", "fetch"] + + git_options + + ["--", git_url, git_branch], + capture_output=True, + cwd=repo, + check=True, + ) + subprocess.run( + ["git", "reset", "-q", "--soft", "FETCH_HEAD"], + capture_output=True, + cwd=repo, + check=True, + ) + else: + subprocess.run( + ["git", "clone"] + + git_options + + ["-n", "-q", "-b", git_branch] + + ["--", git_url, str(repo)], + capture_output=True, + check=True, + ) except subprocess.CalledProcessError as e: if ignore_missing: return @@ -220,6 +246,9 @@ def main(args): verify = False elif less_secure_signed_commits_sufficient: check = "signed-tag-or-commit" + elif args.git_commit: + # user specified pre-verified commit-id + verify = False verify_ref = subprocess.run( ["git", "rev-parse", "-q", "--verify", verify_ref], @@ -250,6 +279,33 @@ def main(args): check=True, env=env, ) + if args.trust_all_keys: + for file in keys_dir.glob("*"): + subprocess.run( + [gpg_client, "--import", str(file)], + check=True, + env=env, + capture_output=True, + ) + list_keys = subprocess.run( + [gpg_client, "--list-keys", "--with-colons"], + capture_output=True, + check=True, + env=env, + ) + for line in list_keys.stdout.splitlines(): + if not line.startswith(b"fpr:"): + continue + keyid = line.decode().split(":")[9] + subprocess.run( + [gpg_client, "--import-ownertrust"], + input=f"{keyid}:6:\n", + capture_output=True, + text=True, + env=env, + check=True, + ) + for keyid in maintainers: key_path = keys_dir / f"{keyid}.asc" if not key_path.exists(): @@ -468,11 +524,20 @@ def get_args(): # optional args parser.add_argument("--git-branch", help="Git branch.", default="main") + parser.add_argument( + "--git-commit", + help="Specific git commit id - full sha. Takes precedence over branch and does not require signed tag.", + ) parser.add_argument( "--clean", action="store_true", help="Remove previous sources (use git up vs git clone).", ) + parser.add_argument( + "--shallow-clone", + action="store_true", + help="Fetch git repo with --depth=1 to reduce amount of data.", + ) parser.add_argument( "--fetch-only", action="store_true", @@ -503,6 +568,11 @@ def get_args(): action="append", help="Allowed maintainer provided as KEYID assumed to be available as KEYID.asc under provided 'keys-dir' directory. Can be used multiple times.", ) + parser.add_argument( + "--trust-all-keys", + action="store_true", + help="Import and trust all keys present in *keys-dir*. Conflicts with --maintainer.", + ) parser.add_argument( "--minimum-distinct-maintainers", help="Minimum of mandatory distinct maintainer signatures.", diff --git a/qubesbuilder/plugins/source_deb/__init__.py b/qubesbuilder/plugins/source_deb/__init__.py index 92735682..5dc732b5 100644 --- a/qubesbuilder/plugins/source_deb/__init__.py +++ b/qubesbuilder/plugins/source_deb/__init__.py @@ -22,7 +22,7 @@ import tempfile from pathlib import Path -from qubesbuilder.common import is_filename_valid +from qubesbuilder.common import is_filename_valid, get_archive_name from qubesbuilder.component import QubesComponent from qubesbuilder.config import Config from qubesbuilder.distribution import QubesDistribution @@ -192,7 +192,7 @@ def run(self, stage: str): source_debian = f"{package_release_name_full}.debian.tar.xz" if parameters.get("files", []): # FIXME: The first file is the source archive. Is it valid for all the cases? - ext = Path(parameters["files"][0]["url"]).suffix + ext = Path(get_archive_name(parameters["files"][0])).suffix msg = f"{self.component}:{self.dist}:{directory}: Invalid extension '{ext}'." if ext not in (".gz", ".bz2", ".xz", ".lzma2"): raise SourceError(msg) @@ -260,7 +260,7 @@ def run(self, stage: str): f"mv {source_dir}/{source_orig} {executor.get_builder_dir()}", ] for file in parameters.get("files", []): - fn = os.path.basename(file["url"]) + fn = get_archive_name(file) cmd.append( f"mv {executor.get_distfiles_dir() / self.component.name / fn} {executor.get_builder_dir()}/{source_orig}" ) diff --git a/qubesbuilder/plugins/source_rpm/__init__.py b/qubesbuilder/plugins/source_rpm/__init__.py index 291ef7dd..a3d5ea6a 100644 --- a/qubesbuilder/plugins/source_rpm/__init__.py +++ b/qubesbuilder/plugins/source_rpm/__init__.py @@ -22,7 +22,7 @@ import tempfile from pathlib import Path -from qubesbuilder.common import is_filename_valid +from qubesbuilder.common import is_filename_valid, get_archive_name from qubesbuilder.component import QubesComponent from qubesbuilder.config import Config from qubesbuilder.distribution import QubesDistribution @@ -125,6 +125,10 @@ def run(self, stage: str): shutil.rmtree(artifacts_dir.as_posix()) artifacts_dir.mkdir(parents=True) + # Create archive only if no external files are provided or if explicitly requested. + create_archive = not parameters.get("files", []) + create_archive = parameters.get("create-archive", create_archive) + for build in parameters["build"]: # Temporary dir for temporary copied-out files temp_dir = Path(tempfile.mkdtemp()) @@ -177,15 +181,19 @@ def run(self, stage: str): raise SourceError(msg) source_rpm = f"{data[0]}.src.rpm" - # Source0 may contain a URL. - source_orig = os.path.basename(data[1]) if not is_filename_valid( source_rpm, forbidden_filename=artifacts_info_filename - ) or not is_filename_valid( - source_orig, forbidden_filename=artifacts_info_filename ): - msg = f"{self.component}:{self.dist}:{build}: Invalid source names." + msg = f"{self.component}:{self.dist}:{build}: Invalid source rpm name." raise SourceError(msg) + if create_archive: + # Source0 may contain a URL. + source_orig = os.path.basename(data[1]) + if not is_filename_valid( + source_orig, forbidden_filename=artifacts_info_filename + ): + msg = f"{self.component}:{self.dist}:{build}: Invalid source names." + raise SourceError(msg) # Read packages list packages_list = [] @@ -247,9 +255,6 @@ def run(self, stage: str): f"--outfile {source_dir}/Makefile.vars -- " f"{executor.get_plugins_dir()}/source/salt/FORMULA-DEFAULTS {source_dir}/FORMULA" ] - # Create archive only if no external files are provided or if explicitly requested. - create_archive = not parameters.get("files", []) - create_archive = parameters.get("create-archive", create_archive) if create_archive: # If no Source0 is provided, we expect 'source' from query-spec. if source_orig != "source": @@ -258,9 +263,7 @@ def run(self, stage: str): ] for file in parameters.get("files", []): - fn = os.path.basename(file["url"]) - if file.get("uncompress", False): - fn = Path(fn).with_suffix("").name + fn = get_archive_name(file) cmd.append( f"mv {executor.get_distfiles_dir() / self.component.name / fn} {source_dir}" ) diff --git a/tests/builder-ci.yml b/tests/builder-ci.yml index 38a06e9c..b763d36b 100644 --- a/tests/builder-ci.yml +++ b/tests/builder-ci.yml @@ -89,6 +89,10 @@ components: - 0064428F455451B3EBE78A7F063938BA42CFA724 - 9FA64B92F95E706BF28E2CA6484010B5CDC576E2 - C4A2E4615A16BD191110DEE17320B2D2134763F3 + - linux-gbulb: + url: https://github.com/marmarek/qubes-linux-gbulb + branch: files-git + verification-mode: less-secure-signed-commits-sufficient templates: - fedora-40-xfce: diff --git a/tests/test_cli.py b/tests/test_cli.py index 9ad9ca80..24ce15a1 100644 --- a/tests/test_cli.py +++ b/tests/test_cli.py @@ -191,6 +191,21 @@ def test_common_component_fetch(artifacts_dir): artifacts_dir / "distfiles/desktop-linux-xfce4-xfwm4/xfwm4-4.16.1.tar.bz2" ).exists() + assert ( + artifacts_dir + / "distfiles/linux-gbulb/gbulb-0.6.3.tar.gz" + ).exists() + # verify files layout inside + subprocess.run( + ["tar", + "tf", + artifacts_dir / "distfiles/linux-gbulb/gbulb-0.6.3.tar.gz", + "gbulb-0.6.3/README.rst" + ], + check=True, + capture_output=True, + text=True, + ) for component in [ "core-qrexec", diff --git a/tests/test_functions.py b/tests/test_functions.py index 7b31a098..55186756 100644 --- a/tests/test_functions.py +++ b/tests/test_functions.py @@ -1,7 +1,8 @@ import pytest import tempfile from pathlib import Path -from qubesbuilder.common import is_filename_valid, deep_check, sed +from qubesbuilder.common import is_filename_valid, deep_check, sed, \ + get_archive_name from qubesbuilder.cli.cli_main import parse_config_from_cli @@ -264,3 +265,49 @@ def test_sed_without_destination(): # Clean up the temporary file source_file.close() Path(source_file.name).unlink() + + +def test_get_archive_name_url(): + file = { + "url": "https://example.com/some-file.tar.gz", + "signature": "https://example.com/some-file.tar.gz.asc", + "pubkeys": ["pubkey1.asc", "pubkey2.asc"], + } + fn = get_archive_name(file) + assert fn == "some-file.tar.gz" + + +def test_get_archive_name_url_uncompress(): + file = { + "url": "https://example.com/some-file.tar.gz", + "signature": "https://example.com/some-file.tar.gz.asc", + "uncompress": True, + "pubkeys": ["pubkey1.asc", "pubkey2.asc"], + } + fn = get_archive_name(file) + assert fn == "some-file.tar" + + +def test_get_archive_name_git_url(): + file = { + "git-url": "https://github.com/owner/repo.git", + "tag": "v1.0.0", + "pubkeys": ["pubkey1.asc", "pubkey2.asc"], + } + fn = get_archive_name(file) + assert fn == "repo-v1.0.0.tar.gz" + + file = { + "git-url": "https://github.com/owner/repo", + "commit-id": "0011223344556677889900112233445566778899", + } + fn = get_archive_name(file) + assert fn == "repo-0011223344556677889900112233445566778899.tar.gz" + + file = { + "git-url": "https://github.com/owner/repo", + "tag": "v2.0.0", + "git-basename": "repo-2.0.0" + } + fn = get_archive_name(file) + assert fn == "repo-2.0.0.tar.gz" diff --git a/tests/test_scripts.py b/tests/test_scripts.py index 37b3bd4c..f5e520fb 100644 --- a/tests/test_scripts.py +++ b/tests/test_scripts.py @@ -117,6 +117,8 @@ def create_dummy_args( component_repository, component_directory, git_branch="main", + git_commit=None, + shallow_clone=False, keys_dir=str(PROJECT_PATH / "qubesbuilder/plugins/fetch/keys"), fetch_only=False, fetch_versions_only=False, @@ -126,17 +128,21 @@ def create_dummy_args( less_secure_signed_commits_sufficient=False, maintainers=None, minimum_distinct_maintainers=1, + trust_all_keys=False, ): args = Namespace() - maintainers = ( - maintainers if maintainers and isinstance(maintainers, list) else [] - ) - # Add default maintainers being Marek and Simon - maintainers += [ - "0064428F455451B3EBE78A7F063938BA42CFA724", - "274E12AB03F2FE293765FC06DA0434BC706E1FCF", - ] + if maintainers != []: + # Add default maintainers being Marek and Simon if maintainers are + # not explicitly an empty list + maintainers = (maintainers or []) + [ + "0064428F455451B3EBE78A7F063938BA42CFA724", + "274E12AB03F2FE293765FC06DA0434BC706E1FCF", + ] + else: + maintainers = ( + maintainers if maintainers and isinstance(maintainers, list) else [] + ) args.component_repository = component_repository args.component_directory = str(component_directory) @@ -144,6 +150,8 @@ def create_dummy_args( args.git_keyring_dir = str(component_directory / ".keyring") args.git_branch = git_branch + args.git_commit = git_commit + args.shallow_clone = shallow_clone args.fetch_only = fetch_only args.fetch_versions_only = fetch_versions_only args.ignore_missing = ignore_missing @@ -154,6 +162,7 @@ def create_dummy_args( ) args.maintainer = maintainers args.minimum_distinct_maintainers = minimum_distinct_maintainers + args.trust_all_keys = trust_all_keys return args @@ -605,3 +614,64 @@ def test_repository_fetch_version_tag_earlier(capsys, temp_directory): check=True, ).stdout.strip() assert vtag.startswith("v") + + +def test_repository_fetch_specific_commit(capsys, temp_directory): + component_repository = "https://github.com/fepitre/qubes-core-qrexec" + # Fresh clone on specific commit id, without tags + commit = "de25326d41b1abf519e9a30c5760d56a1c816b3a" + args = create_dummy_args( + component_repository=component_repository, + component_directory=temp_directory, + git_commit="de25326d41b1abf519e9a30c5760d56a1c816b3a", + ) + get_and_verify_source(args) + + # Check if we have a version tag on HEAD + actual_commit = subprocess.run( + ["git", "show", "--format=%H", "-s"], + capture_output=True, + text=True, + cwd=temp_directory, + check=True, + ).stdout.strip() + assert commit == actual_commit + + +def test_repository_trust_all_keys(temp_directory): + args = create_dummy_args( + component_repository="https://github.com/qubesos/qubes-core-vchan-xen", + component_directory=temp_directory, + trust_all_keys=True, + maintainers=[], + ) + get_and_verify_source(args) + assert ( + subprocess.run( + ["git", "rev-parse", "--abbrev-ref", "HEAD"], + capture_output=True, + cwd=str(temp_directory), + text=True, + ).stdout.strip() + == "main" + ) + assert (temp_directory / "version").exists() + + +def test_repository_shallow_clone(temp_directory): + args = create_dummy_args( + component_repository="https://github.com/qubesos/qubes-core-vchan-xen", + component_directory=temp_directory, + shallow_clone=True, + ) + get_and_verify_source(args) + assert ( + subprocess.run( + ["git", "rev-parse", "--is-shallow-repository"], + capture_output=True, + cwd=str(temp_directory), + text=True, + ).stdout.strip() + == "true" + ) + assert (temp_directory / "version").exists()