diff --git a/.github/workflows/build.yaml b/.github/workflows/build.yaml index cbe2a0fe..88a25f84 100644 --- a/.github/workflows/build.yaml +++ b/.github/workflows/build.yaml @@ -120,13 +120,71 @@ jobs: PYTHON_VERSION=${{ matrix.python-version }} VERSION=${{ steps.version.outputs.version }} push: ${{ github.event_name != 'pull_request' }} + # NOTE: So examples can build in next job + load: ${{ github.event_name == 'pull_request' }} tags: ${{ steps.meta.outputs.tags }} labels: ${{ steps.meta.outputs.labels }} cache-from: type=gha,scope=build-py${{ matrix.python-version }} platforms: ${{ github.event_name != 'pull_request' && 'linux/amd64,linux/arm64' || 'linux/amd64' }} + # NOTE: Test that above built image functions as a base image (no push) + examples-latest-sdk: + if: ${{ github.event_name == 'pull_request' }} + # NOTE: We want the silverback image built above to base ours on + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: read + steps: + - uses: actions/checkout@v5 + + - name: Setup Python tool + uses: astral-sh/setup-uv@v7 + + # NOTE: Temporary until build action supports `--sdk` + - name: Build Examples (using 'latest' SDK from above) + run: | + uv run silverback build \ + --generate \ + --sdk latest \ + --tag-base $(echo '${{ env.REGISTRY }}/${{ env.IMAGE_NAME }}' | tr '[:upper:]' '[:lower:]') \ + --version pr-${{ github.event.pull_request.number }} + + # Build actual image using stable as base + # NOTE: In PR, tests that build works with `stable` as well + examples-stable-sdk: + # NOTE: We want the silverback image built above to base ours on + needs: build + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + steps: + - uses: actions/checkout@v5 + + - name: Build Examples (Using official Github Action) + # uses: SilverbackLtd/build-action@v1 + uses: SilverbackLtd/build-action@main + with: + push: ${{ github.event_name != 'pull_request' }} + tag: latest + registry: ghcr.io + username: ${{ github.actor }} + password: ${{ secrets.GITHUB_TOKEN }} + + prune-registry: + needs: build + if: ${{ github.ref == 'refs/heads/main' }} + runs-on: ubuntu-latest + permissions: + contents: read + packages: write + id-token: write + + steps: - name: Run container retention policy - if: ${{ github.event_name != 'pull_request' }} uses: snok/container-retention-policy@v3.0.1 with: account: ApeWorX @@ -138,23 +196,3 @@ jobs: tag-selection: both cut-off: 4w dry-run: true - - example: - # NOTE: We want the silverback image built above to base ours on - needs: build - runs-on: ubuntu-latest - permissions: - contents: read - packages: write - id-token: write - steps: - - uses: actions/checkout@v4 - - - name: Build Examples - uses: SilverbackLtd/build-action@v1 - with: - push: ${{ github.event_name != 'pull_request' }} - tag: latest - registry: ghcr.io - username: ${{ github.actor }} - password: ${{ secrets.GITHUB_TOKEN }} diff --git a/Dockerfile b/Dockerfile index e47b748c..8f9d6ee7 100644 --- a/Dockerfile +++ b/Dockerfile @@ -6,7 +6,7 @@ ARG BASE_APE_IMAGE="ghcr.io/apeworx/ape:python${PYTHON_VERSION}-stable-slim" # Stage 1: Build dependencies # NOTE: Build with builder image to reduce image size -FROM ${BASE_APE_IMAGE} as slim-builder +FROM ${BASE_APE_IMAGE} AS slim-builder # NOTE: Switch back to root for building USER root diff --git a/silverback/_build_utils.py b/silverback/_build_utils.py index b9fe78ae..2b449e62 100644 --- a/silverback/_build_utils.py +++ b/silverback/_build_utils.py @@ -15,50 +15,56 @@ def containerfile_template( has_pyproject_toml: bool = False, has_ape_config_yaml: bool = False, contracts_folder: str | None = None, - include_bot_dir: bool = False, ): - containerfile = [ + steps = [ f"FROM ghcr.io/apeworx/silverback:{sdk_version}", - "USER root", - "WORKDIR /app", - "RUN chown harambe:harambe /app", - "USER harambe", ] + if sdk_version == "v0.7.36": + # NOTE: Some versions of the base image (using Ape v0.8.{46,47}) do not ensure this + # see: https://github.com/ApeWorX/ape/pull/2753 + # TODO: Remove in Silverback v0.8.x (no longer maintaining v0.7.36) + steps.extend( + [ + "USER root", + "RUN chown harambe:harambe .", + "USER harambe", + ] + ) + if requirements_txt_fname: - containerfile.append(f"COPY {requirements_txt_fname} requirements.txt") + steps.append(f"COPY --chown=harambe:harambe {requirements_txt_fname} requirements.txt") if has_pyproject_toml: - containerfile.append("COPY pyproject.toml .") + steps.append("COPY --chown=harambe:harambe pyproject.toml .") if has_ape_config_yaml: - containerfile.append("COPY ape-config.yaml .") + steps.append("COPY --chown=harambe:harambe ape-config.yaml .") if requirements_txt_fname or has_pyproject_toml: - containerfile.append("RUN pip install --upgrade pip") + steps.append("COPY --from=ghcr.io/astral-sh/uv:latest /uv /bin/") - # NOTE: Only install project via `pyproject.toml` if `requirements-bot].txt` DNE + # NOTE: Only install project via `pyproject.toml` if `requirements-bot.txt` DNE install_arg = "-r requirements.txt" if requirements_txt_fname else "." - containerfile.append(f"RUN pip install {install_arg}") + steps.append(f"RUN uv pip install {install_arg}") if has_pyproject_toml or has_ape_config_yaml: - containerfile.append("RUN ape plugins install -U .") + steps.append("RUN ape plugins install -U .") if contracts_folder: - containerfile.append(f"COPY {contracts_folder} /app/{contracts_folder}") - containerfile.append("RUN ape compile") + steps.append(f"COPY --chown=harambe:harambe {contracts_folder} {contracts_folder}") + steps.append("RUN ape compile") - bot_src = f"{bot_path.parent}/{bot_path.name}" if include_bot_dir else bot_path.name - bot_dst = "/app/bot" if bot_path.is_dir() else "/app/bot.py" - containerfile.append(f"COPY {bot_src} {bot_dst}") + bot_dest = "bot/" if bot_path.is_dir() else "bot.py" + steps.append(f"COPY --chown=harambe:harambe {bot_path} {bot_dest}") - return "\n".join(containerfile) + return "\n".join(steps) def generate_containerfiles(path: Path, sdk_version: str = "stable"): (Path.cwd() / IMAGES_FOLDER_NAME).mkdir(exist_ok=True) - contracts_folder: str | None = "contracts" + contracts_folder: str = "contracts" if has_ape_config_yaml := (ape_config_path := Path.cwd() / "ape-config.yaml").exists(): contracts_folder = ( yaml.safe_load(ape_config_path.read_text()) @@ -85,22 +91,19 @@ def generate_containerfiles(path: Path, sdk_version: str = "stable"): # NOTE: Doesn't exist so make it not be `requirements.txt` requirements_txt_fname = None - assert contracts_folder # make mypy happy - if not (Path.cwd() / contracts_folder).exists(): - contracts_folder = None - if path.is_dir() and path.name == "bots": for bot in path.glob("*.py"): bot = bot.relative_to(Path.cwd()) (Path.cwd() / IMAGES_FOLDER_NAME / f"Dockerfile.{bot.stem}").write_text( containerfile_template( bot, - include_bot_dir=True, sdk_version=sdk_version, requirements_txt_fname=requirements_txt_fname, has_pyproject_toml=has_pyproject_toml, has_ape_config_yaml=has_ape_config_yaml, - contracts_folder=contracts_folder, + contracts_folder=( + contracts_folder if (Path.cwd() / contracts_folder).exists() else None + ), ) ) @@ -112,7 +115,9 @@ def generate_containerfiles(path: Path, sdk_version: str = "stable"): requirements_txt_fname=requirements_txt_fname, has_pyproject_toml=has_pyproject_toml, has_ape_config_yaml=has_ape_config_yaml, - contracts_folder=contracts_folder, + contracts_folder=( + contracts_folder if (Path.cwd() / contracts_folder).exists() else None + ), ) ) diff --git a/silverback/_cli.py b/silverback/_cli.py index 4b8043d3..f22222ae 100644 --- a/silverback/_cli.py +++ b/silverback/_cli.py @@ -173,8 +173,9 @@ def run(cli_ctx, account, runner_class, record, recorder_class, max_exceptions, "--tag-base", default=None, help=( - "The base to use to tag the image. " - "The bot name (or 'bot') is appended to it, following a '-' separator. " + "The base to use to tag the final bot image(s). " + "If multiple bots are in the project, the name of the bot is appended to it, following a '-' separator. " + "If only one bot, then `-bot` is used. " "Defaults to using the name of the folder you are building from." ), ) @@ -182,7 +183,14 @@ def run(cli_ctx, account, runner_class, record, recorder_class, max_exceptions, "--version", default="latest", metavar="VERSION", - help="Version to use in tag. Defaults to 'latest'.", + help="Version label to use for tagging final bot images. Defaults to 'latest'.", +) +@click.option( + "--sdk", + "sdk_version", + default="stable", + metavar="VERSION", + help="Version of Silverback SDK to use as base image. Defaults to 'stable'.", ) @click.option( "--push", @@ -191,7 +199,7 @@ def run(cli_ctx, account, runner_class, record, recorder_class, max_exceptions, help="Push image to logged-in registry. Defaults to false.", ) @click.argument("path", required=False, default=None) -def build(use_docker, generate, tag_base, version, push, path): +def build(use_docker, generate, tag_base, version, sdk_version, push, path): """ Generate Dockerfiles and build bot container images @@ -226,7 +234,7 @@ def build(use_docker, generate, tag_base, version, push, path): ", or process all '*.py' bots in 'bots/' folder." ) - generate_containerfiles(path) + generate_containerfiles(path, sdk_version=sdk_version) if not (Path.cwd() / IMAGES_FOLDER_NAME).exists(): raise click.ClickException( diff --git a/tests/test_utils.py b/tests/test_utils.py index 091941c6..a0391087 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -20,6 +20,27 @@ def test_topic_encoding(topics): assert decode_topics_from_string(encode_topics_to_string(topics)) == topics +EXAMPLE_BOT_PATH = Path(__file__).parent.parent / "bots" / "example.py" + + +@pytest.fixture(scope="module", params=["file", "folder"]) +def bot_path(request): + if request.param == "file": + yield EXAMPLE_BOT_PATH + + else: # Make `bot/` as a module and copy example over to it + folder = Path(__file__).parent.parent / "bot" + folder.mkdir(exist_ok=True) + (folder / "__init__.py").write_text(EXAMPLE_BOT_PATH.read_text()) + + try: + yield folder + + finally: + (folder / "__init__.py").unlink(missing_ok=True) + folder.rmdir() + + @pytest.mark.parametrize( "build_args", [ @@ -30,14 +51,12 @@ def test_topic_encoding(topics): dict(has_pyproject_toml=True), dict(has_ape_config_yaml=True), dict(contracts_folder="src"), - dict(include_bot_dir=True), ], ) -def test_containerfile_generation(build_args): - containerfile = containerfile_template( - Path(__file__).parent.parent / "bots" / "example.py", **build_args - ) - assert "example.py" in containerfile +def test_containerfile_generation(bot_path, build_args): + containerfile = containerfile_template(bot_path, **build_args) + + assert bot_path.name in containerfile assert build_args.get("sdk_version", "stable") in containerfile if requirements_txt_fname := build_args.get("requirements_txt_fname"): assert requirements_txt_fname in containerfile @@ -47,5 +66,3 @@ def test_containerfile_generation(build_args): assert "ape-config.yaml" in containerfile if contracts_folder := build_args.get("contracts_folder"): assert contracts_folder in containerfile - if build_args.get("include_bot_dir"): - assert "bots/" in containerfile