diff --git a/projects/fal/pyproject.toml b/projects/fal/pyproject.toml index 89aaef704..6f37ab58c 100644 --- a/projects/fal/pyproject.toml +++ b/projects/fal/pyproject.toml @@ -113,3 +113,6 @@ keep-runtime-typing = true [tool.pytest.ini_options] timeout = 300 +markers = [ + "shark_smoke: lightweight staging smoke coverage for the shark post-deploy workflow", +] diff --git a/projects/fal/src/fal/app.py b/projects/fal/src/fal/app.py index 8fd2b8b86..4c1d1101f 100644 --- a/projects/fal/src/fal/app.py +++ b/projects/fal/src/fal/app.py @@ -198,6 +198,14 @@ class AppClientError(FalServerlessException): headers: dict[str, str] = field(default_factory=dict) +def _should_retry_health_check_response(status_code: int, headers: Any) -> bool: + if status_code in (404, 500): + return True + + retry_hint = headers.get("x-fal-needs-retry", "") + return str(retry_hint).strip().lower() in {"1", "true", "yes"} + + class AppSpawnInfo: def __init__(self, info: SpawnInfo): self.info = info @@ -266,7 +274,9 @@ def wait( if resp.is_success: return - if resp.status_code in (500, 404): + if _should_retry_health_check_response( + resp.status_code, resp.headers + ): last_error = f"Server not ready (HTTP {resp.status_code})" else: raise AppClientError( diff --git a/projects/fal/tests/integration/test_environments.py b/projects/fal/tests/integration/test_environments.py index 79ca84a46..103fdb32a 100644 --- a/projects/fal/tests/integration/test_environments.py +++ b/projects/fal/tests/integration/test_environments.py @@ -27,6 +27,7 @@ def client(): @pytest.mark.flaky(max_runs=3) +@pytest.mark.shark_smoke def test_environment_lifecycle(client: SyncServerlessClient, test_env_name: str): """Test creating, listing, and deleting environments.""" # Create environment diff --git a/projects/fal/tests/integration/test_files.py b/projects/fal/tests/integration/test_files.py index 3123ee09d..93919a74e 100644 --- a/projects/fal/tests/integration/test_files.py +++ b/projects/fal/tests/integration/test_files.py @@ -1,9 +1,12 @@ import posixpath import uuid +import pytest + from fal.files import FalFileSystem +@pytest.mark.shark_smoke def test_fal_fs(tmp_path): (tmp_path / "myfile").write_text("myfile") (tmp_path / "mydir").mkdir() diff --git a/projects/fal/tests/integration/test_stability.py b/projects/fal/tests/integration/test_stability.py index c850fead8..5c52201fb 100644 --- a/projects/fal/tests/integration/test_stability.py +++ b/projects/fal/tests/integration/test_stability.py @@ -41,6 +41,8 @@ def test2(): test2() +@pytest.mark.flaky(max_runs=3) +@pytest.mark.shark_smoke def test_regular_function(isolated_client): @isolated_client("virtualenv") def regular_function(): diff --git a/projects/fal/tests/integration/toolkit/test_image.py b/projects/fal/tests/integration/toolkit/test_image.py index e9480dc8c..2a2b0e1c6 100644 --- a/projects/fal/tests/integration/toolkit/test_image.py +++ b/projects/fal/tests/integration/toolkit/test_image.py @@ -86,6 +86,7 @@ def fal_image_from_bytes_remote(): assert fal_image_content_matches(fal_image, get_image(as_bytes=True)) +@pytest.mark.shark_smoke def test_fal_image_from_bytes(isolated_client): @isolated_client(requirements=["pillow", f"pydantic=={pydantic_version}", "tomli"]) def fal_image_from_bytes_remote(): diff --git a/projects/fal/tests/unit/test_app.py b/projects/fal/tests/unit/test_app.py index 82e69ee8a..98ec07391 100644 --- a/projects/fal/tests/unit/test_app.py +++ b/projects/fal/tests/unit/test_app.py @@ -157,6 +157,49 @@ async def fake_serve(self, *, limit_max_requests: int | None = None): assert called_limit_max_requests == 1 +def test_app_spawn_info_wait_retries_health_checks_when_server_requests_retry(): + from fal.app import AppSpawnInfo + + info = AppSpawnInfo(MagicMock(url="https://example.com")) + + retry_response = MagicMock( + is_success=False, + status_code=503, + text='{"detail":"Runner connection failed"}', + headers={"x-fal-needs-retry": "1"}, + ) + success_response = MagicMock(is_success=True) + mock_client = MagicMock() + mock_client.get.side_effect = [retry_response, success_response] + + with patch("httpx.Client") as mock_httpx_client: + mock_httpx_client.return_value.__enter__.return_value = mock_client + with patch("fal.app.time.sleep"): + info.wait(startup_timeout=1) + + assert mock_client.get.call_count == 2 + + +def test_app_spawn_info_wait_fails_fast_on_non_retryable_health_error(): + from fal.app import AppClientError, AppSpawnInfo + + info = AppSpawnInfo(MagicMock(url="https://example.com")) + + response = MagicMock( + is_success=False, + status_code=503, + text='{"detail":"bad request"}', + headers={}, + ) + mock_client = MagicMock() + mock_client.get.return_value = response + + with patch("httpx.Client") as mock_httpx_client: + mock_httpx_client.return_value.__enter__.return_value = mock_client + with pytest.raises(AppClientError, match="non-retryable error: 503"): + info.wait(startup_timeout=1) + + def test_wrap_app_raises_for_virtualenv_only_keys_with_conda_kind(): from fal.app import wrap_app