fix(fal): stabilize shark staging smoke tests

projects/fal/pyproject.toml

@@ -113,3 +113,6 @@ keep-runtime-typing = true
 
 [tool.pytest.ini_options]
 timeout = 300
+markers = [
+    "shark_smoke: lightweight staging smoke coverage for the shark post-deploy workflow",
+]

projects/fal/src/fal/app.py

@@ -198,6 +198,14 @@ class AppClientError(FalServerlessException):
     headers: dict[str, str] = field(default_factory=dict)
 
 
+def _should_retry_health_check_response(status_code: int, headers: Any) -> bool:
+    if status_code in (404, 500):
+        return True
+
+    retry_hint = headers.get("x-fal-needs-retry", "")
+    return str(retry_hint).strip().lower() in {"1", "true", "yes"}
+
+
 class AppSpawnInfo:
     def __init__(self, info: SpawnInfo):
         self.info = info
@@ -266,7 +274,9 @@ def wait(
                     if resp.is_success:
                         return
 
-                    if resp.status_code in (500, 404):
+                    if _should_retry_health_check_response(
+                        resp.status_code, resp.headers
+                    ):
                         last_error = f"Server not ready (HTTP {resp.status_code})"
                     else:
                         raise AppClientError(

projects/fal/tests/integration/test_environments.py

@@ -27,6 +27,7 @@ def client():
 
 
 @pytest.mark.flaky(max_runs=3)
+@pytest.mark.shark_smoke
 def test_environment_lifecycle(client: SyncServerlessClient, test_env_name: str):
     """Test creating, listing, and deleting environments."""
     # Create environment

projects/fal/tests/integration/test_files.py

@@ -1,9 +1,12 @@
 import posixpath
 import uuid
 
+import pytest
+
 from fal.files import FalFileSystem
 
 
+@pytest.mark.shark_smoke
 def test_fal_fs(tmp_path):
     (tmp_path / "myfile").write_text("myfile")
     (tmp_path / "mydir").mkdir()

projects/fal/tests/integration/test_stability.py

@@ -41,6 +41,8 @@ def test2():
         test2()
 
 
+@pytest.mark.flaky(max_runs=3)
+@pytest.mark.shark_smoke
 def test_regular_function(isolated_client):
     @isolated_client("virtualenv")
     def regular_function():

projects/fal/tests/integration/toolkit/test_image.py

@@ -86,6 +86,7 @@ def fal_image_from_bytes_remote():
     assert fal_image_content_matches(fal_image, get_image(as_bytes=True))
 
 
+@pytest.mark.shark_smoke
 def test_fal_image_from_bytes(isolated_client):
     @isolated_client(requirements=["pillow", f"pydantic=={pydantic_version}", "tomli"])
     def fal_image_from_bytes_remote():

projects/fal/tests/unit/test_app.py

@@ -157,6 +157,49 @@ async def fake_serve(self, *, limit_max_requests: int | None = None):
     assert called_limit_max_requests == 1
 
 
+def test_app_spawn_info_wait_retries_health_checks_when_server_requests_retry():
+    from fal.app import AppSpawnInfo
+
+    info = AppSpawnInfo(MagicMock(url="https://example.com"))
+
+    retry_response = MagicMock(
+        is_success=False,
+        status_code=503,
+        text='{"detail":"Runner connection failed"}',
+        headers={"x-fal-needs-retry": "1"},
+    )
+    success_response = MagicMock(is_success=True)
+    mock_client = MagicMock()
+    mock_client.get.side_effect = [retry_response, success_response]
+
+    with patch("httpx.Client") as mock_httpx_client:
+        mock_httpx_client.return_value.__enter__.return_value = mock_client
+        with patch("fal.app.time.sleep"):
+            info.wait(startup_timeout=1)
+
+    assert mock_client.get.call_count == 2
+
+
+def test_app_spawn_info_wait_fails_fast_on_non_retryable_health_error():
+    from fal.app import AppClientError, AppSpawnInfo
+
+    info = AppSpawnInfo(MagicMock(url="https://example.com"))
+
+    response = MagicMock(
+        is_success=False,
+        status_code=503,
+        text='{"detail":"bad request"}',
+        headers={},
+    )
+    mock_client = MagicMock()
+    mock_client.get.return_value = response
+
+    with patch("httpx.Client") as mock_httpx_client:
+        mock_httpx_client.return_value.__enter__.return_value = mock_client
+        with pytest.raises(AppClientError, match="non-retryable error: 503"):
+            info.wait(startup_timeout=1)
+
+
 def test_wrap_app_raises_for_virtualenv_only_keys_with_conda_kind():
     from fal.app import wrap_app