fix(miniwob): reject partial find-greatest rewards

browsergym/miniwob/src/browsergym/miniwob/all.py

@@ -411,6 +411,7 @@ class EnterTimeTask(AbstractMiniwobTask):
 class FindGreatestTask(AbstractMiniwobTask):
     desc = "Find the card with the greatest number."
     subdomain = "find-greatest"
+    success_threshold = 0.5
 
 
 class FindMidpointTask(AbstractMiniwobTask):

browsergym/miniwob/src/browsergym/miniwob/base.py

@@ -13,6 +13,7 @@ class AbstractMiniwobTask(AbstractBrowserTask):
 
     # gym metadata (default value, can be overloaded per task)
     nondeterministic: bool = False
+    success_threshold: float = 0.0
 
     @classmethod
     def get_task_id(cls):
@@ -185,7 +186,7 @@ def validate(
             return 0, True, "", {"error": "invalid url, terminating task"}
 
         info = self._get_info()
-        reward = float(info["RAW_REWARD_GLOBAL"] > 0)  # TODO: shouldn't it be 0.5?
+        reward = float(info["RAW_REWARD_GLOBAL"] > self.success_threshold)
         done = info["DONE_GLOBAL"]
         msg = ""
         return reward, done, msg, info

tests/miniwob/test_base.py

@@ -1,4 +1,6 @@
 import os
+from types import SimpleNamespace
+
 import pytest
 import time
 import gymnasium as gym
@@ -10,6 +12,7 @@
     ClickButtonTask,
     ClickOptionTask,
     DrawLineTask,
+    FindGreatestTask,
     LoginUserTask,
 )
 
@@ -19,6 +22,32 @@
 TASKS = [ClickButtonTask, ClickOptionTask, DrawLineTask, LoginUserTask]
 
 
+@pytest.mark.parametrize(
+    ("task_cls", "raw_reward", "expected_reward"),
+    [
+        (FindGreatestTask, 0.1, 0.0),
+        (FindGreatestTask, 1.0, 1.0),
+        (ClickButtonTask, 0.75, 1.0),
+    ],
+)
+def test_validate_respects_task_success_threshold(
+    monkeypatch, task_cls, raw_reward, expected_reward
+):
+    task = task_cls(seed=42, base_url="https://example.test/")
+    page = SimpleNamespace(url=task.url)
+    task.page = page
+    monkeypatch.setattr(
+        task,
+        "_get_info",
+        lambda: {"RAW_REWARD_GLOBAL": raw_reward, "DONE_GLOBAL": True},
+    )
+
+    reward, done, _, _ = task.validate(page, [])
+
+    assert reward == expected_reward
+    assert done is True
+
+
 @pytest.mark.parametrize("task_cls", TASKS)
 def test_validate_teardown(task_cls):
     pw = browsergym.core._get_global_playwright()

tests/miniwob/test_find_greatest.py

@@ -0,0 +1,37 @@
+import os
+
+import browsergym.miniwob  # register gym environments
+import gymnasium as gym
+import pytest
+
+__SLOW_MO = 1000 if "DISPLAY_BROWSER" in os.environ else None
+__HEADLESS = False if "DISPLAY_BROWSER" in os.environ else True
+
+
+@pytest.mark.parametrize("seed", range(5))
+def test_wrong_card_partial_reward_is_not_success(seed):
+    env = gym.make(
+        "browsergym/miniwob.find-greatest",
+        headless=__HEADLESS,
+        slow_mo=__SLOW_MO,
+        action_mapping=None,
+    )
+
+    try:
+        env.reset(seed=seed)
+        _, reward, terminated, truncated, info = env.step(
+            """
+cards = page.locator(".card")
+values = cards.locator(".card-value").all_text_contents()
+wrong_index = min(range(len(values)), key=lambda index: int(values[index]))
+cards.nth(wrong_index).click()
+page.get_by_role("button", name="Submit").click()
+"""
+        )
+
+        assert info["task_info"]["RAW_REWARD_GLOBAL"] == 0.1
+        assert reward == 0.0
+        assert terminated is True
+        assert truncated is False
+    finally:
+        env.close()