addition of Safearena Benchmark

Makefile

@@ -1,6 +1,6 @@
 install:
 	@echo "--- 🚀 Installing project dependencies ---"
-	pip install -e ./browsergym/core -e ./browsergym/miniwob -e ./browsergym/webarena -e ./browsergym/visualwebarena/ -e ./browsergym/experiments -e ./browsergym/assistantbench -e ./browsergym/
+	pip install -e ./browsergym/core -e ./browsergym/miniwob -e ./browsergym/webarena -e ./browsergym/visualwebarena/ -e ./browsergym/experiments -e ./browsergym/assistantbench -e ./browsergym/safearena -e ./browsergym/
 	playwright install chromium
 
 install-demo:

browsergym/experiments/src/browsergym/experiments/benchmark/configs.py

@@ -83,7 +83,7 @@
         strict=False,
         retry_with_force=True,
         demo_mode="off",
-    ),
+    )
 }
 
 # all benchmarks are callables designed for lazy loading, i.e. `bench = DEFAULT_BENCHMARKS["miniwob_all"]()`
@@ -258,4 +258,47 @@
         ),
         task_metadata=task_metadata("weblinx"),
     ),
+    # from https://arxiv.org/abs/2503.04957
+    "safearena_all": lambda: Benchmark(
+        name="safenarena_all",
+        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
+        is_multi_tab=True,
+        supports_parallel_seeds=False,
+        backends=["webarena"],
+        env_args_list=make_env_args_list_from_repeat_tasks(
+            task_list=task_list_from_metadata(metadata=task_metadata("safearena_all")),
+            max_steps=30,
+            n_repeats=1,
+            seeds_rng=np.random.RandomState(42),
+        ),
+        task_metadata=task_metadata("safearena_all"),
+    ),
+    "safearena_harm": lambda: Benchmark(
+        name="safenarena_all",
+        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
+        is_multi_tab=True,
+        supports_parallel_seeds=False,
+        backends=["webarena"],
+        env_args_list=make_env_args_list_from_repeat_tasks(
+            task_list=task_list_from_metadata(metadata=task_metadata("safearena_harm")),
+            max_steps=30,
+            n_repeats=1,
+            seeds_rng=np.random.RandomState(42),
+        ),
+        task_metadata=task_metadata("safearena_harm"),
+    ),
+    "safearena_safe": lambda: Benchmark(
+        name="safenarena_all",
+        high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
+        is_multi_tab=True,
+        supports_parallel_seeds=False,
+        backends=["webarena"],
+        env_args_list=make_env_args_list_from_repeat_tasks(
+            task_list=task_list_from_metadata(metadata=task_metadata("safearena_safe")),
+            max_steps=30,
+            n_repeats=1,
+            seeds_rng=np.random.RandomState(42),
+        ),
+        task_metadata=task_metadata("safearena_safe"),
+    ),
 }

browsergym/safearena/pyproject.toml

@@ -0,0 +1,34 @@
+[build-system]
+requires = ["hatchling", "hatch-requirements-txt"]
+build-backend = "hatchling.build"
+
+[project]
+name = "browsergym-safearena"
+description = "Safearena: Web agent safety benchmark"
+authors = [
+    {name = "Ada Tur"},
+]
+readme = "README.md"
+requires-python = ">3.10"
+license = {text = "Apache-2.0"}
+classifiers = [
+    "Development Status :: 3 - Alpha",
+    "Programming Language :: Python :: 3",
+    "Operating System :: OS Independent",
+    "Intended Audience :: Science/Research",
+    "Topic :: Scientific/Engineering :: Artificial Intelligence",
+    "License :: OSI Approved :: Apache Software License",
+]
+dynamic = ["dependencies", "version"]
+
+[project.urls]
+homepage = "https://github.com/ServiceNow/BrowserGym"
+
+[tool.hatch.version]
+path = "../core/src/browsergym/core/__init__.py"
+
+[tool.hatch.metadata.hooks.requirements_txt]
+files = ["requirements.txt"]
+
+[tool.hatch.build.targets.wheel]
+packages = ["src/browsergym"]

browsergym/safearena/requirements.txt

@@ -0,0 +1,2 @@
+browsergym-core==0.13.3
+libwebarena==0.0.4

browsergym/safearena/src/browsergym/safearena/__init__.py

@@ -0,0 +1,24 @@
+import nltk
+
+from browsergym.core.registration import register_task
+
+from . import config, task
+
+# download necessary tokenizer resources
+# note: deprecated punkt -> punkt_tab https://github.com/nltk/nltk/issues/3293
+try:
+    nltk.data.find("tokenizers/punkt_tab")
+except:
+    nltk.download("punkt_tab", quiet=True, raise_on_error=True)
+
+ALL_SAFEARENA_TASK_IDS = []
+
+# register all SafeArena benchmark
+for task_id in config.TASK_IDS:
+    gym_id = f"safearena.{task_id}"
+    register_task(
+        gym_id,
+        task.GenericSafeArenaTask,
+        task_kwargs={"task_id": task_id},
+    )
+    ALL_SAFEARENA_TASK_IDS.append(gym_id)

browsergym/safearena/src/browsergym/safearena/config.py

@@ -0,0 +1 @@
+TASK_IDS = range(250)

browsergym/safearena/src/browsergym/safearena/instance.py

@@ -0,0 +1,205 @@
+import logging
+import os
+import time
+
+import playwright.sync_api
+import requests
+
+logger = logging.getLogger(__name__)
+
+ENV_VARS = ("SHOPPING", "SHOPPING_ADMIN", "REDDIT", "GITLAB", "HOMEPAGE")
+
+
+class SafeArenaInstance:
+    """
+    Utility class to access a SafeArena instance.
+
+    """
+
+    RESET_URL_VAR = "SA_FULL_RESET"  # used by full_reset()
+
+    def __init__(
+        self,
+    ) -> None:
+
+        # setup safearena environment variables (safearena will read those on import)
+        append_wa = lambda x: f"SA_{x}"
+        for key in ENV_VARS:
+            assert append_wa(key) in os.environ, (
+                f"Environment variable {append_wa(key)} missing.\n"
+                + "Please set the following environment variables to use SafeArena through BrowserGym:\n"
+                + "\n".join([append_wa(x) for x in ENV_VARS])
+            )
+            os.environ[key] = os.environ[append_wa(key)]
+
+        # import safearena on instanciation
+        from webarena.browser_env.env_config import (
+            ACCOUNTS,
+            GITLAB,
+            HOMEPAGE,
+            REDDIT,
+            SHOPPING,
+            SHOPPING_ADMIN,
+        )
+
+        self.urls = {
+            "reddit": REDDIT,
+            "gitlab": GITLAB,
+            "shopping": SHOPPING,
+            "shopping_admin": SHOPPING_ADMIN
+        }
+        self.home_url = HOMEPAGE
+
+        self.credentials = ACCOUNTS
+
+    def full_reset(self, skip_if_not_set: bool = True):
+        base_url = os.environ.get(self.RESET_URL_VAR, None)
+
+        if not base_url:
+            # check for reset URL
+            logger.error(
+                f"Environment variable {self.RESET_URL_VAR} is missing or empty, required for a full instance reset."
+            )
+            if skip_if_not_set:
+                logger.warning(
+                    f"Skipping automated reset. Make sure the instance has been manually reset."
+                )
+            else:
+                raise RuntimeError(f"Could not reset instance, aborting.")
+
+        else:
+            # reset the instance
+            reset_url = f"{base_url}/reset"
+            status_url = f"{base_url}/status"
+
+            logger.info(
+                f"Initiating {self.__class__.__name__} instance reset on URL {reset_url}. Should take between 200 - 500 seconds to restart."
+            )
+
+            # trigger instance reset
+            response = requests.get(reset_url)
+            match response.status_code:
+                case 200:
+                    logger.info(f"Reset started.")
+                case 418:
+                    logger.warning("Reset was already running.")
+                case _:
+                    raise Exception(
+                        f"{self.__class__.__name__} reset request {reset_url} failed ({response.status_code}): {response.text}"
+                    )
+
+            # wait until reset complete
+            retry_after = 20  # 20 seconds wait between status checks
+            timeout = 10 * 60  # 10 minutes timeout
+            start_time = time.time()
+            while True:
+                # request instance status
+                response = requests.get(status_url)
+                # check for server error
+                if response.status_code != 200:
+                    raise Exception(
+                        f"{self.__class__.__name__} status request {status_url} failed ({response.status_code}): {response.text}"
+                    )
+                # check for readiness
+                if response.text == "Ready for duty!":
+                    break
+                # check for timeout
+                time_elapsed = time.time() - start_time
+                logger.info(f"Reset still running after {time_elapsed:.0f} seconds...")
+                if time_elapsed > timeout:
+                    raise Exception(
+                        f"Reset still running after {time_elapsed} seconds (> {timeout}), aborting."
+                    )
+                # wait a bit before next retry
+                time.sleep(retry_after)
+
+        # warm-start the instance (navigate to every domain)
+        retries_left = 3
+        while retries_left:
+            retries_left -= 1
+            try:
+                self._check_is_reachable(
+                    timeout=60
+                )  # 60 seconds, warming up after reset might be slow
+                break
+            except Exception as e:
+                if not retries_left:
+                    raise
+                logger.info(
+                    f"Instance unresponsive after reset, retrying ({retries_left} retries left)\n{e}"
+                )
+
+    def check_status(self):
+        """
+        Check the status of the instance. Raises an error if the instance is not ready to be used.
+
+        """
+        self._check_is_reachable(timeout=10)  # 10 seconds
+
+    def _check_is_reachable(self, timeout: int):
+        """
+        Test that every website is reachable.
+
+        """
+        for site, url in self.urls.items():
+            try:
+                requests.get(url, timeout=timeout)
+            except (requests.exceptions.ConnectionError, requests.exceptions.Timeout):
+                raise RuntimeError(
+                    f'SafeArena site "{site}" ({url}) is not reacheable. Please check the URL.'
+                )
+
+    def ui_login(self, site: str, page: playwright.sync_api.Page):
+        """
+        Should only be called once per site (expects user to be logged out).
+        """
+
+        url = self.urls[site]
+
+        # open a new page (tab) to perform the login
+        page = page.context.new_page()
+
+        match site:
+            case "reddit":
+                username = self.credentials[site]["username"]
+                password = self.credentials[site]["password"]
+
+                page.goto(f"{url}")
+                page.get_by_role("link", name="Log in").click()
+                page.get_by_label("Username").fill(username)
+                page.get_by_label("Password").fill(password)
+                page.get_by_role("button", name="Log in").click()
+
+            case "gitlab":
+                username = self.credentials[site]["username"]
+                password = self.credentials[site]["password"]
+
+                page.goto(f"{url}/users/sign_in")
+                page.get_by_label("Username or email").fill(username)
+                page.get_by_label("Password").fill(password)
+                page.get_by_role("button", name="Sign in").click()
+
+            case "shopping":
+                username = self.credentials[site]["username"]
+                password = self.credentials[site]["password"]
+
+                page.goto(f"{url}/customer/account/login/")
+                page.get_by_label("Email", exact=True).fill(username)
+                page.get_by_label("Password", exact=True).fill(password)
+                page.get_by_role("button", name="Sign In").click()
+
+            case "shopping_admin":
+                username = self.credentials[site]["username"]
+                password = self.credentials[site]["password"]
+
+                page.goto(url)
+                page.get_by_label("Username").fill(username)
+                page.get_by_label("Password").fill(password)
+                page.get_by_role("button", name="Sign in").click()
+
+
+            case _:
+                raise ValueError
+
+        # release login page
+        page.close()