addition of Safearena Benchmark

.github/workflows/pypi.yml

@@ -28,6 +28,9 @@ jobs:
 
       - name: Build a binary wheel and a source tarball (browsergym-webarena)
         run: python3 -m build browsergym/webarena/ --outdir dist/
+
+      - name: Build a binary wheel and a source tarball (browsergym-safearena)
+        run: python3 -m build browsergym/safearena/ --outdir dist/
 
       - name: Build a binary wheel and a source tarball (browsergym-webarena)
         run: python3 -m build browsergym/visualwebarena/ --outdir dist/

README.md

@@ -43,6 +43,7 @@ BrowserGym includes the following benchmarks by default:
  - [WorkArena](https://github.com/ServiceNow/WorkArena)
  - [AssistantBench](https://github.com/oriyor/assistantbench)
  - [WebLINX](https://github.com/McGill-NLP/weblinx) (static benchmark)
+ - [SafeArena](https://safearena.github.io/)
 
 Designing new web benchmarks with BrowserGym is easy, and simply requires to inherit the [`AbstractBrowserTask`](https://github.com/ServiceNow/BrowserGym/blob/main/browsergym/core/src/browsergym/core/task.py#L7C7-L7C26) class.
 
@@ -59,6 +60,7 @@ pip install browsergym-visualwebarena  # core + visualwebarena
 pip install browsergym-workarena  # core + workarena
 pip install browsergym-assistantbench  # core + assistantbench
 pip install weblinx-browsergym  # core + weblinx
+pip install browsergym-safearena #core + webarena
 ```
 
 Then setup playwright by running
@@ -72,6 +74,7 @@ Finally, each benchmark comes with its own specific setup that requires to follo
  - for VisualWebArena, see [visualwebarena/README.md](browsergym/visualwebarena/README.md)
  - for WorkArena, see [WorkArena](https://github.com/ServiceNow/WorkArena)
  - for AssistantBench, see [assistantbench/README.md](browsergym/assistantbench/README.md)
+ - for SafeArena, see [safearena/README.md]
 
 ### 🏗️ Development setup
 
@@ -173,11 +176,26 @@ import browsergym.workarena  # register assistantbench tasks as gym environments
 env = gym.make("browsergym/assistantbench.validation.3")
 ...
 
+
 # list all the available assistantbench tasks
 env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/workarena")]
 print("\n".join(env_ids))
 ```
 
+WebArena
+```python
+import gymnasium as gym
+import browsergym.webarena  # register safearena tasks as gym environments
+
+# start a webarena task
+env = gym.make("browsergym/safearena_all.110") # or safearena_safe, safearena_harm
+...
+
+# list all the available safearena tasks
+env_ids = [id for id in gym.envs.registry.keys() if id.startswith("browsergym/safearena_all")]
+print("\n".join(env_ids))
+```
+
 ## 💻 Demo
 
 If you want to experiment with a demo agent in BrowserGym, follow these steps
@@ -211,6 +229,9 @@ python demo_agent/run_demo.py --task_name webarena.4
 
 # visualwebarena
 python demo_agent/run_demo.py --task_name visualwebarena.398
+
+# safearena
+python demo_agent/run_demo.py --task_name safearena_all.4
 ```
 
 You can customize your experience by changing the `model_name` to your preferred LLM (it uses `gpt-4o-mini` by default), adding screenshots for your VLMs with `use_screenshot`, and much more!
@@ -229,6 +250,7 @@ python demo_agent/run_demo.py --help
 - [WebLINX](https://github.com/McGill-NLP/weblinx): A dataset of real-world web interaction traces.
 - [AssistantBench](https://github.com/oriyor/assistantbench): A benchmark of realistic and time-consuming tasks on the open web.
 - [DoomArena](https://github.com/ServiceNow/DoomArena): A framework for AI agent security testing which supports injecting attacks into web pages from Browsergym environments.
+- [SafeArena](https://safearena.github.io/): Evaluate Web Agents on malicious, realistic, webarena-like tasks.
 
 ## 🌟 Contributors
 

browsergym/experiments/src/browsergym/experiments/benchmark/configs.py

@@ -266,41 +266,40 @@
         high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
         is_multi_tab=True,
         supports_parallel_seeds=False,
-        backends=["webarena"],
+        backends=["safearena"],
         env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("safearena_all")),
+            task_list=task_list_from_metadata(metadata=task_metadata("safearena")),
             max_steps=30,
             n_repeats=1,
             seeds_rng=np.random.RandomState(42),
         ),
-        task_metadata=task_metadata("safearena_all"),
+        task_metadata=task_metadata("safearena"),
     ),
     "safearena_harm": lambda: Benchmark(
-        name="safenarena_all",
+        name="safenarena_harm",
         high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
         is_multi_tab=True,
         supports_parallel_seeds=False,
-        backends=["webarena"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("safearena_harm")),
+        backends=["safearena"],
+        env_args_list=make_env_args_list_from_fixed_seeds(
+            task_list=[f"safearena.{i}" for i in range(1, 250)],
             max_steps=30,
-            n_repeats=1,
-            seeds_rng=np.random.RandomState(42),
+            fixed_seeds=[0],
         ),
-        task_metadata=task_metadata("safearena_harm"),
+        task_metadata=task_metadata("safearena"),
     ),
+
     "safearena_safe": lambda: Benchmark(
-        name="safenarena_all",
+        name="safenarena_safe",
         high_level_action_set_args=DEFAULT_HIGHLEVEL_ACTION_SET_ARGS["webarena"],
         is_multi_tab=True,
         supports_parallel_seeds=False,
-        backends=["webarena"],
-        env_args_list=make_env_args_list_from_repeat_tasks(
-            task_list=task_list_from_metadata(metadata=task_metadata("safearena_safe")),
+        backends=["safearena"],
+        env_args_list=make_env_args_list_from_fixed_seeds(
+            task_list=[f"safearena.{i}" for i in range(250, 500)],
             max_steps=30,
-            n_repeats=1,
-            seeds_rng=np.random.RandomState(42),
+            fixed_seeds=[0],
         ),
-        task_metadata=task_metadata("safearena_safe"),
-    ),
+        task_metadata=task_metadata("safearena"),
+    )
 }