diff --git a/.gitignore b/.gitignore index c264db022..ddb08be76 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,9 @@ # tensorboard runs runs/ +#benchmark folder +download_benchmark/ + # videos #*.mp4 notebooks/videos/* diff --git a/docs/api.rst b/docs/api.rst index add89f080..34001d705 100644 --- a/docs/api.rst +++ b/docs/api.rst @@ -166,6 +166,15 @@ Logging Utilities utils.logging.set_level +Benchmarks +---------------- + +.. autosummary:: + :toctree: generated/ + :template: function.rst + + benchmarks.benchmark_utils.download_benchmark_from_SB3_zoo + Environment Wrappers ==================== diff --git a/docs/basics/userguide/benchmarks.md b/docs/basics/userguide/benchmarks.md new file mode 100644 index 000000000..53bcc9cdf --- /dev/null +++ b/docs/basics/userguide/benchmarks.md @@ -0,0 +1,28 @@ +(benchmarks)= + +# How to use benchmarks + +Rlberry has a tool to download some benchmarks if you need to compare your agent with them. + +Currently, the available benchmars are : + - Pre-trained Reinforcement Learning agents using the rl-baselines3-zoo and Stable Baselines3 ([here](https://github.com/DLR-RM/rl-trained-agents)). + +## Download the benchmark +To download the benchmark it's easy, you just have to call the function matching the expected benchmark. +You need to specify the names of the agent and the environment. And If you want overwrite the previous data on this combination. (you can use the `output_dir` parameter if you want to download the benchmark in a specific folder). +You can find the API about this benchmark [here](rlberry.benchmarks.benchmark_utils.download_benchmark_from_SB3_zoo) + +```python +from rlberry.benchmarks.benchmark_utils import download_benchmark_from_SB3_zoo + +agent_name = "dqn" +environment_name = "PongNoFrameskip-v4_1" + +path_with_downloaded_files = download_benchmark_from_SB3_zoo( + agent_name, environment_name, overwrite=True +) +``` + +## How to use these benchmarks to compare your agent + +in construction ... diff --git a/docs/changelog.rst b/docs/changelog.rst index 741dde0f6..c16eff3b9 100644 --- a/docs/changelog.rst +++ b/docs/changelog.rst @@ -7,7 +7,12 @@ Changelog Dev version ----------- -* nothing + + +*PR #449* + + * Add module to download benchmarks + - Pre-trained Reinforcement Learning agents using the rl-baselines3-zoo and Stable Baselines3. Version 0.7.1 ------------- diff --git a/docs/user_guide.md b/docs/user_guide.md index 791e19ba4..4d50b9e3a 100644 --- a/docs/user_guide.md +++ b/docs/user_guide.md @@ -47,3 +47,4 @@ You can find more details about installation [here](installation)! - Transfer Learning (In construction) - [Hypothesis testing for comparison of RL agents](comparison_page) - [Adaptive hypothesis testing for comparison of RL agents with AdaStop](adastop_userguide) +- [Using benchmarks](benchmarks) diff --git a/rlberry/benchmarks/benchmark_utils.py b/rlberry/benchmarks/benchmark_utils.py new file mode 100644 index 000000000..4b1f90022 --- /dev/null +++ b/rlberry/benchmarks/benchmark_utils.py @@ -0,0 +1,151 @@ +import requests +from tempfile import mkdtemp +import os +import shutil + + +# # TODO : convert external benchmark to DataFrame that match the input of rlberry.manager.comparaison.py -> compare_agents_data() +# # TODO : Download the external benchmark to a specific folder (or new rlberrygithub?), except if they are stable (huggingface/github) + +# benchmark_list = { +# "Google Atari bucket": "https://console.cloud.google.com/storage/brow", +# "SB3 zoo": "https://github.com/DLR-RM/rl-baselines3-zoo/tree/master/logs/benchmark", +# "cleanrl": "https://wandb.ai/openrlbenchmark/openrlbenchmark/reportlist", +# } + + +# def import_from_google_atari_bucket(): +# """import benchmark from Google Atari bucket + +# Parameters +# ----------- +# x_vec : numpy.ndarray +# numpy 1d array to be searched in the bins +# bins : list +# list of numpy 1d array, bins[d] = bins of the d-th dimension + + +# Returns +# -------- +# index (int) corresponding to the position of x in the partition +# defined by the bins. +# """ +# print("TODO") + + +# def import_from_cleanrl(): +# print("TODO") + + +# def import_from_hugingface(): +# print("TODO") + + +def download_benchmark_from_SB3_zoo( + agent_name, environment_name, overwrite, output_dir=None +): + """ + Download folder from pre-trained Reinforcement Learning agents using the rl-baselines3-zoo and Stable Baselines3. + https://github.com/DLR-RM/rl-trained-agents + + Parameters + ----------- + agent_name : str + agent name for benchmark to download + environment_name : list + environment name for benchmark to download + overwrite : bool + how to manage if the combination agent_name/environment_name exist : + True : delete the previous folder, then download + False : raise an error + output_dir : str + root path where to download files. (default=None : create temp folder) + + Returns + -------- + Return the path containing the downloaded files (output_dir/agent_name/environment_name) + """ + if not output_dir: + output_dir = mkdtemp() + + GITHUB_URL = "https://raw.githubusercontent.com/DLR-RM/rl-trained-agents/master/" + base_url = GITHUB_URL + agent_name + "/" + environment_name + "/" + + output_folder = os.path.join(output_dir, agent_name, environment_name) + environment_base_name = environment_name.split("_")[0] + + if os.path.exists(output_folder): + if not overwrite: + raise FileExistsError( + "The 'overwrite' bool is false, and the combination %s / %s already exist" + % (agent_name, environment_name) + ) + shutil.rmtree(output_folder) + os.makedirs(output_folder) + + # download CSVs + url_content = None + i = 0 + while url_content != b"404: Not Found": + file_name_to_download = str(i) + ".monitor.csv" + url_csv_to_download = base_url + file_name_to_download + + req = requests.get(url_csv_to_download) + url_content = req.content + + if url_content != b"404: Not Found": + csv_file = open(os.path.join(output_folder, file_name_to_download), "wb") + csv_file.write(url_content) + csv_file.close() + else: + break + i = i + 1 + + # download zip + file_name_to_download = environment_base_name + ".zip" + url_zip_to_download = base_url + file_name_to_download + req = requests.get(url_zip_to_download) + url_content = req.content + csv_file = open(os.path.join(output_folder, file_name_to_download), "wb") + csv_file.write(url_content) + csv_file.close() + + # download evaluations.npz + file_name_to_download = "evaluations.npz" + url_zip_to_download = base_url + file_name_to_download + req = requests.get(url_zip_to_download) + url_content = req.content + csv_file = open(os.path.join(output_folder, file_name_to_download), "wb") + csv_file.write(url_content) + csv_file.close() + + # hyperparameter and config + config_folder = output_folder + "/" + environment_base_name + base_url_config = base_url + environment_base_name + "/" + + os.makedirs(config_folder) + file_name_to_download = "args.yml" + url_zip_to_download = base_url_config + file_name_to_download + req = requests.get(url_zip_to_download) + url_content = req.content + csv_file = open(os.path.join(config_folder, file_name_to_download), "wb") + csv_file.write(url_content) + csv_file.close() + + file_name_to_download = "config.yml" + url_zip_to_download = base_url_config + file_name_to_download + req = requests.get(url_zip_to_download) + url_content = req.content + csv_file = open(os.path.join(config_folder, file_name_to_download), "wb") + csv_file.write(url_content) + csv_file.close() + + file_name_to_download = "vecnormalize.pkl" + url_zip_to_download = base_url_config + file_name_to_download + req = requests.get(url_zip_to_download) + url_content = req.content + csv_file = open(os.path.join(config_folder, file_name_to_download), "wb") + csv_file.write(url_content) + csv_file.close() + + return output_folder diff --git a/rlberry/benchmarks/tests/test_benchmark_utils.py b/rlberry/benchmarks/tests/test_benchmark_utils.py new file mode 100644 index 000000000..6bfbe8ddb --- /dev/null +++ b/rlberry/benchmarks/tests/test_benchmark_utils.py @@ -0,0 +1,88 @@ +import os +import shutil +from rlberry.benchmarks.benchmark_utils import download_benchmark_from_SB3_zoo +import pytest + + +@pytest.mark.parametrize("agent_class", ["dqn"]) +@pytest.mark.parametrize("env", ["PongNoFrameskip-v4_1"]) +def test_download_benchmark_from_SB3_zoo_(agent_class, env): + # remove previous test if existing + test_folder_path = "./tests_dl" + if os.path.exists(test_folder_path): + shutil.rmtree(test_folder_path) + os.makedirs(test_folder_path) + + # download benchmark + ret_value = download_benchmark_from_SB3_zoo( + agent_class, env, overwrite=True, output_dir=test_folder_path + ) + + # tests expected result + environment_base_name = env.split("_")[0] + assert str(os.path.join(test_folder_path, agent_class, env)) == ret_value + assert os.path.exists(os.path.join(test_folder_path, agent_class, env)) + assert os.path.exists( + os.path.join(test_folder_path, agent_class, env, "0.monitor.csv") + ) + assert os.path.exists( + os.path.join(test_folder_path, agent_class, env, environment_base_name + ".zip") + ) + assert os.path.exists( + os.path.join(test_folder_path, agent_class, env, "evaluations.npz") + ) + assert os.path.exists( + os.path.join( + test_folder_path, agent_class, env, environment_base_name, "args.yml" + ) + ) + assert os.path.exists( + os.path.join( + test_folder_path, agent_class, env, environment_base_name, "config.yml" + ) + ) + assert os.path.exists( + os.path.join( + test_folder_path, + agent_class, + env, + environment_base_name, + "vecnormalize.pkl", + ) + ) + + if os.path.exists(test_folder_path): + shutil.rmtree(test_folder_path) + + +@pytest.mark.parametrize("agent_class", ["dqn"]) +@pytest.mark.parametrize("env", ["PongNoFrameskip-v4_1"]) +@pytest.mark.parametrize("overwrite", [True, False]) +def test_download_benchmark_from_SB3_zoo_overwrite_True(agent_class, env, overwrite): + # remove previous test if existing + test_folder_path = "./tests_dl" + if os.path.exists(test_folder_path): + shutil.rmtree(test_folder_path) + os.makedirs(test_folder_path) + + # first call + ret_value = download_benchmark_from_SB3_zoo( + agent_class, env, overwrite=overwrite, output_dir=test_folder_path + ) + + #'overwrite' test + error_was_raised = False + try: + ret_value = download_benchmark_from_SB3_zoo( + agent_class, env, overwrite=overwrite, output_dir=test_folder_path + ) + except FileExistsError: + error_was_raised = True + + if overwrite: + assert not error_was_raised + else: + assert error_was_raised + + if os.path.exists(test_folder_path): + shutil.rmtree(test_folder_path) diff --git a/rlberry/manager/comparison.py b/rlberry/manager/comparison.py index 341046fc6..9a5472078 100644 --- a/rlberry/manager/comparison.py +++ b/rlberry/manager/comparison.py @@ -206,13 +206,11 @@ def compare_agents( Parameters ---------- agent_source : list of :class:`~rlberry.manager.ExperimentManager`, list of str - - If list of ExperimentManager, load data from it (the agents must be fitted). - - If str, each string must be the path of a agent_manager.obj. - - If pandas DataFrame with column agent (containing agent's names) and mean_eval containing the scores, it is used as input data. + **Each agent must have unique name.** method: str in {"tukey_hsd", "permutation"}, default="tukey_hsd" Method used in the test. "tukey_hsd" use scipy implementation [1] and "permutation" use permutation test with Step-Down method for multiple testing [2]. Tukey HSD method suppose Gaussian model on the aggregated evaluations and permutation test is non-parametric and does not make assumption on the distribution. permutation is the safe choice when the reward is likely to be heavy-tailed or multimodal. @@ -239,6 +237,41 @@ def compare_agents( [2]: Testing Statistical Hypotheses by E. L. Lehmann, Joseph P. Romano (Section 15.4.4), https://doi.org/10.1007/0-387-27605-X, Springer """ + + if isinstance(agent_source, list): + df_agent_data = preprocess_agent_data( + agent_source, eval_function, n_simulations + ) + else: + df_agent_data = agent_source + return compare_agents_data(df_agent_data, method, alpha, B, seed) + + +def preprocess_agent_data( + agent_source, + eval_function=None, + n_simulations=50, +): + """ + Compare several trained agents using the mean over n_simulations evaluations for each agent. + + Parameters + ---------- + agent_source : list of :class:`~rlberry.manager.ExperimentManager`, list of str + - If list of ExperimentManager, load data from it (the agents must be fitted). + - If str, each string must be the path of a agent_manager.obj. + **Each agent must have unique name.** + eval_function: callable or None, default = None + Function used the evaluate the agents. lambda manager : ExperimentManager, eval_budget : int or None, agent_id: int -> eval:float + If None, the mean of the eval function of the agent is used over n_simulations evaluations. + n_simulations: int, default = 50 + Number of evaluations to use if eval_function is None. + + Returns + ------- + results: a DataFrame with the evaluation results. + """ + if isinstance(agent_source, list): # Construction of the array of evaluations df = pd.DataFrame() @@ -272,16 +305,55 @@ def compare_agents( ], ignore_index=True, ) + agent_names = df["agent"].unique() assert len(agent_names) == len( - agent_manager_list + agent_source ), "Each agent must have unique name." - else: - df = agent_source - agent_names = df["agent"].unique() + return df + + +def compare_agents_data( + data_to_compare, + method="tukey_hsd", + alpha=0.05, + B=10_000, + seed=None, +): + """ + Compare several trained agents using the mean over n_simulations evaluations for each agent. + + Parameters + ---------- + data_to_compare : DataFrame + Data of the agents to compare (result from their evaluation). The dataframe must have 2 columns : 'mean_eval' and 'agent' + method: str in {"tukey_hsd", "permutation"}, default="tukey_hsd" + Method used in the test. "tukey_hsd" use scipy implementation [1] and "permutation" use permutation test with Step-Down method for multiple testing [2]. Tukey HSD method suppose Gaussian model on the aggregated evaluations and permutation test is non-parametric and does not make assumption on the distribution. permutation is the safe choice when the reward is likely to be heavy-tailed or multimodal. + alpha: float, default = 0.05 + Level of the test, control the Family-wise error. + B: int, default = 10_000 + Number of random permutations used to approximate the permutation test if method = "permutation" + seed: int or None, + The seed of the random number generator from which we sample permutations. If None, create one. + + Returns + ------- + results: a DataFrame summarising the results. + + References + ---------- + [1]: https://scipy.github.io/devdocs/reference/generated/scipy.stats.tukey_hsd.html + + [2]: Testing Statistical Hypotheses by E. L. Lehmann, Joseph P. Romano (Section 15.4.4), https://doi.org/10.1007/0-387-27605-X, Springer + + """ + agent_names = data_to_compare["agent"].unique() data = np.array( - [np.array(df.loc[df["agent"] == name, "mean_eval"]) for name in agent_names] + [ + np.array(data_to_compare.loc[data_to_compare["agent"] == name, "mean_eval"]) + for name in agent_names + ] ) n_agents = len(agent_names) @@ -293,23 +365,33 @@ def compare_agents( ] mean_agent1 = [ - np.mean(df.loc[df["agent"] == vs[i][0], "mean_eval"]) for i in range(len(vs)) + np.mean(data_to_compare.loc[data_to_compare["agent"] == vs[i][0], "mean_eval"]) + for i in range(len(vs)) ] mean_agent2 = [ - np.mean(df.loc[df["agent"] == vs[i][1], "mean_eval"]) for i in range(len(vs)) + np.mean(data_to_compare.loc[data_to_compare["agent"] == vs[i][1], "mean_eval"]) + for i in range(len(vs)) ] mean_diff = [ np.mean( - np.array(df.loc[df["agent"] == vs[i][0], "mean_eval"]) - - np.array(df.loc[df["agent"] == vs[i][1], "mean_eval"]) + np.array( + data_to_compare.loc[data_to_compare["agent"] == vs[i][0], "mean_eval"] + ) + - np.array( + data_to_compare.loc[data_to_compare["agent"] == vs[i][1], "mean_eval"] + ) ) for i in range(len(vs)) ] std_diff = [ np.std( - np.array(df.loc[df["agent"] == vs[i][0], "mean_eval"]) - - np.array(df.loc[df["agent"] == vs[i][1], "mean_eval"]) + np.array( + data_to_compare.loc[data_to_compare["agent"] == vs[i][0], "mean_eval"] + ) + - np.array( + data_to_compare.loc[data_to_compare["agent"] == vs[i][1], "mean_eval"] + ) ) for i in range(len(vs)) ]