diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index dd4e8fb..60ff226 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -8,7 +8,7 @@ exclude: "^LICENSES" repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.4.0 + rev: v6.0.0 hooks: - id: check-merge-conflict - id: end-of-file-fixer @@ -19,14 +19,14 @@ repos: # Sort package imports alphabetically - repo: https://github.com/PyCQA/isort - rev: 5.10.1 + rev: 9.0.0b2 hooks: - id: isort args: ["--profile", "black", "--filter-files"] # Find common spelling mistakes in comments and docstrings - repo: https://github.com/codespell-project/codespell - rev: v2.2.2 + rev: v2.4.3 hooks: - id: codespell args: ['--ignore-regex="\b[A-Z]+\b"'] # Ignore capital case words, e.g. country codes @@ -34,8 +34,8 @@ repos: files: ^(actions|doc)/ # Formatting with "black" coding style -- repo: https://github.com/psf/black - rev: 22.10.0 +- repo: https://github.com/psf/black-pre-commit-mirror + rev: 26.5.1 hooks: # Format Python files - id: black @@ -43,27 +43,27 @@ repos: - id: black-jupyter - repo: https://github.com/nbQA-dev/nbQA - rev: 1.5.3 + rev: 1.9.1 hooks: # Apply isort to Jupyter Python notebooks - id: nbqa-isort # Do YAML formatting (before the linter checks it for misses) - repo: https://github.com/macisamuele/language-formatters-pre-commit-hooks - rev: v2.4.0 + rev: v2.16.0 hooks: - id: pretty-format-yaml args: [--autofix, --indent, '2', --preserve-quotes] # Format Snakemake rule / workflow files - repo: https://github.com/snakemake/snakefmt - rev: v0.7.0 + rev: v2.0.3 hooks: - id: snakefmt # Check for FSFE REUSE compliance (licensing) - repo: https://github.com/fsfe/reuse-tool - rev: v1.0.0 + rev: v6.2.0 hooks: - id: reuse diff --git a/actions/create_network.py.ipynb b/actions/create_network.py.ipynb index f9ebf4e..dc70a42 100644 --- a/actions/create_network.py.ipynb +++ b/actions/create_network.py.ipynb @@ -314,7 +314,6 @@ "def scale_transportation_with_distance(\n", " n, link_types=[\"HVDC overhead\", \"HVDC submarine\", \"pipeline\", \"submarine pipeline\"]\n", "):\n", - "\n", " \"\"\"Scales the cost and efficiency of specific links (transport options) by distance.\n", "\n", " Only implemented for:\n", @@ -752,9 +751,9 @@ " & (loaded_unloaded_ratio[\"from\"] == \"LOHC (loaded)\")\n", " & (loaded_unloaded_ratio[\"to\"] == \"LOHC (used)\")\n", " ][\"efficiency\"].item()\n", - " n.links.loc[\n", - " f\"{ship_bus} trip demand & losses\", \"bus2\"\n", - " ] = f\"{ship_bus} LOHC (used)\"\n", + " n.links.loc[f\"{ship_bus} trip demand & losses\", \"bus2\"] = (\n", + " f\"{ship_bus} LOHC (used)\"\n", + " )\n", " n.links.loc[f\"{ship_bus} trip demand & losses\", \"efficiency2\"] = (\n", " 1 - shipping_efficiency\n", " ) * loaded_unloaded_ratio\n", diff --git a/actions/solve_network.py.ipynb b/actions/solve_network.py.ipynb index 8dd0f8d..8733903 100644 --- a/actions/solve_network.py.ipynb +++ b/actions/solve_network.py.ipynb @@ -144,7 +144,7 @@ " (\"links\", \"pipeline\", \"CAPEX_pipeline\"),\n", " (\"links\", \"methanolisation\", \"CAPEX_MeOHSynthesis\"),\n", " ]\n", - " for (components_name, search_string, modifier_name) in mapping:\n", + " for components_name, search_string, modifier_name in mapping:\n", " c = getattr(n, components_name)\n", " c.loc[c.index.str.contains(search_string), \"capital_cost\"] *= scenario[\n", " \"modifiers\"\n", diff --git a/analysis/plot_paper_figures.ipynb b/analysis/plot_paper_figures.ipynb index d3c33b3..b85162b 100644 --- a/analysis/plot_paper_figures.ipynb +++ b/analysis/plot_paper_figures.ipynb @@ -51,6 +51,8 @@ "source": [ "# Take a list of labels and rename the ESCs from that list\n", "# to more prettier (and shorter) names\n", + "\n", + "\n", "def prettiefy_esc_names(labels):\n", "\n", " prettier_names = {\n", @@ -628,8 +630,8 @@ "fig = plt.figure(figsize=(12, 4))\n", "ax = plt.gca()\n", "\n", - "for (esc, x) in zip(esc_markers.keys(), xs):\n", - " for ((idx, row), xx) in zip(df.loc[esc].sort_values(\"value\").iterrows(), xxs):\n", + "for esc, x in zip(esc_markers.keys(), xs):\n", + " for (idx, row), xx in zip(df.loc[esc].sort_values(\"value\").iterrows(), xxs):\n", " ax.bar(\n", " x + xx * width - (xxs.shape[0] - 1) * width / 2,\n", " row[\"value\"],\n", @@ -694,8 +696,8 @@ "fig = plt.figure(figsize=(10, 6))\n", "ax = plt.gca()\n", "\n", - "for (esc, x) in zip(esc_markers.keys(), xs):\n", - " for (year, xx) in zip(years, xxs):\n", + "for esc, x in zip(esc_markers.keys(), xs):\n", + " for year, xx in zip(years, xxs):\n", " for idx, row in df.loc[esc, year].iterrows():\n", " # Single value dots\n", " ax.scatter(\n", @@ -777,8 +779,8 @@ "fig = plt.figure(figsize=(10, 6))\n", "ax = plt.gca()\n", "\n", - "for (esc, x) in zip(esc_markers.keys(), xs):\n", - " for (year, xx) in zip(years, xxs):\n", + "for esc, x in zip(esc_markers.keys(), xs):\n", + " for year, xx in zip(years, xxs):\n", " for idx, row in df.loc[esc, year].iterrows():\n", " # Individual dots\n", " ax.scatter(\n", @@ -888,8 +890,8 @@ "ax = plt.gca()\n", "\n", "# Plot each vertical year line and each scatter point separately\n", - "for (esc, x) in zip(conversion_factors.keys(), xs):\n", - " for (year, xx) in zip(years, xxs):\n", + "for esc, x in zip(conversion_factors.keys(), xs):\n", + " for year, xx in zip(years, xxs):\n", " for idx, row in df.loc[esc, year].iterrows():\n", " # each scatter/data point. Marked by color of exporter\n", " ax.scatter(\n", @@ -1000,8 +1002,8 @@ "ax = plt.gca()\n", "\n", "# Plot each vertical year line and each scatter point separately\n", - "for (esc, x) in zip(conversion_factors.keys(), xs):\n", - " for (year, xx) in zip(years, xxs):\n", + "for esc, x in zip(conversion_factors.keys(), xs):\n", + " for year, xx in zip(years, xxs):\n", " for idx, row in df.loc[esc, year].iterrows():\n", " # each scatter/data point. Marked by color of exporter\n", " ax.scatter(\n", diff --git a/analysis/results-plotting.ipynb b/analysis/results-plotting.ipynb index 9a5eb8e..dcd5533 100644 --- a/analysis/results-plotting.ipynb +++ b/analysis/results-plotting.ipynb @@ -90,7 +90,7 @@ "source": [ "# Plot all ESCs per exporter\n", "for name, group in df.groupby(df.index.names[:3]):\n", - " (year, wacc, importer) = name\n", + " year, wacc, importer = name\n", "\n", " tmp = group.loc[name].loc[:, :, \"general\", \"Cost per MWh delivered\"]\n", " tmp.reset_index().pivot(index=\"exporter\", columns=\"esc\", values=\"value\").plot(\n", @@ -118,7 +118,7 @@ "source": [ "# Plot per ESC all exporters\n", "for name, group in df.groupby(df.index.names[:3]):\n", - " (year, wacc, importer) = name\n", + " year, wacc, importer = name\n", "\n", " tmp = group.loc[name].loc[:, :, \"general\", \"Cost per MWh delivered\"]\n", " tmp.reset_index().pivot(index=\"esc\", columns=\"exporter\", values=\"value\").plot(\n", @@ -146,7 +146,7 @@ "source": [ "# Energy surplus factor by ESC for each exporter\n", "for name, group in df.groupby(df.index.names[:3]):\n", - " (year, wacc, importer) = name\n", + " year, wacc, importer = name\n", "\n", " tmp = group.loc[name].loc[:, :, \"general\", \"Energy surplus factor\"]\n", " tmp.reset_index().pivot(index=\"exporter\", columns=\"esc\", values=\"value\").plot(\n", @@ -189,7 +189,7 @@ "cutoff = 0.1\n", "\n", "for name, group in df.groupby(df.index.names[:4]):\n", - " (year, wacc, importer, esc) = name\n", + " year, wacc, importer, esc = name\n", "\n", " group = group.loc[name].loc[:, \"cost\", :] # Select cost only\n", "\n", @@ -289,7 +289,7 @@ "}\n", "\n", "for name, group in df.groupby(df.index.names[:4]):\n", - " (year, wacc, importer, esc) = name\n", + " year, wacc, importer, esc = name\n", "\n", " group = group.loc[name].loc[:, \"capacity factor\", :] # Select CFs only\n", "\n", @@ -359,7 +359,7 @@ "brk = False\n", "\n", "for name, group in df.groupby(df.index.names[:4]):\n", - " (year, wacc, importer, esc) = name\n", + " year, wacc, importer, esc = name\n", "\n", " group = group.loc[name].loc[:, \"installed capacity\", :] # Select capacities only\n", "\n", diff --git a/rules/esc_construction.smk b/rules/esc_construction.smk index e9161b1..9f173f1 100644 --- a/rules/esc_construction.smk +++ b/rules/esc_construction.smk @@ -22,9 +22,9 @@ rule download_technology_data: rule create_additional_components: output: additional_components="resources/additional_components.pkl", - threads: 1 log: python="logs/create_additional_components.log", + threads: 1 notebook: "../actions/create_additional_components.py.ipynb" @@ -40,13 +40,13 @@ rule create_network: additional_components="resources/additional_components.pkl", output: network="resources/networks/{scenario}/{year}/{esc}/{from}-{to}/network.nc", + log: + python="logs/{scenario}/{year}/{esc}/{from}-{to}/create_network.log", + notebook="logs/{scenario}/{year}/{esc}/{from}-{to}/create_network.ipynb", threads: 1 params: scenario=lambda w: get_scenario(w["scenario"]), era_year=config["GlobalEnergyGIS"]["era_year"], - log: - python="logs/{scenario}/{year}/{esc}/{from}-{to}/create_network.log", - notebook="logs/{scenario}/{year}/{esc}/{from}-{to}/create_network.ipynb", notebook: "../actions/create_network.py.ipynb" @@ -61,22 +61,22 @@ def get_import_profile_path(wildcards): rule attach_import_profile: - message: - "Attaching import profile ('ip') to network." input: network="resources/networks/{scenario}/{year}/{esc}/{from}-{to}/network.nc", additional_components="resources/additional_components.pkl", import_profile=get_import_profile_path, output: network="resources/networks_ip/{scenario}/{year}/{esc}/{from}-{to}/network.nc", - threads: 1 - params: - scenario=lambda w: get_scenario(w["scenario"]), log: python="logs/{scenario}/{year}/{esc}/{from}-{to}/attach_import_profile.log", notebook=( "logs/{scenario}/{year}/{esc}/{from}-{to}/attach_import_profile.ipynb" ), + threads: 1 + params: + scenario=lambda w: get_scenario(w["scenario"]), + message: + "Attaching import profile ('ip') to network." notebook: "../actions/attach_import_profile.py.ipynb" @@ -95,8 +95,6 @@ def demand_file(wildcards): rule attach_supply: - message: - "Attaching RES supply ('as') to network." input: supply="resources/supply_TRACES_{era_year}.nc".format( era_year=config["GlobalEnergyGIS"]["era_year"] @@ -111,11 +109,13 @@ rule attach_supply: "resources/networks_ip_as/{scenario}/{year}/{esc}/{from}-{to}/network.nc" ), lcoes="resources/networks_ip_as/{scenario}/{year}/{esc}/{from}-{to}/lcoes.csv", - threads: 1 - params: - scenario=lambda w: get_scenario(w["scenario"]), log: python="logs/{scenario}/{year}/{esc}/{from}-{to}/attach_supply.log", notebook="logs/{scenario}/{year}/{esc}/{from}-{to}/attach_supply.ipynb", + threads: 1 + params: + scenario=lambda w: get_scenario(w["scenario"]), + message: + "Attaching RES supply ('as') to network." notebook: "../actions/attach_supply.py.ipynb" diff --git a/rules/results.smk b/rules/results.smk index 5ea975d..f8f1ed0 100644 --- a/rules/results.smk +++ b/rules/results.smk @@ -5,6 +5,7 @@ # Use paramspace to evaluate which scenarios to run scenarios = Paramspace(pd.read_csv("scenarios/default.csv", dtype=str)) + # Custom pattern for formatting Paramspace, as snakemake # does currently not allow for patterns without the wildcard_name included # see: https://stackoverflow.com/questions/71293563/custom-patterns-with-snakemakes-paramspace/71296522#71296522 @@ -24,10 +25,10 @@ rule all_scenario_results: ), output: results="results/results.csv", - threads: 1 log: python="logs/combine_results.log", notebook="logs/combine_results.ipynb", + threads: 1 notebook: "../actions/combine_results.py.ipynb" @@ -37,13 +38,13 @@ rule extract_result: network="results/{scenario}/{year}/{esc}/{exporter}-{importer}/network.nc", output: results="results/{scenario}/{year}/{esc}/{exporter}-{importer}/results.csv", - threads: 1 - params: - scenario=lambda w: get_scenario(w["scenario"]), log: python="logs/{scenario}/{year}/{esc}/{exporter}-{importer}/extract_result.log", notebook=( "logs/{scenario}/{year}/{esc}/{exporter}-{importer}/extract_result.ipynb" ), + threads: 1 + params: + scenario=lambda w: get_scenario(w["scenario"]), notebook: "../actions/extract_result.py.ipynb" diff --git a/rules/solving.smk b/rules/solving.smk index 3775c6c..0f3095f 100644 --- a/rules/solving.smk +++ b/rules/solving.smk @@ -28,14 +28,14 @@ rule solve_network: additional_components="resources/additional_components.pkl", output: network="results/{scenario}/{year}/{esc}/{from}-{to}/network.nc", - threads: config["solver"]["default"]["threads"] - params: - scenario=lambda w: get_scenario(w["scenario"]), - resources: - mem_mb=lambda wildcards, attempt: attempt * 10000, log: python="logs/{scenario}/{year}/{esc}/{from}-{to}/solve_network.log", notebook="logs/{scenario}/{year}/{esc}/{from}-{to}/solve_network.ipynb", + threads: config["solver"]["default"]["threads"] + resources: + mem_mb=lambda wildcards, attempt: attempt * 10000, + params: + scenario=lambda w: get_scenario(w["scenario"]), notebook: "../actions/solve_network.py.ipynb" @@ -46,8 +46,8 @@ rule backup_scenario: costs="../technology-data/outputs/", output: tarchive="results/{scenario}/inputs.tar", - threads: 1 log: python="logs/{scenario}/backup_run.log", + threads: 1 script: "../actions/backup_run.py"