diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index f6232dd80..0b3ba2f51 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -16,7 +16,7 @@ repos: # Run ruff to lint and format - repo: https://github.com/astral-sh/ruff-pre-commit # Ruff version. - rev: v0.15.9 + rev: v0.15.20 hooks: # Run the linter. - id: ruff @@ -50,6 +50,6 @@ repos: # Format Snakemake rule / workflow files - repo: https://github.com/snakemake/snakefmt - rev: v1.0.0 + rev: v2.0.3 hooks: - id: snakefmt diff --git a/Snakefile b/Snakefile index 1499943b1..19d38bef8 100644 --- a/Snakefile +++ b/Snakefile @@ -262,13 +262,11 @@ rule purge: "Delete all files in the folders above? [y/N] " ) if do_purge == "y": - # Remove the directories and recreate them with .gitkeep for dir_path in ["resources/", "results/"]: rmtree(dir_path, ignore_errors=True) Path(dir_path).mkdir(parents=True, exist_ok=True) (Path(dir_path) / ".gitkeep").touch() - rmtree("doc/_build", ignore_errors=True) print( "Purging all generated resources, results and docs. Downloads are kept." @@ -303,7 +301,7 @@ rule rulegraph: r""" # Generate DOT file using nested snakemake with the dumped final config echo "[Rule rulegraph] Using final config file: {input.config_file}" - snakemake --rulegraph --configfile {input.config_file} --quiet | sed -n "/digraph/,\$p" > {output.dot} + snakemake --rulegraph --configfile {input.config_file} --quiet | sed -n "/digraph/,\$p" >{output.dot} # Generate visualizations from the DOT file if [ -s {output.dot} ]; then @@ -340,7 +338,7 @@ rule filegraph: r""" # Generate DOT file using nested snakemake with the dumped final config echo "[Rule filegraph] Using final config file: {input.config_file}" - snakemake --filegraph all --configfile {input.config_file} --quiet | sed -n "/digraph/,\$p" > {output.dot} + snakemake --filegraph all --configfile {input.config_file} --quiet | sed -n "/digraph/,\$p" >{output.dot} # Generate visualizations from the DOT file if [ -s {output.dot} ]; then diff --git a/rules/retrieve.smk b/rules/retrieve.smk index cde7f3215..42da4b319 100755 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -177,7 +177,6 @@ if (POPULATION_COUNT_DATASET := dataset_version("population_count"))["source"] i "Retrieving population count data" run: copy2(input["tif"], output["tif"]) - if POPULATION_COUNT_DATASET["source"] == "primary": import xarray as xr import rioxarray as rio @@ -188,7 +187,6 @@ if (POPULATION_COUNT_DATASET := dataset_version("population_count"))["source"] i ds_reqd.rio.to_raster(file_path) - if (GHG_EMISSIONS_DATASET := dataset_version("ghg_emissions"))["source"] in [ "archive", "primary", @@ -220,7 +218,6 @@ if (GHG_EMISSIONS_DATASET := dataset_version("ghg_emissions"))["source"] in [ copy2(input["ghg"], output["csv"]) - if (GEBCO_DATASET := dataset_version("gebco"))["source"] in ["archive", "primary"]: rule retrieve_gebco: @@ -240,10 +237,8 @@ if (GEBCO_DATASET := dataset_version("gebco"))["source"] in ["archive", "primary import xarray as xr copy2(input[0], output["zip_file"]) - output_folder = Path(output["zip_file"]).parent unpack_archive(output["zip_file"], output_folder) - # Limit extent to Europe to reduce file size ds = xr.open_dataset(output["gebco"]) ds = ds.sel(lat=slice(32, 73), lon=slice(-21, 45)) @@ -252,7 +247,6 @@ if (GEBCO_DATASET := dataset_version("gebco"))["source"] in ["archive", "primary copy2(input[0], output["gebco"]) - if (ATTRIBUTED_PORTS_DATASET := dataset_version("attributed_ports"))["source"] in [ "archive", "primary", @@ -386,13 +380,10 @@ if (BIDDING_ZONES_ENTSOEPY_DATASET := dataset_version("bidding_zones_entsoepy")) except (URLError, TimeoutError) as e: raise Exception(f"Network error retrieving {name}: {e}") shapes = pd.concat(gdfs, ignore_index=True) # type: ignore - logger.info("Downloading entsoe-py zones... Done") - shapes.to_file(output.geojson) - if (CUTOUT_DATASET := dataset_version("cutout"))["source"] in [ "archive", ]: @@ -700,7 +691,6 @@ if (ENERGY_ATLAS_DATASET := dataset_version("jrc_energy_atlas"))["source"] in [ f.write(response.content) - if ( DESNZ_ELECTRICITY_CONSUMPTION_DATASET := dataset_version( "desnz_electricity_consumption" @@ -722,7 +712,6 @@ if ( f.write(response.content) - if (ONS_LAD_DATASET := dataset_version("ons_lad"))["source"] in ["archive"]: rule retrieve_ons_lad: @@ -756,7 +745,6 @@ elif ONS_LAD_DATASET["source"] in ["primary"]: f.write(response.content) - if (SHIP_RASTER_DATASET := dataset_version("ship_raster"))["source"] in [ "archive", "primary", @@ -881,7 +869,6 @@ if (EEZ_DATASET := dataset_version("eez"))["source"] in ["primary"]: name = str(uuid4())[:8] org = str(uuid4())[:8] - response = requests.post( f"{EEZ_DATASET['url']}", params={"name": f"World_EEZ_{EEZ_DATASET['version']}_LR.zip"}, @@ -895,13 +882,11 @@ if (EEZ_DATASET := dataset_version("eez"))["source"] in ["primary"]: "agree": "1", }, ) - with open(output["zip_file"], "wb") as f: f.write(response.content) output_folder = Path(output["zip_file"]).parent unpack_archive(output["zip_file"], output_folder) - elif (EEZ_DATASET := dataset_version("eez"))["source"] in ["archive"]: rule retrieve_eez: @@ -936,7 +921,6 @@ if (WB_URB_POP_DATASET := dataset_version("worldbank_urban_population"))["source run: copy2(input["zip"], output["zip"]) unpack_archive(output["zip"], WB_URB_POP_DATASET["folder"]) - # Filename contains some added numbers when downloaded, # remove them to have a consistent filename across versions target_filename = Path(output["csv"]) @@ -1115,13 +1099,11 @@ if (WDPA_DATASET := dataset_version("wdpa"))["source"] in [ output_folder = Path(output["zip_file"]).parent copy2(input["zip_file"], output["zip_file"]) unpack_archive(output["zip_file"], output_folder) - # Extract {bYYYY} from the input file / URL bYYYY = re.search( r"WDPA_(\w{3}\d{4})_Public_shp.zip", input["zip_file"], ).group(1) - for i in range(3): # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr layer_path = ( @@ -1131,7 +1113,6 @@ if (WDPA_DATASET := dataset_version("wdpa"))["source"] in [ shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}") - if (WDPA_MARINE_DATASET := dataset_version("wdpa_marine"))["source"] in [ "primary", "archive", @@ -1152,13 +1133,11 @@ if (WDPA_MARINE_DATASET := dataset_version("wdpa_marine"))["source"] in [ output_folder = Path(output["zip_file"]).parent copy2(input["zip_file"], output["zip_file"]) unpack_archive(output["zip_file"], output_folder) - # Extract {bYYYY} from the input file / URL bYYYY = re.search( r"WDPA_WDOECM_(\w{3}\d{4})_Public_marine_shp.zip", input["zip_file"], ).group(1) - for i in range(3): # vsizip is special driver for directly working with zipped shapefiles in ogr2ogr layer_path = f"/vsizip/{output_folder}/WDPA_WDOECM_{bYYYY}_Public_marine_shp_{i}.zip" @@ -1166,7 +1145,6 @@ if (WDPA_MARINE_DATASET := dataset_version("wdpa_marine"))["source"] in [ shell("ogr2ogr -f gpkg -update -append {output.gpkg} {layer_path}") - if (INSTRAT_CO2_PRICES_DATASET := dataset_version("instrat_co2_prices"))["source"] in [ "primary", ]: @@ -1190,10 +1168,8 @@ if (INSTRAT_CO2_PRICES_DATASET := dataset_version("instrat_co2_prices"))["source "Accept": "application/json", "Referer": "https://energy.instrat.pl/", } - r = requests.get(url, headers=headers) r.raise_for_status() - df = pd.read_json(r.text) df.to_csv(output["csv"], index=False) @@ -1232,17 +1208,14 @@ if (TYDNP_DATASET := dataset_version("tyndp"))["source"] in ["primary", "archive for key in input.keys(): # Keep zip file copy2(input[key], output[f"{key}_zip"]) - # unzip output_folder = Path(output[f"{key}_zip"]).parent unpack_archive(output[f"{key}_zip"], output_folder) - # Remove __MACOSX directory if it exists macosx_dir = output_folder / "__MACOSX" rmtree(macosx_dir, ignore_errors=True) - def get_osm_archive_files(version): return [ "buses.csv", @@ -1320,7 +1293,6 @@ if OSM_DATASET["source"] in ["archive"]: copy2(input[key], output[key]) - # Only create incumbent rule if it points to a different folder OSM_DATASET_INCUMBENT = get_osm_network_incumbent( version=config.get("osm_network_release", {}) @@ -1362,7 +1334,6 @@ if OSM_DATASET_INCUMBENT["source"] in ["archive"] and OSM_DATASET_INCUMBENT[ copy2(input[key], output[key]) - if OSM_DATASET["source"] == "build": OSM_RAW_JSON = [ "cables_way.json", @@ -1594,7 +1565,6 @@ if (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in [ for key in input.keys(): copy2(input[key], output[key]) - elif (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in ["archive"]: rule retrieve_jrc_ardeco: @@ -1615,7 +1585,6 @@ elif (JRC_ARDECO_DATASET := dataset_version("jrc_ardeco"))["source"] in ["archiv copy2(input[key], output[key]) - if (AQUIFER_DATA_DATASET := dataset_version("aquifer_data"))["source"] in [ "primary", "archive",