From 7f375f4927999d020d1d899352f1d360915506f7 Mon Sep 17 00:00:00 2001 From: energyls Date: Mon, 10 Aug 2026 17:02:44 +0200 Subject: [PATCH 1/4] feat: add rule to download data from zenodo --- Snakefile | 2 +- rules/retrieve.smk | 30 ++++++++++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) create mode 100644 rules/retrieve.smk diff --git a/Snakefile b/Snakefile index ddf52ba..e104c7e 100644 --- a/Snakefile +++ b/Snakefile @@ -87,7 +87,7 @@ wildcard_constraints: sweep="[a-zA-Z]+", rule="(0|[1-9][0-9]?|100)", - +include: "rules/retrieve.smk" include: "rules/supply_curves.smk" include: "rules/preparation.smk" include: "rules/trade_model.smk" diff --git a/rules/retrieve.smk b/rules/retrieve.smk new file mode 100644 index 0000000..a529867 --- /dev/null +++ b/rules/retrieve.smk @@ -0,0 +1,30 @@ +storage: + provider="http", + keep_local=True, + retries=3, + +rule retrieve_data: + input: + data_url=storage( + f"https://zenodo.org/records/" + ), + output: + bus_locations="data/bus_locations.csv", + ip_market_fabrication="data/demand/ip_market__fabrication.csv", + steel_demand_dri_share="data/demand/steel_demands/output_data/country_raw_steel_demand_and_dri_share.csv", + devlin2023_supplementary="data/devlin2023-supplementary.xlsx", + grid_potential_custom="data/grid_potential_custom.csv", + unido_raw="data/labour/unido-raw/data.csv", + iron_ore_production="data/owid-iron-ore/iron-ore-crude-ore-production.csv", + political_stability="data/political-stability/globaleconomy.csv", + renewable_profiles_geojson="data/renewable_profiles_global_merged.geojson", + renewable_profiles_nc="data/renewable_profiles_global_merged.nc", + log: + "logs/retrieve_data.log", + retries: 2 + resources: + mem_mb=10000, + params: + folder=".", + run: + unpack_archive(input.data_url, params.folder) \ No newline at end of file From 6b3108e6190757f24e0726feec663fdd2afedac7 Mon Sep 17 00:00:00 2001 From: energyls Date: Mon, 10 Aug 2026 17:02:59 +0200 Subject: [PATCH 2/4] chore: add shift data zip to gitignore --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 895b7c8..5e4c1f3 100644 --- a/.gitignore +++ b/.gitignore @@ -20,3 +20,5 @@ # pixi environments .pixi/* !.pixi/config.toml + +shift_data.zip \ No newline at end of file From 70f21e995399cecba085e221b0631dd649d130b0 Mon Sep 17 00:00:00 2001 From: energyls Date: Mon, 10 Aug 2026 17:30:47 +0200 Subject: [PATCH 3/4] feat: add zenodo repo --- rules/retrieve.smk | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rules/retrieve.smk b/rules/retrieve.smk index a529867..5d44969 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -6,7 +6,7 @@ storage: rule retrieve_data: input: data_url=storage( - f"https://zenodo.org/records/" + f"https://zenodo.org/records/21875103/files/shift_data.zip" ), output: bus_locations="data/bus_locations.csv", From 69d4cf1c3f4a7e53e182c9b5d77a0d7d6b968235 Mon Sep 17 00:00:00 2001 From: energyls Date: Mon, 10 Aug 2026 17:31:14 +0200 Subject: [PATCH 4/4] chore: add unpack archive function --- Snakefile | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Snakefile b/Snakefile index e104c7e..aa6219a 100644 --- a/Snakefile +++ b/Snakefile @@ -5,7 +5,7 @@ trade optimization, and reporting. """ from pathlib import Path -from shutil import copyfile +from shutil import copyfile, unpack_archive import sys import pandas as pd