From 85e3f73e8250b5c9899332c4e41d4ce3e1110c1d Mon Sep 17 00:00:00 2001 From: energyls Date: Mon, 31 Aug 2026 12:36:11 +0200 Subject: [PATCH 01/11] docs: add demo branch and description --- README.md | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/README.md b/README.md index b2f5c31..e3eae90 100644 --- a/README.md +++ b/README.md @@ -20,12 +20,12 @@ pixi run snakemake model_trade_all Approximate runtimes: - Installation (`pixi install`): ~5 minutes -- Full model (`model_trade_all`): ~5 hours +- Demo model (this branch): ~1 hour -> [!TIP] -> For a quick, low-resolution run, check out the -> [`demo` branch](https://github.com/energyLS/shift/tree/demo), which uses -> fewer regions and quantities and completes in ~1 hour. +> [!NOTE] +> This is the `demo` branch: it uses fewer regions and quantities so it +> runs much faster. For the full-resolution model (~5 hours), see the +> [`main` branch](https://github.com/energyLS/shift/tree/main). ## Key results From 5da0ffe78091c61caa0e7a717425bd2615d431d3 Mon Sep 17 00:00:00 2001 From: energyls Date: Mon, 31 Aug 2026 12:36:39 +0200 Subject: [PATCH 02/11] chore: implement low resolution demo settings --- config/config.yaml | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index c42c3ef..fffa77e 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -135,7 +135,7 @@ scenario: # Absolute steel demand levels (Mt/year) for supply curve sweep # For each level, PyPSA minimizes cost with fixed renewable capacity # Values represent different production scales -steel_demand_levels: [0.1, 0.5, 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] # Mt/year +steel_demand_levels: [0.1, 100, 500, 1000] #[0.1, 0.5, 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] # Mt/year electricity_steel_ratio: 5.25 #TWh/Mt or MWh/t, see notebooks 'analysis-steel.ipynb' embodied_energy_steel: 2.1 #TWh/Mt or MWh/t, see iron oxide reduction @@ -144,19 +144,19 @@ embodied_energy_steel: 2.1 #TWh/Mt or MWh/t, see iron oxide reduction # Region definitions (ISO3 codes matching renewable clusters metadata) regions: "Europe": ["ALB","AUT","BLR","BEL","BIH","BGR","HRV","CYP","CZE","DNK","EST","FIN","DEU","GRC","HUN","ITA","XKX","LVA","LTU","LUX","MKD","MDA","MNE","NLD","NOR","POL","ROU","SRB","SVK","SVN","SWE","CHE","TUR","UKR"] - "Far_West_Europe": ["FRA","GRL","ISL","IRL","PRT","ESP","GBR"] - "Middle_East": ["BHR","IRN","IRQ","ISR","JOR","KWT","LBN","OMN","PSE","QAT","SAU","SYR","ARE","YEM"] - "North_West_Africa": ["BEN","BFA","CIV","GMB","GHA","GIN","GNB","LBR","NGA","SEN","SLE","TGO","DZA","TCD","EGY","ERI","LBY","MLI","MRT","MAR","NER","SDN","TUN","ESH"] - "Subsaharan_Africa": ["AGO","BWA","BDI","CMR","CAF","COD","ETH","KEN","GAB","MDG","MWI","MOZ","NAM","COG","RWA","SOM","ZAF","SSD","TZA","UGA","ZMB","ZWE"] + # "Far_West_Europe": ["FRA","GRL","ISL","IRL","PRT","ESP","GBR"] + # "Middle_East": ["BHR","IRN","IRQ","ISR","JOR","KWT","LBN","OMN","PSE","QAT","SAU","SYR","ARE","YEM"] + # "North_West_Africa": ["BEN","BFA","CIV","GMB","GHA","GIN","GNB","LBR","NGA","SEN","SLE","TGO","DZA","TCD","EGY","ERI","LBY","MLI","MRT","MAR","NER","SDN","TUN","ESH"] + # "Subsaharan_Africa": ["AGO","BWA","BDI","CMR","CAF","COD","ETH","KEN","GAB","MDG","MWI","MOZ","NAM","COG","RWA","SOM","ZAF","SSD","TZA","UGA","ZMB","ZWE"] "North_America": ["CAN","USA"] - "Eurasia": ["ARM","AZE","GEO","KAZ","KGZ","TJK","TKM","UZB","RUS"] - "South_America": ["BOL","BRA","COL","ECU","GUF","GUY","PRY","PER","SUR","VEN"] - "South_South_America": ["ARG","CHL","URY"] - "Central_America": ["CRI","SLV","GTM","HND","MEX","NIC","PAN","CUB","DOM","JAM","HTI"] - "West_Asia": ["AFG","BGD","BTN","IND","NPL","PAK","LKA"] - "East_Asia": ["CHN","HKG","MNG","TWN"] - "Pacific_Asia": ["BRN","KHM","IDN","LAO","MYS","MMR","PNG","PHL","SGP","THA","VNM"] - "East_East_Asia": ["JPN","KOR","PRK"] + # "Eurasia": ["ARM","AZE","GEO","KAZ","KGZ","TJK","TKM","UZB","RUS"] + # "South_America": ["BOL","BRA","COL","ECU","GUF","GUY","PRY","PER","SUR","VEN"] + # "South_South_America": ["ARG","CHL","URY"] + # "Central_America": ["CRI","SLV","GTM","HND","MEX","NIC","PAN","CUB","DOM","JAM","HTI"] + # "West_Asia": ["AFG","BGD","BTN","IND","NPL","PAK","LKA"] + # "East_Asia": ["CHN","HKG","MNG","TWN"] + # "Pacific_Asia": ["BRN","KHM","IDN","LAO","MYS","MMR","PNG","PHL","SGP","THA","VNM"] + # "East_East_Asia": ["JPN","KOR","PRK"] "Oceania": ["AUS","NZL"] region_nice_names: From 8e9eef7ff148aa833e080e62d0df613bdcd89a4c Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 16:35:58 +0200 Subject: [PATCH 03/11] docs: add MIT license and Zenodo DOI badges to README --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index b2f5c31..ce3f1cf 100644 --- a/README.md +++ b/README.md @@ -1,5 +1,8 @@ # SHIFT – Steel & Hydrogen Integrated Freight Trade +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENCE) +[![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.21875103.svg)](https://doi.org/10.5281/zenodo.21875103) + **SHIFT** is a spatially resolved techno-economic optimization of global iron and steel supply chains under decarbonization. It explores how hydrogen-based direct reduced iron (DRI) production and hot-briquetted iron From ea1333e94e18aaac0ec7e35638dbe59cbc02873f Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 16:39:07 +0200 Subject: [PATCH 04/11] docs: note Python/pixi requirement in quick start --- README.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/README.md b/README.md index ce3f1cf..caf9b5c 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,8 @@ orchestrated with [Snakemake](https://snakemake.readthedocs.io/). ## Quick start – installation & execution +SHIFT is written in Python and uses [pixi](https://pixi.sh/latest/installation/) to manage the environment. Install pixi before you continue. + ```sh git clone https://github.com/energyLS/shift.git cd shift From 18058f834ee6a934ce6a618a3c435767e075e390 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 16:40:53 +0200 Subject: [PATCH 05/11] docs: state Python 3.10+ requirement in quick start --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index caf9b5c..390929a 100644 --- a/README.md +++ b/README.md @@ -13,7 +13,7 @@ orchestrated with [Snakemake](https://snakemake.readthedocs.io/). ## Quick start – installation & execution -SHIFT is written in Python and uses [pixi](https://pixi.sh/latest/installation/) to manage the environment. Install pixi before you continue. +SHIFT requires Python 3.10+ and uses [pixi](https://pixi.sh/latest/installation/) to manage the environment. Install pixi before you continue. ```sh git clone https://github.com/energyLS/shift.git From 242adcc44646f85be5f0c52c3d71eb7379abd942 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 16:55:53 +0200 Subject: [PATCH 06/11] fix: default to license-free HiGHS solver, fix missing trade_opt.csv retrieval output --- README.md | 4 ++++ config/config.yaml | 4 ++-- rules/retrieve.smk | 8 ++++---- 3 files changed, 10 insertions(+), 6 deletions(-) diff --git a/README.md b/README.md index e3eae90..014be07 100644 --- a/README.md +++ b/README.md @@ -22,6 +22,10 @@ Approximate runtimes: - Installation (`pixi install`): ~5 minutes - Demo model (this branch): ~1 hour +By default, the model solves with [HiGHS](https://highs.dev/), a license-free +solver. If you have a [Gurobi](https://www.gurobi.com/) license, set +`solver.name: gurobi` in `config/config.yaml` for faster solves. + > [!NOTE] > This is the `demo` branch: it uses fewer regions and quantities so it > runs much faster. For the full-resolution model (~5 hours), see the diff --git a/config/config.yaml b/config/config.yaml index fffa77e..183dfbd 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -254,8 +254,8 @@ plot: bus_size: 2.0e-7 link_width: 3.0e-8 solver: - name: gurobi - options: gurobi-default + name: highs # license-free by default; set to "gurobi" (with a valid license) for faster solves + options: highs-default compute_iis: False # Set to false to skip expensive IIS computation for infeasible models solver_options: highs-default: diff --git a/rules/retrieve.smk b/rules/retrieve.smk index 5d44969..47ca33f 100644 --- a/rules/retrieve.smk +++ b/rules/retrieve.smk @@ -3,11 +3,10 @@ storage: keep_local=True, retries=3, + rule retrieve_data: input: - data_url=storage( - f"https://zenodo.org/records/21875103/files/shift_data.zip" - ), + data_url=storage(f"https://zenodo.org/records/21875103/files/shift_data.zip"), output: bus_locations="data/bus_locations.csv", ip_market_fabrication="data/demand/ip_market__fabrication.csv", @@ -17,6 +16,7 @@ rule retrieve_data: unido_raw="data/labour/unido-raw/data.csv", iron_ore_production="data/owid-iron-ore/iron-ore-crude-ore-production.csv", political_stability="data/political-stability/globaleconomy.csv", + trade_opt="data/trade_opt.csv", renewable_profiles_geojson="data/renewable_profiles_global_merged.geojson", renewable_profiles_nc="data/renewable_profiles_global_merged.nc", log: @@ -27,4 +27,4 @@ rule retrieve_data: params: folder=".", run: - unpack_archive(input.data_url, params.folder) \ No newline at end of file + unpack_archive(input.data_url, params.folder) From 9ff2642a9055fc0e77e98a092bff83c9cd506eb3 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 18:40:51 +0200 Subject: [PATCH 07/11] chore: reduce demo steel demand levels to [1, 10, 100] --- config/config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 183dfbd..eea8e35 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -135,7 +135,7 @@ scenario: # Absolute steel demand levels (Mt/year) for supply curve sweep # For each level, PyPSA minimizes cost with fixed renewable capacity # Values represent different production scales -steel_demand_levels: [0.1, 100, 500, 1000] #[0.1, 0.5, 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] # Mt/year +steel_demand_levels: [1, 10, 100] #[0.1, 0.5, 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] # Mt/year electricity_steel_ratio: 5.25 #TWh/Mt or MWh/t, see notebooks 'analysis-steel.ipynb' embodied_energy_steel: 2.1 #TWh/Mt or MWh/t, see iron oxide reduction From 91d5cb04f967fc63d9639e0b174d2e17dd8c1fd2 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 19:00:53 +0200 Subject: [PATCH 08/11] docs: point license badge to opensource.org/licenses/MIT --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index befc398..0a73e1c 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # SHIFT – Steel & Hydrogen Integrated Freight Trade -[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENCE) +[![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](https://opensource.org/licenses/MIT) [![DOI](https://zenodo.org/badge/DOI/10.5281/zenodo.21875103.svg)](https://doi.org/10.5281/zenodo.21875103) **SHIFT** is a spatially resolved techno-economic optimization of global iron From c775c6e676333a0a0f2a0aaa5a26ea89e70153a0 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 19:21:01 +0200 Subject: [PATCH 09/11] perf: skip crossover for LCOX solves, keep it for the trade LP duals --- config/config.yaml | 17 ++++++++++++++++- workflow/scripts/calculate_lcox.py | 3 ++- 2 files changed, 18 insertions(+), 2 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index eea8e35..4fa22fa 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -255,7 +255,8 @@ plot: link_width: 3.0e-8 solver: name: highs # license-free by default; set to "gurobi" (with a valid license) for faster solves - options: highs-default + options: highs-default # used for the trade LP, where crossover-derived duals (nodal prices) matter + options_lcox: highs-lcox # used for regional LCOX/supply-curve solves, which only consume the primal solution compute_iis: False # Set to false to skip expensive IIS computation for infeasible models solver_options: highs-default: @@ -270,6 +271,20 @@ solver_options: ipm_optimality_tolerance: 1e-6 parallel: "on" random_seed: 123 + highs-lcox: + # No crossover: calculate_lcox.py only reads network.objective and the + # primal capacity mix, never duals, so a basic (vertex) solution is not + # required and skipping crossover saves the simplex clean-up pass. + threads: 4 + solver: "ipm" + run_crossover: "off" + small_matrix_value: 1e-6 + large_matrix_value: 1e15 + primal_feasibility_tolerance: 1e-6 + dual_feasibility_tolerance: 1e-6 + ipm_optimality_tolerance: 1e-5 + parallel: "on" + random_seed: 123 gurobi-default: threads: 4 method: 2 # barrier diff --git a/workflow/scripts/calculate_lcox.py b/workflow/scripts/calculate_lcox.py index c3832b0..a7397ad 100644 --- a/workflow/scripts/calculate_lcox.py +++ b/workflow/scripts/calculate_lcox.py @@ -365,7 +365,8 @@ def solve_network(network, config): solver_cfg = config.get("solver", {}) solver_name = os.getenv("SHIFT_SOLVER", solver_cfg.get("name", "glpk")) solver_options_key = os.getenv( - "SHIFT_SOLVER_OPTIONS", solver_cfg.get("options", "default") + "SHIFT_SOLVER_OPTIONS", + solver_cfg.get("options_lcox", solver_cfg.get("options", "default")), ) solver_options = config.get("solver_options", {}).get(solver_options_key, {}) From fea5e38a0735f55dfcc7acbc73e05be4d0eeb0f0 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Mon, 31 Aug 2026 19:37:41 +0200 Subject: [PATCH 10/11] perf: let HiGHS choose the algorithm for LCOX solves instead of forcing IPM --- config/config.yaml | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/config/config.yaml b/config/config.yaml index 4fa22fa..59c951c 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -275,8 +275,9 @@ solver_options: # No crossover: calculate_lcox.py only reads network.objective and the # primal capacity mix, never duals, so a basic (vertex) solution is not # required and skipping crossover saves the simplex clean-up pass. + # solver left as "choose" so HiGHS can pick simplex over IPM where cheaper. threads: 4 - solver: "ipm" + solver: "choose" run_crossover: "off" small_matrix_value: 1e-6 large_matrix_value: 1e15 From 786aebbb5ccd3bfcd989a4e5ee2bd543dfdae889 Mon Sep 17 00:00:00 2001 From: Jan Tautorus Date: Tue, 1 Sep 2026 16:12:31 +0200 Subject: [PATCH 11/11] Add 1000 to steel demand levels in config --- config/config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/config/config.yaml b/config/config.yaml index 59c951c..f249c56 100644 --- a/config/config.yaml +++ b/config/config.yaml @@ -135,7 +135,7 @@ scenario: # Absolute steel demand levels (Mt/year) for supply curve sweep # For each level, PyPSA minimizes cost with fixed renewable capacity # Values represent different production scales -steel_demand_levels: [1, 10, 100] #[0.1, 0.5, 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] # Mt/year +steel_demand_levels: [1, 10, 100, 1000] #[0.1, 0.5, 1, 5, 10, 25, 50, 75, 100, 150, 200, 250, 300, 400, 500, 600, 700, 800, 900, 1000] # Mt/year electricity_steel_ratio: 5.25 #TWh/Mt or MWh/t, see notebooks 'analysis-steel.ipynb' embodied_energy_steel: 2.1 #TWh/Mt or MWh/t, see iron oxide reduction @@ -376,4 +376,4 @@ colors: # Friendshoring colors bloc_1: "#17679E" # Europe (Western bloc: Europe + Far West Europe + North America) bloc_2: "#8352D8" # Middle_East (Global South bloc) - bloc_3: "#BF9A39" # East_East_Asia (East Asia + Oceania bloc) \ No newline at end of file + bloc_3: "#BF9A39" # East_East_Asia (East Asia + Oceania bloc)