From 5e9b9c3a5e3dd5e4902065ed3af89c91dd99bb6b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 30 Mar 2026 16:37:44 +0200 Subject: [PATCH 01/37] Define mapping and functions for dPower_BusInfo and dPower_Network. --- PypsaReader.py | 332 ++++++++++++++++++++++++++---------------------- pypsa_helper.py | 25 ++-- 2 files changed, 196 insertions(+), 161 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index ea282b7..d33ec20 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -3,12 +3,66 @@ import pypsa_helper as h import numpy as np import os +import yaml +import inspect + +class Conversions: + """Registry of conversion functions for unit transformations.""" + @staticmethod + def EUR_to_MEUR(val, row=None): + return val * 1e-6 + + @staticmethod + def MEUR_to_EUR(val, row=None): + return val * 1e6 + + @staticmethod + def MW_to_kW(val, row=None): + return val * 1e3 + + @staticmethod + def V_to_kV(val, row=None): + return val * 1e-3 + + @staticmethod + def EUR_per_MVA_to_MEUR(val, df): + """Calculates total cost in MEUR: (EUR/unit) * capacity * 1e-6.""" + capacity = df.s_nom + return val * capacity * 1e-6 + + @staticmethod + def bool_to_binary(val): + """Converts boolean values to binary (0/1) integers.""" + return val.astype(int) + + @staticmethod + def year_and_lifetime_to_year_decom(val, df): + """Calculates decommissioning year based on commissioning year and lifetime.""" + # Only return a decom year if build_year and lifetime is available; otherwise return NaN + if df.build_year.isnull().all() and df.lifetiem.isnull().all(): + return np.nan + else: + return df.build_year + df.lifetime + + @staticmethod + def line_carrier_to_tec_repr(val, df): + if df.carrier.isnull().all(): + return 'DC-OPF' + else: + return df.carrier.map({'AC': 'DC-OPF', 'DC': 'TP'}) + class NetworkDataExtractor: - def __init__(self, network: pypsa.Network): + def __init__(self, network: pypsa.Network, config_path: str = None): self.network = network - self.columns = { + if config_path is None: + config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") + + with open(config_path, 'r') as f: + self.config = yaml.safe_load(f) + # The expected columns for each table (used for reordering and filling empties) + self.columns = { "dPower_BusInfo": ['excl', 'id', 'z', 'pBusBaseV', 'pBusMaxV', 'pBusMinV', 'pBusB', 'pBusG', 'pBus_pf', 'YearCom', 'YearDecom', 'lat', 'lon', 'zoi', 'dataPackage', 'dataSource'], @@ -43,181 +97,159 @@ def __init__(self, network: pypsa.Network): "dPower_Inflows": ['Inflow'], } - - self.component_definitions = { - "dPower_BusInfo": { - "source": lambda net: net.buses[net.buses["carrier"] == "AC"], - "index": lambda Buses: Buses.index.rename("i"), - "z": lambda Buses: Buses["country"] , - "pBusBaseV": lambda Buses: Buses["v_nom"], - "pBusMaxV": lambda Buses: 1.1, - "pBusMinV": lambda Buses: 0.9, - "lat": lambda Buses: Buses["y"], - "lon": lambda Buses: Buses["x"], - }, - "dPower_Network": { - "source": lambda net: pd.concat([h.prepare_ac_lines(net), - h.prepare_dc_links(net) - ], ignore_index=True), - "index": lambda df: pd.MultiIndex.from_frame( - df[["bus0", "bus1", "name"]].rename(columns={"bus0": "i", "bus1": "j", "name": "c"}) - ).set_names(["i", "j", "c"]), - "pRline": lambda df: df["r"], - "pXline": lambda df: df["x"], - "pBcline": lambda df: df["b"], - "pMax": lambda df: df["pmax"] - - }, - "dPower_ThermalGen": { - "source": lambda net: h.prepare_thermal_generators(net), - "index": lambda df: df["id"].rename("g"), - "tec": lambda df: df["carrier"], - "i": lambda df: df["bus"], - "MaxProd": lambda df: df["max_prod"], - "MinProd": lambda df: df["min_prod"], - "RampUp": lambda df: df["ramp_up"], - "RampDown": lambda df: df["ramp_down"], - "pStartupCostEUR": lambda df: df["start_up_cost"], - "EnableInvest": lambda df: df["enable_invest"], - "InvestCost": lambda df: df["capital_cost"], - "OMVarCost": lambda df: df["marginal_cost"], - }, - "dPower_VRESProfiles": { - "source": lambda net: h.prepare_renewable_profiles(net), - "Capacity": lambda df: df["Capacity"], - - "index": lambda df: pd.MultiIndex.from_frame( - df[["generator_id", "snapshot"]].rename(columns={"generator_id": "g", "snapshot": "k"})) - }, - "dPower_VRES": { - "source": lambda net: h.prepare_renewable_generators(net), - "index": lambda df: df["id"].rename("g"), - "tec": lambda df: df["carrier"], - "i": lambda df: df["bus"], - "MaxProd": lambda df: df["max_prod"], - "enableinvest": lambda df: df["enable_invest"], - "MaxInvest": lambda df: df["p_nom_max"], - "InvestCost": lambda df: df["capital_cost"], - "OMVarCost": lambda df: df["marginal_cost"] - }, - "dPower_RoR": { - "source": lambda net: h.prepare_ror_generators(net), - "tec": lambda df: df["carrier"], - "index": lambda df: df["id"].rename("g"), - "i": lambda df: df["bus"], - "MaxProd": lambda df: df["max_prod"], - "MinProd": lambda df: df["min_prod"], - "DisEffic": lambda df: df["discharge"], - "IsHydro": lambda df: df["is_hydro"], - "OMVarCost": lambda df: df["marginal_cost"], - "EnableInvest": lambda df: df["enable_invest"], - "MaxInvest": lambda df: df["p_nom_max"], - "InvestCostPerMW": lambda df: df["capital_cost"] - }, - "dPower_Storage": { - "source": lambda net: h.prepare_storage_units(net), - "index": lambda df: df["id"].rename("g"), - "tec": lambda df: df["carrier"], - "i": lambda df: df["bus"], - "MaxProd": lambda df: df["max_prod"], - "MinProd": lambda df: df["min_prod"], - "DisEffic": lambda df: df["discharge"], - "ChEffic": lambda df: df["charge"], - "IniReserve": lambda df: df["ini_reserve"], - "IsHydro": lambda df: df["is_hydro"], - "OMVarCost": lambda df: df["marginal_cost"], - "EnableInvest": lambda df: df["enable_invest"], - "MaxInvest": lambda df: df["p_nom_max"], - "InvestCostPerMWh": lambda df: df["capital_cost"], - "Ene2PowRatio": lambda df: df["max_hours"], - "ShelfLife": lambda df: df["lifetime"] - }, - "dPower_Inflows": { - "source": lambda net: h.prepare_inflow_profiles(net), - "rp": lambda df: df["rp"], - "g": lambda df: df["g"], - "k": lambda df: df["k"], - "Inflow": lambda df: df["Inflow"], - "index": lambda df: pd.MultiIndex.from_frame(df[["rp","k", "g"]]) - }, - "dPower_Demand": { - "source": lambda net: h.prepare_demand_profiles(net), - "rp": lambda df: df["rp"], - "g": lambda df: df["g"], - "k": lambda df: df["k"], - "Demand": lambda df: df["Demand"], - "index": lambda df: pd.MultiIndex.from_frame(df[["rp", "k", "g"]]) - } - } - self.dataframes = self._extract_dataframes() # add empty columns self.dataframes = self._add_empty_columns() # reorder columns self.dataframes = self._reorder_columns() + + def _get_unit_factor(self, pypsa_unit: str, lego_unit: str) -> float: + """Calculates conversion factor based on metric prefixes (e.g., MW to kW).""" + if not pypsa_unit or not lego_unit or pypsa_unit == lego_unit: + return 1.0 + + # Power of 10 mapping for metric prefixes + prefixes = {'T': 12, 'G': 9, 'M': 6, 'k': 3, '': 0, 'm': -3, 'u': -6, 'n': -9} + + def split_unit(u): + if len(u) > 1 and u[0] in prefixes and (u[1:] in ['W', 'V', 'EUR', 'Wh', 'g', 'l']): + return u[0], u[1:] + return '', u + + p_pre, p_base = split_unit(pypsa_unit) + l_pre, l_base = split_unit(lego_unit) + + # Specific handling for EUR/MEUR if they are treated as base units + if pypsa_unit == "MEUR" and lego_unit == "EUR": return 1e6 + if pypsa_unit == "EUR" and lego_unit == "MEUR": return 1e-6 + + if p_base != l_base: + return 1.0 + + return 10** (prefixes[p_pre] - prefixes[l_pre]) def _extract_dataframes(self): df_dict = {} - for name, config in self.component_definitions.items(): - source_df = config["source"](self.network) - - column_data = { - column_name: transform(source_df) - for column_name, transform in config.items() - if column_name not in ("source", "index") - } + for table_name, cfg in self.config.items(): + # 1. Get Source Data + src = cfg['source'] + if src['type'] == 'attribute': + source_df = getattr(self.network, src['name']) + if 'filter' in src: + source_df = source_df.query(src['filter']) + elif src['type'] == 'helper': + source_df = getattr(h, src['name'])(self.network) + else: + continue + + # 2. Process Mapping + column_data = {} + for lego_col, mapping in cfg['mapping'].items(): + if isinstance(mapping, dict): + # Attribute mapping with potential unit conversion or transformation function + attr = mapping.get('attr') + if attr and attr in source_df.columns: + val = source_df[attr] + else: + # Try to get value as series of NaNs or fixed value + val = pd.Series(mapping.get('value', np.nan), index=source_df.index) + + # Priority 1: Named conversion function + if 'conversion' in mapping: + conv_name = mapping['conversion'].replace('()', '') + if hasattr(Conversions, conv_name): + conv_func = getattr(Conversions, conv_name) + # Check function signature + sig = inspect.signature(conv_func) + params = list(sig.parameters.values()) + + if len(params) >= 2: + # Vectorized call: (Series, DataFrame) + val = conv_func(val, source_df) + else: + # Vectorized call: (Series) + val = conv_func(val) + else: + print(f"Warning: Conversion function '{conv_name}' not found in Conversions class.") + + # Priority 2: Explicit unit strings + elif 'pypsa_unit' in mapping and 'lego_unit' in mapping: + factor = self._get_unit_factor(mapping['pypsa_unit'], mapping['lego_unit']) + val = val * factor + + # Priority 3: Simple multiplier factor + elif 'factor' in mapping: + val = val * mapping['factor'] + + column_data[lego_col] = val + elif isinstance(mapping, (int, float)): + # Static value + column_data[lego_col] = mapping + else: + # Direct attribute string mapping + if mapping in source_df.columns: + column_data[lego_col] = source_df[mapping] + else: + column_data[lego_col] = np.nan + + # Todo: Add warning if the column defined in mapping is not available in the LEGO columns!!! df = pd.DataFrame(column_data) - # Set custom index if defined - if "index" in config: - index_values = config["index"](source_df) - - df.index = index_values - - # Drop the columns used in the index if they exist in the DataFrame - if isinstance(index_values, pd.MultiIndex): - df = df.drop(columns=[col for col in index_values.names if col in df.columns], errors="ignore") - elif isinstance(index_values, pd.Index): - if index_values.name in df.columns: - df = df.drop(columns=[index_values.name], errors="ignore") - - # df.index.name = None # Optional: remove index name - - - else: - df = df.reset_index(drop=True) - - df_dict[name] = df + # 3. Handle Indexing + if 'index' in cfg: + # Logic from original PypsaReader for indexing + if table_name == "dPower_BusInfo": + df.index = source_df.index.rename("i") + elif table_name == "dPower_Network": + df.index = pd.MultiIndex.from_frame( + source_df[["bus0", "bus1", "name"]].rename(columns={"bus0": "i", "bus1": "j", "name": "c"}) + ).set_names(["i", "j", "c"]) + elif table_name == "dPower_ThermalGen": + df.index = source_df["id"].rename("g") + elif table_name == "dPower_VRESProfiles": + df.index = pd.MultiIndex.from_frame( + source_df[["generator_id", "k"]].rename(columns={"generator_id": "g"}) + ).set_names(["g", "k"]) + elif table_name in ["dPower_VRES", "dPower_RoR", "dPower_Storage"]: + df.index = source_df["id"].rename("g") + elif table_name in ["dPower_Inflows", "dPower_Demand"]: + df.index = pd.MultiIndex.from_frame(source_df[["rp", "k", "g"]]) + + df_dict[table_name] = df return df_dict def _add_empty_columns(self): for name, df in self.dataframes.items(): - for col in self.columns[name]: - if col not in df.columns: - df[col] = np.nan + if name in self.columns: + for col in self.columns[name]: + if col not in df.columns: + df[col] = np.nan return self.dataframes def _reorder_columns(self): for name, df in self.dataframes.items(): if name in self.columns: cols = self.columns[name] - # Reorder the DataFrame columns df = df.reindex(columns=cols) - # Update the DataFrame in the dictionary self.dataframes[name] = df return self.dataframes def get_dataframes(self): return self.dataframes -filepath = os.path.join(os.path.dirname(__file__), "..", "pypsa-eur/resources/test/networks/base_s_39_elec_1year.nc") -net = pypsa.Network(filepath) -extractor = NetworkDataExtractor(net) -dfs = extractor.get_dataframes() -for name, df in dfs.items(): - print(f"DataFrame: {name}") - print(df.head()) - print("\n") + +if __name__ == "__main__": + filepath = os.path.join(os.path.dirname(__file__), "..", "pypsa-eur/resources/test/networks/base_s_39_elec_1year.nc") + if os.path.exists(filepath): + net = pypsa.Network(filepath) + extractor = NetworkDataExtractor(net) + dfs = extractor.get_dataframes() + for name, df in dfs.items(): + print(f"DataFrame: {name}") + print(df.head()) + print("\n") + else: + print(f"Test file not found: {filepath}") diff --git a/pypsa_helper.py b/pypsa_helper.py index 1fe31d9..ade19b3 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -11,11 +11,10 @@ def prepare_ac_lines(net): lines["b"] = lines.apply( lambda row: row["b"] if row["b"] != 0 else types.loc[row["type"]].x_per_length * row["length"], axis=1) - lines["pmax"] = lines["s_nom"] * lines["s_max_pu"] - lines["id"] = lines.index - lines["name"] = [f"Line_{i}" for i in range(len(lines))] + if "name" not in lines.columns or lines["name"].isnull().all(): + lines["name"] = "c1" - return lines[["bus0", "bus1", "r", "x", "b", "pmax", "id", "name"]] + return lines def prepare_dc_links(net): links = net.links[net.links["carrier"] == "DC"].copy() @@ -28,6 +27,11 @@ def prepare_dc_links(net): return links[["bus0", "bus1", "r", "x", "b", "pmax", "id", "name"]] +def prepare_ac_lines_and_dc_links(net): + ac_lines = prepare_ac_lines(net) + dc_links = prepare_dc_links(net) + return pd.concat([ac_lines, dc_links], ignore_index=True) + def prepare_thermal_generators(net): thermal_types = ['OCGT', 'biomass', 'CCGT', 'nuclear', 'oil', 'coal', 'lignite'] gens = net.generators.copy() @@ -53,10 +57,10 @@ def prepare_renewable_profiles(net): vres_ids = vres_gens.index.to_list() profiles = net.generators_t.p_max_pu[vres_ids].copy() - profiles = profiles.reset_index().melt( - id_vars="snapshot", var_name="generator_id", value_name="Capacity" + # Ensure the index (snapshots) has a known name before resetting + profiles = profiles.rename_axis("k").reset_index().melt( + id_vars="k", var_name="generator_id", value_name="Capacity" ) - profiles = profiles.rename(columns={"index": "k"}) profiles["rp"] = "rp01" # add a dummy column for compatibility return profiles # flat, column-based, no index set yet @@ -126,11 +130,10 @@ def prepare_inflow_profiles(net): combined = pd.concat([inflow_storage, inflow_ror], axis=1) combined = combined.T # index: generator_id, columns: time - # Convert to long format - inflow_long = combined.reset_index().melt( - id_vars="index", var_name="k", value_name="Inflow" + # Convert to long format by renaming index to 'g' before resetting + inflow_long = combined.rename_axis('g').reset_index().melt( + id_vars="g", var_name="k", value_name="Inflow" ) - inflow_long = inflow_long.rename(columns={"index": "g"}) inflow_long["rp"] = "rp01" # add a dummy column for compatibility return inflow_long[["rp", "g", "k", "Inflow"]] From 73f92e5e077248b5487576292832b1dcf3bc9c16 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 30 Mar 2026 17:44:42 +0200 Subject: [PATCH 02/37] Define mapping and functions for dPower_ThermalGen. --- PypsaReader.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/PypsaReader.py b/PypsaReader.py index d33ec20..30a823a 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -30,6 +30,11 @@ def EUR_per_MVA_to_MEUR(val, df): capacity = df.s_nom return val * capacity * 1e-6 + @staticmethod + def pu_to_absolute(val, df): + """Converts per-unit values to absolute values using the base value from the DataFrame.""" + return val * df.p_nom + @staticmethod def bool_to_binary(val): """Converts boolean values to binary (0/1) integers.""" @@ -207,7 +212,7 @@ def _extract_dataframes(self): source_df[["bus0", "bus1", "name"]].rename(columns={"bus0": "i", "bus1": "j", "name": "c"}) ).set_names(["i", "j", "c"]) elif table_name == "dPower_ThermalGen": - df.index = source_df["id"].rename("g") + df.index = source_df.index.rename("g") elif table_name == "dPower_VRESProfiles": df.index = pd.MultiIndex.from_frame( source_df[["generator_id", "k"]].rename(columns={"generator_id": "g"}) From 1f7553f131d23857f1cfa257333e662bc65d6ddc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Wed, 1 Apr 2026 12:28:14 +0300 Subject: [PATCH 03/37] Add mapping_config.yaml. --- mapping_config.yaml | 194 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 194 insertions(+) create mode 100644 mapping_config.yaml diff --git a/mapping_config.yaml b/mapping_config.yaml new file mode 100644 index 0000000..fa463b6 --- /dev/null +++ b/mapping_config.yaml @@ -0,0 +1,194 @@ +Metadata: + source: + type: "metadata" + dataPackage: "PyPSA-Import" + dataSource: "scigrid-de" + +dPower_BusInfo: + source: + type: "attribute" + name: "buses" + filter: "carrier == 'AC'" + index: "i" + mapping: + z: "country" + pBusBaseV: "v_nom" + pBusMaxV: 1.1 # Fixed value used, since v_mag_pu_max is currently not used in PyPSA + pBusMinV: 0.9 # Fixed value used, since v_mag_pu_min is currently not used in PyPSA + lat: "y" + lon: "x" + +dPower_Network: + source: + type: "helper" + name: "prepare_ac_lines_and_dc_links" + index: ["i", "j", "c"] + mapping: + c: "c1" + pRline: "r" + pXline: "x" + pBcline: "b" + pPmax: "s_nom" + pFOMCost: + attr: "fom_cost" + conversion: "EUR_per_MVA_to_MEUR" + pEnableInvest: + attr: "s_nom_extendable" + conversion: "bool_to_binary" + pInvestCost: + attr: "capital_cost" + conversion: "EUR_per_MVA_to_MEUR" + pTecRepr: + attr: "carrier" + conversion: "line_carrier_to_tec_repr" # if no carrier is specified, the technical representation is set to DC-OPF + YearCom: "build_year" + YearDecom: + attr: "decom_year" + conversion: "year_and_lifetime_to_year_decom" + pAngle: 0 + pRatio: 1 + + +dPower_ThermalGen: + source: + type: "attribute" + name: "generators" + filter: "carrier in ['Gas', 'Hard Coal', 'Waste', 'Brown Coal', 'Oil', 'Other', 'Nuclear', 'Geothermal']" + index: "g" + mapping: + tec: "carrier" + i: "bus" + ExisUnits: + attr: "active" + conversion: "bool_to_binary" + MaxProd: "p_nom" + MinProd: + attr: "p_min_pu" + conversion: "pu_to_absolute" + RampUp: + attr: "ramp_limit_up" + conversion: "pu_to_absolute" + RampDown: + attr: "ramp_limit_down" + conversion: "pu_to_absolute" + MinUpTime: "min_up_time" + MinDownTime: "min_down_time" + Efficiency: "efficiency" + CommitConsumption: + attr: "stand_by_cost" + conversion: "EUR_per_hour_to_MWh_per_hour" + OMVarCost: "marginal_cost" + StartupConsumption: + attr: "start_up_consumption" + conversion: "EUR_to_MWh" + EnableInvest: + attr: "p_nom_extendable" + conversion: "bool_to_binary" + InvestCost: "capital_cost" + YearCom: "build_year" + YearDecom: + attr: "decom_year" + conversion: "year_and_lifetime_to_year_decom" + +dPower_VRESProfiles: + source: + type: "helper" + name: "prepare_renewable_profiles" + filter: "carrier in ['Solar', 'Wind Onshore', 'Wind Offshore']" + index: ["rp", "g"] + mapping: + value: "Capacity" + +dPower_VRES: + source: + type: "attribute" + name: "generators" + filter: "carrier in ['Solar', 'Wind Onshore', 'Wind Offshore']" + index: "g" + mapping: + tec: "carrier" + i: "bus" + ExisUnits: + attr: "active" + conversion: "bool_to_binary" + MaxProd: "p_nom" + EnableInvest: + attr: "p_nom_extendable" + conversion: "bool_to_binary" + MaxInvest: + attr: "p_nom_max" + conversion: "total_capacity_to_number_of_units" + InvestCost: "capital_cost" + OMVarCost: "marginal_cost" + YearCom: "build_year" + YearDecom: + attr: "decom_year" + conversion: "year_and_lifetime_to_year_decom" + +dPower_RoR: + source: + type: "helper" + name: "prepare_ror_generators" + index: "g" + mapping: + tec: "carrier" + i: "bus" + MaxProd: "max_prod" + MinProd: "min_prod" + DisEffic: "discharge" + IsHydro: "is_hydro" + OMVarCost: "marginal_cost" + EnableInvest: "enable_invest" + MaxInvest: "p_nom_max" + InvestCostPerMW: "capital_cost" + +dPower_Storage: + source: + type: "attribute" + name: "storage_units" + index: "g" + mapping: + tec: "carrier" + i: "bus" + ExisUnits: 1 + MaxProd: + attr: "p_max_pu" + conversion: "pu_to_absolute" + MinProd: 0 + MaxCons: + attr: "p_max_pu" + conversion: "pu_to_absolute" + DisEffic: "efficiency_dispatch" + ChEffic: "efficiency_store" + Self Discharge: "standing_loss" + IniReserve: "state_of_charge_initial" + OMVarCost: "marginal_cost" + EnableInvest: + attr: "p_nom_extendable" + conversion: "bool_to_binary" + MaxInvest: + attr: "p_nom_max" + conversion: "total_capacity_to_number_of_units" + InvestCostPerMW: "capital_cost" + Ene2PowRatio: "max_hours" + YearCom: "build_year" + YearDecom: + attr: "decom_year" + conversion: "year_and_lifetime_to_year_decom" + +dPower_Inflows: + source: + type: "helper" + name: "prepare_inflow_profiles" + filter: "carrier in ['Pumped Hydro', 'Run of River']" + index: ["rp", "k", "g"] + mapping: + value: "Inflow" + +dPower_Demand: + source: + type: "helper" + name: "prepare_demand_profiles" + index: ["rp", "k", "g"] + mapping: + value: "Demand" From 3bcc8b28816648e3db43f33a6e9d37ba0068d423 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Wed, 1 Apr 2026 12:28:34 +0300 Subject: [PATCH 04/37] Update mapping functions. --- PypsaReader.py | 123 ++++++++++++++++++++++++++++++------------------ pypsa_helper.py | 96 ++++++++++++++++++++----------------- 2 files changed, 130 insertions(+), 89 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 30a823a..1fd2ec2 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -1,13 +1,17 @@ -import pypsa -import pandas as pd -import pypsa_helper as h -import numpy as np +import inspect import os + +import numpy as np +import pandas as pd +import pypsa import yaml -import inspect + +import pypsa_helper as h + class Conversions: """Registry of conversion functions for unit transformations.""" + @staticmethod def EUR_to_MEUR(val, row=None): return val * 1e-6 @@ -56,50 +60,59 @@ def line_carrier_to_tec_repr(val, df): else: return df.carrier.map({'AC': 'DC-OPF', 'DC': 'TP'}) + @staticmethod + def total_capacity_to_number_of_units(val, df): + """Calculates the number of units based on total capacity and nominal capacity per unit.""" + # Check if the input value is all NaN or all infinite, and return a default of 100 in that case + if val.isnull().all() or np.isinf(val).all(): + return 100 + else: + return (np.ceil(val / df.p_nom)).astype(int) + class NetworkDataExtractor: def __init__(self, network: pypsa.Network, config_path: str = None): self.network = network if config_path is None: config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") - + with open(config_path, 'r') as f: self.config = yaml.safe_load(f) # The expected columns for each table (used for reordering and filling empties) self.columns = { "dPower_BusInfo": ['excl', 'id', 'z', 'pBusBaseV', 'pBusMaxV', 'pBusMinV', 'pBusB', - 'pBusG', 'pBus_pf', 'YearCom', 'YearDecom', 'lat', 'lon', 'zoi', - 'dataPackage', 'dataSource'], + 'pBusG', 'pBus_pf', 'YearCom', 'YearDecom', 'lat', 'lon', 'zoi', + 'dataPackage', 'dataSource'], "dPower_Network": ['excl', 'id', 'pRline', 'pXline', 'pBcline', 'pAngle', 'pRatio', - 'pPmax', 'pEnableInvest', 'pFOMCost', 'pInvestCost', 'pTecRepr', - 'YearCom', 'YearDecom', 'dataPackage', 'dataSource'], + 'pPmax', 'pEnableInvest', 'pFOMCost', 'pInvestCost', 'pTecRepr', + 'YearCom', 'YearDecom', 'dataPackage', 'dataSource'], "dPower_ThermalGen": ['excl', 'id', 'tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'RampUp', - 'RampDw', 'MinUpTime', 'MinDownTime', 'Qmax', 'Qmin', 'InertiaConst', - 'FuelCost', 'Efficiency', 'CommitConsumption', 'OMVarCost', - 'StartupConsumption', 'EFOR', 'EnableInvest', 'InvestCost', - 'FirmCapCoef', 'CO2Emis', 'YearCom', 'YearDecom', 'lat', 'long', - 'dataPackage', 'dataSource', 'pSlopeVarCostEUR', 'pInterVarCostEUR', - 'pStartupCostEUR', 'MaxInvest', 'InvestCostEUR'], - "dPower_VRESProfiles": ['Capacity'], + 'RampDw', 'MinUpTime', 'MinDownTime', 'Qmax', 'Qmin', 'InertiaConst', + 'FuelCost', 'Efficiency', 'CommitConsumption', 'OMVarCost', + 'StartupConsumption', 'EFOR', 'EnableInvest', 'InvestCost', + 'FirmCapCoef', 'CO2Emis', 'YearCom', 'YearDecom', 'lat', 'long', + 'dataPackage', 'dataSource', 'pSlopeVarCostEUR', 'pInterVarCostEUR', + 'pStartupCostEUR', 'MaxInvest', 'InvestCostEUR'], + "dPower_VRESProfiles": ['value'], "dPower_VRES": ['excl', 'id', 'tec', 'i', 'ExisUnits', 'MaxProd', 'EnableInvest', 'MaxInvest', 'InvestCost', 'OMVarCost', 'FirmCapCoef', 'Qmax', 'Qmin', 'InertiaConst', 'YearCom', 'YearDecom', 'lat', 'lon', 'dataPackage', 'dataSource', 'MinProd', 'InvestCostEUR'], "dPower_Storage": ['tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'MaxCons', 'DisEffic', - 'ChEffic', 'Qmax', 'Qmin', 'InertiaConst', 'MinReserve', 'IniReserve', - 'IsHydro', 'OMVarCost', 'EnableInvest', 'MaxInvest', 'InvestCostPerMW', - 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', - 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', - 'YearDecom', 'lat', 'long', 'pOMVarCostEUR', 'InvestCostEUR'], - "dPower_RoR": ['tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'MaxCons', 'DisEffic', - 'ChEffic', 'Qmax', 'Qmin', 'InertiaConst', 'MinReserve', 'IniReserve', - 'IsHydro', 'OMVarCost', 'EnableInvest', 'MaxInvest', 'InvestCostPerMW', - 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', - 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', - 'YearDecom', 'lat', 'long', 'InvestCostEUR'], - "dPower_Demand": ['Capacity'], - "dPower_Inflows": ['Inflow'], + 'ChEffic', 'Qmax', 'Qmin', 'InertiaConst', 'MinReserve', 'IniReserve', + 'IsHydro', 'OMVarCost', 'EnableInvest', 'MaxInvest', 'InvestCostPerMW', + 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', + 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', + 'YearDecom', 'lat', 'long', 'pOMVarCostEUR', 'InvestCostEUR', 'dataPackage', 'dataSource'], + # "dPower_RoR": ['tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'MaxCons', 'DisEffic', + # 'ChEffic', 'Qmax', 'Qmin', 'InertiaConst', 'MinReserve', 'IniReserve', + # 'IsHydro', 'OMVarCost', 'EnableInvest', 'MaxInvest', 'InvestCostPerMW', + # 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', + # 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', + # 'YearDecom', 'lat', 'long', 'InvestCostEUR'], + "dPower_Demand": ['value'], + "dPower_Inflows": ['value'], } self.dataframes = self._extract_dataframes() @@ -112,10 +125,10 @@ def _get_unit_factor(self, pypsa_unit: str, lego_unit: str) -> float: """Calculates conversion factor based on metric prefixes (e.g., MW to kW).""" if not pypsa_unit or not lego_unit or pypsa_unit == lego_unit: return 1.0 - + # Power of 10 mapping for metric prefixes prefixes = {'T': 12, 'G': 9, 'M': 6, 'k': 3, '': 0, 'm': -3, 'u': -6, 'n': -9} - + def split_unit(u): if len(u) > 1 and u[0] in prefixes and (u[1:] in ['W', 'V', 'EUR', 'Wh', 'g', 'l']): return u[0], u[1:] @@ -131,8 +144,8 @@ def split_unit(u): if p_base != l_base: return 1.0 - return 10** (prefixes[p_pre] - prefixes[l_pre]) - + return 10 ** (prefixes[p_pre] - prefixes[l_pre]) + def _extract_dataframes(self): df_dict = {} @@ -144,12 +157,13 @@ def _extract_dataframes(self): if 'filter' in src: source_df = source_df.query(src['filter']) elif src['type'] == 'helper': - source_df = getattr(h, src['name'])(self.network) + source_df = getattr(h, src['name'])(self.network, cfg) else: continue # 2. Process Mapping column_data = {} + # if 'mapping' in cfg.keys(): for lego_col, mapping in cfg['mapping'].items(): if isinstance(mapping, dict): # Attribute mapping with potential unit conversion or transformation function @@ -159,7 +173,7 @@ def _extract_dataframes(self): else: # Try to get value as series of NaNs or fixed value val = pd.Series(mapping.get('value', np.nan), index=source_df.index) - + # Priority 1: Named conversion function if 'conversion' in mapping: conv_name = mapping['conversion'].replace('()', '') @@ -168,7 +182,7 @@ def _extract_dataframes(self): # Check function signature sig = inspect.signature(conv_func) params = list(sig.parameters.values()) - + if len(params) >= 2: # Vectorized call: (Series, DataFrame) val = conv_func(val, source_df) @@ -177,16 +191,16 @@ def _extract_dataframes(self): val = conv_func(val) else: print(f"Warning: Conversion function '{conv_name}' not found in Conversions class.") - + # Priority 2: Explicit unit strings elif 'pypsa_unit' in mapping and 'lego_unit' in mapping: factor = self._get_unit_factor(mapping['pypsa_unit'], mapping['lego_unit']) val = val * factor - + # Priority 3: Simple multiplier factor elif 'factor' in mapping: val = val * mapping['factor'] - + column_data[lego_col] = val elif isinstance(mapping, (int, float)): # Static value @@ -215,12 +229,27 @@ def _extract_dataframes(self): df.index = source_df.index.rename("g") elif table_name == "dPower_VRESProfiles": df.index = pd.MultiIndex.from_frame( - source_df[["generator_id", "k"]].rename(columns={"generator_id": "g"}) - ).set_names(["g", "k"]) - elif table_name in ["dPower_VRES", "dPower_RoR", "dPower_Storage"]: - df.index = source_df["id"].rename("g") - elif table_name in ["dPower_Inflows", "dPower_Demand"]: + source_df[["rp", "generator_id", "k"]].rename(columns={"generator_id": "g"}) + ).set_names(["rp", "g", "k"]) + elif table_name in ["dPower_VRES", "dPower_Storage"]: + df.index = source_df.index.rename("g") + elif table_name == "dPower_Inflows": df.index = pd.MultiIndex.from_frame(source_df[["rp", "k", "g"]]) + elif table_name == "dPower_Demand": + df.index = pd.MultiIndex.from_frame( + source_df[["rp", "k", "g"]].rename(columns={"g": "i"}) + ).set_names(["rp", "k", "i"]) + + # Add default dataPackage and dataSource if they are all NaN (from PypsaReader) + if "Metadata" in self.config.keys() and "dataPackage" in self.config["Metadata"]: + df['dataPackage'] = self.config["Metadata"]["dataPackage"] + else: + df['dataPackage'] = 'default-package' + + if "Metadata" in self.config.keys() and "dataSource" in self.config["Metadata"]: + df['dataSource'] = self.config["Metadata"]["dataSource"] + else: + df['dataSource'] = 'default-source' df_dict[table_name] = df @@ -233,7 +262,7 @@ def _add_empty_columns(self): if col not in df.columns: df[col] = np.nan return self.dataframes - + def _reorder_columns(self): for name, df in self.dataframes.items(): if name in self.columns: @@ -241,7 +270,7 @@ def _reorder_columns(self): df = df.reindex(columns=cols) self.dataframes[name] = df return self.dataframes - + def get_dataframes(self): return self.dataframes diff --git a/pypsa_helper.py b/pypsa_helper.py index ade19b3..f49f760 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -1,6 +1,7 @@ import pandas as pd -def prepare_ac_lines(net): + +def prepare_ac_lines(net, config: dict): lines = net.lines.copy() types = net.line_types @@ -16,7 +17,8 @@ def prepare_ac_lines(net): return lines -def prepare_dc_links(net): + +def prepare_dc_links(net, config: dict): links = net.links[net.links["carrier"] == "DC"].copy() links["r"] = 0.0 links["x"] = 0.0 @@ -27,12 +29,14 @@ def prepare_dc_links(net): return links[["bus0", "bus1", "r", "x", "b", "pmax", "id", "name"]] -def prepare_ac_lines_and_dc_links(net): - ac_lines = prepare_ac_lines(net) - dc_links = prepare_dc_links(net) + +def prepare_ac_lines_and_dc_links(net, config: dict): + ac_lines = prepare_ac_lines(net, config) + dc_links = prepare_dc_links(net, config) return pd.concat([ac_lines, dc_links], ignore_index=True) -def prepare_thermal_generators(net): + +def prepare_thermal_generators(net, config: dict): thermal_types = ['OCGT', 'biomass', 'CCGT', 'nuclear', 'oil', 'coal', 'lignite'] gens = net.generators.copy() gens = gens[gens.carrier.isin(thermal_types)] @@ -42,7 +46,7 @@ def prepare_thermal_generators(net): gens["ramp_up"] = gens["ramp_limit_up"] * gens["p_nom"] gens["ramp_down"] = gens["ramp_limit_down"] * gens["p_nom"] gens["enable_invest"] = gens["p_nom_extendable"].astype(int) - + gens["id"] = gens.index return gens[[ "id", "carrier", "bus", "max_prod", "min_prod", @@ -50,10 +54,11 @@ def prepare_thermal_generators(net): "enable_invest", "capital_cost", "marginal_cost" ]] -def prepare_renewable_profiles(net): - renewable_types = ['solar-hsat', 'onwind', 'solar'] + +def prepare_renewable_profiles(net, config: dict): + # renewable_types = ['Solar', 'Wind Onshore', 'Wind Offshore'] gens = net.generators.copy() - vres_gens = gens[gens.carrier.isin(renewable_types)] + vres_gens = gens.query(config["source"]["filter"]) vres_ids = vres_gens.index.to_list() profiles = net.generators_t.p_max_pu[vres_ids].copy() @@ -65,7 +70,8 @@ def prepare_renewable_profiles(net): return profiles # flat, column-based, no index set yet -def prepare_renewable_generators(net): + +def prepare_renewable_generators(net, config: dict): renewable_types = ['solar-hsat', 'onwind', 'solar'] gens = net.generators.copy() vres = gens[gens.carrier.isin(renewable_types)].copy() @@ -79,7 +85,8 @@ def prepare_renewable_generators(net): "enable_invest", "p_nom_max", "capital_cost", "marginal_cost" ]] -def prepare_ror_generators(net): + +def prepare_ror_generators(net, config: dict): ror = net.generators[net.generators.carrier == "ror"].copy() ror["id"] = ror.index @@ -94,37 +101,43 @@ def prepare_ror_generators(net): "marginal_cost", "enable_invest", "p_nom_max", "capital_cost" ]] -def prepare_storage_units(net): - su = net.storage_units.copy() - - su["id"] = su.index - su["max_prod"] = su["p_nom"] * su["p_max_pu"] - su["min_prod"] = su["p_nom"] * su["p_min_pu"] # note: often negative - su["discharge"] = su["efficiency_dispatch"] - su["charge"] = su["efficiency_store"] - su["ini_reserve"] = su["state_of_charge_initial"] - su["is_hydro"] = su["carrier"].isin(["PHS", "hydro"]).astype(int) - su["enable_invest"] = su["p_nom_extendable"].astype(int) - - # Note: "min_reserve" is not present in PyPSA by default — we'll skip it - return su[[ - "id", "carrier", "bus", "max_prod", "min_prod", "discharge", "charge", - "ini_reserve", "is_hydro", "marginal_cost", "enable_invest", "p_nom_max", - "capital_cost", "max_hours", "lifetime" - ]] -def prepare_inflow_profiles(net): +def prepare_inflow_profiles(net, config: dict): # Get hydro storage inflows - hydro_ids = net.storage_units[net.storage_units["carrier"] == "hydro"].index.to_list() - inflow_storage = net.storage_units_t.inflow[hydro_ids].copy() + hydro_ids = net.storage_units.query(config["source"]["filter"]).index.to_list() + set_hydro_ids = set(hydro_ids) + set_inflow_columns = set(net.storage_units_t.inflow.columns.to_list()) + + # check if inflows are specified for hydro storage units + existing_inflows = list(set_hydro_ids & set_inflow_columns) + missing_inflows = list(set_hydro_ids - set_inflow_columns) + + if len(existing_inflows) == 0: + print("Warning: No hydro storage units have inflow data. Storage inflow profiles will be empty.") + inflow_storage = net.storage_units_t.inflow.copy() + else: + inflow_storage = net.storage_units_t.inflow[hydro_ids].copy() + if len(missing_inflows) > 0: + print(f"Warning: The following hydro storage units are missing inflow data and will be skipped: {missing_inflows}") # Get RoR generator inflows - ror = net.generators[net.generators.carrier == "ror"] - ror_ids = ror.index.to_list() - p_nom = ror["p_nom"] - - inflow_ror = net.generators_t.p_max_pu[ror_ids].copy() - inflow_ror = inflow_ror.mul(p_nom, axis=1) + ror_ids = net.generators.query(config["source"]["filter"]).index.to_list() + set_ror_ids = set(ror_ids) + set_ror_inflow_columns = set(net.generators_t.p_max_pu.columns.to_list()) + + # check if inflows are specified for RoR generators + existing_ror_inflows = list(set_ror_ids & set_ror_inflow_columns) + missing_ror_inflows = list(set_ror_ids - set_ror_inflow_columns) + + if len(existing_ror_inflows) == 0: + print("Warning: No RoR generators have inflow data. RoR inflow profiles will be empty.") + inflow_ror = pd.DataFrame() + else: + ror = net.generators[ror_ids].copy() + inflow_ror = net.generators_t.p_max_pu[ror_ids].copy() + inflow_ror = inflow_ror.mul(ror["p_nom"], axis=1) + if len(missing_ror_inflows) > 0: + print(f"Warning: The following RoR generators are missing inflow data and will be skipped: {missing_ror_inflows}") # Concatenate both: hydro + RoR inflows → [time, generator] combined = pd.concat([inflow_storage, inflow_ror], axis=1) @@ -138,12 +151,11 @@ def prepare_inflow_profiles(net): return inflow_long[["rp", "g", "k", "Inflow"]] -def prepare_demand_profiles(net): + +def prepare_demand_profiles(net, config: dict): df = net.loads_t.p_set.copy() # shape: [time, load_id] df = df.rename_axis("k").reset_index() # 'k' = time demand_long = df.melt(id_vars="k", var_name="g", value_name="Demand") demand_long["rp"] = "rp01" return demand_long[["rp", "g", "k", "Demand"]] - - From 6889f937e26ba7399d0940ef8975cb354dc1ef10 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 2 Apr 2026 12:23:31 +0300 Subject: [PATCH 05/37] Add functionality of fuel cost for thermal generators. --- PypsaReader.py | 190 +++++++++++++++++++++++++++----------------- mapping_config.yaml | 93 +++++++++++++++------- pypsa_helper.py | 10 +-- 3 files changed, 187 insertions(+), 106 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 1fd2ec2..a7e57ec 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -13,61 +13,101 @@ class Conversions: """Registry of conversion functions for unit transformations.""" @staticmethod - def EUR_to_MEUR(val, row=None): + def EUR_to_MEUR(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: + """Converts € to Mio.€""" return val * 1e-6 @staticmethod - def MEUR_to_EUR(val, row=None): + def MEUR_to_EUR(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: + """Converts Mio.€ to € """ return val * 1e6 @staticmethod - def MW_to_kW(val, row=None): + def MW_to_kW(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: return val * 1e3 @staticmethod - def V_to_kV(val, row=None): + def V_to_kV(val, df: pd.DataFrame = None) -> pd.Series: return val * 1e-3 @staticmethod - def EUR_per_MVA_to_MEUR(val, df): + def EUR_per_MVA_to_MEUR(val, df: pd.DataFrame = None) -> pd.Series: """Calculates total cost in MEUR: (EUR/unit) * capacity * 1e-6.""" capacity = df.s_nom return val * capacity * 1e-6 @staticmethod - def pu_to_absolute(val, df): + def pu_to_absolute(val, df: pd.DataFrame) -> pd.Series: """Converts per-unit values to absolute values using the base value from the DataFrame.""" return val * df.p_nom @staticmethod - def bool_to_binary(val): + def bool_to_binary(val, df: pd.DataFrame = None) -> pd.Series: """Converts boolean values to binary (0/1) integers.""" - return val.astype(int) + if val.isnull().all(): + return 0 + else: + val = val.fillna(False) # Treat NaN as False for binary conversion + return val.astype(int) @staticmethod - def year_and_lifetime_to_year_decom(val, df): + def year_and_lifetime_to_year_decom(val, df: pd.DataFrame) -> pd.Series: """Calculates decommissioning year based on commissioning year and lifetime.""" # Only return a decom year if build_year and lifetime is available; otherwise return NaN - if df.build_year.isnull().all() and df.lifetiem.isnull().all(): + if df.build_year.isnull().all() and df.lifetime.isnull().all(): return np.nan else: return df.build_year + df.lifetime @staticmethod - def line_carrier_to_tec_repr(val, df): + def line_carrier_to_tec_repr(val, df: pd.DataFrame) -> pd.Series: if df.carrier.isnull().all(): return 'DC-OPF' else: return df.carrier.map({'AC': 'DC-OPF', 'DC': 'TP'}) @staticmethod - def total_capacity_to_number_of_units(val, df): + def total_capacity_to_number_of_units(val, df: pd.DataFrame) -> pd.Series: """Calculates the number of units based on total capacity and nominal capacity per unit.""" # Check if the input value is all NaN or all infinite, and return a default of 100 in that case if val.isnull().all() or np.isinf(val).all(): return 100 else: - return (np.ceil(val / df.p_nom)).astype(int) + val = val.fillna(0).replace(np.inf, 1000) # Treat NaN as 0 for investment calculation + p_nom = df.p_nom.fillna(1).replace(np.inf, 100).replace(0, 1) # Avoid division by zero and treat NaN as 1 for unit calculation + return (np.ceil(val / p_nom)).astype(int) + + @staticmethod + def _get_fuel_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: + """Helper to get fuel costs for each row in the DataFrame based on carrier.""" + fuel_mapping = {} + meta_config = metadata.get('Metadata', {}) + for key, fuel_info in meta_config.items(): + if isinstance(fuel_info, dict) and 'filter' in fuel_info and 'cost' in fuel_info: + for carrier in fuel_info['filter']: + fuel_mapping[carrier] = fuel_info['cost'] + + # Map carriers to costs; default to NaN if not found + return df['carrier'].map(fuel_mapping) + + @staticmethod + def EUR_per_hour_to_MWh_per_hour(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + """Converts costs per hour of thermal generation (e.g. stand_by_cost) to costs per MWh based on the fuel cost specified in the metadata.""" + fuel_costs = Conversions._get_fuel_costs(df, metadata) + # Result is MWh/h = (EUR/h) / (EUR/MWh) + # Avoid division by zero, handle NaN/Inf + with np.errstate(divide='ignore', invalid='ignore'): + res = val / fuel_costs + return res.replace([np.inf, -np.inf], 0).fillna(0) + + @staticmethod + def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + """Converts costs to costs per MWh based on the fuel cost specified in the metadata.""" + fuel_costs = Conversions._get_fuel_costs(df, metadata) + # Result is MWh = EUR / (EUR/MWh) + with np.errstate(divide='ignore', invalid='ignore'): + res = val / fuel_costs + return res.replace([np.inf, -np.inf], 0).fillna(0) class NetworkDataExtractor: @@ -105,12 +145,6 @@ def __init__(self, network: pypsa.Network, config_path: str = None): 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', 'YearDecom', 'lat', 'long', 'pOMVarCostEUR', 'InvestCostEUR', 'dataPackage', 'dataSource'], - # "dPower_RoR": ['tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'MaxCons', 'DisEffic', - # 'ChEffic', 'Qmax', 'Qmin', 'InertiaConst', 'MinReserve', 'IniReserve', - # 'IsHydro', 'OMVarCost', 'EnableInvest', 'MaxInvest', 'InvestCostPerMW', - # 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', - # 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', - # 'YearDecom', 'lat', 'long', 'InvestCostEUR'], "dPower_Demand": ['value'], "dPower_Inflows": ['value'], } @@ -121,37 +155,46 @@ def __init__(self, network: pypsa.Network, config_path: str = None): # reorder columns self.dataframes = self._reorder_columns() - def _get_unit_factor(self, pypsa_unit: str, lego_unit: str) -> float: - """Calculates conversion factor based on metric prefixes (e.g., MW to kW).""" - if not pypsa_unit or not lego_unit or pypsa_unit == lego_unit: - return 1.0 - - # Power of 10 mapping for metric prefixes - prefixes = {'T': 12, 'G': 9, 'M': 6, 'k': 3, '': 0, 'm': -3, 'u': -6, 'n': -9} - def split_unit(u): - if len(u) > 1 and u[0] in prefixes and (u[1:] in ['W', 'V', 'EUR', 'Wh', 'g', 'l']): - return u[0], u[1:] - return '', u + def _extract_dataframes(self): + df_dict = {} - p_pre, p_base = split_unit(pypsa_unit) - l_pre, l_base = split_unit(lego_unit) + for table_name, cfg in self.config.items(): + if table_name == "Metadata": + continue - # Specific handling for EUR/MEUR if they are treated as base units - if pypsa_unit == "MEUR" and lego_unit == "EUR": return 1e6 - if pypsa_unit == "EUR" and lego_unit == "MEUR": return 1e-6 + # Resolve filter from category if defined + if 'category' in cfg: + category = cfg['category'] + meta_data = self.config.get('Metadata', {}) + meta_cat = meta_data.get(category, {}) - if p_base != l_base: - return 1.0 + # Combine filters from listed technologies or use direct filter + cat_filter = meta_cat.get('filter', []) + if isinstance(cat_filter, (str, int, float)): + cat_filter = [cat_filter] + else: + cat_filter = list(cat_filter) - return 10 ** (prefixes[p_pre] - prefixes[l_pre]) + for tech in meta_cat.get('technologies', []): + tech_filter = meta_data.get(tech, {}).get('filter', []) + if isinstance(tech_filter, list): + cat_filter.extend(tech_filter) + else: + cat_filter.append(tech_filter) - def _extract_dataframes(self): - df_dict = {} + if cat_filter: + if 'source' not in cfg: + cfg['source'] = {} + # Ensure uniqueness and format as query string + unique_filter = list(set(cat_filter)) + cfg['source']['filter'] = f"carrier in {unique_filter}" - for table_name, cfg in self.config.items(): # 1. Get Source Data - src = cfg['source'] + src = cfg.get('source') + if not src: + continue + if src['type'] == 'attribute': source_df = getattr(self.network, src['name']) if 'filter' in src: @@ -183,20 +226,21 @@ def _extract_dataframes(self): sig = inspect.signature(conv_func) params = list(sig.parameters.values()) - if len(params) >= 2: - # Vectorized call: (Series, DataFrame) - val = conv_func(val, source_df) - else: - # Vectorized call: (Series) - val = conv_func(val) + try: + if len(params) == 3: + # Vectorized call: (Series, DataFrame, config) + val = conv_func(val, source_df, self.config) + elif len(params) == 2: + # Vectorized call: (Series, DataFrame) + val = conv_func(val, source_df) + else: + # Vectorized call: (Series) + val = conv_func(val) + except Exception as e: + raise ValueError(f"Error applying conversion '{conv_name}' to column '{lego_col}': {e}") else: print(f"Warning: Conversion function '{conv_name}' not found in Conversions class.") - # Priority 2: Explicit unit strings - elif 'pypsa_unit' in mapping and 'lego_unit' in mapping: - factor = self._get_unit_factor(mapping['pypsa_unit'], mapping['lego_unit']) - val = val * factor - # Priority 3: Simple multiplier factor elif 'factor' in mapping: val = val * mapping['factor'] @@ -218,27 +262,25 @@ def _extract_dataframes(self): # 3. Handle Indexing if 'index' in cfg: - # Logic from original PypsaReader for indexing - if table_name == "dPower_BusInfo": - df.index = source_df.index.rename("i") - elif table_name == "dPower_Network": - df.index = pd.MultiIndex.from_frame( - source_df[["bus0", "bus1", "name"]].rename(columns={"bus0": "i", "bus1": "j", "name": "c"}) - ).set_names(["i", "j", "c"]) - elif table_name == "dPower_ThermalGen": - df.index = source_df.index.rename("g") - elif table_name == "dPower_VRESProfiles": - df.index = pd.MultiIndex.from_frame( - source_df[["rp", "generator_id", "k"]].rename(columns={"generator_id": "g"}) - ).set_names(["rp", "g", "k"]) - elif table_name in ["dPower_VRES", "dPower_Storage"]: - df.index = source_df.index.rename("g") - elif table_name == "dPower_Inflows": - df.index = pd.MultiIndex.from_frame(source_df[["rp", "k", "g"]]) - elif table_name == "dPower_Demand": - df.index = pd.MultiIndex.from_frame( - source_df[["rp", "k", "g"]].rename(columns={"g": "i"}) - ).set_names(["rp", "k", "i"]) + idx_cfg = cfg['index'] + if isinstance(idx_cfg, dict): + # MultiIndex from specified columns/attributes + index_data = {} + for lego_idx, pypsa_source in idx_cfg.items(): + if pypsa_source in source_df.columns: + index_data[lego_idx] = source_df[pypsa_source] + elif pypsa_source == "index": + index_data[lego_idx] = source_df.index + else: + index_data[lego_idx] = np.nan + df.index = pd.MultiIndex.from_frame(pd.DataFrame(index_data, index=source_df.index)) + elif isinstance(idx_cfg, str): + # Simple index renaming + df.index = source_df.index.rename(idx_cfg) + elif isinstance(idx_cfg, list): + # Fallback for current list style: assumes columns match LEGO names + df.index = pd.MultiIndex.from_frame(source_df[idx_cfg]).set_names(idx_cfg) + # Add default dataPackage and dataSource if they are all NaN (from PypsaReader) if "Metadata" in self.config.keys() and "dataPackage" in self.config["Metadata"]: diff --git a/mapping_config.yaml b/mapping_config.yaml index fa463b6..6cd0176 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -2,7 +2,50 @@ Metadata: source: type: "metadata" dataPackage: "PyPSA-Import" - dataSource: "scigrid-de" + dataSource: "Eur_2050_1h" + ThermalGen: + technologies: ['naturalGas', 'coal', 'oil', 'nuclear', 'biomass', 'geothermal', 'waste'] + VRES: + technologies: ['solar', 'onshoreWind', 'offshoreWind',] + Storage: + technologies: ['phs', 'hydro'] + VRESProfiles: + technologies: ['solar', 'onshoreWind', 'offshoreWind'] + Inflows: + technologies: ['phs', 'hydro', 'ror'] + naturalGas: + filter: ['Gas', 'OCGT', 'CCGT'] + cost: 100 # EUR/MWh + coal: + filter: ['coal', 'lignite', 'Hard coal', 'Brown coal'] + cost: 50 # EUR/MWh + oil: + filter: ['oil', 'Oil'] + cost: 150 # EUR/MWh + nuclear: + filter: ['nuclear'] + cost: 10 # EUR/MWh + biomass: + filter: ['biomass'] + cost: 70 # EUR/MWh + geothermal: + filter: ['geothermal'] + cost: 60 # EUR/MWh + waste: + filter: ['waste', 'Waste'] + cost: 40 # EUR/MWh + solar: + filter: ['solar', 'Solar', 'solar-hsat'] + onshoreWind: + filter: ['onwind', 'onshore-wind'] + offshoreWind: + filter: ['offwind-float', 'offwind-dc', 'offwind-ac'] + ror: + filter: ['ror', 'Run of River'] + hydro: + filter: ['hydro'] + phs: + filter: ['PHS'] dPower_BusInfo: source: @@ -22,7 +65,10 @@ dPower_Network: source: type: "helper" name: "prepare_ac_lines_and_dc_links" - index: ["i", "j", "c"] + index: + i: "bus0" + j: "bus1" + c: "name" mapping: c: "c1" pRline: "r" @@ -50,10 +96,10 @@ dPower_Network: dPower_ThermalGen: + category: "ThermalGen" source: type: "attribute" name: "generators" - filter: "carrier in ['Gas', 'Hard Coal', 'Waste', 'Brown Coal', 'Oil', 'Other', 'Nuclear', 'Geothermal']" index: "g" mapping: tec: "carrier" @@ -91,19 +137,22 @@ dPower_ThermalGen: conversion: "year_and_lifetime_to_year_decom" dPower_VRESProfiles: + category: "VRESProfiles" source: type: "helper" name: "prepare_renewable_profiles" - filter: "carrier in ['Solar', 'Wind Onshore', 'Wind Offshore']" - index: ["rp", "g"] + index: + rp: "rp" + g: "generator_id" + k: "k" mapping: value: "Capacity" -dPower_VRES: +dPower_VRES: # including ROR + category: "VRES" source: type: "attribute" name: "generators" - filter: "carrier in ['Solar', 'Wind Onshore', 'Wind Offshore']" index: "g" mapping: tec: "carrier" @@ -125,24 +174,8 @@ dPower_VRES: attr: "decom_year" conversion: "year_and_lifetime_to_year_decom" -dPower_RoR: - source: - type: "helper" - name: "prepare_ror_generators" - index: "g" - mapping: - tec: "carrier" - i: "bus" - MaxProd: "max_prod" - MinProd: "min_prod" - DisEffic: "discharge" - IsHydro: "is_hydro" - OMVarCost: "marginal_cost" - EnableInvest: "enable_invest" - MaxInvest: "p_nom_max" - InvestCostPerMW: "capital_cost" - dPower_Storage: + category: "Storage" source: type: "attribute" name: "storage_units" @@ -177,11 +210,14 @@ dPower_Storage: conversion: "year_and_lifetime_to_year_decom" dPower_Inflows: + category: "Inflows" source: type: "helper" name: "prepare_inflow_profiles" - filter: "carrier in ['Pumped Hydro', 'Run of River']" - index: ["rp", "k", "g"] + index: + rp: "rp" + k: "k" + g: "g" mapping: value: "Inflow" @@ -189,6 +225,9 @@ dPower_Demand: source: type: "helper" name: "prepare_demand_profiles" - index: ["rp", "k", "g"] + index: + rp: "rp" + k: "k" + i: "g" mapping: value: "Demand" diff --git a/pypsa_helper.py b/pypsa_helper.py index f49f760..abd6bc8 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -116,9 +116,9 @@ def prepare_inflow_profiles(net, config: dict): print("Warning: No hydro storage units have inflow data. Storage inflow profiles will be empty.") inflow_storage = net.storage_units_t.inflow.copy() else: - inflow_storage = net.storage_units_t.inflow[hydro_ids].copy() + inflow_storage = net.storage_units_t.inflow[existing_inflows].copy() if len(missing_inflows) > 0: - print(f"Warning: The following hydro storage units are missing inflow data and will be skipped: {missing_inflows}") + print(f"Warning: The following hydro storage units are missing inflow data and will not be defined: {missing_inflows}") # Get RoR generator inflows ror_ids = net.generators.query(config["source"]["filter"]).index.to_list() @@ -133,11 +133,11 @@ def prepare_inflow_profiles(net, config: dict): print("Warning: No RoR generators have inflow data. RoR inflow profiles will be empty.") inflow_ror = pd.DataFrame() else: - ror = net.generators[ror_ids].copy() - inflow_ror = net.generators_t.p_max_pu[ror_ids].copy() + ror = net.generators.loc[existing_ror_inflows].copy() + inflow_ror = net.generators_t.p_max_pu[existing_ror_inflows].copy() inflow_ror = inflow_ror.mul(ror["p_nom"], axis=1) if len(missing_ror_inflows) > 0: - print(f"Warning: The following RoR generators are missing inflow data and will be skipped: {missing_ror_inflows}") + print(f"Warning: The following RoR generators are missing inflow data and will not be defined: {missing_ror_inflows}") # Concatenate both: hydro + RoR inflows → [time, generator] combined = pd.concat([inflow_storage, inflow_ror], axis=1) From 8a35990d21e3e25f34697447a667dd48026be01e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 2 Apr 2026 13:32:28 +0300 Subject: [PATCH 06/37] Improve code structure. --- PypsaReader.py | 38 +++++++++++++++++++++++++++++-------- pypsa_helper.py | 50 ++++++++++--------------------------------------- 2 files changed, 40 insertions(+), 48 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index a7e57ec..9db346e 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -47,7 +47,8 @@ def bool_to_binary(val, df: pd.DataFrame = None) -> pd.Series: if val.isnull().all(): return 0 else: - val = val.fillna(False) # Treat NaN as False for binary conversion + # Use infer_objects to explicitly handle the type conversion from object to bool + val = val.fillna(False).infer_objects(copy=False) return val.astype(int) @staticmethod @@ -88,7 +89,15 @@ def _get_fuel_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: fuel_mapping[carrier] = fuel_info['cost'] # Map carriers to costs; default to NaN if not found - return df['carrier'].map(fuel_mapping) + fuel_costs = df['carrier'].map(fuel_mapping) + + # Performance check: only check for missing carriers if DataFrame isn't empty + if not fuel_costs.empty: + missing_carriers = df.loc[fuel_costs.isnull(), 'carrier'].unique() + if len(missing_carriers) > 0: + print(f"Warning: No fuel cost defined in Metadata for carrier(s): {missing_carriers.tolist()}") + + return fuel_costs @staticmethod def EUR_per_hour_to_MWh_per_hour(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: @@ -113,6 +122,7 @@ def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: class NetworkDataExtractor: def __init__(self, network: pypsa.Network, config_path: str = None): self.network = network + self._conv_params_cache = {} # Performance: Cache function signatures if config_path is None: config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") @@ -222,15 +232,19 @@ def _extract_dataframes(self): conv_name = mapping['conversion'].replace('()', '') if hasattr(Conversions, conv_name): conv_func = getattr(Conversions, conv_name) - # Check function signature - sig = inspect.signature(conv_func) - params = list(sig.parameters.values()) + + # Performance Improvement: Cache function parameter counts + if conv_name not in self._conv_params_cache: + sig = inspect.signature(conv_func) + self._conv_params_cache[conv_name] = len(sig.parameters) + + num_params = self._conv_params_cache[conv_name] try: - if len(params) == 3: + if num_params == 3: # Vectorized call: (Series, DataFrame, config) val = conv_func(val, source_df, self.config) - elif len(params) == 2: + elif num_params == 2: # Vectorized call: (Series, DataFrame) val = conv_func(val, source_df) else: @@ -299,6 +313,14 @@ def _extract_dataframes(self): def _add_empty_columns(self): for name, df in self.dataframes.items(): + # Add scenario column + if 'scenario' not in df.columns: + df['scenario'] = 'ScenarioA' + + # Add id column if missing + if 'id' not in df.columns: + df['id'] = np.nan + if name in self.columns: for col in self.columns[name]: if col not in df.columns: @@ -318,7 +340,7 @@ def get_dataframes(self): if __name__ == "__main__": - filepath = os.path.join(os.path.dirname(__file__), "..", "pypsa-eur/resources/test/networks/base_s_39_elec_1year.nc") + filepath = os.path.join(r"C:\BeSt\PyPSA-LEGO-Translator\scigrid-de.nc") if os.path.exists(filepath): net = pypsa.Network(filepath) extractor = NetworkDataExtractor(net) diff --git a/pypsa_helper.py b/pypsa_helper.py index abd6bc8..bbcf8c5 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -5,12 +5,14 @@ def prepare_ac_lines(net, config: dict): lines = net.lines.copy() types = net.line_types - lines["r"] = lines.apply( - lambda row: row["r"] if row["r"] != 0 else types.loc[row["type"]].r_per_length * row["length"], axis=1) - lines["x"] = lines.apply( - lambda row: row["x"] if row["x"] != 0 else types.loc[row["type"]].x_per_length * row["length"], axis=1) - lines["b"] = lines.apply( - lambda row: row["b"] if row["b"] != 0 else types.loc[row["type"]].x_per_length * row["length"], axis=1) + # Map type attributes to lines for vectorized calculation + r_per_len = lines["type"].map(types["r_per_length"]) + x_per_len = lines["type"].map(types["x_per_length"]) + + # Vectorized calculation: replace 0 values with type-based defaults + lines["r"] = lines["r"].where(lines["r"] != 0, r_per_len * lines["length"]) + lines["x"] = lines["x"].where(lines["x"] != 0, x_per_len * lines["length"]) + lines["b"] = lines["b"].where(lines["b"] != 0, x_per_len * lines["length"]) if "name" not in lines.columns or lines["name"].isnull().all(): lines["name"] = "c1" @@ -25,7 +27,8 @@ def prepare_dc_links(net, config: dict): links["b"] = 0.0 links["pmax"] = links["p_nom"] links["id"] = links.index - links["name"] = [f"DC_Link_{i}" for i in range(len(links))] + # Vectorized name generation + links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype(str) return links[["bus0", "bus1", "r", "x", "b", "pmax", "id", "name"]] @@ -36,25 +39,6 @@ def prepare_ac_lines_and_dc_links(net, config: dict): return pd.concat([ac_lines, dc_links], ignore_index=True) -def prepare_thermal_generators(net, config: dict): - thermal_types = ['OCGT', 'biomass', 'CCGT', 'nuclear', 'oil', 'coal', 'lignite'] - gens = net.generators.copy() - gens = gens[gens.carrier.isin(thermal_types)] - - gens["max_prod"] = gens["p_max_pu"] * gens["p_nom"] - gens["min_prod"] = gens["p_min_pu"] * gens["p_nom"] - gens["ramp_up"] = gens["ramp_limit_up"] * gens["p_nom"] - gens["ramp_down"] = gens["ramp_limit_down"] * gens["p_nom"] - gens["enable_invest"] = gens["p_nom_extendable"].astype(int) - - gens["id"] = gens.index - return gens[[ - "id", "carrier", "bus", "max_prod", "min_prod", - "ramp_up", "ramp_down", "start_up_cost", - "enable_invest", "capital_cost", "marginal_cost" - ]] - - def prepare_renewable_profiles(net, config: dict): # renewable_types = ['Solar', 'Wind Onshore', 'Wind Offshore'] gens = net.generators.copy() @@ -71,20 +55,6 @@ def prepare_renewable_profiles(net, config: dict): return profiles # flat, column-based, no index set yet -def prepare_renewable_generators(net, config: dict): - renewable_types = ['solar-hsat', 'onwind', 'solar'] - gens = net.generators.copy() - vres = gens[gens.carrier.isin(renewable_types)].copy() - - vres["max_prod"] = vres["p_max_pu"] * vres["p_nom"] - vres["enable_invest"] = vres["p_nom_extendable"].astype(int) - - vres["id"] = vres.index.values - return vres[[ - "id", "carrier", "bus", "max_prod", - "enable_invest", "p_nom_max", "capital_cost", "marginal_cost" - ]] - def prepare_ror_generators(net, config: dict): ror = net.generators[net.generators.carrier == "ror"].copy() From bb30bab7a17980aca7fea91426b8a8e99a929261 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 2 Apr 2026 14:27:32 +0300 Subject: [PATCH 07/37] Add transformers to line conversions. --- PypsaReader.py | 272 ++++++++++++++++++++++++-------------------- mapping_config.yaml | 29 +++-- pypsa_helper.py | 41 +++---- 3 files changed, 186 insertions(+), 156 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 9db346e..f0ee6dc 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -167,149 +167,169 @@ def __init__(self, network: pypsa.Network, config_path: str = None): def _extract_dataframes(self): + """Extracts and transforms data from the PyPSA network into LEGO DataFrames.""" df_dict = {} for table_name, cfg in self.config.items(): if table_name == "Metadata": continue - # Resolve filter from category if defined - if 'category' in cfg: - category = cfg['category'] - meta_data = self.config.get('Metadata', {}) - meta_cat = meta_data.get(category, {}) + # 1. Resolve Filters from Category + self._resolve_category_filters(cfg) - # Combine filters from listed technologies or use direct filter - cat_filter = meta_cat.get('filter', []) - if isinstance(cat_filter, (str, int, float)): - cat_filter = [cat_filter] - else: - cat_filter = list(cat_filter) - - for tech in meta_cat.get('technologies', []): - tech_filter = meta_data.get(tech, {}).get('filter', []) - if isinstance(tech_filter, list): - cat_filter.extend(tech_filter) - else: - cat_filter.append(tech_filter) - - if cat_filter: - if 'source' not in cfg: - cfg['source'] = {} - # Ensure uniqueness and format as query string - unique_filter = list(set(cat_filter)) - cfg['source']['filter'] = f"carrier in {unique_filter}" - - # 1. Get Source Data - src = cfg.get('source') - if not src: + # 2. Get Source Data + source_df = self._get_source_df(cfg) + if source_df is None: continue - if src['type'] == 'attribute': - source_df = getattr(self.network, src['name']) - if 'filter' in src: - source_df = source_df.query(src['filter']) - elif src['type'] == 'helper': - source_df = getattr(h, src['name'])(self.network, cfg) - else: - continue + # 3. Process Column Mapping + df = self._map_columns(source_df, cfg) - # 2. Process Mapping - column_data = {} - # if 'mapping' in cfg.keys(): - for lego_col, mapping in cfg['mapping'].items(): - if isinstance(mapping, dict): - # Attribute mapping with potential unit conversion or transformation function - attr = mapping.get('attr') - if attr and attr in source_df.columns: - val = source_df[attr] - else: - # Try to get value as series of NaNs or fixed value - val = pd.Series(mapping.get('value', np.nan), index=source_df.index) - - # Priority 1: Named conversion function - if 'conversion' in mapping: - conv_name = mapping['conversion'].replace('()', '') - if hasattr(Conversions, conv_name): - conv_func = getattr(Conversions, conv_name) - - # Performance Improvement: Cache function parameter counts - if conv_name not in self._conv_params_cache: - sig = inspect.signature(conv_func) - self._conv_params_cache[conv_name] = len(sig.parameters) - - num_params = self._conv_params_cache[conv_name] - - try: - if num_params == 3: - # Vectorized call: (Series, DataFrame, config) - val = conv_func(val, source_df, self.config) - elif num_params == 2: - # Vectorized call: (Series, DataFrame) - val = conv_func(val, source_df) - else: - # Vectorized call: (Series) - val = conv_func(val) - except Exception as e: - raise ValueError(f"Error applying conversion '{conv_name}' to column '{lego_col}': {e}") - else: - print(f"Warning: Conversion function '{conv_name}' not found in Conversions class.") - - # Priority 3: Simple multiplier factor - elif 'factor' in mapping: - val = val * mapping['factor'] - - column_data[lego_col] = val - elif isinstance(mapping, (int, float)): - # Static value - column_data[lego_col] = mapping - else: - # Direct attribute string mapping - if mapping in source_df.columns: - column_data[lego_col] = source_df[mapping] - else: - column_data[lego_col] = np.nan + # 4. Handle Indexing + if 'index' in cfg: + self._apply_indexing(df, source_df, cfg['index']) - # Todo: Add warning if the column defined in mapping is not available in the LEGO columns!!! + # 5. Normalize for LEGO Format + df = self._add_scenario_columns(df) - df = pd.DataFrame(column_data) + df_dict[table_name] = df - # 3. Handle Indexing - if 'index' in cfg: - idx_cfg = cfg['index'] - if isinstance(idx_cfg, dict): - # MultiIndex from specified columns/attributes - index_data = {} - for lego_idx, pypsa_source in idx_cfg.items(): - if pypsa_source in source_df.columns: - index_data[lego_idx] = source_df[pypsa_source] - elif pypsa_source == "index": - index_data[lego_idx] = source_df.index - else: - index_data[lego_idx] = np.nan - df.index = pd.MultiIndex.from_frame(pd.DataFrame(index_data, index=source_df.index)) - elif isinstance(idx_cfg, str): - # Simple index renaming - df.index = source_df.index.rename(idx_cfg) - elif isinstance(idx_cfg, list): - # Fallback for current list style: assumes columns match LEGO names - df.index = pd.MultiIndex.from_frame(source_df[idx_cfg]).set_names(idx_cfg) - - - # Add default dataPackage and dataSource if they are all NaN (from PypsaReader) - if "Metadata" in self.config.keys() and "dataPackage" in self.config["Metadata"]: - df['dataPackage'] = self.config["Metadata"]["dataPackage"] + return df_dict + + + def _resolve_category_filters(self, cfg): + """Resolves technology filters from Metadata if a category is defined.""" + if 'category' not in cfg: + return + + category = cfg['category'] + meta_data = self.config.get('Metadata', {}) + meta_cat = meta_data.get(category, {}) + + # Combine filters from listed technologies or use direct filter + cat_filter = meta_cat.get('filter', []) + if isinstance(cat_filter, (str, int, float)): + cat_filter = [cat_filter] + else: + cat_filter = list(cat_filter) + + for tech in meta_cat.get('technologies', []): + tech_filter = meta_data.get(tech, {}).get('filter', []) + if isinstance(tech_filter, list): + cat_filter.extend(tech_filter) else: - df['dataPackage'] = 'default-package' + cat_filter.append(tech_filter) + + if cat_filter: + if 'source' not in cfg: + cfg['source'] = {} + # Ensure uniqueness and format as query string + unique_filter = list(set(cat_filter)) + cfg['source']['filter'] = f"carrier in {unique_filter}" + + + def _get_source_df(self, cfg): + """Retrieves the source DataFrame based on the configuration.""" + src = cfg.get('source') + if not src: + return None + + if src['type'] == 'attribute': + source_df = getattr(self.network, src['name']) + if 'filter' in src: + source_df = source_df.query(src['filter']) + return source_df + elif src['type'] == 'helper': + return getattr(h, src['name'])(self.network, cfg) + return None + + + def _map_columns(self, source_df, cfg): + """Maps PyPSA attributes to LEGO columns using the mapping configuration.""" + column_data = {} + for lego_col, mapping in cfg.get('mapping', {}).items(): + if isinstance(mapping, dict): + # Attribute mapping with potential unit conversion or transformation function + attr = mapping.get('attr') + if attr and attr in source_df.columns: + val = source_df[attr] + else: + # Try to get value as series of NaNs or fixed value + val = pd.Series(mapping.get('value', np.nan), index=source_df.index) + + # Priority 1: Named conversion function + if 'conversion' in mapping: + conv_name = mapping['conversion'].replace('()', '') + if hasattr(Conversions, conv_name): + conv_func = getattr(Conversions, conv_name) + + # Performance Improvement: Cache function parameter counts + if conv_name not in self._conv_params_cache: + sig = inspect.signature(conv_func) + self._conv_params_cache[conv_name] = len(sig.parameters) + + num_params = self._conv_params_cache[conv_name] + + try: + if num_params == 3: + # Vectorized call: (Series, DataFrame, config) + val = conv_func(val, source_df, self.config) + elif num_params == 2: + # Vectorized call: (Series, DataFrame) + val = conv_func(val, source_df) + else: + # Vectorized call: (Series) + val = conv_func(val) + except Exception as e: + raise ValueError(f"Error applying conversion '{conv_name}' to column '{lego_col}': {e}") + else: + print(f"Warning: Conversion function '{conv_name}' not found in Conversions class.") + + # Priority 3: Simple multiplier factor + elif 'factor' in mapping: + val = val * mapping['factor'] - if "Metadata" in self.config.keys() and "dataSource" in self.config["Metadata"]: - df['dataSource'] = self.config["Metadata"]["dataSource"] + column_data[lego_col] = val + elif isinstance(mapping, (int, float)): + # Static value + column_data[lego_col] = mapping else: - df['dataSource'] = 'default-source' + # Direct attribute string mapping + if mapping in source_df.columns: + column_data[lego_col] = source_df[mapping] + else: + column_data[lego_col] = np.nan + return pd.DataFrame(column_data) - df_dict[table_name] = df + @staticmethod + def _apply_indexing(df, source_df, idx_cfg): + """Applies the indexing logic to the DataFrame.""" + if isinstance(idx_cfg, dict): + # MultiIndex from specified columns/attributes + index_data = {} + for lego_idx, pypsa_source in idx_cfg.items(): + if pypsa_source in source_df.columns: + index_data[lego_idx] = source_df[pypsa_source] + elif pypsa_source == "index": + index_data[lego_idx] = source_df.index + else: + index_data[lego_idx] = np.nan + df.index = pd.MultiIndex.from_frame(pd.DataFrame(index_data, index=source_df.index)) + elif isinstance(idx_cfg, str): + # Simple index renaming + df.index = source_df.index.rename(idx_cfg) + elif isinstance(idx_cfg, list): + # Fallback for current list style: assumes columns match LEGO names + df.index = pd.MultiIndex.from_frame(source_df[idx_cfg]).set_names(idx_cfg) + + + def _add_scenario_columns(self, df) -> pd.DataFrame: + """Adds dataPackage and dataSource columns based on config or defaults.""" + meta = self.config.get('Metadata', {}) + df['dataPackage'] = meta.get('dataPackage', 'default-package') + df['dataSource'] = meta.get('dataSource', 'default-source') + return df - return df_dict def _add_empty_columns(self): for name, df in self.dataframes.items(): @@ -327,6 +347,7 @@ def _add_empty_columns(self): df[col] = np.nan return self.dataframes + def _reorder_columns(self): for name, df in self.dataframes.items(): if name in self.columns: @@ -335,6 +356,7 @@ def _reorder_columns(self): self.dataframes[name] = df return self.dataframes + def get_dataframes(self): return self.dataframes diff --git a/mapping_config.yaml b/mapping_config.yaml index 6cd0176..046147d 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -47,6 +47,7 @@ Metadata: phs: filter: ['PHS'] + dPower_BusInfo: source: type: "attribute" @@ -61,6 +62,7 @@ dPower_BusInfo: lat: "y" lon: "x" + dPower_Network: source: type: "helper" @@ -136,17 +138,6 @@ dPower_ThermalGen: attr: "decom_year" conversion: "year_and_lifetime_to_year_decom" -dPower_VRESProfiles: - category: "VRESProfiles" - source: - type: "helper" - name: "prepare_renewable_profiles" - index: - rp: "rp" - g: "generator_id" - k: "k" - mapping: - value: "Capacity" dPower_VRES: # including ROR category: "VRES" @@ -174,6 +165,7 @@ dPower_VRES: # including ROR attr: "decom_year" conversion: "year_and_lifetime_to_year_decom" + dPower_Storage: category: "Storage" source: @@ -209,6 +201,7 @@ dPower_Storage: attr: "decom_year" conversion: "year_and_lifetime_to_year_decom" + dPower_Inflows: category: "Inflows" source: @@ -221,6 +214,7 @@ dPower_Inflows: mapping: value: "Inflow" + dPower_Demand: source: type: "helper" @@ -231,3 +225,16 @@ dPower_Demand: i: "g" mapping: value: "Demand" + + +dPower_VRESProfiles: + category: "VRESProfiles" + source: + type: "helper" + name: "prepare_renewable_profiles" + index: + rp: "rp" + g: "generator_id" + k: "k" + mapping: + value: "Capacity" diff --git a/pypsa_helper.py b/pypsa_helper.py index bbcf8c5..3be9012 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -14,6 +14,10 @@ def prepare_ac_lines(net, config: dict): lines["x"] = lines["x"].where(lines["x"] != 0, x_per_len * lines["length"]) lines["b"] = lines["b"].where(lines["b"] != 0, x_per_len * lines["length"]) + # Add tap ratios and phase shifts with default values (if not already present) + lines['tap_ratio'] = 1 + lines['phase_shift'] = 0 + if "name" not in lines.columns or lines["name"].isnull().all(): lines["name"] = "c1" @@ -25,18 +29,32 @@ def prepare_dc_links(net, config: dict): links["r"] = 0.0 links["x"] = 0.0 links["b"] = 0.0 - links["pmax"] = links["p_nom"] + links["s_nom"] = links["p_nom"] + links["s_nom_extendable"] = links["p_nom_extendable"] links["id"] = links.index # Vectorized name generation links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype(str) - return links[["bus0", "bus1", "r", "x", "b", "pmax", "id", "name"]] + # Add tap ratios and phase shifts with default values (if not already present) + links['tap_ratio'] = 1 + links['phase_shift'] = 0 + + return links + + +def prepare_transformers(net, config: dict) -> pd.DataFrame: + transformers = net.transformers.copy() + if "name" not in transformers.columns or transformers["name"].isnull().all(): + transformers["name"] = "c1" + + return transformers def prepare_ac_lines_and_dc_links(net, config: dict): ac_lines = prepare_ac_lines(net, config) dc_links = prepare_dc_links(net, config) - return pd.concat([ac_lines, dc_links], ignore_index=True) + transformers = prepare_transformers(net, config) + return pd.concat([ac_lines, dc_links, transformers], ignore_index=True) def prepare_renewable_profiles(net, config: dict): @@ -55,23 +73,6 @@ def prepare_renewable_profiles(net, config: dict): return profiles # flat, column-based, no index set yet - -def prepare_ror_generators(net, config: dict): - ror = net.generators[net.generators.carrier == "ror"].copy() - - ror["id"] = ror.index - ror["max_prod"] = ror["p_max_pu"] * ror["p_nom"] - ror["min_prod"] = ror["p_min_pu"] * ror["p_nom"] - ror["discharge"] = ror["efficiency"] - ror["is_hydro"] = 1 - ror["enable_invest"] = ror["p_nom_extendable"].astype(int) - - return ror[[ - "id", "carrier", "bus", "max_prod", "min_prod", "discharge", "is_hydro", - "marginal_cost", "enable_invest", "p_nom_max", "capital_cost" - ]] - - def prepare_inflow_profiles(net, config: dict): # Get hydro storage inflows hydro_ids = net.storage_units.query(config["source"]["filter"]).index.to_list() From 79dda9550e41ae5aff2a6b8fcab436b9b2e20f7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 2 Apr 2026 17:09:11 +0300 Subject: [PATCH 08/37] Use predefined tables for dataframes. --- PypsaReader.py | 94 ++++++++++++++++++++++++++++----------------- mapping_config.yaml | 80 +++++++++++++++++++++++++------------- pypsa_helper.py | 38 +++++++++++++++--- 3 files changed, 144 insertions(+), 68 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index f0ee6dc..d1a1b86 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -54,11 +54,17 @@ def bool_to_binary(val, df: pd.DataFrame = None) -> pd.Series: @staticmethod def year_and_lifetime_to_year_decom(val, df: pd.DataFrame) -> pd.Series: """Calculates decommissioning year based on commissioning year and lifetime.""" - # Only return a decom year if build_year and lifetime is available; otherwise return NaN - if df.build_year.isnull().all() and df.lifetime.isnull().all(): - return np.nan - else: - return df.build_year + df.lifetime + # Only return a decom year if build_year and lifetime is available and build_year is not 0; otherwise return NaN + mask = df.build_year.notnull() & (df.build_year != 0) & df.lifetime.notnull() + res = pd.Series(np.nan, index=df.index) + res[mask] = df.loc[mask, 'build_year'] + df.loc[mask, 'lifetime'] + return res + + @staticmethod + def is_ldes(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + """Sets binary to 1 if energy-to-power ratio (max_hours) is greater than the threshold in metadata.""" + threshold = metadata.get('Metadata', {}).get('LDES_threshold', 2160) + return (val > threshold).astype(int) @staticmethod def line_carrier_to_tec_repr(val, df: pd.DataFrame) -> pd.Series: @@ -99,6 +105,11 @@ def _get_fuel_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: return fuel_costs + @staticmethod + def get_fuel_cost_from_metadata(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + """Retrieves fuel costs based on the carrier and Metadata configuration.""" + return Conversions._get_fuel_costs(df, metadata) + @staticmethod def EUR_per_hour_to_MWh_per_hour(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: """Converts costs per hour of thermal generation (e.g. stand_by_cost) to costs per MWh based on the fuel cost specified in the metadata.""" @@ -120,50 +131,55 @@ def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: class NetworkDataExtractor: - def __init__(self, network: pypsa.Network, config_path: str = None): + def __init__(self, network: pypsa.Network, config_path: str = None, table_definitions_path: str = None): self.network = network self._conv_params_cache = {} # Performance: Cache function signatures + if config_path is None: config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") + + if table_definitions_path is None: + table_definitions_path = os.path.join(os.path.dirname(__file__), "TableDefinitions.xml") with open(config_path, 'r') as f: self.config = yaml.safe_load(f) - # The expected columns for each table (used for reordering and filling empties) - self.columns = { - "dPower_BusInfo": ['excl', 'id', 'z', 'pBusBaseV', 'pBusMaxV', 'pBusMinV', 'pBusB', - 'pBusG', 'pBus_pf', 'YearCom', 'YearDecom', 'lat', 'lon', 'zoi', - 'dataPackage', 'dataSource'], - "dPower_Network": ['excl', 'id', 'pRline', 'pXline', 'pBcline', 'pAngle', 'pRatio', - 'pPmax', 'pEnableInvest', 'pFOMCost', 'pInvestCost', 'pTecRepr', - 'YearCom', 'YearDecom', 'dataPackage', 'dataSource'], - "dPower_ThermalGen": ['excl', 'id', 'tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'RampUp', - 'RampDw', 'MinUpTime', 'MinDownTime', 'Qmax', 'Qmin', 'InertiaConst', - 'FuelCost', 'Efficiency', 'CommitConsumption', 'OMVarCost', - 'StartupConsumption', 'EFOR', 'EnableInvest', 'InvestCost', - 'FirmCapCoef', 'CO2Emis', 'YearCom', 'YearDecom', 'lat', 'long', - 'dataPackage', 'dataSource', 'pSlopeVarCostEUR', 'pInterVarCostEUR', - 'pStartupCostEUR', 'MaxInvest', 'InvestCostEUR'], - "dPower_VRESProfiles": ['value'], - "dPower_VRES": ['excl', 'id', 'tec', 'i', 'ExisUnits', 'MaxProd', 'EnableInvest', - 'MaxInvest', 'InvestCost', 'OMVarCost', 'FirmCapCoef', 'Qmax', 'Qmin', - 'InertiaConst', 'YearCom', 'YearDecom', 'lat', 'lon', 'dataPackage', - 'dataSource', 'MinProd', 'InvestCostEUR'], - "dPower_Storage": ['tec', 'i', 'ExisUnits', 'MaxProd', 'MinProd', 'MaxCons', 'DisEffic', - 'ChEffic', 'Qmax', 'Qmin', 'InertiaConst', 'MinReserve', 'IniReserve', - 'IsHydro', 'OMVarCost', 'EnableInvest', 'MaxInvest', 'InvestCostPerMW', - 'InvestCostPerMWh', 'Ene2PowRatio', 'ReplaceCost', 'ShelfLife', - 'FirmCapCoef', 'CDSF_alpha', 'CDSF_beta', 'PPName', 'YearCom', - 'YearDecom', 'lat', 'long', 'pOMVarCostEUR', 'InvestCostEUR', 'dataPackage', 'dataSource'], - "dPower_Demand": ['value'], - "dPower_Inflows": ['value'], - } + # Automatically load expected columns from TableDefinitions.xml + self.columns = self._load_table_definitions(table_definitions_path) self.dataframes = self._extract_dataframes() # add empty columns self.dataframes = self._add_empty_columns() # reorder columns - self.dataframes = self._reorder_columns() + # self.dataframes = self._reorder_columns() + + @staticmethod + def _load_table_definitions(xml_path): + """Parses TableDefinitions.xml to determine expected LEGO columns for each table.""" + if not os.path.exists(xml_path): + print(f"Warning: Table definitions file not found at {xml_path}. Using fallback column lists.") + return {} + + import xml.etree.ElementTree as ET + try: + tree = ET.parse(xml_path) + root = tree.getroot() + table_cols = {} + for table in root.findall(".//TableDefinition"): + # Map XML ID (Power_BusInfo) to internal ID (dPower_BusInfo) + table_id = "d" + table.get("id") + cols = [] + columns_node = table.find("Columns") + if columns_node is not None: + for col in columns_node: + col_id = col.get("id") + if col_id: + cols.append(col_id) + table_cols[table_id] = cols + return table_cols + except Exception as e: + print(f"Error parsing TableDefinitions.xml: {e}") + return {} def _extract_dataframes(self): @@ -190,6 +206,12 @@ def _extract_dataframes(self): self._apply_indexing(df, source_df, cfg['index']) # 5. Normalize for LEGO Format + # Drop columns that are now in the index to avoid "already exists" error on reset_index + for idx_name in df.index.names: + if idx_name in df.columns: + df = df.drop(columns=[idx_name]) + + df = df.reset_index() df = self._add_scenario_columns(df) df_dict[table_name] = df diff --git a/mapping_config.yaml b/mapping_config.yaml index 046147d..b1d1e43 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -3,49 +3,71 @@ Metadata: type: "metadata" dataPackage: "PyPSA-Import" dataSource: "Eur_2050_1h" - ThermalGen: - technologies: ['naturalGas', 'coal', 'oil', 'nuclear', 'biomass', 'geothermal', 'waste'] - VRES: - technologies: ['solar', 'onshoreWind', 'offshoreWind',] - Storage: - technologies: ['phs', 'hydro'] - VRESProfiles: - technologies: ['solar', 'onshoreWind', 'offshoreWind'] - Inflows: - technologies: ['phs', 'hydro', 'ror'] + LDES_threshold: 3 # hours + + # Define names of carriers and their corresponding fuel costs (if thermal generator) for every technology + # [technology]: + # filter: list of strings to filter the carrier name in the PyPSA network + # cost: fuel cost in EUR/MWh (only for thermal generators, not needed naturalGas: - filter: ['Gas', 'OCGT', 'CCGT'] + filter: [ 'Gas', 'gas' ] + cost: 100 # EUR/MWh + openCycleGasTurbine: + filter: [ 'OCGT' ] + cost: 100 # EUR/MWh + closedCycleGasTurbine: + filter: [ 'CCGT' ] cost: 100 # EUR/MWh coal: - filter: ['coal', 'lignite', 'Hard coal', 'Brown coal'] + filter: [ 'coal' ] cost: 50 # EUR/MWh + hardCoal: + filter: [ 'Hard coal' ] + cost: 15 # EUR/MWh + brownCoal_lignite: + filter: [ 'Brown coal', 'lignite' ] + cost: 22 # EUR/MWh oil: - filter: ['oil', 'Oil'] - cost: 150 # EUR/MWh + filter: [ 'oil', 'Oil' ] + cost: 70 # EUR/MWh nuclear: - filter: ['nuclear'] - cost: 10 # EUR/MWh + filter: [ 'Nuclear', 'nuclear' ] + cost: 50 # EUR/MWh biomass: - filter: ['biomass'] + filter: [ 'biomass' ] cost: 70 # EUR/MWh geothermal: - filter: ['geothermal'] + filter: [ 'Geothermal', 'geothermal' ] cost: 60 # EUR/MWh waste: - filter: ['waste', 'Waste'] + filter: [ 'waste', 'Waste' ] cost: 40 # EUR/MWh solar: - filter: ['solar', 'Solar', 'solar-hsat'] + filter: [ 'solar', 'Solar', 'solar-hsat' ] onshoreWind: - filter: ['onwind', 'onshore-wind'] + filter: [ 'Wind Onshore', 'onwind', 'onshore-wind' ] offshoreWind: - filter: ['offwind-float', 'offwind-dc', 'offwind-ac'] + filter: [ 'Wind Offshore', 'offwind-float', 'offwind-dc', 'offwind-ac' ] ror: - filter: ['ror', 'Run of River'] + filter: [ 'ror', 'Run of River' ] hydro: - filter: ['hydro'] + filter: [ 'Storage Hydro', 'hydro' ] phs: - filter: ['PHS'] + filter: [ 'PHS', 'Pumped Hydro' ] + + # Define categories of technologies and their corresponding technologies. + # The technologies must have the same name as specified in the mapping above. + ThermalGen: + technologies: [ 'naturalGas', 'openCycleGasTurbine', 'closedCycleGasTurbine', 'coal', 'hardCoal', 'brownCoal_lignite', 'oil', 'nuclear', 'biomass', 'geothermal', 'waste' ] + VRES: + technologies: [ 'solar', 'onshoreWind', 'offshoreWind', ] + Storage: + technologies: [ 'phs', 'hydro' ] + VRESProfiles: + technologies: [ 'solar', 'onshoreWind', 'offshoreWind' ] + Inflows: + technologies: [ 'phs', 'hydro', 'ror' ] + dPower_BusInfo: @@ -122,6 +144,9 @@ dPower_ThermalGen: MinUpTime: "min_up_time" MinDownTime: "min_down_time" Efficiency: "efficiency" + FuelCost: + attr: "carrier" + conversion: "get_fuel_cost_from_metadata" CommitConsumption: attr: "stand_by_cost" conversion: "EUR_per_hour_to_MWh_per_hour" @@ -187,6 +212,9 @@ dPower_Storage: ChEffic: "efficiency_store" Self Discharge: "standing_loss" IniReserve: "state_of_charge_initial" + IsLDES: + attr: "max_hours" + conversion: "is_ldes" OMVarCost: "marginal_cost" EnableInvest: attr: "p_nom_extendable" @@ -222,7 +250,7 @@ dPower_Demand: index: rp: "rp" k: "k" - i: "g" + i: "n" mapping: value: "Demand" diff --git a/pypsa_helper.py b/pypsa_helper.py index 3be9012..db0f10a 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -1,11 +1,12 @@ import pandas as pd +import numpy as np def prepare_ac_lines(net, config: dict): lines = net.lines.copy() types = net.line_types - # Map type attributes to lines for vectorized calculation + # Define the line parameters r, x, b if only line type is specified r_per_len = lines["type"].map(types["r_per_length"]) x_per_len = lines["type"].map(types["x_per_length"]) @@ -21,20 +22,30 @@ def prepare_ac_lines(net, config: dict): if "name" not in lines.columns or lines["name"].isnull().all(): lines["name"] = "c1" + # Todo: Add robust checking if carrier of line is defined! + # if carrier is not defined, set to AC for all lines to get defined as DC-OPF + lines.carrier = lines.carrier.fillna('AC') + return lines def prepare_dc_links(net, config: dict): links = net.links[net.links["carrier"] == "DC"].copy() - links["r"] = 0.0 - links["x"] = 0.0 - links["b"] = 0.0 + + # Define line parameters as nan for DC links, as they are not relevant for DC-OPF + links["r"] = np.nan + links["x"] = np.nan + links["b"] = np.nan + + # Define s_nom and s_nom_extendable to be consistent with lines and transformers links["s_nom"] = links["p_nom"] links["s_nom_extendable"] = links["p_nom_extendable"] links["id"] = links.index # Vectorized name generation links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype(str) + links.carrier = links.carrier.fillna('DC') + # Add tap ratios and phase shifts with default values (if not already present) links['tap_ratio'] = 1 links['phase_shift'] = 0 @@ -44,6 +55,21 @@ def prepare_dc_links(net, config: dict): def prepare_transformers(net, config: dict) -> pd.DataFrame: transformers = net.transformers.copy() + types = net.transformer_types + + # Calculate r, x, b, based on transformer type if specified + vsc = transformers["type"].map(types["vsc"]) + nlc = transformers["type"].map(types["i0"]) + pfe = transformers["type"].map(types["pfe"]) + g = pfe / (1000 * transformers.s_nom) + + transformers["r"] = transformers["r"].where(transformers["r"] != 0, vsc / 100) + transformers["x"] = transformers["x"].where(transformers["x"] != 0, np.sqrt((vsc/100) ** 2 - transformers.r ** 2)) + transformers["b"] = transformers["b"].where(transformers["b"] != 0, - np.sqrt((nlc / 100) ** 2 - g ** 2)) + + # Set carrier to AC for all transformers to get defined as DC-OPF + transformers["carrier"] = "AC" + if "name" not in transformers.columns or transformers["name"].isnull().all(): transformers["name"] = "c1" @@ -127,6 +153,6 @@ def prepare_demand_profiles(net, config: dict): df = net.loads_t.p_set.copy() # shape: [time, load_id] df = df.rename_axis("k").reset_index() # 'k' = time - demand_long = df.melt(id_vars="k", var_name="g", value_name="Demand") + demand_long = df.melt(id_vars="k", var_name="n", value_name="Demand") demand_long["rp"] = "rp01" - return demand_long[["rp", "g", "k", "Demand"]] + return demand_long[["rp", "n", "k", "Demand"]] From 05abafdb73a1a1a1e8aa7c1d093942f6a243a8e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 3 Apr 2026 16:58:48 +0300 Subject: [PATCH 09/37] Add functionality to translate whole case studies in the file PypsaReader.py. --- PypsaReader.py | 157 +++++++++++++++++++++++++------------------- mapping_config.yaml | 2 +- pypsa_helper.py | 8 +-- 3 files changed, 96 insertions(+), 71 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index d1a1b86..1978c7b 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -6,6 +6,7 @@ import pypsa import yaml +from ExcelWriter import ExcelWriter import pypsa_helper as h @@ -24,10 +25,12 @@ def MEUR_to_EUR(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: @staticmethod def MW_to_kW(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: + """Converts MW to kW.""" return val * 1e3 @staticmethod def V_to_kV(val, df: pd.DataFrame = None) -> pd.Series: + """Converts V to kV.""" return val * 1e-3 @staticmethod @@ -68,6 +71,7 @@ def is_ldes(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: @staticmethod def line_carrier_to_tec_repr(val, df: pd.DataFrame) -> pd.Series: + """Maps line carriers to LEGO's technology representations (e.g., DC-OPF for AC lines).""" if df.carrier.isnull().all(): return 'DC-OPF' else: @@ -132,6 +136,13 @@ def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: class NetworkDataExtractor: def __init__(self, network: pypsa.Network, config_path: str = None, table_definitions_path: str = None): + """ + Initializes the extractor with a PyPSA network and configuration files. + + :param network: The PyPSA network instance to extract data from. + :param config_path: Path to the mapping configuration YAML file. + :param table_definitions_path: Path to the TableDefinitions XML file. + """ self.network = network self._conv_params_cache = {} # Performance: Cache function signatures @@ -144,43 +155,42 @@ def __init__(self, network: pypsa.Network, config_path: str = None, table_defini with open(config_path, 'r') as f: self.config = yaml.safe_load(f) - # Automatically load expected columns from TableDefinitions.xml - self.columns = self._load_table_definitions(table_definitions_path) + # Initialize ExcelWriter to get definitions for checking/normalization + self.writer = ExcelWriter(table_definitions_path) + self.excel_definitions = self.writer.excel_definitions self.dataframes = self._extract_dataframes() - # add empty columns - self.dataframes = self._add_empty_columns() - # reorder columns - # self.dataframes = self._reorder_columns() + # Normalize and check dataframes (moving checking functionality from testing script) + self.dataframes = self._normalize_dataframes() - @staticmethod - def _load_table_definitions(xml_path): - """Parses TableDefinitions.xml to determine expected LEGO columns for each table.""" - if not os.path.exists(xml_path): - print(f"Warning: Table definitions file not found at {xml_path}. Using fallback column lists.") - return {} + def _normalize_dataframes(self): + """Checks and normalizes dataframes to ensure they are ready for LEGO output.""" + normalized_dfs = {} + for name, df in self.dataframes.items(): + # Only write tables that are defined in TableDefinitions.xml + table_id = name[1:] if name.startswith('d') else name + + if table_id not in self.excel_definitions: + print(f" Skipping {name} (not defined in TableDefinitions.xml)") + continue - import xml.etree.ElementTree as ET - try: - tree = ET.parse(xml_path) - root = tree.getroot() - table_cols = {} - for table in root.findall(".//TableDefinition"): - # Map XML ID (Power_BusInfo) to internal ID (dPower_BusInfo) - table_id = "d" + table.get("id") - cols = [] - columns_node = table.find("Columns") - if columns_node is not None: - for col in columns_node: - col_id = col.get("id") - if col_id: - cols.append(col_id) - table_cols[table_id] = cols - return table_cols - except Exception as e: - print(f"Error parsing TableDefinitions.xml: {e}") - return {} + # Add mandatory 'scenario' column for LEGO format if missing + if 'scenario' not in df.columns: + df['scenario'] = 'ScenarioA' + # Add 'id' column if missing + if 'id' not in df.columns: + df['id'] = np.nan + + # Ensure all columns from TableDefinition are present (at least as NaN) + definition = self.excel_definitions[table_id] + for col_def in definition.columns: + if col_def.db_name not in df.columns and col_def.db_name != "NOEXCL": + df[col_def.db_name] = np.nan + + normalized_dfs[name] = df + + return normalized_dfs def _extract_dataframes(self): """Extracts and transforms data from the PyPSA network into LEGO DataFrames.""" @@ -353,45 +363,60 @@ def _add_scenario_columns(self, df) -> pd.DataFrame: return df - def _add_empty_columns(self): - for name, df in self.dataframes.items(): - # Add scenario column - if 'scenario' not in df.columns: - df['scenario'] = 'ScenarioA' - - # Add id column if missing - if 'id' not in df.columns: - df['id'] = np.nan - - if name in self.columns: - for col in self.columns[name]: - if col not in df.columns: - df[col] = np.nan - return self.dataframes - - - def _reorder_columns(self): - for name, df in self.dataframes.items(): - if name in self.columns: - cols = self.columns[name] - df = df.reindex(columns=cols) - self.dataframes[name] = df - return self.dataframes - - def get_dataframes(self): + """Returns the dictionary of extracted and normalized DataFrames.""" return self.dataframes if __name__ == "__main__": - filepath = os.path.join(r"C:\BeSt\PyPSA-LEGO-Translator\scigrid-de.nc") - if os.path.exists(filepath): - net = pypsa.Network(filepath) + """ + Main execution block for converting a PyPSA network to LEGO-formatted Excel files. + + To use this: + 1. Update the 'directory' and 'input_file' variables to point to your .nc PyPSA network. + 2. Set 'output_directory' and 'output_folder_name' for the resulting Excel files. + 3. Run the script: `python PypsaReader.py` + """ + # Define the path to the PyPSA network (similar than implemented in PyPSA-LEGO-Translator_testing.py) + directory = r"C:\BeSt\PyPSA-LEGO-Translator" + output_directory = r"C:\BeSt\PyPSA-LEGO-Translator" + input_file = r"scigrid-de.nc" + output_folder_name = "scigrid-de" + filepath = os.path.join(directory, input_file) + + if not os.path.exists(filepath): + print(f"Error: File not found at {filepath}") + else: + # Load the network + print(f"Loading PyPSA network from {filepath}...") + try: + net = pypsa.Network(filepath) + except Exception as e: + print(f"Could not load network: {e}") + exit(1) + + # Extract data using NetworkDataExtractor + print("Extracting data into LEGO format...") extractor = NetworkDataExtractor(net) dfs = extractor.get_dataframes() + + # Initialize ExcelWriter + writer = ExcelWriter() + + # Define output directory + output_dir = os.path.join(output_directory, output_folder_name) + if not os.path.exists(output_dir): + os.makedirs(output_dir, exist_ok=True) + + # Writing Excel files (filtering/checking functionality is now inside NetworkDataExtractor) + print("Writing Excel files...") for name, df in dfs.items(): - print(f"DataFrame: {name}") - print(df.head()) - print("\n") - else: - print(f"Test file not found: {filepath}") + table_id = name[1:] if name.startswith('d') else name + + print(f" Writing {name} to {output_dir}...") + try: + writer._write_Excel_from_definition(df, output_dir, table_id) + except Exception as e: + print(f" Error writing {name}: {e}") + + print("\nConversion complete. Output files are in:", output_dir) diff --git a/mapping_config.yaml b/mapping_config.yaml index b1d1e43..6143601 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -3,7 +3,7 @@ Metadata: type: "metadata" dataPackage: "PyPSA-Import" dataSource: "Eur_2050_1h" - LDES_threshold: 3 # hours + LDES_threshold: 2160 # hours # Define names of carriers and their corresponding fuel costs (if thermal generator) for every technology # [technology]: diff --git a/pypsa_helper.py b/pypsa_helper.py index db0f10a..5abd1cd 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -22,9 +22,8 @@ def prepare_ac_lines(net, config: dict): if "name" not in lines.columns or lines["name"].isnull().all(): lines["name"] = "c1" - # Todo: Add robust checking if carrier of line is defined! - # if carrier is not defined, set to AC for all lines to get defined as DC-OPF - lines.carrier = lines.carrier.fillna('AC') + # if carrier is nan or empty string (''), set to AC for all lines to get defined as DC-OPF + lines.carrier = lines.carrier.fillna('AC').where(lines.carrier != '', 'AC') return lines @@ -44,7 +43,8 @@ def prepare_dc_links(net, config: dict): # Vectorized name generation links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype(str) - links.carrier = links.carrier.fillna('DC') + # if carrier is nan or empty string (''), set to DC for all links to get defined as transport problem (TP) + links.carrier = links.carrier.fillna('DC').where(links.carrier != '', 'DC') # Add tap ratios and phase shifts with default values (if not already present) links['tap_ratio'] = 1 From 0e77ea6aa8976f2a4b19b1910b8d9c0c003dcb45 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 3 Apr 2026 17:03:36 +0300 Subject: [PATCH 10/37] Move main functions to standalone function. --- PypsaReader.py | 48 +++++++++++++++++++++++------------------------- 1 file changed, 23 insertions(+), 25 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 1978c7b..9439de7 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -6,8 +6,8 @@ import pypsa import yaml -from ExcelWriter import ExcelWriter import pypsa_helper as h +from ExcelWriter import ExcelWriter class Conversions: @@ -145,10 +145,10 @@ def __init__(self, network: pypsa.Network, config_path: str = None, table_defini """ self.network = network self._conv_params_cache = {} # Performance: Cache function signatures - + if config_path is None: config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") - + if table_definitions_path is None: table_definitions_path = os.path.join(os.path.dirname(__file__), "TableDefinitions.xml") @@ -169,7 +169,7 @@ def _normalize_dataframes(self): for name, df in self.dataframes.items(): # Only write tables that are defined in TableDefinitions.xml table_id = name[1:] if name.startswith('d') else name - + if table_id not in self.excel_definitions: print(f" Skipping {name} (not defined in TableDefinitions.xml)") continue @@ -187,9 +187,9 @@ def _normalize_dataframes(self): for col_def in definition.columns: if col_def.db_name not in df.columns and col_def.db_name != "NOEXCL": df[col_def.db_name] = np.nan - + normalized_dfs[name] = df - + return normalized_dfs def _extract_dataframes(self): @@ -228,7 +228,6 @@ def _extract_dataframes(self): return df_dict - def _resolve_category_filters(self, cfg): """Resolves technology filters from Metadata if a category is defined.""" if 'category' not in cfg: @@ -259,7 +258,6 @@ def _resolve_category_filters(self, cfg): unique_filter = list(set(cat_filter)) cfg['source']['filter'] = f"carrier in {unique_filter}" - def _get_source_df(self, cfg): """Retrieves the source DataFrame based on the configuration.""" src = cfg.get('source') @@ -275,7 +273,6 @@ def _get_source_df(self, cfg): return getattr(h, src['name'])(self.network, cfg) return None - def _map_columns(self, source_df, cfg): """Maps PyPSA attributes to LEGO columns using the mapping configuration.""" column_data = {} @@ -354,7 +351,6 @@ def _apply_indexing(df, source_df, idx_cfg): # Fallback for current list style: assumes columns match LEGO names df.index = pd.MultiIndex.from_frame(source_df[idx_cfg]).set_names(idx_cfg) - def _add_scenario_columns(self, df) -> pd.DataFrame: """Adds dataPackage and dataSource columns based on config or defaults.""" meta = self.config.get('Metadata', {}) @@ -362,27 +358,25 @@ def _add_scenario_columns(self, df) -> pd.DataFrame: df['dataSource'] = meta.get('dataSource', 'default-source') return df - def get_dataframes(self): """Returns the dictionary of extracted and normalized DataFrames.""" return self.dataframes -if __name__ == "__main__": - """ - Main execution block for converting a PyPSA network to LEGO-formatted Excel files. - - To use this: - 1. Update the 'directory' and 'input_file' variables to point to your .nc PyPSA network. - 2. Set 'output_directory' and 'output_folder_name' for the resulting Excel files. - 3. Run the script: `python PypsaReader.py` +def translate_pypsa_to_lego(input_directory: str = r"./input_data", + input_file: str = r"pypsa-model.nc", + output_directory: str = r"./output_data", + output_folder_name: str = "LEGO-Model"): """ + Main execution block for converting a PyPSA network to LEGO-formatted Excel files. + + To use this: + 1. Update the 'directory' and 'input_file' variables to point to your .nc PyPSA network. + 2. Set 'output_directory' and 'output_folder_name' for the resulting Excel files. + 3. Run the script: `python PypsaReader.py` + """ # Define the path to the PyPSA network (similar than implemented in PyPSA-LEGO-Translator_testing.py) - directory = r"C:\BeSt\PyPSA-LEGO-Translator" - output_directory = r"C:\BeSt\PyPSA-LEGO-Translator" - input_file = r"scigrid-de.nc" - output_folder_name = "scigrid-de" - filepath = os.path.join(directory, input_file) + filepath = os.path.join(input_directory, input_file) if not os.path.exists(filepath): print(f"Error: File not found at {filepath}") @@ -412,7 +406,7 @@ def get_dataframes(self): print("Writing Excel files...") for name, df in dfs.items(): table_id = name[1:] if name.startswith('d') else name - + print(f" Writing {name} to {output_dir}...") try: writer._write_Excel_from_definition(df, output_dir, table_id) @@ -420,3 +414,7 @@ def get_dataframes(self): print(f" Error writing {name}: {e}") print("\nConversion complete. Output files are in:", output_dir) + + +if __name__ == "__main__": + translate_pypsa_to_lego() From 7df6bac2d043983a8ac0524078a044f40cbac5b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 3 Apr 2026 17:07:01 +0300 Subject: [PATCH 11/37] Add docstring comments to pypsa_helper.py. --- pypsa_helper.py | 28 ++++++++++++++++++++++++++++ 1 file changed, 28 insertions(+) diff --git a/pypsa_helper.py b/pypsa_helper.py index 5abd1cd..fb2de5f 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -3,6 +3,10 @@ def prepare_ac_lines(net, config: dict): + """ + Prepares AC line data by calculating missing parameters (r, x, b) from line types + and ensuring consistent naming and carrier definitions. + """ lines = net.lines.copy() types = net.line_types @@ -29,6 +33,10 @@ def prepare_ac_lines(net, config: dict): def prepare_dc_links(net, config: dict): + """ + Extracts and prepares DC link data, initializing parameters for DC-OPF compatibility + and generating unique names. + """ links = net.links[net.links["carrier"] == "DC"].copy() # Define line parameters as nan for DC links, as they are not relevant for DC-OPF @@ -54,6 +62,10 @@ def prepare_dc_links(net, config: dict): def prepare_transformers(net, config: dict) -> pd.DataFrame: + """ + Calculates transformer electrical parameters (r, x, b) based on their types + and sets default values for LEGO compatibility. + """ transformers = net.transformers.copy() types = net.transformer_types @@ -77,6 +89,10 @@ def prepare_transformers(net, config: dict) -> pd.DataFrame: def prepare_ac_lines_and_dc_links(net, config: dict): + """ + Combines AC lines, DC links, and transformers into a single DataFrame + for comprehensive network mapping. + """ ac_lines = prepare_ac_lines(net, config) dc_links = prepare_dc_links(net, config) transformers = prepare_transformers(net, config) @@ -84,6 +100,10 @@ def prepare_ac_lines_and_dc_links(net, config: dict): def prepare_renewable_profiles(net, config: dict): + """ + Extracts renewable generation profiles (p_max_pu) for specified carriers + and formats them for LEGO input. + """ # renewable_types = ['Solar', 'Wind Onshore', 'Wind Offshore'] gens = net.generators.copy() vres_gens = gens.query(config["source"]["filter"]) @@ -100,6 +120,10 @@ def prepare_renewable_profiles(net, config: dict): def prepare_inflow_profiles(net, config: dict): + """ + Aggregates inflow data from hydro storage units and Run-of-River generators + into a unified profile format. + """ # Get hydro storage inflows hydro_ids = net.storage_units.query(config["source"]["filter"]).index.to_list() set_hydro_ids = set(hydro_ids) @@ -150,6 +174,10 @@ def prepare_inflow_profiles(net, config: dict): def prepare_demand_profiles(net, config: dict): + """ + Extracts load demand profiles and formats them into a long-form DataFrame + for LEGO representation. + """ df = net.loads_t.p_set.copy() # shape: [time, load_id] df = df.rename_axis("k").reset_index() # 'k' = time From be895768bb1c21c7dff92a10fbd0540b9839f0e3 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 3 Apr 2026 17:19:30 +0300 Subject: [PATCH 12/37] Add checker for parallel lines and transformers. --- pypsa_helper.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pypsa_helper.py b/pypsa_helper.py index fb2de5f..5cc2691 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -23,8 +23,8 @@ def prepare_ac_lines(net, config: dict): lines['tap_ratio'] = 1 lines['phase_shift'] = 0 - if "name" not in lines.columns or lines["name"].isnull().all(): - lines["name"] = "c1" + # Ensure every line has an individual circuit identifier (c1, c2, ...) for parallel lines + lines["name"] = "c" + (lines.groupby(["bus0", "bus1"]).cumcount() + 1).astype(str) # if carrier is nan or empty string (''), set to AC for all lines to get defined as DC-OPF lines.carrier = lines.carrier.fillna('AC').where(lines.carrier != '', 'AC') @@ -82,8 +82,8 @@ def prepare_transformers(net, config: dict) -> pd.DataFrame: # Set carrier to AC for all transformers to get defined as DC-OPF transformers["carrier"] = "AC" - if "name" not in transformers.columns or transformers["name"].isnull().all(): - transformers["name"] = "c1" + # Ensure every transformer has an individual circuit identifier (c1, c2, ...) for parallel transformers + transformers["name"] = "c" + (transformers.groupby(["bus0", "bus1"]).cumcount() + 1).astype(str) return transformers From 2b13602382b431125eca685abe656c0ca42aa0d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Wed, 15 Apr 2026 16:10:31 +0300 Subject: [PATCH 13/37] Add 'DC' to bus carriers. --- mapping_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapping_config.yaml b/mapping_config.yaml index 6143601..00e993e 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -74,7 +74,7 @@ dPower_BusInfo: source: type: "attribute" name: "buses" - filter: "carrier == 'AC'" + filter: "carrier in ['AC', 'DC']" index: "i" mapping: z: "country" From cc1e86d17d8ec74eaf3fac18d9619a86fcdecbfe Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 16 Apr 2026 11:04:12 +0300 Subject: [PATCH 14/37] Add demand of 0 for buses without specified demand. --- pypsa_helper.py | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/pypsa_helper.py b/pypsa_helper.py index 5cc2691..f64c440 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -176,9 +176,22 @@ def prepare_inflow_profiles(net, config: dict): def prepare_demand_profiles(net, config: dict): """ Extracts load demand profiles and formats them into a long-form DataFrame - for LEGO representation. + for LEGO representation. Make sure all buses are included, even those without loads, to ensure a complete demand profile. """ df = net.loads_t.p_set.copy() # shape: [time, load_id] + + # Map load IDs to bus IDs and aggregate demand per bus + df.columns = df.columns.map(net.loads.bus) + if not df.empty: + df = df.groupby(level=0, axis=1).sum() + + # Ensure all buses are present in the demand DataFrame + missing_buses = net.buses.index.difference(df.columns) + if not missing_buses.empty: + index = df.index if not df.empty else net.snapshots + zero_demand = pd.DataFrame(0.0, index=index, columns=missing_buses) + df = pd.concat([df, zero_demand], axis=1) + df = df.rename_axis("k").reset_index() # 'k' = time demand_long = df.melt(id_vars="k", var_name="n", value_name="Demand") From e020ef4dcef48425eca19fb8fad8d1e481920a64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 16 Apr 2026 11:14:52 +0300 Subject: [PATCH 15/37] Change k index to standard format of k0001 -> k0002. --- pypsa_helper.py | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/pypsa_helper.py b/pypsa_helper.py index f64c440..3a5460b 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -110,6 +110,11 @@ def prepare_renewable_profiles(net, config: dict): vres_ids = vres_gens.index.to_list() profiles = net.generators_t.p_max_pu[vres_ids].copy() + + # Change the index to k0001, k0002, ... + num_timesteps = len(profiles) + profiles.index = [f"k{i + 1:04d}" for i in range(num_timesteps)] + # Ensure the index (snapshots) has a known name before resetting profiles = profiles.rename_axis("k").reset_index().melt( id_vars="k", var_name="generator_id", value_name="Capacity" @@ -162,6 +167,11 @@ def prepare_inflow_profiles(net, config: dict): # Concatenate both: hydro + RoR inflows → [time, generator] combined = pd.concat([inflow_storage, inflow_ror], axis=1) + + # Change the index to k0001, k0002, ... + num_timesteps = len(combined) + combined.index = [f"k{i + 1:04d}" for i in range(num_timesteps)] + combined = combined.T # index: generator_id, columns: time # Convert to long format by renaming index to 'g' before resetting @@ -176,7 +186,7 @@ def prepare_inflow_profiles(net, config: dict): def prepare_demand_profiles(net, config: dict): """ Extracts load demand profiles and formats them into a long-form DataFrame - for LEGO representation. Make sure all buses are included, even those without loads, to ensure a complete demand profile. + for LEGO representation. """ df = net.loads_t.p_set.copy() # shape: [time, load_id] @@ -192,8 +202,12 @@ def prepare_demand_profiles(net, config: dict): zero_demand = pd.DataFrame(0.0, index=index, columns=missing_buses) df = pd.concat([df, zero_demand], axis=1) - df = df.rename_axis("k").reset_index() # 'k' = time + # Change the index to k0001, k0002, ... + num_timesteps = len(df) + df.index = [f"k{i + 1:04d}" for i in range(num_timesteps)] + df = df.rename_axis("k").reset_index() demand_long = df.melt(id_vars="k", var_name="n", value_name="Demand") demand_long["rp"] = "rp01" return demand_long[["rp", "n", "k", "Demand"]] + From a1270e1ce445b069e4fb04faebba13de6c6d537c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 16 Apr 2026 17:05:14 +0300 Subject: [PATCH 16/37] Add PypsaReaderDocumentation.md. --- PypsaReaderDocumentation.md | 80 +++++++++++++++++++++++++++++++++++++ 1 file changed, 80 insertions(+) create mode 100644 PypsaReaderDocumentation.md diff --git a/PypsaReaderDocumentation.md b/PypsaReaderDocumentation.md new file mode 100644 index 0000000..65ad054 --- /dev/null +++ b/PypsaReaderDocumentation.md @@ -0,0 +1,80 @@ +# PyPSA Reader Documentation + +The `PypsaReader` module provides a specialized pipeline for converting PyPSA (Python for Power System Analysis) networks into the Excel-based data +format required by the LEGO model. + +## General Workflow + +The conversion process follows a structured workflow: + +1. **Network Loading**: A PyPSA network is loaded from a NetCDF (`.nc`) file. +2. **Configuration Parsing**: The `NetworkDataExtractor` reads `mapping_config.yaml` to determine how PyPSA components (buses, lines, generators, + etc.) map to LEGO tables. +3. **Data Extraction & Transformation**: + - **Source Retrieval**: Data is pulled either directly from PyPSA network attributes (e.g., `net.buses`) or via complex processing in + `pypsa_helper.py`. + - **Filtering**: Technologies are filtered based on their `carrier` attribute using definitions in the `Metadata` section of the config. + - **Column Mapping**: PyPSA attributes are mapped to LEGO column names. + - **Unit Conversion**: The `Conversions` class applies transformations (e.g., MW to kW, EUR to MEUR, or calculating decommissioning years). +4. **Normalization**: DataFrames are augmented with mandatory LEGO columns (`scenario`, `id`, `dataPackage`, `dataSource`) and aligned with + definitions in `TableDefinitions.xml`. +5. **Excel Export**: The `ExcelWriter` saves each processed DataFrame into individual `.xlsx` files.
**Warning:** This can take up to several hours for very + large networks! + +## Configuration File (`mapping_config.yaml`) + +The YAML configuration acts as the "translation map" between the two models. + +### Metadata Section + +* **Technology Filters**: Defines list of `carrier` strings that identify specific technologies (e.g., `solar: filter: ['solar', 'Solar']`). +* **Fuel Costs**: Specifies fuel prices in EUR/MWh for thermal technologies. +* **Categories**: Groups individual technologies into broader LEGO categories like `ThermalGen` or `VRES` for batch processing. +* **Global Settings**: Defines `dataPackage`, `dataSource`, and thresholds like `LDES_threshold`. + +### Table Mapping Section + +Each entry (e.g., `dPower_ThermalGen`) defines: + +* **source**: The data origin (`type: attribute` for direct PyPSA access or `type: helper` for function calls). +* **category**: (Optional) References a metadata category to automatically filter the source data. +* **index**: Defines the LEGO index columns (e.g., `i` for bus, `g` for generator). +* **mapping**: A dictionary where keys are LEGO columns and values are either: + - A direct PyPSA attribute name. + - A static value (int/float). + - A dictionary specifying an `attr`, a `conversion` function, or a `factor`. + +## Helper Functions (`pypsa_helper.py`) + +When simple attribute mapping is insufficient, helper functions handle complex data aggregation and profile generation: + +* **Network Topology**: + - `prepare_ac_lines_and_dc_links`: Combines PyPSA `lines`, `links` (filtered for DC), and `transformers` into a single LEGO `Power_Network` table. + - `prepare_ac_lines` / `prepare_transformers`: Calculate missing electrical parameters ($r, x, b$) based on standard type definitions if they are + not explicitly set. +* **Time-Series Profiles**: + - `prepare_renewable_profiles`: Extracts `p_max_pu` for VRES generators and reshapes it into a long-form LEGO format. + - `prepare_inflow_profiles`: Aggregates inflow data from both `storage_units` (hydro) and `generators` (Run-of-River) into a unified time-series. + - `prepare_demand_profiles`: Sums PyPSA load data per bus and formats it for the LEGO demand table. + +## Conversions & Logic + +The `Conversions` class in `PypsaReader.py` contains static methods for specialized logic: + +* **Unit Scaling**: `EUR_to_MEUR`, `MW_to_kW`, `V_to_kV`. +* **Calculated Values**: `year_and_lifetime_to_year_decom` (derives decommissioning date) and `total_capacity_to_number_of_units`. +* **LEGO Specifics**: `is_ldes` (Long Duration Energy Storage detection) and `line_carrier_to_tec_repr` (mapping carriers to LEGO technical + representations like `DC-OPF` or `TP`). + +## Current Limitations + +Users should be aware of the following architectural and data gaps: + +1. **Direct-to-File Output**: The reader currently saves data directly to Excel files. It **does not return a LEGO CaseStudy object** in memory, + meaning it cannot be used for immediate programmatic manipulation within a Python script without re-reading the exported files. +2. **Missing LEGO Tables**: The current mapping does not generate several files required for a complete LEGO run: + - `Power_Parameters` + - `Global_Parameters` + - `Power_Hindex` (Time mapping/Horizon index) + - `Power_WeightsK` & `Power_WeightsRP` (Representative period weighting) +3. **Scenario Logic**: The `scenario` column is currently hardcoded to `ScenarioA` during normalization. From 144074386f57d137e58e33afe470ae1e039dffc9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 17 Apr 2026 10:04:17 +0300 Subject: [PATCH 17/37] Add bus filter to demand consider same buses, when creating the demand. --- mapping_config.yaml | 1 + pypsa_helper.py | 17 ++++++++++++++--- 2 files changed, 15 insertions(+), 3 deletions(-) diff --git a/mapping_config.yaml b/mapping_config.yaml index 00e993e..506e03d 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -247,6 +247,7 @@ dPower_Demand: source: type: "helper" name: "prepare_demand_profiles" + filter: "carrier in ['AC', 'DC']" # Should be the same filter as for the buses in dPower_BusInfo index: rp: "rp" k: "k" diff --git a/pypsa_helper.py b/pypsa_helper.py index 3a5460b..2d8bdc9 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -190,13 +190,24 @@ def prepare_demand_profiles(net, config: dict): """ df = net.loads_t.p_set.copy() # shape: [time, load_id] - # Map load IDs to bus IDs and aggregate demand per bus + # Map load IDs to bus IDs df.columns = df.columns.map(net.loads.bus) + + # Filter buses based on the filter defined in the config (e.g. from dPower_BusInfo) + bus_filter = config.get("source", {}).get("filter") + if bus_filter: + valid_buses = net.buses.query(bus_filter).index + # Only keep demands at valid buses + df = df[df.columns.intersection(valid_buses)] + else: + valid_buses = net.buses.index + + # Aggregate demand per bus if not df.empty: df = df.groupby(level=0, axis=1).sum() - # Ensure all buses are present in the demand DataFrame - missing_buses = net.buses.index.difference(df.columns) + # Ensure all valid buses are present in the demand DataFrame + missing_buses = valid_buses.difference(df.columns) if not missing_buses.empty: index = df.index if not df.empty else net.snapshots zero_demand = pd.DataFrame(0.0, index=index, columns=missing_buses) From 7e9daf2ee1752ff72417acab8ce20a2fd36efa8e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 17 Apr 2026 14:35:25 +0300 Subject: [PATCH 18/37] Add a default_storage_capacity_cost to avoid NaN values in the optimization model. --- PypsaReader.py | 20 ++++++++++++++++++++ mapping_config.yaml | 4 ++++ 2 files changed, 24 insertions(+) diff --git a/PypsaReader.py b/PypsaReader.py index 9439de7..562e730 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -114,6 +114,26 @@ def get_fuel_cost_from_metadata(val: pd.Series, df: pd.DataFrame, metadata: dict """Retrieves fuel costs based on the carrier and Metadata configuration.""" return Conversions._get_fuel_costs(df, metadata) + @staticmethod + def _get_storage_capacity_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: + """Helper to get storage capacity costs for each row in the DataFrame based on carrier.""" + cost_mapping = {} + meta_config = metadata.get('Metadata', {}) + for key, info in meta_config.items(): + if isinstance(info, dict) and 'filter' in info and 'default_storage_capacity_cost' in info: + for carrier in info['filter']: + cost_mapping[carrier] = info['default_storage_capacity_cost'] + + # Map carriers to costs; default to NaN if not found + costs = df['carrier'].map(cost_mapping) + + return costs + + @staticmethod + def get_storage_capacity_cost_from_metadata(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + """Retrieves storage capacity costs based on the carrier and Metadata configuration.""" + return Conversions._get_storage_capacity_costs(df, metadata) + @staticmethod def EUR_per_hour_to_MWh_per_hour(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: """Converts costs per hour of thermal generation (e.g. stand_by_cost) to costs per MWh based on the fuel cost specified in the metadata.""" diff --git a/mapping_config.yaml b/mapping_config.yaml index 506e03d..7058478 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -53,6 +53,7 @@ Metadata: hydro: filter: [ 'Storage Hydro', 'hydro' ] phs: + default_storage_capacity_cost: 1 # EUR/MWh, PyPSA does not have a separation between power and energy capacity for investment costs filter: [ 'PHS', 'Pumped Hydro' ] # Define categories of technologies and their corresponding technologies. @@ -223,6 +224,9 @@ dPower_Storage: attr: "p_nom_max" conversion: "total_capacity_to_number_of_units" InvestCostPerMW: "capital_cost" + InvestCostPerMWh: + attr: "carrier" + conversion: "get_storage_capacity_cost_from_metadata" Ene2PowRatio: "max_hours" YearCom: "build_year" YearDecom: From 70ecec21a5f3850ddc51500281caa5071f929aa5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 20 Apr 2026 08:06:01 +0300 Subject: [PATCH 19/37] Set MinReserve of storage in mapping_config.yaml to 0. --- PypsaReader.py | 5 +++++ mapping_config.yaml | 9 +++++++-- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 562e730..b7df163 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -153,6 +153,11 @@ def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: res = val / fuel_costs return res.replace([np.inf, -np.inf], 0).fillna(0) + @staticmethod + def greater_or_equal_to_one(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + """Ensures that all values are greater or equal to 1, replacing values less than 1 with 1.""" + return val.apply(lambda x: max(x, 1) if pd.notnull(x) else x) + class NetworkDataExtractor: def __init__(self, network: pypsa.Network, config_path: str = None, table_definitions_path: str = None): diff --git a/mapping_config.yaml b/mapping_config.yaml index 7058478..7b4b608 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -142,8 +142,12 @@ dPower_ThermalGen: RampDown: attr: "ramp_limit_down" conversion: "pu_to_absolute" - MinUpTime: "min_up_time" - MinDownTime: "min_down_time" + MinUpTime: + attr: "min_up_time" + conversion: "greater_or_equal_to_one" + MinDownTime: + attr: "min_down_time" + conversion: "greater_or_equal_to_one" Efficiency: "efficiency" FuelCost: attr: "carrier" @@ -212,6 +216,7 @@ dPower_Storage: DisEffic: "efficiency_dispatch" ChEffic: "efficiency_store" Self Discharge: "standing_loss" + MinReserve: 0 IniReserve: "state_of_charge_initial" IsLDES: attr: "max_hours" From f667840f046671cd422806d47e9b5c516c79f311 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 20 Apr 2026 09:11:15 +0300 Subject: [PATCH 20/37] Fill NaN values in inflow profiles. --- pypsa_helper.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/pypsa_helper.py b/pypsa_helper.py index 2d8bdc9..078623c 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -168,6 +168,9 @@ def prepare_inflow_profiles(net, config: dict): # Concatenate both: hydro + RoR inflows → [time, generator] combined = pd.concat([inflow_storage, inflow_ror], axis=1) + # sanitize combined inflow profiles for LEGO model + combined.fillna(0, inplace=True) # fill missing inflows with 0 (if any) to avoid NaNs in the final profiles + # Change the index to k0001, k0002, ... num_timesteps = len(combined) combined.index = [f"k{i + 1:04d}" for i in range(num_timesteps)] From e8f9a2c5293a7b0c629bf504d42623d008fe5c34 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 20 Apr 2026 11:08:22 +0300 Subject: [PATCH 21/37] Fill pRampUp and pRampDown with pmax if not specified. --- PypsaReader.py | 5 +++++ mapping_config.yaml | 4 ++-- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index b7df163..51afe6a 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -158,6 +158,11 @@ def greater_or_equal_to_one(val: pd.Series, df: pd.DataFrame, metadata: dict) -> """Ensures that all values are greater or equal to 1, replacing values less than 1 with 1.""" return val.apply(lambda x: max(x, 1) if pd.notnull(x) else x) + @staticmethod + def pu_to_absolute_and_p_nom_if_nan_or_zero(val: pd.Series, df: pd.DataFrame) -> pd.Series: + val = val * df.p_nom + return val.where((val.notnull() & (val != 0)), df.p_nom) + class NetworkDataExtractor: def __init__(self, network: pypsa.Network, config_path: str = None, table_definitions_path: str = None): diff --git a/mapping_config.yaml b/mapping_config.yaml index 7b4b608..5087d6f 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -138,10 +138,10 @@ dPower_ThermalGen: conversion: "pu_to_absolute" RampUp: attr: "ramp_limit_up" - conversion: "pu_to_absolute" + conversion: "pu_to_absolute_and_p_nom_if_nan_or_zero" RampDown: attr: "ramp_limit_down" - conversion: "pu_to_absolute" + conversion: "pu_to_absolute_and_p_nom_if_nan_or_zero" MinUpTime: attr: "min_up_time" conversion: "greater_or_equal_to_one" From 6f0a394ed2ecad135214f1169ae51029e78b3b4e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Wed, 22 Apr 2026 15:45:16 +0300 Subject: [PATCH 22/37] Add handling of units with zero installed capacity. --- PypsaReader.py | 16 ++++++++++++++++ mapping_config.yaml | 6 ++++-- 2 files changed, 20 insertions(+), 2 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 51afe6a..ab5663f 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -163,6 +163,22 @@ def pu_to_absolute_and_p_nom_if_nan_or_zero(val: pd.Series, df: pd.DataFrame) -> val = val * df.p_nom return val.where((val.notnull() & (val != 0)), df.p_nom) + @staticmethod + def one_if_nan_or_zero(val: pd.Series,) -> pd.Series: + """Replaces NaN or zero values with 1, keeping other values unchanged.""" + return val.where((val.notnull() | (val != 0)), 1) + + @staticmethod + def bool_to_binary_zero_if_p_nom_is_zero(val: pd.Series, df: pd.DataFrame) -> pd.Series: + """Converts boolean values to binary (0/1) integers and sets to 0, if p_nom is zero.""" + if val.isnull().all(): + return 0 + else: + # Use infer_objects to explicitly handle the type conversion from object to bool + val = val.fillna(False).infer_objects(copy=False) + val = val.where(df.p_nom != 0, 0) # set to 0 if p_nom is zero + return val.astype(int) + class NetworkDataExtractor: def __init__(self, network: pypsa.Network, config_path: str = None, table_definitions_path: str = None): diff --git a/mapping_config.yaml b/mapping_config.yaml index 5087d6f..58a3087 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -180,8 +180,10 @@ dPower_VRES: # including ROR i: "bus" ExisUnits: attr: "active" - conversion: "bool_to_binary" - MaxProd: "p_nom" + conversion: "bool_to_binary_zero_if_p_nom_is_zero" + MaxProd: + attr: "p_nom" + conversion: "one_if_nan_or_zero" EnableInvest: attr: "p_nom_extendable" conversion: "bool_to_binary" From e5d0eebf44fa98369bc3367a2aa36bc8ac1f899e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 23 Apr 2026 08:39:05 +0300 Subject: [PATCH 23/37] Adapt handling of units with zero installed capacity. --- PypsaReader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/PypsaReader.py b/PypsaReader.py index ab5663f..817af46 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -166,7 +166,7 @@ def pu_to_absolute_and_p_nom_if_nan_or_zero(val: pd.Series, df: pd.DataFrame) -> @staticmethod def one_if_nan_or_zero(val: pd.Series,) -> pd.Series: """Replaces NaN or zero values with 1, keeping other values unchanged.""" - return val.where((val.notnull() | (val != 0)), 1) + return val.where((val.notnull() & (val != 0)), 1) @staticmethod def bool_to_binary_zero_if_p_nom_is_zero(val: pd.Series, df: pd.DataFrame) -> pd.Series: From a04f71ce336535197ca4ccf960035f851eee5515 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 23 Apr 2026 08:42:10 +0300 Subject: [PATCH 24/37] Apply ruff formatting and checking. --- PypsaReader.py | 199 +++++++++++++++++++++++++++++------------------- pypsa_helper.py | 75 +++++++++++------- 2 files changed, 169 insertions(+), 105 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 817af46..3fa688d 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -20,7 +20,7 @@ def EUR_to_MEUR(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: @staticmethod def MEUR_to_EUR(val: pd.Series, df: pd.DataFrame = None) -> pd.Series: - """Converts Mio.€ to € """ + """Converts Mio.€ to €""" return val * 1e6 @staticmethod @@ -60,22 +60,22 @@ def year_and_lifetime_to_year_decom(val, df: pd.DataFrame) -> pd.Series: # Only return a decom year if build_year and lifetime is available and build_year is not 0; otherwise return NaN mask = df.build_year.notnull() & (df.build_year != 0) & df.lifetime.notnull() res = pd.Series(np.nan, index=df.index) - res[mask] = df.loc[mask, 'build_year'] + df.loc[mask, 'lifetime'] + res[mask] = df.loc[mask, "build_year"] + df.loc[mask, "lifetime"] return res @staticmethod def is_ldes(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: """Sets binary to 1 if energy-to-power ratio (max_hours) is greater than the threshold in metadata.""" - threshold = metadata.get('Metadata', {}).get('LDES_threshold', 2160) + threshold = metadata.get("Metadata", {}).get("LDES_threshold", 2160) return (val > threshold).astype(int) @staticmethod def line_carrier_to_tec_repr(val, df: pd.DataFrame) -> pd.Series: """Maps line carriers to LEGO's technology representations (e.g., DC-OPF for AC lines).""" if df.carrier.isnull().all(): - return 'DC-OPF' + return "DC-OPF" else: - return df.carrier.map({'AC': 'DC-OPF', 'DC': 'TP'}) + return df.carrier.map({"AC": "DC-OPF", "DC": "TP"}) @staticmethod def total_capacity_to_number_of_units(val, df: pd.DataFrame) -> pd.Series: @@ -84,33 +84,45 @@ def total_capacity_to_number_of_units(val, df: pd.DataFrame) -> pd.Series: if val.isnull().all() or np.isinf(val).all(): return 100 else: - val = val.fillna(0).replace(np.inf, 1000) # Treat NaN as 0 for investment calculation - p_nom = df.p_nom.fillna(1).replace(np.inf, 100).replace(0, 1) # Avoid division by zero and treat NaN as 1 for unit calculation + val = val.fillna(0).replace( + np.inf, 1000 + ) # Treat NaN as 0 for investment calculation + p_nom = ( + df.p_nom.fillna(1).replace(np.inf, 100).replace(0, 1) + ) # Avoid division by zero and treat NaN as 1 for unit calculation return (np.ceil(val / p_nom)).astype(int) @staticmethod def _get_fuel_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: """Helper to get fuel costs for each row in the DataFrame based on carrier.""" fuel_mapping = {} - meta_config = metadata.get('Metadata', {}) + meta_config = metadata.get("Metadata", {}) for key, fuel_info in meta_config.items(): - if isinstance(fuel_info, dict) and 'filter' in fuel_info and 'cost' in fuel_info: - for carrier in fuel_info['filter']: - fuel_mapping[carrier] = fuel_info['cost'] + if ( + isinstance(fuel_info, dict) + and "filter" in fuel_info + and "cost" in fuel_info + ): + for carrier in fuel_info["filter"]: + fuel_mapping[carrier] = fuel_info["cost"] # Map carriers to costs; default to NaN if not found - fuel_costs = df['carrier'].map(fuel_mapping) + fuel_costs = df["carrier"].map(fuel_mapping) # Performance check: only check for missing carriers if DataFrame isn't empty if not fuel_costs.empty: - missing_carriers = df.loc[fuel_costs.isnull(), 'carrier'].unique() + missing_carriers = df.loc[fuel_costs.isnull(), "carrier"].unique() if len(missing_carriers) > 0: - print(f"Warning: No fuel cost defined in Metadata for carrier(s): {missing_carriers.tolist()}") + print( + f"Warning: No fuel cost defined in Metadata for carrier(s): {missing_carriers.tolist()}" + ) return fuel_costs @staticmethod - def get_fuel_cost_from_metadata(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + def get_fuel_cost_from_metadata( + val: pd.Series, df: pd.DataFrame, metadata: dict + ) -> pd.Series: """Retrieves fuel costs based on the carrier and Metadata configuration.""" return Conversions._get_fuel_costs(df, metadata) @@ -118,29 +130,37 @@ def get_fuel_cost_from_metadata(val: pd.Series, df: pd.DataFrame, metadata: dict def _get_storage_capacity_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: """Helper to get storage capacity costs for each row in the DataFrame based on carrier.""" cost_mapping = {} - meta_config = metadata.get('Metadata', {}) + meta_config = metadata.get("Metadata", {}) for key, info in meta_config.items(): - if isinstance(info, dict) and 'filter' in info and 'default_storage_capacity_cost' in info: - for carrier in info['filter']: - cost_mapping[carrier] = info['default_storage_capacity_cost'] + if ( + isinstance(info, dict) + and "filter" in info + and "default_storage_capacity_cost" in info + ): + for carrier in info["filter"]: + cost_mapping[carrier] = info["default_storage_capacity_cost"] # Map carriers to costs; default to NaN if not found - costs = df['carrier'].map(cost_mapping) + costs = df["carrier"].map(cost_mapping) return costs @staticmethod - def get_storage_capacity_cost_from_metadata(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + def get_storage_capacity_cost_from_metadata( + val: pd.Series, df: pd.DataFrame, metadata: dict + ) -> pd.Series: """Retrieves storage capacity costs based on the carrier and Metadata configuration.""" return Conversions._get_storage_capacity_costs(df, metadata) @staticmethod - def EUR_per_hour_to_MWh_per_hour(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + def EUR_per_hour_to_MWh_per_hour( + val: pd.Series, df: pd.DataFrame, metadata: dict + ) -> pd.Series: """Converts costs per hour of thermal generation (e.g. stand_by_cost) to costs per MWh based on the fuel cost specified in the metadata.""" fuel_costs = Conversions._get_fuel_costs(df, metadata) # Result is MWh/h = (EUR/h) / (EUR/MWh) # Avoid division by zero, handle NaN/Inf - with np.errstate(divide='ignore', invalid='ignore'): + with np.errstate(divide="ignore", invalid="ignore"): res = val / fuel_costs return res.replace([np.inf, -np.inf], 0).fillna(0) @@ -149,27 +169,35 @@ def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: """Converts costs to costs per MWh based on the fuel cost specified in the metadata.""" fuel_costs = Conversions._get_fuel_costs(df, metadata) # Result is MWh = EUR / (EUR/MWh) - with np.errstate(divide='ignore', invalid='ignore'): + with np.errstate(divide="ignore", invalid="ignore"): res = val / fuel_costs return res.replace([np.inf, -np.inf], 0).fillna(0) @staticmethod - def greater_or_equal_to_one(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: + def greater_or_equal_to_one( + val: pd.Series + ) -> pd.Series: """Ensures that all values are greater or equal to 1, replacing values less than 1 with 1.""" return val.apply(lambda x: max(x, 1) if pd.notnull(x) else x) @staticmethod - def pu_to_absolute_and_p_nom_if_nan_or_zero(val: pd.Series, df: pd.DataFrame) -> pd.Series: + def pu_to_absolute_and_p_nom_if_nan_or_zero( + val: pd.Series, df: pd.DataFrame + ) -> pd.Series: val = val * df.p_nom return val.where((val.notnull() & (val != 0)), df.p_nom) @staticmethod - def one_if_nan_or_zero(val: pd.Series,) -> pd.Series: + def one_if_nan_or_zero( + val: pd.Series, + ) -> pd.Series: """Replaces NaN or zero values with 1, keeping other values unchanged.""" return val.where((val.notnull() & (val != 0)), 1) @staticmethod - def bool_to_binary_zero_if_p_nom_is_zero(val: pd.Series, df: pd.DataFrame) -> pd.Series: + def bool_to_binary_zero_if_p_nom_is_zero( + val: pd.Series, df: pd.DataFrame + ) -> pd.Series: """Converts boolean values to binary (0/1) integers and sets to 0, if p_nom is zero.""" if val.isnull().all(): return 0 @@ -181,7 +209,12 @@ def bool_to_binary_zero_if_p_nom_is_zero(val: pd.Series, df: pd.DataFrame) -> pd class NetworkDataExtractor: - def __init__(self, network: pypsa.Network, config_path: str = None, table_definitions_path: str = None): + def __init__( + self, + network: pypsa.Network, + config_path: str = None, + table_definitions_path: str = None, + ): """ Initializes the extractor with a PyPSA network and configuration files. @@ -196,9 +229,11 @@ def __init__(self, network: pypsa.Network, config_path: str = None, table_defini config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") if table_definitions_path is None: - table_definitions_path = os.path.join(os.path.dirname(__file__), "TableDefinitions.xml") + table_definitions_path = os.path.join( + os.path.dirname(__file__), "TableDefinitions.xml" + ) - with open(config_path, 'r') as f: + with open(config_path, "r") as f: self.config = yaml.safe_load(f) # Initialize ExcelWriter to get definitions for checking/normalization @@ -214,19 +249,19 @@ def _normalize_dataframes(self): normalized_dfs = {} for name, df in self.dataframes.items(): # Only write tables that are defined in TableDefinitions.xml - table_id = name[1:] if name.startswith('d') else name + table_id = name[1:] if name.startswith("d") else name if table_id not in self.excel_definitions: print(f" Skipping {name} (not defined in TableDefinitions.xml)") continue # Add mandatory 'scenario' column for LEGO format if missing - if 'scenario' not in df.columns: - df['scenario'] = 'ScenarioA' + if "scenario" not in df.columns: + df["scenario"] = "ScenarioA" # Add 'id' column if missing - if 'id' not in df.columns: - df['id'] = np.nan + if "id" not in df.columns: + df["id"] = np.nan # Ensure all columns from TableDefinition are present (at least as NaN) definition = self.excel_definitions[table_id] @@ -258,8 +293,8 @@ def _extract_dataframes(self): df = self._map_columns(source_df, cfg) # 4. Handle Indexing - if 'index' in cfg: - self._apply_indexing(df, source_df, cfg['index']) + if "index" in cfg: + self._apply_indexing(df, source_df, cfg["index"]) # 5. Normalize for LEGO Format # Drop columns that are now in the index to avoid "already exists" error on reset_index @@ -276,65 +311,65 @@ def _extract_dataframes(self): def _resolve_category_filters(self, cfg): """Resolves technology filters from Metadata if a category is defined.""" - if 'category' not in cfg: + if "category" not in cfg: return - category = cfg['category'] - meta_data = self.config.get('Metadata', {}) + category = cfg["category"] + meta_data = self.config.get("Metadata", {}) meta_cat = meta_data.get(category, {}) # Combine filters from listed technologies or use direct filter - cat_filter = meta_cat.get('filter', []) + cat_filter = meta_cat.get("filter", []) if isinstance(cat_filter, (str, int, float)): cat_filter = [cat_filter] else: cat_filter = list(cat_filter) - for tech in meta_cat.get('technologies', []): - tech_filter = meta_data.get(tech, {}).get('filter', []) + for tech in meta_cat.get("technologies", []): + tech_filter = meta_data.get(tech, {}).get("filter", []) if isinstance(tech_filter, list): cat_filter.extend(tech_filter) else: cat_filter.append(tech_filter) if cat_filter: - if 'source' not in cfg: - cfg['source'] = {} + if "source" not in cfg: + cfg["source"] = {} # Ensure uniqueness and format as query string unique_filter = list(set(cat_filter)) - cfg['source']['filter'] = f"carrier in {unique_filter}" + cfg["source"]["filter"] = f"carrier in {unique_filter}" def _get_source_df(self, cfg): """Retrieves the source DataFrame based on the configuration.""" - src = cfg.get('source') + src = cfg.get("source") if not src: return None - if src['type'] == 'attribute': - source_df = getattr(self.network, src['name']) - if 'filter' in src: - source_df = source_df.query(src['filter']) + if src["type"] == "attribute": + source_df = getattr(self.network, src["name"]) + if "filter" in src: + source_df = source_df.query(src["filter"]) return source_df - elif src['type'] == 'helper': - return getattr(h, src['name'])(self.network, cfg) + elif src["type"] == "helper": + return getattr(h, src["name"])(self.network, cfg) return None def _map_columns(self, source_df, cfg): """Maps PyPSA attributes to LEGO columns using the mapping configuration.""" column_data = {} - for lego_col, mapping in cfg.get('mapping', {}).items(): + for lego_col, mapping in cfg.get("mapping", {}).items(): if isinstance(mapping, dict): # Attribute mapping with potential unit conversion or transformation function - attr = mapping.get('attr') + attr = mapping.get("attr") if attr and attr in source_df.columns: val = source_df[attr] else: # Try to get value as series of NaNs or fixed value - val = pd.Series(mapping.get('value', np.nan), index=source_df.index) + val = pd.Series(mapping.get("value", np.nan), index=source_df.index) # Priority 1: Named conversion function - if 'conversion' in mapping: - conv_name = mapping['conversion'].replace('()', '') + if "conversion" in mapping: + conv_name = mapping["conversion"].replace("()", "") if hasattr(Conversions, conv_name): conv_func = getattr(Conversions, conv_name) @@ -356,13 +391,17 @@ def _map_columns(self, source_df, cfg): # Vectorized call: (Series) val = conv_func(val) except Exception as e: - raise ValueError(f"Error applying conversion '{conv_name}' to column '{lego_col}': {e}") + raise ValueError( + f"Error applying conversion '{conv_name}' to column '{lego_col}': {e}" + ) else: - print(f"Warning: Conversion function '{conv_name}' not found in Conversions class.") + print( + f"Warning: Conversion function '{conv_name}' not found in Conversions class." + ) # Priority 3: Simple multiplier factor - elif 'factor' in mapping: - val = val * mapping['factor'] + elif "factor" in mapping: + val = val * mapping["factor"] column_data[lego_col] = val elif isinstance(mapping, (int, float)): @@ -389,7 +428,9 @@ def _apply_indexing(df, source_df, idx_cfg): index_data[lego_idx] = source_df.index else: index_data[lego_idx] = np.nan - df.index = pd.MultiIndex.from_frame(pd.DataFrame(index_data, index=source_df.index)) + df.index = pd.MultiIndex.from_frame( + pd.DataFrame(index_data, index=source_df.index) + ) elif isinstance(idx_cfg, str): # Simple index renaming df.index = source_df.index.rename(idx_cfg) @@ -399,9 +440,9 @@ def _apply_indexing(df, source_df, idx_cfg): def _add_scenario_columns(self, df) -> pd.DataFrame: """Adds dataPackage and dataSource columns based on config or defaults.""" - meta = self.config.get('Metadata', {}) - df['dataPackage'] = meta.get('dataPackage', 'default-package') - df['dataSource'] = meta.get('dataSource', 'default-source') + meta = self.config.get("Metadata", {}) + df["dataPackage"] = meta.get("dataPackage", "default-package") + df["dataSource"] = meta.get("dataSource", "default-source") return df def get_dataframes(self): @@ -409,18 +450,20 @@ def get_dataframes(self): return self.dataframes -def translate_pypsa_to_lego(input_directory: str = r"./input_data", - input_file: str = r"pypsa-model.nc", - output_directory: str = r"./output_data", - output_folder_name: str = "LEGO-Model"): +def translate_pypsa_to_lego( + input_directory: str = r"./input_data", + input_file: str = r"pypsa-model.nc", + output_directory: str = r"./output_data", + output_folder_name: str = "LEGO-Model", +): """ - Main execution block for converting a PyPSA network to LEGO-formatted Excel files. + Main execution block for converting a PyPSA network to LEGO-formatted Excel files. - To use this: - 1. Update the 'directory' and 'input_file' variables to point to your .nc PyPSA network. - 2. Set 'output_directory' and 'output_folder_name' for the resulting Excel files. - 3. Run the script: `python PypsaReader.py` - """ + To use this: + 1. Update the 'directory' and 'input_file' variables to point to your .nc PyPSA network. + 2. Set 'output_directory' and 'output_folder_name' for the resulting Excel files. + 3. Run the script: `python PypsaReader.py` + """ # Define the path to the PyPSA network (similar than implemented in PyPSA-LEGO-Translator_testing.py) filepath = os.path.join(input_directory, input_file) @@ -451,7 +494,7 @@ def translate_pypsa_to_lego(input_directory: str = r"./input_data", # Writing Excel files (filtering/checking functionality is now inside NetworkDataExtractor) print("Writing Excel files...") for name, df in dfs.items(): - table_id = name[1:] if name.startswith('d') else name + table_id = name[1:] if name.startswith("d") else name print(f" Writing {name} to {output_dir}...") try: diff --git a/pypsa_helper.py b/pypsa_helper.py index 078623c..0fe7e68 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -4,7 +4,7 @@ def prepare_ac_lines(net, config: dict): """ - Prepares AC line data by calculating missing parameters (r, x, b) from line types + Prepares AC line data by calculating missing parameters (r, x, b) from line types and ensuring consistent naming and carrier definitions. """ lines = net.lines.copy() @@ -20,21 +20,21 @@ def prepare_ac_lines(net, config: dict): lines["b"] = lines["b"].where(lines["b"] != 0, x_per_len * lines["length"]) # Add tap ratios and phase shifts with default values (if not already present) - lines['tap_ratio'] = 1 - lines['phase_shift'] = 0 + lines["tap_ratio"] = 1 + lines["phase_shift"] = 0 # Ensure every line has an individual circuit identifier (c1, c2, ...) for parallel lines lines["name"] = "c" + (lines.groupby(["bus0", "bus1"]).cumcount() + 1).astype(str) # if carrier is nan or empty string (''), set to AC for all lines to get defined as DC-OPF - lines.carrier = lines.carrier.fillna('AC').where(lines.carrier != '', 'AC') + lines.carrier = lines.carrier.fillna("AC").where(lines.carrier != "", "AC") return lines def prepare_dc_links(net, config: dict): """ - Extracts and prepares DC link data, initializing parameters for DC-OPF compatibility + Extracts and prepares DC link data, initializing parameters for DC-OPF compatibility and generating unique names. """ links = net.links[net.links["carrier"] == "DC"].copy() @@ -49,21 +49,23 @@ def prepare_dc_links(net, config: dict): links["s_nom_extendable"] = links["p_nom_extendable"] links["id"] = links.index # Vectorized name generation - links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype(str) + links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype( + str + ) # if carrier is nan or empty string (''), set to DC for all links to get defined as transport problem (TP) - links.carrier = links.carrier.fillna('DC').where(links.carrier != '', 'DC') + links.carrier = links.carrier.fillna("DC").where(links.carrier != "", "DC") # Add tap ratios and phase shifts with default values (if not already present) - links['tap_ratio'] = 1 - links['phase_shift'] = 0 + links["tap_ratio"] = 1 + links["phase_shift"] = 0 return links def prepare_transformers(net, config: dict) -> pd.DataFrame: """ - Calculates transformer electrical parameters (r, x, b) based on their types + Calculates transformer electrical parameters (r, x, b) based on their types and sets default values for LEGO compatibility. """ transformers = net.transformers.copy() @@ -76,21 +78,27 @@ def prepare_transformers(net, config: dict) -> pd.DataFrame: g = pfe / (1000 * transformers.s_nom) transformers["r"] = transformers["r"].where(transformers["r"] != 0, vsc / 100) - transformers["x"] = transformers["x"].where(transformers["x"] != 0, np.sqrt((vsc/100) ** 2 - transformers.r ** 2)) - transformers["b"] = transformers["b"].where(transformers["b"] != 0, - np.sqrt((nlc / 100) ** 2 - g ** 2)) + transformers["x"] = transformers["x"].where( + transformers["x"] != 0, np.sqrt((vsc / 100) ** 2 - transformers.r**2) + ) + transformers["b"] = transformers["b"].where( + transformers["b"] != 0, -np.sqrt((nlc / 100) ** 2 - g**2) + ) # Set carrier to AC for all transformers to get defined as DC-OPF transformers["carrier"] = "AC" # Ensure every transformer has an individual circuit identifier (c1, c2, ...) for parallel transformers - transformers["name"] = "c" + (transformers.groupby(["bus0", "bus1"]).cumcount() + 1).astype(str) + transformers["name"] = "c" + ( + transformers.groupby(["bus0", "bus1"]).cumcount() + 1 + ).astype(str) return transformers def prepare_ac_lines_and_dc_links(net, config: dict): """ - Combines AC lines, DC links, and transformers into a single DataFrame + Combines AC lines, DC links, and transformers into a single DataFrame for comprehensive network mapping. """ ac_lines = prepare_ac_lines(net, config) @@ -101,7 +109,7 @@ def prepare_ac_lines_and_dc_links(net, config: dict): def prepare_renewable_profiles(net, config: dict): """ - Extracts renewable generation profiles (p_max_pu) for specified carriers + Extracts renewable generation profiles (p_max_pu) for specified carriers and formats them for LEGO input. """ # renewable_types = ['Solar', 'Wind Onshore', 'Wind Offshore'] @@ -116,8 +124,10 @@ def prepare_renewable_profiles(net, config: dict): profiles.index = [f"k{i + 1:04d}" for i in range(num_timesteps)] # Ensure the index (snapshots) has a known name before resetting - profiles = profiles.rename_axis("k").reset_index().melt( - id_vars="k", var_name="generator_id", value_name="Capacity" + profiles = ( + profiles.rename_axis("k") + .reset_index() + .melt(id_vars="k", var_name="generator_id", value_name="Capacity") ) profiles["rp"] = "rp01" # add a dummy column for compatibility @@ -126,7 +136,7 @@ def prepare_renewable_profiles(net, config: dict): def prepare_inflow_profiles(net, config: dict): """ - Aggregates inflow data from hydro storage units and Run-of-River generators + Aggregates inflow data from hydro storage units and Run-of-River generators into a unified profile format. """ # Get hydro storage inflows @@ -139,12 +149,16 @@ def prepare_inflow_profiles(net, config: dict): missing_inflows = list(set_hydro_ids - set_inflow_columns) if len(existing_inflows) == 0: - print("Warning: No hydro storage units have inflow data. Storage inflow profiles will be empty.") + print( + "Warning: No hydro storage units have inflow data. Storage inflow profiles will be empty." + ) inflow_storage = net.storage_units_t.inflow.copy() else: inflow_storage = net.storage_units_t.inflow[existing_inflows].copy() if len(missing_inflows) > 0: - print(f"Warning: The following hydro storage units are missing inflow data and will not be defined: {missing_inflows}") + print( + f"Warning: The following hydro storage units are missing inflow data and will not be defined: {missing_inflows}" + ) # Get RoR generator inflows ror_ids = net.generators.query(config["source"]["filter"]).index.to_list() @@ -156,20 +170,26 @@ def prepare_inflow_profiles(net, config: dict): missing_ror_inflows = list(set_ror_ids - set_ror_inflow_columns) if len(existing_ror_inflows) == 0: - print("Warning: No RoR generators have inflow data. RoR inflow profiles will be empty.") + print( + "Warning: No RoR generators have inflow data. RoR inflow profiles will be empty." + ) inflow_ror = pd.DataFrame() else: ror = net.generators.loc[existing_ror_inflows].copy() inflow_ror = net.generators_t.p_max_pu[existing_ror_inflows].copy() inflow_ror = inflow_ror.mul(ror["p_nom"], axis=1) if len(missing_ror_inflows) > 0: - print(f"Warning: The following RoR generators are missing inflow data and will not be defined: {missing_ror_inflows}") + print( + f"Warning: The following RoR generators are missing inflow data and will not be defined: {missing_ror_inflows}" + ) # Concatenate both: hydro + RoR inflows → [time, generator] combined = pd.concat([inflow_storage, inflow_ror], axis=1) # sanitize combined inflow profiles for LEGO model - combined.fillna(0, inplace=True) # fill missing inflows with 0 (if any) to avoid NaNs in the final profiles + combined.fillna( + 0, inplace=True + ) # fill missing inflows with 0 (if any) to avoid NaNs in the final profiles # Change the index to k0001, k0002, ... num_timesteps = len(combined) @@ -178,8 +198,10 @@ def prepare_inflow_profiles(net, config: dict): combined = combined.T # index: generator_id, columns: time # Convert to long format by renaming index to 'g' before resetting - inflow_long = combined.rename_axis('g').reset_index().melt( - id_vars="g", var_name="k", value_name="Inflow" + inflow_long = ( + combined.rename_axis("g") + .reset_index() + .melt(id_vars="g", var_name="k", value_name="Inflow") ) inflow_long["rp"] = "rp01" # add a dummy column for compatibility @@ -188,7 +210,7 @@ def prepare_inflow_profiles(net, config: dict): def prepare_demand_profiles(net, config: dict): """ - Extracts load demand profiles and formats them into a long-form DataFrame + Extracts load demand profiles and formats them into a long-form DataFrame for LEGO representation. """ df = net.loads_t.p_set.copy() # shape: [time, load_id] @@ -224,4 +246,3 @@ def prepare_demand_profiles(net, config: dict): demand_long = df.melt(id_vars="k", var_name="n", value_name="Demand") demand_long["rp"] = "rp01" return demand_long[["rp", "n", "k", "Demand"]] - From a4890769498c0c9e7787937f9a9a4a450e6af024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 23 Apr 2026 08:55:08 +0300 Subject: [PATCH 25/37] Add default_storage_capacity_cost for hydro units. --- mapping_config.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/mapping_config.yaml b/mapping_config.yaml index 58a3087..81148d7 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -51,6 +51,7 @@ Metadata: ror: filter: [ 'ror', 'Run of River' ] hydro: + default_storage_capacity_cost: 1 # EUR/MWh, PyPSA does not have a separation between power and energy capacity for investment costs filter: [ 'Storage Hydro', 'hydro' ] phs: default_storage_capacity_cost: 1 # EUR/MWh, PyPSA does not have a separation between power and energy capacity for investment costs From 2aa081be2932e7a1fbcf91e5ee6cf50f50425a90 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 23 Apr 2026 16:07:02 +0300 Subject: [PATCH 26/37] Correct name of RampDw. --- mapping_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapping_config.yaml b/mapping_config.yaml index 81148d7..da9018e 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -140,7 +140,7 @@ dPower_ThermalGen: RampUp: attr: "ramp_limit_up" conversion: "pu_to_absolute_and_p_nom_if_nan_or_zero" - RampDown: + RampDw: attr: "ramp_limit_down" conversion: "pu_to_absolute_and_p_nom_if_nan_or_zero" MinUpTime: From 41e1143fdb22d3a4abeb2bc3f546cf42f94702cb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Thu, 23 Apr 2026 17:21:47 +0300 Subject: [PATCH 27/37] Add ExisUnits validation to ThermalGen and Storage. --- mapping_config.yaml | 18 +++++++++++------- 1 file changed, 11 insertions(+), 7 deletions(-) diff --git a/mapping_config.yaml b/mapping_config.yaml index da9018e..b8d2e03 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -132,8 +132,10 @@ dPower_ThermalGen: i: "bus" ExisUnits: attr: "active" - conversion: "bool_to_binary" - MaxProd: "p_nom" + conversion: "bool_to_binary_zero_if_p_nom_is_zero" + MaxProd: + attr: "p_nom" + conversion: "one_if_nan_or_zero" MinProd: attr: "p_min_pu" conversion: "pu_to_absolute" @@ -208,14 +210,16 @@ dPower_Storage: mapping: tec: "carrier" i: "bus" - ExisUnits: 1 + ExisUnits: + attr: "active" + conversion: "bool_to_binary_zero_if_p_nom_is_zero" MaxProd: - attr: "p_max_pu" - conversion: "pu_to_absolute" + attr: "p_nom" + conversion: "one_if_nan_or_zero" MinProd: 0 MaxCons: - attr: "p_max_pu" - conversion: "pu_to_absolute" + attr: "p_nom" + conversion: "one_if_nan_or_zero" DisEffic: "efficiency_dispatch" ChEffic: "efficiency_store" Self Discharge: "standing_loss" From cef4f571c2f08fb49c8d3adb886dab85c5b4fb69 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 27 Apr 2026 15:07:17 +0300 Subject: [PATCH 28/37] Add per unit conversion for AC line parameters. --- mapping_config.yaml | 3 ++- pypsa_helper.py | 25 ++++++++++++++++++++++--- 2 files changed, 24 insertions(+), 4 deletions(-) diff --git a/mapping_config.yaml b/mapping_config.yaml index b8d2e03..74458e3 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -3,7 +3,7 @@ Metadata: type: "metadata" dataPackage: "PyPSA-Import" dataSource: "Eur_2050_1h" - LDES_threshold: 2160 # hours + LDES_threshold: 2160 # defines the threshold in hours for storage units to be classified as LDES (Long Duration Energy Storage) # Define names of carriers and their corresponding fuel costs (if thermal generator) for every technology # [technology]: @@ -88,6 +88,7 @@ dPower_BusInfo: dPower_Network: + BasePower: 100 source: type: "helper" name: "prepare_ac_lines_and_dc_links" diff --git a/pypsa_helper.py b/pypsa_helper.py index 0fe7e68..4d965bb 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -4,20 +4,39 @@ def prepare_ac_lines(net, config: dict): """ - Prepares AC line data by calculating missing parameters (r, x, b) from line types + Prepares AC line data by calculating missing parameters (r, x, b) from line types, + converting them to per-unit values using BasePower and v_nom, and ensuring consistent naming and carrier definitions. """ lines = net.lines.copy() types = net.line_types + s_base = config.get("BasePower", 100) # Define the line parameters r, x, b if only line type is specified r_per_len = lines["type"].map(types["r_per_length"]) x_per_len = lines["type"].map(types["x_per_length"]) - # Vectorized calculation: replace 0 values with type-based defaults + # Determine b_per_length, assuming 50 Hz + # b_per_len [uS/km] = 2 * pi * 50 * C [nF/km] * 1e-3 + b_per_len = lines["type"].map(types["c_per_length"]) * 2 * np.pi * 50 * 1e-3 + + # Vectorized calculation: replace 0 values with type-based defaults (SI values) + # Note: b_per_length in PyPSA line_types is typically in uS/km, so we multiply by 1e-6 to get Siemens lines["r"] = lines["r"].where(lines["r"] != 0, r_per_len * lines["length"]) lines["x"] = lines["x"].where(lines["x"] != 0, x_per_len * lines["length"]) - lines["b"] = lines["b"].where(lines["b"] != 0, x_per_len * lines["length"]) + lines["b"] = lines["b"].where(lines["b"] != 0, b_per_len * lines["length"] * 1e-6) + + # Get v_nom from buses (kV) for each line (using bus0 as reference) + v_nom = lines.bus0.map(net.buses.v_nom) + + # Calculate Z_base = V_nom^2 / S_base [Ohm] + # Since V_nom is in kV and S_base is in MW, (kV^2 / MW) results in Ohms. + z_base = (v_nom**2) / s_base + + # Convert electrical parameters to per-unit values + lines["r"] = lines["r"] / z_base + lines["x"] = lines["x"] / z_base + lines["b"] = lines["b"] * z_base # Add tap ratios and phase shifts with default values (if not already present) lines["tap_ratio"] = 1 From 5e189d2d6b3818702c67412e2d2963b58c6ade0e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 27 Apr 2026 15:31:04 +0300 Subject: [PATCH 29/37] Add per unit conversion for trafos. --- pypsa_helper.py | 62 ++++++++++++++++++++++++++++++++++++++----------- 1 file changed, 49 insertions(+), 13 deletions(-) diff --git a/pypsa_helper.py b/pypsa_helper.py index 4d965bb..7cd173f 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -85,24 +85,60 @@ def prepare_dc_links(net, config: dict): def prepare_transformers(net, config: dict) -> pd.DataFrame: """ Calculates transformer electrical parameters (r, x, b) based on their types - and sets default values for LEGO compatibility. + if they are not already defined, converts them to per-unit values based + on the system BasePower and v_nom, and sets default values for LEGO compatibility. """ transformers = net.transformers.copy() types = net.transformer_types + s_base = config.get("BasePower", 100) - # Calculate r, x, b, based on transformer type if specified - vsc = transformers["type"].map(types["vsc"]) - nlc = transformers["type"].map(types["i0"]) - pfe = transformers["type"].map(types["pfe"]) - g = pfe / (1000 * transformers.s_nom) + # Define which values are considered "not defined" (0 or NaN) + r_missing = (transformers["r"] == 0) | transformers["r"].isna() + x_missing = (transformers["x"] == 0) | transformers["x"].isna() + b_missing = (transformers["b"] == 0) | transformers["b"].isna() - transformers["r"] = transformers["r"].where(transformers["r"] != 0, vsc / 100) - transformers["x"] = transformers["x"].where( - transformers["x"] != 0, np.sqrt((vsc / 100) ** 2 - transformers.r**2) - ) - transformers["b"] = transformers["b"].where( - transformers["b"] != 0, -np.sqrt((nlc / 100) ** 2 - g**2) - ) + # Only perform type-based calculation if there are missing values + if r_missing.any() or x_missing.any() or b_missing.any(): + vsc = transformers["type"].map(types["vsc"]) + nlc = transformers["type"].map(types["i0"]) + pfe = transformers["type"].map(types["pfe"]) + + # Avoid division by zero for s_nom + s_nom_safe = transformers.s_nom.where(transformers.s_nom != 0, 1) + + if r_missing.any(): + transformers.loc[r_missing, "r"] = vsc.loc[r_missing] / 100 + + if x_missing.any(): + r_val = transformers["r"] + transformers.loc[x_missing, "x"] = np.sqrt( + np.maximum((vsc.loc[x_missing] / 100) ** 2 - r_val.loc[x_missing] ** 2, 0) + ) + + if b_missing.any(): + g = pfe / (1000 * s_nom_safe) + transformers.loc[b_missing, "b"] = -np.sqrt( + np.maximum((nlc.loc[b_missing] / 100) ** 2 - g.loc[b_missing] ** 2, 0) + ) + + # Get v_nom from buses (kV) for each transformer (using bus0 as reference) + v_nom = transformers.bus0.map(net.buses.v_nom) + + # Calculate Z_base = V_nom^2 / S_base [Ohm] + z_base = (v_nom**2) / s_base + + # Convert electrical parameters to per-unit values + # Transformers r, x, b in PyPSA are usually p.u. on transformer base (s_nom) + # To convert to system base (s_base): + # Z_pu_sys = Z_pu_trans * (S_base / S_trans) + # Y_pu_sys = Y_pu_trans * (S_trans / S_base) + s_nom = transformers.s_nom.where(transformers.s_nom != 0, 1) + scaling_factor_z = s_base / s_nom + scaling_factor_y = s_nom / s_base + + transformers["r"] = transformers["r"] * scaling_factor_z + transformers["x"] = transformers["x"] * scaling_factor_z + transformers["b"] = transformers["b"] * scaling_factor_y # Set carrier to AC for all transformers to get defined as DC-OPF transformers["carrier"] = "AC" From 1c94a0add0646a38119b290c308a451d8539651f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 8 May 2026 17:26:25 +0300 Subject: [PATCH 30/37] Add ror to VRES technologies category. --- mapping_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mapping_config.yaml b/mapping_config.yaml index 74458e3..f06b842 100644 --- a/mapping_config.yaml +++ b/mapping_config.yaml @@ -62,7 +62,7 @@ Metadata: ThermalGen: technologies: [ 'naturalGas', 'openCycleGasTurbine', 'closedCycleGasTurbine', 'coal', 'hardCoal', 'brownCoal_lignite', 'oil', 'nuclear', 'biomass', 'geothermal', 'waste' ] VRES: - technologies: [ 'solar', 'onshoreWind', 'offshoreWind', ] + technologies: [ 'solar', 'onshoreWind', 'offshoreWind', 'ror'] Storage: technologies: [ 'phs', 'hydro' ] VRESProfiles: From c53c6b5ad4eafadc61e93133c72fe486f52d6025 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Fri, 8 May 2026 17:55:29 +0300 Subject: [PATCH 31/37] Add clipping of generator name to less than 100 characters. --- PypsaReader.py | 17 ++++++++++++++++- pypsa_helper.py | 2 +- 2 files changed, 17 insertions(+), 2 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 3fa688d..5bc5fdc 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -302,6 +302,16 @@ def _extract_dataframes(self): if idx_name in df.columns: df = df.drop(columns=[idx_name]) + # Reduces length of index to less than 100 characters + # This is only implemented because the LEGO database currently cannot handle more than 100 characters in the index! + # This can be deleted once the LEGO database is updated to handle longer index values + if isinstance(df.index, pd.MultiIndex): + df.index = pd.MultiIndex.from_frame( + df.index.to_frame().astype(str).apply(lambda x: x.str[:99]) + ) + else: + df.index = df.index.astype(str).str[:99] + df = df.reset_index() df = self._add_scenario_columns(df) @@ -506,4 +516,9 @@ def translate_pypsa_to_lego( if __name__ == "__main__": - translate_pypsa_to_lego() + translate_pypsa_to_lego( + input_directory=r"C:\BeSt\VA-GA\PyPSA-Eur-all_v2026.02.0\networks", + input_file="base_s_all_elec_.nc", + output_directory=r"C:\BeSt\PyPSA-LEGO-Translator", + output_folder_name="LEGO_PyPSA-Eur_2050", + ) diff --git a/pypsa_helper.py b/pypsa_helper.py index 7cd173f..9937cfb 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -212,7 +212,7 @@ def prepare_inflow_profiles(net, config: dict): inflow_storage = net.storage_units_t.inflow[existing_inflows].copy() if len(missing_inflows) > 0: print( - f"Warning: The following hydro storage units are missing inflow data and will not be defined: {missing_inflows}" + f"Warning: The following hydro storage units are missing inflow data and therefore inflows will not be defined: {missing_inflows}" ) # Get RoR generator inflows From 14c20504853db80da095da0c065e48ba081ced7d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 11 May 2026 21:29:50 +0300 Subject: [PATCH 32/37] Define line, link & transformer names individual to ensure compatibility with LEGO database. --- PypsaReader.py | 4 +--- pypsa_helper.py | 31 ++++++++++++++++++++++++++----- 2 files changed, 27 insertions(+), 8 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 5bc5fdc..6bafd0b 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -174,9 +174,7 @@ def EUR_to_MWh(val: pd.Series, df: pd.DataFrame, metadata: dict) -> pd.Series: return res.replace([np.inf, -np.inf], 0).fillna(0) @staticmethod - def greater_or_equal_to_one( - val: pd.Series - ) -> pd.Series: + def greater_or_equal_to_one(val: pd.Series) -> pd.Series: """Ensures that all values are greater or equal to 1, replacing values less than 1 with 1.""" return val.apply(lambda x: max(x, 1) if pd.notnull(x) else x) diff --git a/pypsa_helper.py b/pypsa_helper.py index 9937cfb..719777b 100644 --- a/pypsa_helper.py +++ b/pypsa_helper.py @@ -42,8 +42,15 @@ def prepare_ac_lines(net, config: dict): lines["tap_ratio"] = 1 lines["phase_shift"] = 0 + ### INFO: Currently can the LEGO Database not handle similar line names (e.g. c1, c1, c2, c2, ...) + # 11-05-2026 # Ensure every line has an individual circuit identifier (c1, c2, ...) for parallel lines - lines["name"] = "c" + (lines.groupby(["bus0", "bus1"]).cumcount() + 1).astype(str) + # lines["name"] = "c" + (lines.groupby(["bus0", "bus1"]).cumcount() + 1).astype(str) + + # Define individual line names until LEGO database can handle similar line names + lines["name"] = "AC_line_" + pd.Series(range(len(lines)), index=lines.index).astype( + str + ) # if carrier is nan or empty string (''), set to AC for all lines to get defined as DC-OPF lines.carrier = lines.carrier.fillna("AC").where(lines.carrier != "", "AC") @@ -68,7 +75,13 @@ def prepare_dc_links(net, config: dict): links["s_nom_extendable"] = links["p_nom_extendable"] links["id"] = links.index # Vectorized name generation - links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype( + ### INFO: Currently can the LEGO Database not handle similar line names (e.g. c1, c1, c2, c2, ...) + # 11-05-2026 + # links["name"] = "DC_Link_" + pd.Series(range(len(links)), index=links.index).astype( + # str + # ) + + links["name"] = "DC_link_" + pd.Series(range(len(links)), index=links.index).astype( str ) @@ -112,7 +125,9 @@ def prepare_transformers(net, config: dict) -> pd.DataFrame: if x_missing.any(): r_val = transformers["r"] transformers.loc[x_missing, "x"] = np.sqrt( - np.maximum((vsc.loc[x_missing] / 100) ** 2 - r_val.loc[x_missing] ** 2, 0) + np.maximum( + (vsc.loc[x_missing] / 100) ** 2 - r_val.loc[x_missing] ** 2, 0 + ) ) if b_missing.any(): @@ -143,9 +158,15 @@ def prepare_transformers(net, config: dict) -> pd.DataFrame: # Set carrier to AC for all transformers to get defined as DC-OPF transformers["carrier"] = "AC" + ### INFO: Currently can the LEGO Database not handle similar line names (e.g. c1, c1, c2, c2, ...) + # 11-05-2026 # Ensure every transformer has an individual circuit identifier (c1, c2, ...) for parallel transformers - transformers["name"] = "c" + ( - transformers.groupby(["bus0", "bus1"]).cumcount() + 1 + # transformers["name"] = "c" + ( + # transformers.groupby(["bus0", "bus1"]).cumcount() + 1 + # ).astype(str) + + transformers["name"] = "Transformer_" + pd.Series( + range(len(transformers)), index=transformers.index ).astype(str) return transformers From 1356b2684c07df932e2a23b77e226f0001a1030e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Mon, 11 May 2026 21:46:46 +0300 Subject: [PATCH 33/37] Update the documentation and rename mapping file for improved understanding. --- PypsaReader.py | 2 +- PypsaReaderDocumentation.md | 28 +++++++++---------- ...fig.yaml => pypsa_lego_mapping_config.yaml | 0 3 files changed, 14 insertions(+), 16 deletions(-) rename mapping_config.yaml => pypsa_lego_mapping_config.yaml (100%) diff --git a/PypsaReader.py b/PypsaReader.py index 6bafd0b..58bf65e 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -224,7 +224,7 @@ def __init__( self._conv_params_cache = {} # Performance: Cache function signatures if config_path is None: - config_path = os.path.join(os.path.dirname(__file__), "mapping_config.yaml") + config_path = os.path.join(os.path.dirname(__file__), "pypsa_lego_mapping_config.yaml") if table_definitions_path is None: table_definitions_path = os.path.join( diff --git a/PypsaReaderDocumentation.md b/PypsaReaderDocumentation.md index 65ad054..d2eb573 100644 --- a/PypsaReaderDocumentation.md +++ b/PypsaReaderDocumentation.md @@ -3,12 +3,22 @@ The `PypsaReader` module provides a specialized pipeline for converting PyPSA (Python for Power System Analysis) networks into the Excel-based data format required by the LEGO model. +## Purpose & limitations + +- This reader only converts PyPSA networks to LEGO input files (EXCEL). It does not return a LEGO CaseStudy object. +- For running the converted case study in LEGO, the at least the files `Global_Paramters.xlsx`, `Power_Parameters.xlsx`, `Power_Hindex.xlsx`, + `Power_WeightsK.xlsx` and `Power_WeightsRP.xlsx` must be defined. +- Currently, only the power sector is supported by the converter. +- For importing the generated Excel files into the LEGO database, the files `Global_Scenarios.xlsx`, `Data_Packages.xlsx` and `Data_Sources.xlsx` must + be defined. + ## General Workflow The conversion process follows a structured workflow: 1. **Network Loading**: A PyPSA network is loaded from a NetCDF (`.nc`) file. -2. **Configuration Parsing**: The `NetworkDataExtractor` reads `mapping_config.yaml` to determine how PyPSA components (buses, lines, generators, +2. **Configuration Parsing**: The `NetworkDataExtractor` reads `pypsa_lego_mapping_config.yaml` to determine how PyPSA components (buses, lines, + generators, etc.) map to LEGO tables. 3. **Data Extraction & Transformation**: - **Source Retrieval**: Data is pulled either directly from PyPSA network attributes (e.g., `net.buses`) or via complex processing in @@ -18,7 +28,8 @@ The conversion process follows a structured workflow: - **Unit Conversion**: The `Conversions` class applies transformations (e.g., MW to kW, EUR to MEUR, or calculating decommissioning years). 4. **Normalization**: DataFrames are augmented with mandatory LEGO columns (`scenario`, `id`, `dataPackage`, `dataSource`) and aligned with definitions in `TableDefinitions.xml`. -5. **Excel Export**: The `ExcelWriter` saves each processed DataFrame into individual `.xlsx` files.
**Warning:** This can take up to several hours for very +5. **Excel Export**: The `ExcelWriter` saves each processed DataFrame into individual `.xlsx` files.
**Warning:** This can take up to several + hours for very large networks! ## Configuration File (`mapping_config.yaml`) @@ -65,16 +76,3 @@ The `Conversions` class in `PypsaReader.py` contains static methods for speciali * **Calculated Values**: `year_and_lifetime_to_year_decom` (derives decommissioning date) and `total_capacity_to_number_of_units`. * **LEGO Specifics**: `is_ldes` (Long Duration Energy Storage detection) and `line_carrier_to_tec_repr` (mapping carriers to LEGO technical representations like `DC-OPF` or `TP`). - -## Current Limitations - -Users should be aware of the following architectural and data gaps: - -1. **Direct-to-File Output**: The reader currently saves data directly to Excel files. It **does not return a LEGO CaseStudy object** in memory, - meaning it cannot be used for immediate programmatic manipulation within a Python script without re-reading the exported files. -2. **Missing LEGO Tables**: The current mapping does not generate several files required for a complete LEGO run: - - `Power_Parameters` - - `Global_Parameters` - - `Power_Hindex` (Time mapping/Horizon index) - - `Power_WeightsK` & `Power_WeightsRP` (Representative period weighting) -3. **Scenario Logic**: The `scenario` column is currently hardcoded to `ScenarioA` during normalization. diff --git a/mapping_config.yaml b/pypsa_lego_mapping_config.yaml similarity index 100% rename from mapping_config.yaml rename to pypsa_lego_mapping_config.yaml From 0189a5519012f5679a807f437e8ca9dd75d371c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Benjamin=20St=C3=B6ckl?= Date: Tue, 2 Jun 2026 12:58:34 +0200 Subject: [PATCH 34/37] Increase character length of the index from 100 to 450. --- PypsaReader.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/PypsaReader.py b/PypsaReader.py index 58bf65e..1e52c60 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -300,15 +300,14 @@ def _extract_dataframes(self): if idx_name in df.columns: df = df.drop(columns=[idx_name]) - # Reduces length of index to less than 100 characters - # This is only implemented because the LEGO database currently cannot handle more than 100 characters in the index! - # This can be deleted once the LEGO database is updated to handle longer index values + # Reduces length of index to less than 450 characters + # This is only implemented because the LEGO database currently cannot handle more than 450 characters in the index! if isinstance(df.index, pd.MultiIndex): df.index = pd.MultiIndex.from_frame( - df.index.to_frame().astype(str).apply(lambda x: x.str[:99]) + df.index.to_frame().astype(str).apply(lambda x: x.str[:449]) ) else: - df.index = df.index.astype(str).str[:99] + df.index = df.index.astype(str).str[:449] df = df.reset_index() df = self._add_scenario_columns(df) From 690dfe95dea72d4c3392355e4bfed82556c328f2 Mon Sep 17 00:00:00 2001 From: Orhidea Shatri <140660448+scrappie1@users.noreply.github.com> Date: Mon, 6 Jul 2026 16:25:55 +0200 Subject: [PATCH 35/37] Add guide on how to translate from pypsa to lego, add to environment, and update PypsaReader to now run with command line arguments --- PypsaReader.py | 41 ++++++++++++++++++++---- README.md | 5 ++- README_pypsa_to_lego.md | 69 +++++++++++++++++++++++++++++++++++++++++ environment.yml | 2 ++ 4 files changed, 110 insertions(+), 7 deletions(-) create mode 100644 README_pypsa_to_lego.md diff --git a/PypsaReader.py b/PypsaReader.py index 1e52c60..f3bc55d 100644 --- a/PypsaReader.py +++ b/PypsaReader.py @@ -467,9 +467,9 @@ def translate_pypsa_to_lego( Main execution block for converting a PyPSA network to LEGO-formatted Excel files. To use this: - 1. Update the 'directory' and 'input_file' variables to point to your .nc PyPSA network. + 1. Point 'input_directory' and 'input_file' to your .nc PyPSA network. 2. Set 'output_directory' and 'output_folder_name' for the resulting Excel files. - 3. Run the script: `python PypsaReader.py` + 3. Run the script with command-line arguments. See: `python PypsaReader.py --help` """ # Define the path to the PyPSA network (similar than implemented in PyPSA-LEGO-Translator_testing.py) filepath = os.path.join(input_directory, input_file) @@ -513,9 +513,38 @@ def translate_pypsa_to_lego( if __name__ == "__main__": + import argparse + from rich_argparse import RichHelpFormatter + + parser = argparse.ArgumentParser( + description="Convert a PyPSA network file to LEGO formatted Excel files.", + formatter_class=RichHelpFormatter, + ) + parser.add_argument( + "inputDirectory", + type=str, + help="Path to the folder containing the PyPSA .nc network file.", + ) + parser.add_argument( + "inputFile", + type=str, + help="Name of the PyPSA .nc network file.", + ) + parser.add_argument( + "outputDirectory", + type=str, + help="Path to the folder where the LEGO output folder should be created.", + ) + parser.add_argument( + "outputFolderName", + type=str, + help="Name of the output folder for the generated LEGO Excel files.", + ) + args = parser.parse_args() + translate_pypsa_to_lego( - input_directory=r"C:\BeSt\VA-GA\PyPSA-Eur-all_v2026.02.0\networks", - input_file="base_s_all_elec_.nc", - output_directory=r"C:\BeSt\PyPSA-LEGO-Translator", - output_folder_name="LEGO_PyPSA-Eur_2050", + input_directory=args.inputDirectory, + input_file=args.inputFile, + output_directory=args.outputDirectory, + output_folder_name=args.outputFolderName, ) diff --git a/README.md b/README.md index 6c553be..4dd0eb2 100644 --- a/README.md +++ b/README.md @@ -1 +1,4 @@ -# InOutModule \ No newline at end of file +# Translating a PyPSA-Eur dataset to LEGO + +For a guide on how to translate a PyPSA-Eur dataset to LEGO Excel files, see +[`README_pypsa_to_lego.md`](README_pypsa_to_lego.md). diff --git a/README_pypsa_to_lego.md b/README_pypsa_to_lego.md new file mode 100644 index 0000000..3d462c5 --- /dev/null +++ b/README_pypsa_to_lego.md @@ -0,0 +1,69 @@ +# How to translate a PyPSA-Eur dataset to LEGO + +This guide explains how to convert an existing PyPSA-Eur network file (`.nc`) to LEGO Excel files. + +To build the PyPSA-Eur dataset first, follow +[How to create a PyPSA-Eur dataset](https://github.com/IEE-TUGraz/pypsa-eur-multivoltage/blob/fix/corine-bool-and-offshore-keyerror/README_pypsa_dataset.md) +in the `pypsa-eur-multivoltage` repository. + +It is recommended to connect to an IEE workstation via +[Remote Desktop](https://gitlab.tugraz.at/iee/iee4u/-/wikis/IT-Support/Remote-Desktop), +since the conversion can require a lot of memory and may also take some time. + +## 1. Set the mapping metadata + +In `pypsa_lego_mapping_config.yaml`, update the dataset metadata fields `dataPackage` and `dataSource`. + +Use the name of the PyPSA-Eur dataset for `dataSource`. + +Example: + +```yaml +Metadata: + dataPackage: "PyPSA-Eur" + dataSource: "PyPSA-Eur_2025_CY2013" +``` + +## 2. Translate to LEGO + +Run the converter with: + +``` +python PypsaReader.py +``` + +Example: + +``` +python PypsaReader.py "../pypsa-eur-multivoltage/resources/PyPSA-Eur_2025_CY2013/networks" "base_s_all_elec.nc" "../" "LEGO_PyPSA-Eur_2025_CY2013" +``` + +This will write the LEGO Excel files to: + +``` +../LEGO_PyPSA-Eur_2025_CY2013 +``` + +## About the generated files + +The converter generates the following LEGO files: + +- `Power_BusInfo.xlsx` +- `Power_Demand.xlsx` +- `Power_Inflows.xlsx` +- `Power_Network.xlsx` +- `Power_Storage.xlsx` +- `Power_ThermalGen.xlsx` +- `Power_VRES.xlsx` +- `Power_VRESProfiles.xlsx` + +The following files are not generated and have to be added and adapted manually: + +- `Data_Packages.xlsx` +- `Data_Sources.xlsx` +- `Global_Parameters.xlsx` +- `Global_Scenarios.xlsx` +- `Power_Hindex.xlsx` +- `Power_Parameters.xlsx` +- `Power_WeightsK.xlsx` +- `Power_WeightsRP.xlsx` diff --git a/environment.yml b/environment.yml index c066c5b..b204fbc 100644 --- a/environment.yml +++ b/environment.yml @@ -15,5 +15,7 @@ dependencies: - rich=13.9.4 - rich-argparse=1.7.1 - tsam=2.3.9 + - pypsa=1.1.2 + - pyyaml=6.0.3 - pip: - pulp==2.9.0 From f37e5d696d3c68d44d2ac20b25d2a119ea95aa24 Mon Sep 17 00:00:00 2001 From: Orhidea Shatri <140660448+scrappie1@users.noreply.github.com> Date: Wed, 8 Jul 2026 15:31:36 +0200 Subject: [PATCH 36/37] Move pypsa files into dedicated PyPSA folder, replace plain prints, merge the two pypsa readmes --- PypsaReader.py => PyPSA/PypsaReader.py | 78 +++++---- PyPSA/README.md | 165 ++++++++++++++++++ pypsa_helper.py => PyPSA/pypsa_helper.py | 26 +-- .../pypsa_lego_mapping_config.yaml | 4 +- PypsaReaderDocumentation.md | 78 --------- README.md | 9 +- README_pypsa_to_lego.md | 69 -------- pyproject.toml | 6 +- 8 files changed, 236 insertions(+), 199 deletions(-) rename PypsaReader.py => PyPSA/PypsaReader.py (90%) create mode 100644 PyPSA/README.md rename pypsa_helper.py => PyPSA/pypsa_helper.py (94%) rename pypsa_lego_mapping_config.yaml => PyPSA/pypsa_lego_mapping_config.yaml (98%) delete mode 100644 PypsaReaderDocumentation.md delete mode 100644 README_pypsa_to_lego.md diff --git a/PypsaReader.py b/PyPSA/PypsaReader.py similarity index 90% rename from PypsaReader.py rename to PyPSA/PypsaReader.py index f3bc55d..075618e 100644 --- a/PypsaReader.py +++ b/PyPSA/PypsaReader.py @@ -1,13 +1,21 @@ import inspect import os +import sys import numpy as np import pandas as pd import pypsa import yaml +PYPSA_DIR: str = os.path.dirname(os.path.abspath(__file__)) +PARENT_DIR: str = os.path.dirname(PYPSA_DIR) +sys.path.insert(0, PARENT_DIR) + import pypsa_helper as h from ExcelWriter import ExcelWriter +from printer import Printer + +printer = Printer.getInstance() class Conversions: @@ -99,9 +107,9 @@ def _get_fuel_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: meta_config = metadata.get("Metadata", {}) for key, fuel_info in meta_config.items(): if ( - isinstance(fuel_info, dict) - and "filter" in fuel_info - and "cost" in fuel_info + isinstance(fuel_info, dict) + and "filter" in fuel_info + and "cost" in fuel_info ): for carrier in fuel_info["filter"]: fuel_mapping[carrier] = fuel_info["cost"] @@ -113,15 +121,15 @@ def _get_fuel_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: if not fuel_costs.empty: missing_carriers = df.loc[fuel_costs.isnull(), "carrier"].unique() if len(missing_carriers) > 0: - print( - f"Warning: No fuel cost defined in Metadata for carrier(s): {missing_carriers.tolist()}" + printer.warning( + f"No fuel cost defined in Metadata for carrier(s): {missing_carriers.tolist()}" ) return fuel_costs @staticmethod def get_fuel_cost_from_metadata( - val: pd.Series, df: pd.DataFrame, metadata: dict + val: pd.Series, df: pd.DataFrame, metadata: dict ) -> pd.Series: """Retrieves fuel costs based on the carrier and Metadata configuration.""" return Conversions._get_fuel_costs(df, metadata) @@ -133,9 +141,9 @@ def _get_storage_capacity_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: meta_config = metadata.get("Metadata", {}) for key, info in meta_config.items(): if ( - isinstance(info, dict) - and "filter" in info - and "default_storage_capacity_cost" in info + isinstance(info, dict) + and "filter" in info + and "default_storage_capacity_cost" in info ): for carrier in info["filter"]: cost_mapping[carrier] = info["default_storage_capacity_cost"] @@ -147,14 +155,14 @@ def _get_storage_capacity_costs(df: pd.DataFrame, metadata: dict) -> pd.Series: @staticmethod def get_storage_capacity_cost_from_metadata( - val: pd.Series, df: pd.DataFrame, metadata: dict + val: pd.Series, df: pd.DataFrame, metadata: dict ) -> pd.Series: """Retrieves storage capacity costs based on the carrier and Metadata configuration.""" return Conversions._get_storage_capacity_costs(df, metadata) @staticmethod def EUR_per_hour_to_MWh_per_hour( - val: pd.Series, df: pd.DataFrame, metadata: dict + val: pd.Series, df: pd.DataFrame, metadata: dict ) -> pd.Series: """Converts costs per hour of thermal generation (e.g. stand_by_cost) to costs per MWh based on the fuel cost specified in the metadata.""" fuel_costs = Conversions._get_fuel_costs(df, metadata) @@ -180,21 +188,21 @@ def greater_or_equal_to_one(val: pd.Series) -> pd.Series: @staticmethod def pu_to_absolute_and_p_nom_if_nan_or_zero( - val: pd.Series, df: pd.DataFrame + val: pd.Series, df: pd.DataFrame ) -> pd.Series: val = val * df.p_nom return val.where((val.notnull() & (val != 0)), df.p_nom) @staticmethod def one_if_nan_or_zero( - val: pd.Series, + val: pd.Series, ) -> pd.Series: """Replaces NaN or zero values with 1, keeping other values unchanged.""" return val.where((val.notnull() & (val != 0)), 1) @staticmethod def bool_to_binary_zero_if_p_nom_is_zero( - val: pd.Series, df: pd.DataFrame + val: pd.Series, df: pd.DataFrame ) -> pd.Series: """Converts boolean values to binary (0/1) integers and sets to 0, if p_nom is zero.""" if val.isnull().all(): @@ -208,10 +216,10 @@ def bool_to_binary_zero_if_p_nom_is_zero( class NetworkDataExtractor: def __init__( - self, - network: pypsa.Network, - config_path: str = None, - table_definitions_path: str = None, + self, + network: pypsa.Network, + config_path: str = None, + table_definitions_path: str = None, ): """ Initializes the extractor with a PyPSA network and configuration files. @@ -224,11 +232,11 @@ def __init__( self._conv_params_cache = {} # Performance: Cache function signatures if config_path is None: - config_path = os.path.join(os.path.dirname(__file__), "pypsa_lego_mapping_config.yaml") + config_path = os.path.join(PYPSA_DIR, "pypsa_lego_mapping_config.yaml") if table_definitions_path is None: table_definitions_path = os.path.join( - os.path.dirname(__file__), "TableDefinitions.xml" + PARENT_DIR, "TableDefinitions.xml" ) with open(config_path, "r") as f: @@ -250,7 +258,7 @@ def _normalize_dataframes(self): table_id = name[1:] if name.startswith("d") else name if table_id not in self.excel_definitions: - print(f" Skipping {name} (not defined in TableDefinitions.xml)") + printer.warning(f"Skipping {name} (not defined in TableDefinitions.xml)") continue # Add mandatory 'scenario' column for LEGO format if missing @@ -402,8 +410,8 @@ def _map_columns(self, source_df, cfg): f"Error applying conversion '{conv_name}' to column '{lego_col}': {e}" ) else: - print( - f"Warning: Conversion function '{conv_name}' not found in Conversions class." + printer.warning( + f"Conversion function '{conv_name}' not found in Conversions class." ) # Priority 3: Simple multiplier factor @@ -458,10 +466,10 @@ def get_dataframes(self): def translate_pypsa_to_lego( - input_directory: str = r"./input_data", - input_file: str = r"pypsa-model.nc", - output_directory: str = r"./output_data", - output_folder_name: str = "LEGO-Model", + input_directory: str = r"./input_data", + input_file: str = r"pypsa-model.nc", + output_directory: str = r"./output_data", + output_folder_name: str = "LEGO-Model", ): """ Main execution block for converting a PyPSA network to LEGO-formatted Excel files. @@ -475,18 +483,18 @@ def translate_pypsa_to_lego( filepath = os.path.join(input_directory, input_file) if not os.path.exists(filepath): - print(f"Error: File not found at {filepath}") + printer.error(f"File not found at {filepath}") else: # Load the network - print(f"Loading PyPSA network from {filepath}...") + printer.information(f"Loading PyPSA network from {filepath}...") try: net = pypsa.Network(filepath) except Exception as e: - print(f"Could not load network: {e}") + printer.error(f"Could not load network: {e}") exit(1) # Extract data using NetworkDataExtractor - print("Extracting data into LEGO format...") + printer.information("Extracting data into LEGO format...") extractor = NetworkDataExtractor(net) dfs = extractor.get_dataframes() @@ -499,17 +507,17 @@ def translate_pypsa_to_lego( os.makedirs(output_dir, exist_ok=True) # Writing Excel files (filtering/checking functionality is now inside NetworkDataExtractor) - print("Writing Excel files...") + printer.information("Writing Excel files...") for name, df in dfs.items(): table_id = name[1:] if name.startswith("d") else name - print(f" Writing {name} to {output_dir}...") + printer.information(f"Writing {name} to {output_dir}...") try: writer._write_Excel_from_definition(df, output_dir, table_id) except Exception as e: - print(f" Error writing {name}: {e}") + printer.error(f"Error writing {name}: {e}") - print("\nConversion complete. Output files are in:", output_dir) + printer.information(f"Conversion complete. Output files are in: {output_dir}") if __name__ == "__main__": diff --git a/PyPSA/README.md b/PyPSA/README.md new file mode 100644 index 0000000..4a54f24 --- /dev/null +++ b/PyPSA/README.md @@ -0,0 +1,165 @@ +# How to translate a PyPSA-Eur dataset to LEGO + +This guide explains how to convert an existing PyPSA-Eur network file (`.nc`) to LEGO Excel files. + +To build the PyPSA-Eur dataset first, follow +[How to create a PyPSA-Eur dataset](https://github.com/IEE-TUGraz/pypsa-eur-multivoltage/blob/fix/corine-bool-and-offshore-keyerror/README_pypsa_dataset.md) +in the `pypsa-eur-multivoltage` repository. + +It is recommended to connect to an IEE workstation via +[Remote Desktop](https://gitlab.tugraz.at/iee/iee4u/-/wikis/IT-Support/Remote-Desktop), +since the conversion can require a lot of memory and may also take some time. + +## 1. Set the mapping metadata + +In `pypsa_lego_mapping_config.yaml`, update the dataset metadata fields `dataPackage` and `dataSource`. + +Use the name of the PyPSA-Eur dataset for `dataSource`. + +Example: + +```yaml +Metadata: + dataPackage: "PyPSA-Eur" + dataSource: "PyPSA-Eur_2025_CY2013" +``` + +## 2. Translate to LEGO + +From the `PyPSA/` folder, run the converter with: + +``` +python PypsaReader.py +``` + +Example: + +``` +python PypsaReader.py "../../pypsa-eur-multivoltage/resources/PyPSA-Eur_2025_CY2013/networks" "base_s_all_elec.nc" "../../" "LEGO_PyPSA-Eur_2025_CY2013" +``` + +This will write the LEGO Excel files to: + +``` +../../LEGO_PyPSA-Eur_2025_CY2013 +``` + +## About the generated files + +The converter generates the following LEGO files: + +- `Power_BusInfo.xlsx` +- `Power_Demand.xlsx` +- `Power_Inflows.xlsx` +- `Power_Network.xlsx` +- `Power_Storage.xlsx` +- `Power_ThermalGen.xlsx` +- `Power_VRES.xlsx` +- `Power_VRESProfiles.xlsx` + +The following files are not generated and have to be added and adapted manually: + +- `Data_Packages.xlsx` +- `Data_Sources.xlsx` +- `Global_Parameters.xlsx` +- `Global_Scenarios.xlsx` +- `Power_Hindex.xlsx` +- `Power_Parameters.xlsx` +- `Power_WeightsK.xlsx` +- `Power_WeightsRP.xlsx` + +# Reader Documentation + +The `PypsaReader` module provides a specialized pipeline for converting PyPSA (Python for Power System Analysis) +networks into the Excel-based data +format required by the LEGO model. + +## Purpose & limitations + +- This reader only converts PyPSA networks to LEGO input files (EXCEL). It does not return a LEGO CaseStudy object. +- For running the converted case study in LEGO, at least the files `Global_Parameters.xlsx`, `Power_Parameters.xlsx`, + `Power_Hindex.xlsx`, + `Power_WeightsK.xlsx` and `Power_WeightsRP.xlsx` must be defined. +- Currently, only the power sector is supported by the converter. +- For importing the generated Excel files into the LEGO database, the files `Global_Scenarios.xlsx`, + `Data_Packages.xlsx` and `Data_Sources.xlsx` must + be defined. + +## General Workflow + +The conversion process follows a structured workflow: + +1. **Network Loading**: A PyPSA network is loaded from a NetCDF (`.nc`) file. +2. **Configuration Parsing**: The `NetworkDataExtractor` reads `pypsa_lego_mapping_config.yaml` to determine how PyPSA + components (buses, lines, + generators, + etc.) map to LEGO tables. +3. **Data Extraction & Transformation**: + - **Source Retrieval**: Data is pulled either directly from PyPSA network attributes (e.g., `net.buses`) or via + complex processing in + `pypsa_helper.py`. + - **Filtering**: Technologies are filtered based on their `carrier` attribute using definitions in the `Metadata` + section of the config. + - **Column Mapping**: PyPSA attributes are mapped to LEGO column names. + - **Unit Conversion**: The `Conversions` class applies transformations (e.g., MW to kW, EUR to MEUR, or calculating + decommissioning years). +4. **Normalization**: DataFrames are augmented with mandatory LEGO columns (`scenario`, `id`, `dataPackage`, + `dataSource`) and aligned with + definitions in `TableDefinitions.xml`. +5. **Excel Export**: The `ExcelWriter` saves each processed DataFrame into individual `.xlsx` files.
**Warning:** + This can take up to several + hours for very + large networks! + +## Configuration File (`pypsa_lego_mapping_config.yaml`) + +The YAML configuration acts as the "translation map" between the two models. + +### Metadata Section + +* **Technology Filters**: Defines list of `carrier` strings that identify specific technologies (e.g., + `solar: filter: ['solar', 'Solar']`). +* **Fuel Costs**: Specifies fuel prices in EUR/MWh for thermal technologies. +* **Categories**: Groups individual technologies into broader LEGO categories like `ThermalGen` or `VRES` for batch + processing. +* **Global Settings**: Defines `dataPackage`, `dataSource`, and thresholds like `LDES_threshold`. + +### Table Mapping Section + +Each entry (e.g., `dPower_ThermalGen`) defines: + +* **source**: The data origin (`type: attribute` for direct PyPSA access or `type: helper` for function calls). +* **category**: (Optional) References a metadata category to automatically filter the source data. +* **index**: Defines the LEGO index columns (e.g., `i` for bus, `g` for generator). +* **mapping**: A dictionary where keys are LEGO columns and values are either: + - A direct PyPSA attribute name. + - A static value (int/float). + - A dictionary specifying an `attr`, a `conversion` function, or a `factor`. + +## Helper Functions (`pypsa_helper.py`) + +When simple attribute mapping is insufficient, helper functions handle complex data aggregation and profile generation: + +* **Network Topology**: + - `prepare_ac_lines_and_dc_links`: Combines PyPSA `lines`, `links` (filtered for DC), and `transformers` into a + single LEGO `Power_Network` table. + - `prepare_ac_lines` / `prepare_transformers`: Calculate missing electrical parameters ($r, x, b$) based on standard + type definitions if they are + not explicitly set. +* **Time-Series Profiles**: + - `prepare_renewable_profiles`: Extracts `p_max_pu` for VRES generators and reshapes it into a long-form LEGO + format. + - `prepare_inflow_profiles`: Aggregates inflow data from both `storage_units` (hydro) and `generators` ( + Run-of-River) into a unified time-series. + - `prepare_demand_profiles`: Sums PyPSA load data per bus and formats it for the LEGO demand table. + +## Conversions & Logic + +The `Conversions` class in `PypsaReader.py` contains static methods for specialized logic: + +* **Unit Scaling**: `EUR_to_MEUR`, `MW_to_kW`, `V_to_kV`. +* **Calculated Values**: `year_and_lifetime_to_year_decom` (derives decommissioning date) and + `total_capacity_to_number_of_units`. +* **LEGO Specifics**: `is_ldes` (Long Duration Energy Storage detection) and `line_carrier_to_tec_repr` (mapping + carriers to LEGO technical + representations like `DC-OPF` or `TP`). diff --git a/pypsa_helper.py b/PyPSA/pypsa_helper.py similarity index 94% rename from pypsa_helper.py rename to PyPSA/pypsa_helper.py index 719777b..6c21214 100644 --- a/pypsa_helper.py +++ b/PyPSA/pypsa_helper.py @@ -1,5 +1,9 @@ -import pandas as pd import numpy as np +import pandas as pd + +from printer import Printer + +printer = Printer.getInstance() def prepare_ac_lines(net, config: dict): @@ -31,7 +35,7 @@ def prepare_ac_lines(net, config: dict): # Calculate Z_base = V_nom^2 / S_base [Ohm] # Since V_nom is in kV and S_base is in MW, (kV^2 / MW) results in Ohms. - z_base = (v_nom**2) / s_base + z_base = (v_nom ** 2) / s_base # Convert electrical parameters to per-unit values lines["r"] = lines["r"] / z_base @@ -140,7 +144,7 @@ def prepare_transformers(net, config: dict) -> pd.DataFrame: v_nom = transformers.bus0.map(net.buses.v_nom) # Calculate Z_base = V_nom^2 / S_base [Ohm] - z_base = (v_nom**2) / s_base + z_base = (v_nom ** 2) / s_base # Convert electrical parameters to per-unit values # Transformers r, x, b in PyPSA are usually p.u. on transformer base (s_nom) @@ -225,15 +229,15 @@ def prepare_inflow_profiles(net, config: dict): missing_inflows = list(set_hydro_ids - set_inflow_columns) if len(existing_inflows) == 0: - print( - "Warning: No hydro storage units have inflow data. Storage inflow profiles will be empty." + printer.warning( + "No hydro storage units have inflow data. Storage inflow profiles will be empty." ) inflow_storage = net.storage_units_t.inflow.copy() else: inflow_storage = net.storage_units_t.inflow[existing_inflows].copy() if len(missing_inflows) > 0: - print( - f"Warning: The following hydro storage units are missing inflow data and therefore inflows will not be defined: {missing_inflows}" + printer.warning( + f"The following hydro storage units are missing inflow data and therefore inflows will not be defined: {missing_inflows}" ) # Get RoR generator inflows @@ -246,8 +250,8 @@ def prepare_inflow_profiles(net, config: dict): missing_ror_inflows = list(set_ror_ids - set_ror_inflow_columns) if len(existing_ror_inflows) == 0: - print( - "Warning: No RoR generators have inflow data. RoR inflow profiles will be empty." + printer.warning( + "No RoR generators have inflow data. RoR inflow profiles will be empty." ) inflow_ror = pd.DataFrame() else: @@ -255,8 +259,8 @@ def prepare_inflow_profiles(net, config: dict): inflow_ror = net.generators_t.p_max_pu[existing_ror_inflows].copy() inflow_ror = inflow_ror.mul(ror["p_nom"], axis=1) if len(missing_ror_inflows) > 0: - print( - f"Warning: The following RoR generators are missing inflow data and will not be defined: {missing_ror_inflows}" + printer.warning( + f"The following RoR generators are missing inflow data and will not be defined: {missing_ror_inflows}" ) # Concatenate both: hydro + RoR inflows → [time, generator] diff --git a/pypsa_lego_mapping_config.yaml b/PyPSA/pypsa_lego_mapping_config.yaml similarity index 98% rename from pypsa_lego_mapping_config.yaml rename to PyPSA/pypsa_lego_mapping_config.yaml index f06b842..0d8aca7 100644 --- a/pypsa_lego_mapping_config.yaml +++ b/PyPSA/pypsa_lego_mapping_config.yaml @@ -8,7 +8,7 @@ Metadata: # Define names of carriers and their corresponding fuel costs (if thermal generator) for every technology # [technology]: # filter: list of strings to filter the carrier name in the PyPSA network - # cost: fuel cost in EUR/MWh (only for thermal generators, not needed + # cost: fuel cost in EUR/MWh (only needed for thermal generators) naturalGas: filter: [ 'Gas', 'gas' ] cost: 100 # EUR/MWh @@ -62,7 +62,7 @@ Metadata: ThermalGen: technologies: [ 'naturalGas', 'openCycleGasTurbine', 'closedCycleGasTurbine', 'coal', 'hardCoal', 'brownCoal_lignite', 'oil', 'nuclear', 'biomass', 'geothermal', 'waste' ] VRES: - technologies: [ 'solar', 'onshoreWind', 'offshoreWind', 'ror'] + technologies: [ 'solar', 'onshoreWind', 'offshoreWind', 'ror' ] Storage: technologies: [ 'phs', 'hydro' ] VRESProfiles: diff --git a/PypsaReaderDocumentation.md b/PypsaReaderDocumentation.md deleted file mode 100644 index d2eb573..0000000 --- a/PypsaReaderDocumentation.md +++ /dev/null @@ -1,78 +0,0 @@ -# PyPSA Reader Documentation - -The `PypsaReader` module provides a specialized pipeline for converting PyPSA (Python for Power System Analysis) networks into the Excel-based data -format required by the LEGO model. - -## Purpose & limitations - -- This reader only converts PyPSA networks to LEGO input files (EXCEL). It does not return a LEGO CaseStudy object. -- For running the converted case study in LEGO, the at least the files `Global_Paramters.xlsx`, `Power_Parameters.xlsx`, `Power_Hindex.xlsx`, - `Power_WeightsK.xlsx` and `Power_WeightsRP.xlsx` must be defined. -- Currently, only the power sector is supported by the converter. -- For importing the generated Excel files into the LEGO database, the files `Global_Scenarios.xlsx`, `Data_Packages.xlsx` and `Data_Sources.xlsx` must - be defined. - -## General Workflow - -The conversion process follows a structured workflow: - -1. **Network Loading**: A PyPSA network is loaded from a NetCDF (`.nc`) file. -2. **Configuration Parsing**: The `NetworkDataExtractor` reads `pypsa_lego_mapping_config.yaml` to determine how PyPSA components (buses, lines, - generators, - etc.) map to LEGO tables. -3. **Data Extraction & Transformation**: - - **Source Retrieval**: Data is pulled either directly from PyPSA network attributes (e.g., `net.buses`) or via complex processing in - `pypsa_helper.py`. - - **Filtering**: Technologies are filtered based on their `carrier` attribute using definitions in the `Metadata` section of the config. - - **Column Mapping**: PyPSA attributes are mapped to LEGO column names. - - **Unit Conversion**: The `Conversions` class applies transformations (e.g., MW to kW, EUR to MEUR, or calculating decommissioning years). -4. **Normalization**: DataFrames are augmented with mandatory LEGO columns (`scenario`, `id`, `dataPackage`, `dataSource`) and aligned with - definitions in `TableDefinitions.xml`. -5. **Excel Export**: The `ExcelWriter` saves each processed DataFrame into individual `.xlsx` files.
**Warning:** This can take up to several - hours for very - large networks! - -## Configuration File (`mapping_config.yaml`) - -The YAML configuration acts as the "translation map" between the two models. - -### Metadata Section - -* **Technology Filters**: Defines list of `carrier` strings that identify specific technologies (e.g., `solar: filter: ['solar', 'Solar']`). -* **Fuel Costs**: Specifies fuel prices in EUR/MWh for thermal technologies. -* **Categories**: Groups individual technologies into broader LEGO categories like `ThermalGen` or `VRES` for batch processing. -* **Global Settings**: Defines `dataPackage`, `dataSource`, and thresholds like `LDES_threshold`. - -### Table Mapping Section - -Each entry (e.g., `dPower_ThermalGen`) defines: - -* **source**: The data origin (`type: attribute` for direct PyPSA access or `type: helper` for function calls). -* **category**: (Optional) References a metadata category to automatically filter the source data. -* **index**: Defines the LEGO index columns (e.g., `i` for bus, `g` for generator). -* **mapping**: A dictionary where keys are LEGO columns and values are either: - - A direct PyPSA attribute name. - - A static value (int/float). - - A dictionary specifying an `attr`, a `conversion` function, or a `factor`. - -## Helper Functions (`pypsa_helper.py`) - -When simple attribute mapping is insufficient, helper functions handle complex data aggregation and profile generation: - -* **Network Topology**: - - `prepare_ac_lines_and_dc_links`: Combines PyPSA `lines`, `links` (filtered for DC), and `transformers` into a single LEGO `Power_Network` table. - - `prepare_ac_lines` / `prepare_transformers`: Calculate missing electrical parameters ($r, x, b$) based on standard type definitions if they are - not explicitly set. -* **Time-Series Profiles**: - - `prepare_renewable_profiles`: Extracts `p_max_pu` for VRES generators and reshapes it into a long-form LEGO format. - - `prepare_inflow_profiles`: Aggregates inflow data from both `storage_units` (hydro) and `generators` (Run-of-River) into a unified time-series. - - `prepare_demand_profiles`: Sums PyPSA load data per bus and formats it for the LEGO demand table. - -## Conversions & Logic - -The `Conversions` class in `PypsaReader.py` contains static methods for specialized logic: - -* **Unit Scaling**: `EUR_to_MEUR`, `MW_to_kW`, `V_to_kV`. -* **Calculated Values**: `year_and_lifetime_to_year_decom` (derives decommissioning date) and `total_capacity_to_number_of_units`. -* **LEGO Specifics**: `is_ldes` (Long Duration Energy Storage detection) and `line_carrier_to_tec_repr` (mapping carriers to LEGO technical - representations like `DC-OPF` or `TP`). diff --git a/README.md b/README.md index 4dd0eb2..2fd18ec 100644 --- a/README.md +++ b/README.md @@ -1,4 +1,7 @@ -# Translating a PyPSA-Eur dataset to LEGO +# InOutModule -For a guide on how to translate a PyPSA-Eur dataset to LEGO Excel files, see -[`README_pypsa_to_lego.md`](README_pypsa_to_lego.md). +This repository contains the LEGO input/output utilities. + +PyPSA-related files are located in the folder [`PyPSA`](PyPSA). This folder contains the PyPSA reader, helper functions, +the mapping configuration, and a [`README.md`](PyPSA/README.md) with documentation and instructions for translating a +PyPSA-Eur dataset to LEGO Excel files. diff --git a/README_pypsa_to_lego.md b/README_pypsa_to_lego.md deleted file mode 100644 index 3d462c5..0000000 --- a/README_pypsa_to_lego.md +++ /dev/null @@ -1,69 +0,0 @@ -# How to translate a PyPSA-Eur dataset to LEGO - -This guide explains how to convert an existing PyPSA-Eur network file (`.nc`) to LEGO Excel files. - -To build the PyPSA-Eur dataset first, follow -[How to create a PyPSA-Eur dataset](https://github.com/IEE-TUGraz/pypsa-eur-multivoltage/blob/fix/corine-bool-and-offshore-keyerror/README_pypsa_dataset.md) -in the `pypsa-eur-multivoltage` repository. - -It is recommended to connect to an IEE workstation via -[Remote Desktop](https://gitlab.tugraz.at/iee/iee4u/-/wikis/IT-Support/Remote-Desktop), -since the conversion can require a lot of memory and may also take some time. - -## 1. Set the mapping metadata - -In `pypsa_lego_mapping_config.yaml`, update the dataset metadata fields `dataPackage` and `dataSource`. - -Use the name of the PyPSA-Eur dataset for `dataSource`. - -Example: - -```yaml -Metadata: - dataPackage: "PyPSA-Eur" - dataSource: "PyPSA-Eur_2025_CY2013" -``` - -## 2. Translate to LEGO - -Run the converter with: - -``` -python PypsaReader.py -``` - -Example: - -``` -python PypsaReader.py "../pypsa-eur-multivoltage/resources/PyPSA-Eur_2025_CY2013/networks" "base_s_all_elec.nc" "../" "LEGO_PyPSA-Eur_2025_CY2013" -``` - -This will write the LEGO Excel files to: - -``` -../LEGO_PyPSA-Eur_2025_CY2013 -``` - -## About the generated files - -The converter generates the following LEGO files: - -- `Power_BusInfo.xlsx` -- `Power_Demand.xlsx` -- `Power_Inflows.xlsx` -- `Power_Network.xlsx` -- `Power_Storage.xlsx` -- `Power_ThermalGen.xlsx` -- `Power_VRES.xlsx` -- `Power_VRESProfiles.xlsx` - -The following files are not generated and have to be added and adapted manually: - -- `Data_Packages.xlsx` -- `Data_Sources.xlsx` -- `Global_Parameters.xlsx` -- `Global_Scenarios.xlsx` -- `Power_Hindex.xlsx` -- `Power_Parameters.xlsx` -- `Power_WeightsK.xlsx` -- `Power_WeightsRP.xlsx` diff --git a/pyproject.toml b/pyproject.toml index a72a436..b5f2033 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -8,4 +8,8 @@ version = "0.0.0-dev" readme = "README.md" [tool.setuptools] -py-modules = ['CaseStudy', 'ExcelReader', 'ExcelWriter', 'printer', 'PypsaReader', 'pypsa_helper', 'SQLiteWriter', 'TableDefinition'] \ No newline at end of file +py-modules = ['CaseStudy', 'ExcelReader', 'ExcelWriter', 'printer', 'SQLiteWriter', 'TableDefinition'] +packages = ['PyPSA'] + +[tool.setuptools.package-data] +PyPSA = ['pypsa_lego_mapping_config.yaml', 'README.md'] From 9dd037ca792e3a07f8969417776a99e5bc67dbf7 Mon Sep 17 00:00:00 2001 From: Orhidea Shatri <140660448+scrappie1@users.noreply.github.com> Date: Thu, 9 Jul 2026 12:35:00 +0200 Subject: [PATCH 37/37] Remove link to remote desktop from pypsa readme --- PyPSA/README.md | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/PyPSA/README.md b/PyPSA/README.md index 4a54f24..c360500 100644 --- a/PyPSA/README.md +++ b/PyPSA/README.md @@ -6,9 +6,8 @@ To build the PyPSA-Eur dataset first, follow [How to create a PyPSA-Eur dataset](https://github.com/IEE-TUGraz/pypsa-eur-multivoltage/blob/fix/corine-bool-and-offshore-keyerror/README_pypsa_dataset.md) in the `pypsa-eur-multivoltage` repository. -It is recommended to connect to an IEE workstation via -[Remote Desktop](https://gitlab.tugraz.at/iee/iee4u/-/wikis/IT-Support/Remote-Desktop), -since the conversion can require a lot of memory and may also take some time. +Since the conversion can require a lot of memory and may also take some time, it is recommended to use a powerful +machine. ## 1. Set the mapping metadata