From a7f57fc388987285a04cbf76fd64e6f0cf887919 Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Wed, 13 May 2026 15:39:13 +0200 Subject: [PATCH 01/11] Move country data out of utils --- pycoupler/countries.json | 946 +++++++++++++++++++++++++++++++++++++++ pycoupler/utils.py | 300 +------------ 2 files changed, 951 insertions(+), 295 deletions(-) create mode 100644 pycoupler/countries.json diff --git a/pycoupler/countries.json b/pycoupler/countries.json new file mode 100644 index 0000000..6cb2e5b --- /dev/null +++ b/pycoupler/countries.json @@ -0,0 +1,946 @@ +{ + "Afghanistan": { + "name": "Afghanistan", + "code": "AFG" + }, + "Aland Islands": { + "name": "Aland Islands", + "code": "ALA" + }, + "Albania": { + "name": "Albania", + "code": "ALB" + }, + "Algeria": { + "name": "Algeria", + "code": "DZA" + }, + "American Samoa": { + "name": "American Samoa", + "code": "ASM" + }, + "Angola": { + "name": "Angola", + "code": "AGO" + }, + "Anguilla": { + "name": "Anguilla", + "code": "AIA" + }, + "Antigua and Barbuda": { + "name": "Antigua and Barbuda", + "code": "ATG" + }, + "Argentina": { + "name": "Argentina", + "code": "ARG" + }, + "Armenia": { + "name": "Armenia", + "code": "ARM" + }, + "Austria": { + "name": "Austria", + "code": "AUT" + }, + "Azerbaijan": { + "name": "Azerbaijan", + "code": "AZE" + }, + "Bahamas The": { + "name": "Bahamas, The", + "code": "BHS" + }, + "Bahrain": { + "name": "Bahrain", + "code": "BHR" + }, + "Bangladesh": { + "name": "Bangladesh", + "code": "BGD" + }, + "Barbados": { + "name": "Barbados", + "code": "BRB" + }, + "Belgium": { + "name": "Belgium", + "code": "BEL" + }, + "Belize": { + "name": "Belize", + "code": "BLZ" + }, + "Benin": { + "name": "Benin", + "code": "BEN" + }, + "Bermuda": { + "name": "Bermuda", + "code": "BMU" + }, + "Bhutan": { + "name": "Bhutan", + "code": "BTN" + }, + "Bolivia": { + "name": "Bolivia", + "code": "BOL" + }, + "Bosnia and Herzegovina": { + "name": "Bosnia and Herzegovina", + "code": "BIH" + }, + "Botswana": { + "name": "Botswana", + "code": "BWA" + }, + "British Indian Ocean Territory": { + "name": "British Indian Ocean Territory", + "code": "IOT" + }, + "Brunei": { + "name": "Brunei", + "code": "BRN" + }, + "Bulgaria": { + "name": "Bulgaria", + "code": "BGR" + }, + "Burkina Faso": { + "name": "Burkina Faso", + "code": "BFA" + }, + "Burundi": { + "name": "Burundi", + "code": "BDI" + }, + "Byelarus": { + "name": "Byelarus", + "code": "BLR" + }, + "Cambodia": { + "name": "Cambodia", + "code": "KHM" + }, + "Cameroon": { + "name": "Cameroon", + "code": "CMR" + }, + "Cape Verde": { + "name": "Cape Verde", + "code": "CPV" + }, + "Cayman Islands": { + "name": "Cayman Islands", + "code": "CYM" + }, + "Central African Republic": { + "name": "Central African Republic", + "code": "CAF" + }, + "Chad": { + "name": "Chad", + "code": "TCD" + }, + "Chile": { + "name": "Chile", + "code": "CHL" + }, + "Christmas Island": { + "name": "Christmas Island", + "code": "CXR" + }, + "Cocos Keeling Islands": { + "name": "Cocos Keeling Islands", + "code": "CCK" + }, + "Colombia": { + "name": "Colombia", + "code": "COL" + }, + "Comoros": { + "name": "Comoros", + "code": "COM" + }, + "Congo Brazzaville": { + "name": "Congo-Brazzaville", + "code": "COG" + }, + "Cook Islands": { + "name": "Cook Islands", + "code": "COK" + }, + "Costa Rica": { + "name": "Costa Rica", + "code": "CRI" + }, + "Croatia": { + "name": "Croatia", + "code": "HRV" + }, + "Cuba": { + "name": "Cuba", + "code": "CUB" + }, + "Curacao": { + "name": "Curacao", + "code": "CUW" + }, + "Cyprus": { + "name": "Cyprus", + "code": "CYP" + }, + "Czech Republic": { + "name": "Czech Republic", + "code": "CZE" + }, + "Denmark": { + "name": "Denmark", + "code": "DNK" + }, + "Djibouti": { + "name": "Djibouti", + "code": "DJI" + }, + "Dominica": { + "name": "Dominica", + "code": "DMA" + }, + "Dominican Republic": { + "name": "Dominican Republic", + "code": "DOM" + }, + "Ecuador": { + "name": "Ecuador", + "code": "ECU" + }, + "Egypt": { + "name": "Egypt", + "code": "EGY" + }, + "El Salvador": { + "name": "El Salvador", + "code": "SLV" + }, + "Equatorial Guinea": { + "name": "Equatorial Guinea", + "code": "GNQ" + }, + "Eritrea": { + "name": "Eritrea", + "code": "ERI" + }, + "Estonia": { + "name": "Estonia", + "code": "EST" + }, + "Ethiopia": { + "name": "Ethiopia", + "code": "ETH" + }, + "Falkland Islands or Islas Malvinas": { + "name": "Falkland Islands or Islas Malvinas", + "code": "FLK" + }, + "Faroe Islands": { + "name": "Faroe Islands", + "code": "FRO" + }, + "Federated States of Micronesia": { + "name": "Federated States of Micronesia", + "code": "FSM" + }, + "Fiji": { + "name": "Fiji", + "code": "FJI" + }, + "Finland": { + "name": "Finland", + "code": "FIN" + }, + "France": { + "name": "France", + "code": "FRA" + }, + "French Guiana": { + "name": "French Guiana", + "code": "GUF" + }, + "French Polynesia": { + "name": "French Polynesia", + "code": "PYF" + }, + "French Southern and Antarctica Lands": { + "name": "French Southern and Antarctica Lands", + "code": "NOC" + }, + "Gabon": { + "name": "Gabon", + "code": "GAB" + }, + "Gambia The": { + "name": "Gambia,The", + "code": "GMB" + }, + "Georgia": { + "name": "Georgia", + "code": "GEO" + }, + "Germany": { + "name": "Germany", + "code": "DEU" + }, + "Ghana": { + "name": "Ghana", + "code": "GHA" + }, + "Greece": { + "name": "Greece", + "code": "GRC" + }, + "Greenland": { + "name": "Greenland", + "code": "GRL" + }, + "Grenada": { + "name": "Grenada", + "code": "GRD" + }, + "Guadeloupe": { + "name": "Guadeloupe", + "code": "GLP" + }, + "Guam": { + "name": "Guam", + "code": "GUM" + }, + "Guatemala": { + "name": "Guatemala", + "code": "GTM" + }, + "Guernsey": { + "name": "Guernsey", + "code": "GGY" + }, + "Guinea Bissau": { + "name": "Guinea-Bissau", + "code": "GNB" + }, + "Guinea": { + "name": "Guinea", + "code": "GIN" + }, + "Guyana": { + "name": "Guyana", + "code": "GUY" + }, + "Haiti": { + "name": "Haiti", + "code": "HTI" + }, + "Heard Island and McDonald Islands": { + "name": "Heard Island and McDonald Islands", + "code": "HMD" + }, + "Honduras": { + "name": "Honduras", + "code": "HND" + }, + "Hong Kong": { + "name": "Hong Kong", + "code": "HKG" + }, + "Hungary": { + "name": "Hungary", + "code": "HUN" + }, + "Iceland": { + "name": "Iceland", + "code": "ISL" + }, + "Indonesia": { + "name": "Indonesia", + "code": "IDN" + }, + "Iran": { + "name": "Iran", + "code": "IRN" + }, + "Iraq": { + "name": "Iraq", + "code": "IRQ" + }, + "Ireland": { + "name": "Ireland", + "code": "IRL" + }, + "Isle of Man": { + "name": "Isle of Man", + "code": "IMN" + }, + "Israel": { + "name": "Israel", + "code": "ISR" + }, + "Italy": { + "name": "Italy", + "code": "ITA" + }, + "Ivory Coast": { + "name": "Ivory Coast", + "code": "CIV" + }, + "Jamaica": { + "name": "Jamaica", + "code": "JAM" + }, + "Japan": { + "name": "Japan", + "code": "JPN" + }, + "Jersey": { + "name": "Jersey", + "code": "JEY" + }, + "Jordan": { + "name": "Jordan", + "code": "JOR" + }, + "Kazakhstan": { + "name": "Kazakhstan", + "code": "KAZ" + }, + "Kenya": { + "name": "Kenya", + "code": "KEN" + }, + "Kiribati": { + "name": "Kiribati", + "code": "KIR" + }, + "Kosovo": { + "name": "Kosovo", + "code": "KO-" + }, + "Kuwait": { + "name": "Kuwait", + "code": "KWT" + }, + "Kyrgyzstan": { + "name": "Kyrgyzstan", + "code": "KGZ" + }, + "Laos": { + "name": "Laos", + "code": "LAO" + }, + "Latvia": { + "name": "Latvia", + "code": "LVA" + }, + "Lebanon": { + "name": "Lebanon", + "code": "LBN" + }, + "Lesotho": { + "name": "Lesotho", + "code": "LSO" + }, + "Liberia": { + "name": "Liberia", + "code": "LBR" + }, + "Libya": { + "name": "Libya", + "code": "LBY" + }, + "Lithuania": { + "name": "Lithuania", + "code": "LTU" + }, + "Luxembourg": { + "name": "Luxembourg", + "code": "LUX" + }, + "Macedonia": { + "name": "Macedonia", + "code": "MKD" + }, + "Madagascar": { + "name": "Madagascar", + "code": "MDG" + }, + "Malawi": { + "name": "Malawi", + "code": "MWI" + }, + "Malaysia": { + "name": "Malaysia", + "code": "MYS" + }, + "Maldives": { + "name": "Maldives", + "code": "MDV" + }, + "Mali": { + "name": "Mali", + "code": "MLI" + }, + "Malta": { + "name": "Malta", + "code": "MLT" + }, + "Marshall Islands": { + "name": "Marshall Islands", + "code": "MHL" + }, + "Martinique": { + "name": "Martinique", + "code": "MTQ" + }, + "Mauritania": { + "name": "Mauritania", + "code": "MRT" + }, + "Mauritius": { + "name": "Mauritius", + "code": "MUS" + }, + "Mayotte": { + "name": "Mayotte", + "code": "MYT" + }, + "Mexico": { + "name": "Mexico", + "code": "MEX" + }, + "Moldova": { + "name": "Moldova", + "code": "MDA" + }, + "Mongolia": { + "name": "Mongolia", + "code": "MNG" + }, + "Montenegro": { + "name": "Montenegro", + "code": "MNE" + }, + "Montserrat": { + "name": "Montserrat", + "code": "MSR" + }, + "Morocco": { + "name": "Morocco", + "code": "MAR" + }, + "Mozambique": { + "name": "Mozambique", + "code": "MOZ" + }, + "Myanmar or Burma": { + "name": "Myanmar or Burma", + "code": "MMR" + }, + "Namibia": { + "name": "Namibia", + "code": "NAM" + }, + "Nauru": { + "name": "Nauru", + "code": "NRU" + }, + "Nepal": { + "name": "Nepal", + "code": "NPL" + }, + "Netherlands": { + "name": "Netherlands", + "code": "NLD" + }, + "New Caledonia": { + "name": "New Caledonia", + "code": "NCL" + }, + "New Zealand": { + "name": "New Zealand", + "code": "NZL" + }, + "Nicaragua": { + "name": "Nicaragua", + "code": "NIC" + }, + "Niger": { + "name": "Niger", + "code": "NER" + }, + "Nigeria": { + "name": "Nigeria", + "code": "NGA" + }, + "Niue": { + "name": "Niue", + "code": "NIU" + }, + "No Land": { + "name": "No Land", + "code": "XNL" + }, + "Norfolk Island": { + "name": "Norfolk Island", + "code": "NFK" + }, + "North Korea": { + "name": "North Korea", + "code": "PRK" + }, + "Northern Mariana Islands": { + "name": "Northern Mariana Islands", + "code": "MNP" + }, + "Norway": { + "name": "Norway", + "code": "NOR" + }, + "Oman": { + "name": "Oman", + "code": "OMN" + }, + "Pakistan": { + "name": "Pakistan", + "code": "PAK" + }, + "Palau": { + "name": "Palau", + "code": "PLW" + }, + "Panama": { + "name": "Panama", + "code": "PAN" + }, + "Papua New Guinea": { + "name": "Papua New Guinea", + "code": "PNG" + }, + "Paraguay": { + "name": "Paraguay", + "code": "PRY" + }, + "Peru": { + "name": "Peru", + "code": "PER" + }, + "Philippines": { + "name": "Philippines", + "code": "PHL" + }, + "Pitcairn Islands": { + "name": "Pitcairn Islands", + "code": "PCN" + }, + "Poland": { + "name": "Poland", + "code": "POL" + }, + "Portugal": { + "name": "Portugal", + "code": "PRT" + }, + "Puerto Rico": { + "name": "Puerto Rico", + "code": "PRI" + }, + "Qatar": { + "name": "Qatar", + "code": "QAT" + }, + "Reunion": { + "name": "Reunion", + "code": "REU" + }, + "Romania": { + "name": "Romania", + "code": "ROU" + }, + "Rwanda": { + "name": "Rwanda", + "code": "RWA" + }, + "Saint Helena Ascension and Tristan da Cunha": { + "name": "Saint Helena Ascension and Tristan da Cunha", + "code": "SHN" + }, + "Saint Kitts and Nevis": { + "name": "Saint Kitts and Nevis", + "code": "KNA" + }, + "Saint Lucia": { + "name": "Saint Lucia", + "code": "LCA" + }, + "Saint Pierre and Miquelon": { + "name": "Saint Pierre and Miquelon", + "code": "SPM" + }, + "Sao Tome and Principe": { + "name": "Sao Tome and Principe", + "code": "STP" + }, + "Saudi Arabia": { + "name": "Saudi Arabia", + "code": "SAU" + }, + "Senegal": { + "name": "Senegal", + "code": "SEN" + }, + "Serbia": { + "name": "Serbia", + "code": "SRB" + }, + "Seychelles": { + "name": "Seychelles", + "code": "SYC" + }, + "Sierra Leone": { + "name": "Sierra Leone", + "code": "SLE" + }, + "Singapore": { + "name": "Singapore", + "code": "SGP" + }, + "Slovakia": { + "name": "Slovakia", + "code": "SVK" + }, + "Slovenia": { + "name": "Slovenia", + "code": "SVN" + }, + "Solomon Islands": { + "name": "Solomon Islands", + "code": "SLB" + }, + "Somalia": { + "name": "Somalia", + "code": "SOM" + }, + "South Africa": { + "name": "South Africa", + "code": "ZAF" + }, + "South Georgia and the South Sandwich Islands": { + "name": "South Georgia and the South Sandwich Islands", + "code": "SGS" + }, + "South Korea": { + "name": "South Korea", + "code": "KOR" + }, + "South Sudan": { + "name": "South Sudan", + "code": "SSD" + }, + "Spain": { + "name": "Spain", + "code": "ESP" + }, + "Sri Lanka": { + "name": "Sri Lanka", + "code": "LKA" + }, + "St Vincent and the Grenadines": { + "name": "St. Vincent and the Grenadines", + "code": "VCT" + }, + "Sudan": { + "name": "Sudan", + "code": "SDN" + }, + "Suriname": { + "name": "Suriname", + "code": "SUR" + }, + "Svalbard": { + "name": "Svalbard", + "code": "SJM" + }, + "Swaziland": { + "name": "Swaziland", + "code": "SWZ" + }, + "Sweden": { + "name": "Sweden", + "code": "SWE" + }, + "Switzerland": { + "name": "Switzerland", + "code": "CHE" + }, + "Syria": { + "name": "Syria", + "code": "SYR" + }, + "Taiwan": { + "name": "Taiwan", + "code": "TWN" + }, + "Tajikistan": { + "name": "Tajikistan", + "code": "TJK" + }, + "Tanzania United Republic of": { + "name": "Tanzania, United Republic of", + "code": "TZA" + }, + "Thailand": { + "name": "Thailand", + "code": "THA" + }, + "Timor Leste": { + "name": "Timor Leste", + "code": "TLS" + }, + "Togo": { + "name": "Togo", + "code": "TGO" + }, + "Tokelau": { + "name": "Tokelau", + "code": "TKL" + }, + "Tonga": { + "name": "Tonga", + "code": "TON" + }, + "Trinidad and Tobago": { + "name": "Trinidad and Tobago", + "code": "TTO" + }, + "Tunisia": { + "name": "Tunisia", + "code": "TUN" + }, + "Turkey": { + "name": "Turkey", + "code": "TUR" + }, + "Turkmenistan": { + "name": "Turkmenistan", + "code": "TKM" + }, + "Turks and Caicos Islands": { + "name": "Turks and Caicos Islands", + "code": "TCA" + }, + "Tuvalu": { + "name": "Tuvalu", + "code": "TUV" + }, + "Uganda": { + "name": "Uganda", + "code": "UGA" + }, + "Ukraine": { + "name": "Ukraine", + "code": "UKR" + }, + "United Arab Emirates": { + "name": "United Arab Emirates", + "code": "ARE" + }, + "United Kingdom": { + "name": "United Kingdom", + "code": "GBR" + }, + "United States Minor Outlying Islands": { + "name": "United States Minor Outlying Islands", + "code": "UMI" + }, + "Uruguay": { + "name": "Uruguay", + "code": "URY" + }, + "Uzbekistan": { + "name": "Uzbekistan", + "code": "UZB" + }, + "Vanuatu": { + "name": "Vanuatu", + "code": "VUT" + }, + "Venezuela": { + "name": "Venezuela", + "code": "VEN" + }, + "Vietnam": { + "name": "Vietnam", + "code": "VNM" + }, + "Virgin Islands": { + "name": "Virgin Islands", + "code": "VGB" + }, + "Wallis and Futuna": { + "name": "Wallis and Futuna", + "code": "WLF" + }, + "West Bank": { + "name": "West Bank", + "code": "PSE" + }, + "Western Sahara": { + "name": "Western Sahara", + "code": "ESH" + }, + "Western Samoa": { + "name": "Western Samoa", + "code": "WSM" + }, + "Yemen": { + "name": "Yemen", + "code": "YEM" + }, + "Zaire DR Congo": { + "name": "DR Congo, former Zaire", + "code": "COD" + }, + "Zambia": { + "name": "Zambia", + "code": "ZMB" + }, + "Zimbabwe": { + "name": "Zimbabwe", + "code": "ZWE" + }, + "Australia": { + "name": "Australia", + "code": "AUS" + }, + "Brazil": { + "name": "Brazil", + "code": "BRA" + }, + "Canada": { + "name": "Canada", + "code": "CAN" + }, + "China": { + "name": "China", + "code": "CHN" + }, + "India": { + "name": "India", + "code": "IND" + }, + "Russia": { + "name": "Russia", + "code": "RUS" + }, + "United States": { + "name": "United States of America", + "code": "USA" + }, +} \ No newline at end of file diff --git a/pycoupler/utils.py b/pycoupler/utils.py index 82a766d..a3d7051 100644 --- a/pycoupler/utils.py +++ b/pycoupler/utils.py @@ -1,3 +1,4 @@ +import importlib.resources import os import json from fuzzywuzzy import fuzz, process @@ -11,301 +12,10 @@ def get_countries(): dict Dictionary with countries and their codes. """ - return { - "Afghanistan": {"name": "Afghanistan", "code": "AFG"}, - "Aland Islands": {"name": "Aland Islands", "code": "ALA"}, - "Albania": {"name": "Albania", "code": "ALB"}, - "Algeria": {"name": "Algeria", "code": "DZA"}, - "American Samoa": {"name": "American Samoa", "code": "ASM"}, - "Angola": {"name": "Angola", "code": "AGO"}, - "Anguilla": {"name": "Anguilla", "code": "AIA"}, - "Antigua and Barbuda": { - "name": "Antigua and Barbuda", - "code": "ATG", - }, - "Argentina": {"name": "Argentina", "code": "ARG"}, - "Armenia": {"name": "Armenia", "code": "ARM"}, - "Austria": {"name": "Austria", "code": "AUT"}, - "Azerbaijan": {"name": "Azerbaijan", "code": "AZE"}, - "Bahamas The": {"name": "Bahamas, The", "code": "BHS"}, - "Bahrain": {"name": "Bahrain", "code": "BHR"}, - "Bangladesh": {"name": "Bangladesh", "code": "BGD"}, - "Barbados": {"name": "Barbados", "code": "BRB"}, - "Belgium": {"name": "Belgium", "code": "BEL"}, - "Belize": {"name": "Belize", "code": "BLZ"}, - "Benin": {"name": "Benin", "code": "BEN"}, - "Bermuda": {"name": "Bermuda", "code": "BMU"}, - "Bhutan": {"name": "Bhutan", "code": "BTN"}, - "Bolivia": {"name": "Bolivia", "code": "BOL"}, - "Bosnia and Herzegovina": { - "name": "Bosnia and Herzegovina", - "code": "BIH", - }, - "Botswana": {"name": "Botswana", "code": "BWA"}, - "British Indian Ocean Territory": { - "name": "British Indian Ocean Territory", - "code": "IOT", - }, - "Brunei": {"name": "Brunei", "code": "BRN"}, - "Bulgaria": {"name": "Bulgaria", "code": "BGR"}, - "Burkina Faso": {"name": "Burkina Faso", "code": "BFA"}, - "Burundi": {"name": "Burundi", "code": "BDI"}, - "Byelarus": {"name": "Byelarus", "code": "BLR"}, - "Cambodia": {"name": "Cambodia", "code": "KHM"}, - "Cameroon": {"name": "Cameroon", "code": "CMR"}, - "Cape Verde": {"name": "Cape Verde", "code": "CPV"}, - "Cayman Islands": {"name": "Cayman Islands", "code": "CYM"}, - "Central African Republic": { - "name": "Central African Republic", - "code": "CAF", - }, - "Chad": {"name": "Chad", "code": "TCD"}, - "Chile": {"name": "Chile", "code": "CHL"}, - "Christmas Island": {"name": "Christmas Island", "code": "CXR"}, - "Cocos Keeling Islands": { - "name": "Cocos Keeling Islands", - "code": "CCK", - }, - "Colombia": {"name": "Colombia", "code": "COL"}, - "Comoros": {"name": "Comoros", "code": "COM"}, - "Congo Brazzaville": {"name": "Congo-Brazzaville", "code": "COG"}, - "Cook Islands": {"name": "Cook Islands", "code": "COK"}, - "Costa Rica": {"name": "Costa Rica", "code": "CRI"}, - "Croatia": {"name": "Croatia", "code": "HRV"}, - "Cuba": {"name": "Cuba", "code": "CUB"}, - "Curacao": {"name": "Curacao", "code": "CUW"}, - "Cyprus": {"name": "Cyprus", "code": "CYP"}, - "Czech Republic": {"name": "Czech Republic", "code": "CZE"}, - "Denmark": {"name": "Denmark", "code": "DNK"}, - "Djibouti": {"name": "Djibouti", "code": "DJI"}, - "Dominica": {"name": "Dominica", "code": "DMA"}, - "Dominican Republic": {"name": "Dominican Republic", "code": "DOM"}, - "Ecuador": {"name": "Ecuador", "code": "ECU"}, - "Egypt": {"name": "Egypt", "code": "EGY"}, - "El Salvador": {"name": "El Salvador", "code": "SLV"}, - "Equatorial Guinea": {"name": "Equatorial Guinea", "code": "GNQ"}, - "Eritrea": {"name": "Eritrea", "code": "ERI"}, - "Estonia": {"name": "Estonia", "code": "EST"}, - "Ethiopia": {"name": "Ethiopia", "code": "ETH"}, - "Falkland Islands or Islas Malvinas": { - "name": "Falkland Islands or Islas Malvinas", - "code": "FLK", - }, - "Faroe Islands": {"name": "Faroe Islands", "code": "FRO"}, - "Federated States of Micronesia": { - "name": "Federated States of Micronesia", - "code": "FSM", - }, - "Fiji": {"name": "Fiji", "code": "FJI"}, - "Finland": {"name": "Finland", "code": "FIN"}, - "France": {"name": "France", "code": "FRA"}, - "French Guiana": {"name": "French Guiana", "code": "GUF"}, - "French Polynesia": {"name": "French Polynesia", "code": "PYF"}, - "French Southern and Antarctica Lands": { - "name": "French Southern and Antarctica Lands", - "code": "NOC", - }, - "Gabon": {"name": "Gabon", "code": "GAB"}, - "Gambia The": {"name": "Gambia,The", "code": "GMB"}, - "Georgia": {"name": "Georgia", "code": "GEO"}, - "Germany": {"name": "Germany", "code": "DEU"}, - "Ghana": {"name": "Ghana", "code": "GHA"}, - "Greece": {"name": "Greece", "code": "GRC"}, - "Greenland": {"name": "Greenland", "code": "GRL"}, - "Grenada": {"name": "Grenada", "code": "GRD"}, - "Guadeloupe": {"name": "Guadeloupe", "code": "GLP"}, - "Guam": {"name": "Guam", "code": "GUM"}, - "Guatemala": {"name": "Guatemala", "code": "GTM"}, - "Guernsey": {"name": "Guernsey", "code": "GGY"}, - "Guinea Bissau": {"name": "Guinea-Bissau", "code": "GNB"}, - "Guinea": {"name": "Guinea", "code": "GIN"}, - "Guyana": {"name": "Guyana", "code": "GUY"}, - "Haiti": {"name": "Haiti", "code": "HTI"}, - "Heard Island and McDonald Islands": { - "name": "Heard Island and McDonald Islands", - "code": "HMD", - }, - "Honduras": {"name": "Honduras", "code": "HND"}, - "Hong Kong": {"name": "Hong Kong", "code": "HKG"}, - "Hungary": {"name": "Hungary", "code": "HUN"}, - "Iceland": {"name": "Iceland", "code": "ISL"}, - "Indonesia": {"name": "Indonesia", "code": "IDN"}, - "Iran": {"name": "Iran", "code": "IRN"}, - "Iraq": {"name": "Iraq", "code": "IRQ"}, - "Ireland": {"name": "Ireland", "code": "IRL"}, - "Isle of Man": {"name": "Isle of Man", "code": "IMN"}, - "Israel": {"name": "Israel", "code": "ISR"}, - "Italy": {"name": "Italy", "code": "ITA"}, - "Ivory Coast": {"name": "Ivory Coast", "code": "CIV"}, - "Jamaica": {"name": "Jamaica", "code": "JAM"}, - "Japan": {"name": "Japan", "code": "JPN"}, - "Jersey": {"name": "Jersey", "code": "JEY"}, - "Jordan": {"name": "Jordan", "code": "JOR"}, - "Kazakhstan": {"name": "Kazakhstan", "code": "KAZ"}, - "Kenya": {"name": "Kenya", "code": "KEN"}, - "Kiribati": {"name": "Kiribati", "code": "KIR"}, - "Kosovo": {"name": "Kosovo", "code": "KO-"}, - "Kuwait": {"name": "Kuwait", "code": "KWT"}, - "Kyrgyzstan": {"name": "Kyrgyzstan", "code": "KGZ"}, - "Laos": {"name": "Laos", "code": "LAO"}, - "Latvia": {"name": "Latvia", "code": "LVA"}, - "Lebanon": {"name": "Lebanon", "code": "LBN"}, - "Lesotho": {"name": "Lesotho", "code": "LSO"}, - "Liberia": {"name": "Liberia", "code": "LBR"}, - "Libya": {"name": "Libya", "code": "LBY"}, - "Lithuania": {"name": "Lithuania", "code": "LTU"}, - "Luxembourg": {"name": "Luxembourg", "code": "LUX"}, - "Macedonia": {"name": "Macedonia", "code": "MKD"}, - "Madagascar": {"name": "Madagascar", "code": "MDG"}, - "Malawi": {"name": "Malawi", "code": "MWI"}, - "Malaysia": {"name": "Malaysia", "code": "MYS"}, - "Maldives": {"name": "Maldives", "code": "MDV"}, - "Mali": {"name": "Mali", "code": "MLI"}, - "Malta": {"name": "Malta", "code": "MLT"}, - "Marshall Islands": {"name": "Marshall Islands", "code": "MHL"}, - "Martinique": {"name": "Martinique", "code": "MTQ"}, - "Mauritania": {"name": "Mauritania", "code": "MRT"}, - "Mauritius": {"name": "Mauritius", "code": "MUS"}, - "Mayotte": {"name": "Mayotte", "code": "MYT"}, - "Mexico": {"name": "Mexico", "code": "MEX"}, - "Moldova": {"name": "Moldova", "code": "MDA"}, - "Mongolia": {"name": "Mongolia", "code": "MNG"}, - "Montenegro": {"name": "Montenegro", "code": "MNE"}, - "Montserrat": {"name": "Montserrat", "code": "MSR"}, - "Morocco": {"name": "Morocco", "code": "MAR"}, - "Mozambique": {"name": "Mozambique", "code": "MOZ"}, - "Myanmar or Burma": {"name": "Myanmar or Burma", "code": "MMR"}, - "Namibia": {"name": "Namibia", "code": "NAM"}, - "Nauru": {"name": "Nauru", "code": "NRU"}, - "Nepal": {"name": "Nepal", "code": "NPL"}, - "Netherlands": {"name": "Netherlands", "code": "NLD"}, - "New Caledonia": {"name": "New Caledonia", "code": "NCL"}, - "New Zealand": {"name": "New Zealand", "code": "NZL"}, - "Nicaragua": {"name": "Nicaragua", "code": "NIC"}, - "Niger": {"name": "Niger", "code": "NER"}, - "Nigeria": {"name": "Nigeria", "code": "NGA"}, - "Niue": {"name": "Niue", "code": "NIU"}, - "No Land": {"name": "No Land", "code": "XNL"}, - "Norfolk Island": {"name": "Norfolk Island", "code": "NFK"}, - "North Korea": {"name": "North Korea", "code": "PRK"}, - "Northern Mariana Islands": { - "name": "Northern Mariana Islands", - "code": "MNP", - }, - "Norway": {"name": "Norway", "code": "NOR"}, - "Oman": {"name": "Oman", "code": "OMN"}, - "Pakistan": {"name": "Pakistan", "code": "PAK"}, - "Palau": {"name": "Palau", "code": "PLW"}, - "Panama": {"name": "Panama", "code": "PAN"}, - "Papua New Guinea": {"name": "Papua New Guinea", "code": "PNG"}, - "Paraguay": {"name": "Paraguay", "code": "PRY"}, - "Peru": {"name": "Peru", "code": "PER"}, - "Philippines": {"name": "Philippines", "code": "PHL"}, - "Pitcairn Islands": {"name": "Pitcairn Islands", "code": "PCN"}, - "Poland": {"name": "Poland", "code": "POL"}, - "Portugal": {"name": "Portugal", "code": "PRT"}, - "Puerto Rico": {"name": "Puerto Rico", "code": "PRI"}, - "Qatar": {"name": "Qatar", "code": "QAT"}, - "Reunion": {"name": "Reunion", "code": "REU"}, - "Romania": {"name": "Romania", "code": "ROU"}, - "Rwanda": {"name": "Rwanda", "code": "RWA"}, - "Saint Helena Ascension and Tristan da Cunha": { - "name": "Saint Helena Ascension and Tristan da Cunha", - "code": "SHN", - }, - "Saint Kitts and Nevis": { - "name": "Saint Kitts and Nevis", - "code": "KNA", - }, - "Saint Lucia": {"name": "Saint Lucia", "code": "LCA"}, - "Saint Pierre and Miquelon": { - "name": "Saint Pierre and Miquelon", - "code": "SPM", - }, - "Sao Tome and Principe": { - "name": "Sao Tome and Principe", - "code": "STP", - }, - "Saudi Arabia": {"name": "Saudi Arabia", "code": "SAU"}, - "Senegal": {"name": "Senegal", "code": "SEN"}, - "Serbia": {"name": "Serbia", "code": "SRB"}, - "Seychelles": {"name": "Seychelles", "code": "SYC"}, - "Sierra Leone": {"name": "Sierra Leone", "code": "SLE"}, - "Singapore": {"name": "Singapore", "code": "SGP"}, - "Slovakia": {"name": "Slovakia", "code": "SVK"}, - "Slovenia": {"name": "Slovenia", "code": "SVN"}, - "Solomon Islands": {"name": "Solomon Islands", "code": "SLB"}, - "Somalia": {"name": "Somalia", "code": "SOM"}, - "South Africa": {"name": "South Africa", "code": "ZAF"}, - "South Georgia and the South Sandwich Islands": { - "name": "South Georgia and the South Sandwich Islands", - "code": "SGS", - }, - "South Korea": {"name": "South Korea", "code": "KOR"}, - "South Sudan": {"name": "South Sudan", "code": "SSD"}, - "Spain": {"name": "Spain", "code": "ESP"}, - "Sri Lanka": {"name": "Sri Lanka", "code": "LKA"}, - "St Vincent and the Grenadines": { - "name": "St. Vincent and the Grenadines", - "code": "VCT", - }, - "Sudan": {"name": "Sudan", "code": "SDN"}, - "Suriname": {"name": "Suriname", "code": "SUR"}, - "Svalbard": {"name": "Svalbard", "code": "SJM"}, - "Swaziland": {"name": "Swaziland", "code": "SWZ"}, - "Sweden": {"name": "Sweden", "code": "SWE"}, - "Switzerland": {"name": "Switzerland", "code": "CHE"}, - "Syria": {"name": "Syria", "code": "SYR"}, - "Taiwan": {"name": "Taiwan", "code": "TWN"}, - "Tajikistan": {"name": "Tajikistan", "code": "TJK"}, - "Tanzania United Republic of": { - "name": "Tanzania, United Republic of", - "code": "TZA", - }, - "Thailand": {"name": "Thailand", "code": "THA"}, - "Timor Leste": {"name": "Timor Leste", "code": "TLS"}, - "Togo": {"name": "Togo", "code": "TGO"}, - "Tokelau": {"name": "Tokelau", "code": "TKL"}, - "Tonga": {"name": "Tonga", "code": "TON"}, - "Trinidad and Tobago": {"name": "Trinidad and Tobago", "code": "TTO"}, - "Tunisia": {"name": "Tunisia", "code": "TUN"}, - "Turkey": {"name": "Turkey", "code": "TUR"}, - "Turkmenistan": {"name": "Turkmenistan", "code": "TKM"}, - "Turks and Caicos Islands": { - "name": "Turks and Caicos Islands", - "code": "TCA", - }, - "Tuvalu": {"name": "Tuvalu", "code": "TUV"}, - "Uganda": {"name": "Uganda", "code": "UGA"}, - "Ukraine": {"name": "Ukraine", "code": "UKR"}, - "United Arab Emirates": {"name": "United Arab Emirates", "code": "ARE"}, - "United Kingdom": {"name": "United Kingdom", "code": "GBR"}, - "United States Minor Outlying Islands": { - "name": "United States Minor Outlying Islands", - "code": "UMI", - }, - "Uruguay": {"name": "Uruguay", "code": "URY"}, - "Uzbekistan": {"name": "Uzbekistan", "code": "UZB"}, - "Vanuatu": {"name": "Vanuatu", "code": "VUT"}, - "Venezuela": {"name": "Venezuela", "code": "VEN"}, - "Vietnam": {"name": "Vietnam", "code": "VNM"}, - "Virgin Islands": {"name": "Virgin Islands", "code": "VGB"}, - "Wallis and Futuna": {"name": "Wallis and Futuna", "code": "WLF"}, - "West Bank": {"name": "West Bank", "code": "PSE"}, - "Western Sahara": {"name": "Western Sahara", "code": "ESH"}, - "Western Samoa": {"name": "Western Samoa", "code": "WSM"}, - "Yemen": {"name": "Yemen", "code": "YEM"}, - "Zaire DR Congo": {"name": "DR Congo, former Zaire", "code": "COD"}, - "Zambia": {"name": "Zambia", "code": "ZMB"}, - "Zimbabwe": {"name": "Zimbabwe", "code": "ZWE"}, - "Australia": {"name": "Australia", "code": "AUS"}, - "Brazil": {"name": "Brazil", "code": "BRA"}, - "Canada": {"name": "Canada", "code": "CAN"}, - "China": {"name": "China", "code": "CHN"}, - "India": {"name": "India", "code": "IND"}, - "Russia": {"name": "Russia", "code": "RUS"}, - "United States": {"name": "United States of America", "code": "USA"}, - } + with (importlib.resources.files(__package__) / "countries.json").open( + "r" + ) as countries: + return json.load(countries) def search_country(query): From decc86d81428f13480a1c2cd481b39c7b9e87ea9 Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Wed, 13 May 2026 15:39:43 +0200 Subject: [PATCH 02/11] Ignore line length in linting --- .flake8 | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.flake8 b/.flake8 index 0f2c19d..56ef4c9 100644 --- a/.flake8 +++ b/.flake8 @@ -1,4 +1,4 @@ [flake8] max-line-length = 88 extend-exclude = pycoupler/_version.py -ignore = F841,F541,W503,E226 \ No newline at end of file +ignore = F841,F541,W503,E226,E501 \ No newline at end of file From 6710bb04f40a19e593c98bf5fa0894eda8ceef7f Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Wed, 13 May 2026 15:41:36 +0200 Subject: [PATCH 03/11] Abstract out the subprocess spawning for LPJmL utilities --- pycoupler/config.py | 252 +++++++++++++++++++++++++++--------------- pycoupler/coupler.py | 19 ++-- pycoupler/run.py | 255 ++++++++++++++++++++++++------------------- 3 files changed, 311 insertions(+), 215 deletions(-) diff --git a/pycoupler/config.py b/pycoupler/config.py index d005366..3aa39ef 100644 --- a/pycoupler/config.py +++ b/pycoupler/config.py @@ -1,15 +1,19 @@ """Classes and functions to handle LPJmL configurations and related operations""" +import logging import os +import shutil import sys -import subprocess import json -from subprocess import run +from subprocess import DEVNULL, CompletedProcess, Popen, run as run_subprocess +from typing import Any, override from ruamel.yaml import YAML from pycoupler.utils import read_json, get_countries, create_subdirs, detect_io_type from pycoupler.data import read_header +logger = logging.getLogger(__name__) + class SubConfig: """ @@ -120,6 +124,115 @@ def __init__(self, sub_config): sub_config.__dict__["changed"] = [] self.__dict__.update(sub_config.__dict__) + def run_model_bin( + self, + binary: str, + *args: str, + detach: bool = False, + # pyright: ignore[reportExplicitAny] + subprocess_args: dict[str, Any] = {}, + ) -> CompletedProcess[bytes] | Popen[str]: + """Runs the LPJmL model binaries in the configured environment + + Parameters + ---------- + binary + Name of the binary to run. It should be available + in the PATH or in the `bin` folder at `self.model_path`. + subprocess_args, optional + Commands to pass to subprocess.run or subprocess.Popen. Defaults: + - `capture_output`: False + - `check`: True + - `env`: Passes the current runtime environment as well as the + LPJmL related runtime variables LPJINPATH, LPJOUTPATH, LPJROOT, + LPJRESTARTPATH (from :func:`~get_runtime_env`) + detach + TODO + *args + Arguments to pass to the binary as strings + + Returns + ------- + The CompletedProcess object of the spawned subprocess. + """ + + default_args: dict[str, Any] = { + "env": os.environ | self.get_runtime_env(ensure_paths=False), + "capture_output": False, + "check": True, + } + + if self.model_path: + if not os.path.exists(self.model_path): + raise FileNotFoundError("The given model_path does not exist.") + command = os.path.join(self.model_path, "bin", binary) + else: + # If the model_path was not set, we expect the binaries to be added to the PATH + command = shutil.which(binary) + logger.debug(f'Using {binary} found at "{command}"') + if not command: + raise ValueError( + f"The tool '{binary}' is not available in the PATH and no model_path was given." + ) + + if detach: + return Popen([command, *args], **subprocess_args) + else: + # pyright: ignore[reportExplicitAny] + return run_subprocess([command, *args], **(default_args | subprocess_args)) + + def get_runtime_env(self, ensure_paths=True): + return { + "LPJROOT": self.model_path, + "LPJINPATH": self.get_input_folder(), + "LPJOUTPATH": self.get_output_folder(ensure=ensure_paths), + "LPJRESTARTPATH": self.get_restart_folder(ensure=ensure_paths), + } + + def get_output_folder(self, ensure: bool = False) -> str: + output_folder = os.path.join(self.sim_path, "output", self.sim_name) + if ensure: + os.makedirs(output_folder, exist_ok=True) + return output_folder + + def get_input_filepath(self, input_file_name: str) -> str: + return ( + input_file_name + if os.path.isfile(input_file_name) + else os.path.join(self.get_input_folder(), input_file_name) + ) + + def get_input_folder(self) -> str: + input_path = "" + + if self.inpath: + if not os.path.isdir(self.inpath): + raise FileNotFoundError( + "The input path, set in `inpath` does not exist." + ) + input_path = self.inpath + logger.debug(f"Using config.inpath '{self.inpath}' as input path.") + elif os.environ["LPJINPATH"]: + if not os.path.isdir(os.environ["LPJINPATH"]): + raise FileNotFoundError( + "The input path, set in `LPJINPATH` does not exist." + ) + input_path = os.environ["LPJINPATH"] + logger.debug( + f"Using LPJINPATH '{os.environ['LPJINPATH']}' from runtime environtment as input path." + ) + else: + logger.warning( + "No default input path available in config file or environment." + ) + return input_path + + def get_restart_folder(self, ensure: bool = False) -> str: + restart_folder = os.path.join(self.sim_path, "restart") + if ensure: + os.makedirs(restart_folder, exist_ok=True) + return restart_folder + def get_output_avail(self, id_only=True, to_dict=False): """ Get available output (outputvar) names or objects. @@ -203,13 +316,14 @@ def set_spinup(self, sim_path, sim_name="spinup"): Name of the simulation. """ self.sim_name = sim_name + # TODO: Remove self.sim_path = create_subdirs(sim_path, self.sim_name) - output_path = f"{sim_path}/output/{self.sim_name}" + output_path = self.get_output_folder(ensure=True) # set output writing self._set_outputpath(output_path) # set restart directory to restart from in subsequent historic run - self._set_restart(path=f"{sim_path}/restart") + self._set_restart(path=self.get_restart_folder(ensure=True)) def set_transient( self, @@ -253,7 +367,7 @@ def set_transient( """ self.sim_name = sim_name self.sim_path = create_subdirs(sim_path, self.sim_name) - output_path = f"{sim_path}/output/{self.sim_name}" + output_path = self.get_output_folder(ensure=True) # set time range for historic run self._set_timerange( start_year=start_year, end_year=end_year, write_start_year=start_year @@ -267,9 +381,11 @@ def set_transient( append_output=append_output, ) # set start from directory to start from spinup run - self._set_startfrom(path=f"{sim_path}/restart", dependency=dependency) + self._set_startfrom( + path=self.get_restart_folder(ensure=True), dependency=dependency + ) # set restart directory to restart from in subsequent transient run - self._set_restart(path=f"{sim_path}/restart") + self._set_restart(path=self.get_restart_folder(ensure=True)) def set_coupled( self, @@ -328,7 +444,7 @@ def set_coupled( """ self.sim_name = sim_name self.sim_path = create_subdirs(sim_path, self.sim_name) - output_path = f"{sim_path}/output/{self.sim_name}" + output_path = self.get_output_folder(ensure=True) # set time range for coupled run self._set_timerange( @@ -356,7 +472,9 @@ def set_coupled( model_name=model_name, ) # set start from directory to start from historic run - self._set_startfrom(path=f"{sim_path}/restart", dependency=dependency) + self._set_startfrom( + path=self.get_restart_folder(ensure=True), dependency=dependency + ) def _set_output( self, @@ -517,23 +635,13 @@ def _set_grid_explicitly(self, only_all=True): self.endgrid = 2 elif self.input.soil.fmt in ["json", "meta"]: self.endgrid = ( - read_json( - ( - self.input.soil.name - if os.path.isfile(self.input.soil.name) - else f"{self.inpath}/{self.input.soil.name}" - ) - )["ncell"] + read_json(self.get_input_filepath(self.input.soil.name))["ncell"] - 1 ) else: self.endgrid = ( read_header( - ( - self.input.soil.name - if os.path.isfile(self.input.soil.name) - else f"{self.inpath}/{self.input.soil.name}" - ), + self.get_input_filepath(self.input.soil.name), to_dict=True, )["header"]["ncell"] - 1 @@ -645,7 +753,7 @@ def add_config(self, file_name): """ self.coupled_config = read_yaml(file_name, CoupledConfig) - def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False): + def regrid(self, sim_path, country_code="BEL", overwrite=False): """ Regrid LPJmL configuration file to a new country. @@ -654,9 +762,6 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) sim_path : str Directory to check whether required subfolders exist. If not, create corresponding folders (input, output, restart). - model_path : str, optional - Path to LPJmL_internal (lpjml repository). If None, uses - self.model_path. country_code : str, default "BEL" Country code of country to regrid to. overwrite : bool, default False @@ -664,18 +769,13 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) Raises ------ - OSError - If sim_path or model_path do not exist, or if required grid files + FileNotFoundError + If sim_path does not exist, or if required grid files are missing. """ if not os.path.exists(sim_path): - raise OSError(f"Path '{sim_path}' does not exist.") - - if hasattr(self, "model_path"): - model_path = self.model_path - elif not model_path or not os.path.exists(model_path): - raise OSError(f"Path '{model_path}' does not exist.") + raise FileNotFoundError(f"Path '{sim_path}' does not exist.") # get available countries of LPJmL countries = get_countries() @@ -690,17 +790,13 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) None, ).lower() - grid_file = ( - self.input.coord.name - if os.path.isfile(self.input.coord.name) - or hasattr(sys, "_called_from_test") - else f"{self.inpath}/{self.input.coord.name}" - ) + grid_file = self.get_input_filepath(self.input.coord.name) # proxy check if regrid was already performed if country in self.input.coord.name: return + # TODO: Mount in container! country_grid_file = ( f"{sim_path}/input/{country}_{os.path.basename(self.input.coord.name)}" ) @@ -710,22 +806,15 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) ): if not os.path.isfile(grid_file): - raise OSError(f"Grid file '{grid_file}' does not exist.") + raise FileNotFoundError(f"Grid file '{grid_file}' does not exist.") # extract country specific grid - run( - [ - f"{model_path}/bin/getcountry", - ( - self.input.countrycode.name - if os.path.isfile(self.input.countrycode.name) - else f"{self.inpath}/{self.input.countrycode.name}" - ), - grid_file, - country_grid_file, - country_code, - ], - check=True, + self.run_model_bin( + "getcountry", + self.get_input_filepath(self.input.countrycode.name), + grid_file, + country_grid_file, + country_code, ) self.input.coord.fmt = ( @@ -735,12 +824,7 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) ) self.input.coord.name = country_grid_file - lakes_fn_string = ( - self.input.lakes.name - if os.path.isfile(self.input.lakes.name) - or hasattr(sys, "_called_from_test") - else f"{self.inpath}/{self.input.lakes.name}" - ) + lakes_fn_string = self.get_input_filepath(self.input.lakes.name) # extract country specific lakes file from meta file if self.input.lakes.fmt == "meta" and not hasattr(sys, "_called_from_test"): lakes_filename = read_json(lakes_fn_string)["filename"] @@ -765,16 +849,13 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) raise OSError(f"Lakes file '{lakes_file}' does not exist.") # regrid lakes file to country specific grid - run( - [ - f"{model_path}/bin/regridsoil", - grid_file, - country_grid_file, - lakes_file, - country_lakes_file, - ], - check=True, - stdout=open(os.devnull, "wb"), + self.run_model_bin( + "regridsoil", + grid_file, + country_grid_file, + lakes_file, + country_lakes_file, + stdout=DEVNULL, ) self.input.lakes.fmt = ( @@ -795,12 +876,7 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) ): continue - input_file = ( - config_input.name - if os.path.isfile(config_input.name) - or hasattr(sys, "_called_from_test") - else f"{self.inpath}/{config_input.name}" - ) + input_file = self.get_input_filepath(config_input.name) country_input_file = ( f"{sim_path}/input/{country}_{os.path.basename(input_file)}" @@ -822,16 +898,16 @@ def regrid(self, sim_path, model_path=None, country_code="BEL", overwrite=False) regrid_func = "regridclm" # regrid all other input files to country specific grid - regrid_cmd = [ - f"{model_path}/bin/{regrid_func}", + self.run_model_bin( + "regrid_func", grid_file, self.input.coord.name, input_file, country_input_file, - ] + stdout=DEVNULL, + ) # if additional_arg: # regrid_cmd.insert(1, additional_arg) - run(regrid_cmd, check=True, stdout=open(os.devnull, "wb")) config_input.fmt = ( detect_io_type(country_input_file) @@ -851,12 +927,7 @@ def convert_cdf_to_raw(self, output_id=None): output_dir = f"{self.sim_path}/output/{self.sim_name}" - grid_file = ( - self.input.coord.name - if os.path.isfile(self.input.coord.name) - or hasattr(sys, "_called_from_test") - else f"{self.inpath}/{self.input.coord.name}" - ) + grid_file = self.get_input_filepath(self.input.coord.name) grid_name = os.path.basename(grid_file) @@ -884,8 +955,7 @@ def convert_cdf_to_raw(self, output_id=None): for output in output_details: # convert netcdf output to netcdf files - conversion_cmd = [ - f"{self.model_path}/bin/cdf2bin", + conversion_args = [ # "-units", output.unit, "-var", output.var, @@ -896,11 +966,11 @@ def convert_cdf_to_raw(self, output_id=None): f"{output_dir}/{output.name}.nc4", ] - if None in conversion_cmd: - conversion_cmd.remove(None) + if None in conversion_args: + conversion_args.remove(None) if not hasattr(sys, "_called_from_test"): - run(conversion_cmd) + self.run_model_bin("cdf2bin", *conversion_args) nc4_meta_dict = read_json(f"{output_dir}/{output.name}.nc4.json") @@ -1039,7 +1109,7 @@ def parse_config( cmd.append(file_name) # Subprocess call of cmd - return stdout - json_str = subprocess.run(cmd, capture_output=True) + json_str = run_subprocess(cmd, capture_output=True) # Convert to dict lpjml_config = json.loads(json_str.stdout, object_hook=config_class) diff --git a/pycoupler/coupler.py b/pycoupler/coupler.py index c968036..6256356 100644 --- a/pycoupler/coupler.py +++ b/pycoupler/coupler.py @@ -920,26 +920,24 @@ def _copy_input(self, start_year, end_year): cut_start_year = start_year cut_end = cut_end_year = min(meta_data.lastyear, end_year) - cut_clm_start = [ - f"{self._config.model_path}/bin/cutclm", + cut_clm_start_args = [ str(cut_start_year), sock_inputs[key]["name"], f"{temp_dir}/1_{file_name_tmp}", ] if not hasattr(sys, "_called_from_test"): - run(cut_clm_start, stdout=open(os.devnull, "wb")) + self.config.run_model_bin("cutclm", *cut_clm_start_args) # predefine cut clm command for reusage # cannot deal with overwriting a temp file with same name - cut_clm_end = [ - f"{self._config.model_path}/bin/cutclm", + cut_clm_end_args = [ "-end", str(cut_end_year), f"{temp_dir}/1_{file_name_tmp}", f"{temp_dir}/2_{file_name_tmp}", ] if not hasattr(sys, "_called_from_test"): - run(cut_clm_end, stdout=open(os.devnull, "wb")) + self.config.run_model_bin("cutclm", *cut_clm_end_args) # a flag for multi (categorical) band input - if true, set # "-landuse" @@ -960,8 +958,7 @@ def _copy_input(self, start_year, end_year): else: grid_file = f"{self.config.inpath}/{self.config.input.coord.name}" # convert clm input to netcdf files - conversion_cmd = [ - f"{self._config.model_path}/bin/clm2cdf", + conversion_cmd_args = [ is_int, is_multiband, key, @@ -970,11 +967,11 @@ def _copy_input(self, start_year, end_year): f"{input_path}/{key}.nc", ] - if None in conversion_cmd: - conversion_cmd.remove(None) + if None in conversion_cmd_args: + conversion_cmd_args.remove(None) if not hasattr(sys, "_called_from_test"): - run(conversion_cmd) + self.config.run_model_bin("clm2cdf", *conversion_cmd_args) else: return "tested" # remove the temporary clm (binary) files, 1_* is not created in diff --git a/pycoupler/run.py b/pycoupler/run.py index 4299842..3d48077 100644 --- a/pycoupler/run.py +++ b/pycoupler/run.py @@ -1,12 +1,12 @@ import os from datetime import datetime -from subprocess import run, Popen, PIPE, CalledProcessError +from subprocess import STDOUT, run, Popen, CalledProcessError +from typing import cast from pycoupler.config import read_config +import warnings -import multiprocessing as mp - -def operate_lpjml(config_file, std_to_file=False): +def operate_lpjml(config_file, std_to_file=False, wait_for_exit=True): """Run LPJmL using a generated (class LpjmlConfig) config file. Similar to R function `lpjmlKit::run_lpjml`. @@ -17,11 +17,13 @@ def operate_lpjml(config_file, std_to_file=False): std_to_file : bool, optional If True, stdout and stderr are written to files in the output folder. Defaults to False. + wait_for_exit + Whether to block the thread until the process exits. """ config = read_config(config_file) - if not os.path.isdir(config.model_path): + if config.model_path and not os.path.isdir(config.model_path): raise ValueError(f"Folder of model_path '{config.model_path}' does not exist!") output_path = f"{config.sim_path}/output/{config.sim_name}" @@ -34,44 +36,51 @@ def operate_lpjml(config_file, std_to_file=False): os.makedirs(output_path) print(f"Created output_path '{output_path}'") - cmd = [f"{config.model_path}/bin/lpjml", config_file] - # environment settings to be used for interartive LPJmL sessions - # MPI settings conflict with (e.g. on login node) - os.environ["I_MPI_DAPL_UD"] = "disable" - os.environ["I_MPI_FABRICS"] = "shm:shm" - os.environ["I_MPI_DAPL_FABRIC"] = "shm:sh" + subprocess_args = { + "env": os.environ + | { + # environment settings to be used for interactive LPJmL sessions + # MPI settings conflict with (e.g. on login node) + "I_MPI_DAPL_UD": "disable", + "I_MPI_FABRICS": "shm:shm", + "I_MPI_DAPL_FABRIC": "shm:sh", + } + | config.get_runtime_env(), + # This might be None, running in the current directory: + "cwd": config.model_path, + "text": True, + } + if std_to_file: - with open(stdout_file, "w") as f_out, open(stderr_file, "w") as f_err: - with Popen( - cmd, - stdout=f_out, - stderr=f_err, - bufsize=1, - universal_newlines=True, - cwd=config.model_path, - ) as p: - p.wait() + subprocess_args |= { + "stdout": open(stdout_file, "w"), + "stderr": open(stderr_file, "w"), + "bufsize": 1, + } else: - with Popen( - cmd, - stdout=PIPE, - stderr=PIPE, - bufsize=1, - universal_newlines=True, - cwd=config.model_path, - ) as p: - for line in p.stdout: - print(line, end="") - for line in p.stderr: - print(line, end="") - - # reset default MPI settings to be able to submit jobs in parallel again - os.environ["I_MPI_DAPL_UD"] = "enable" - os.environ["I_MPI_FABRICS"] = "shm:dapl" - del os.environ["I_MPI_DAPL_FABRIC"] - # raise error if returncode does not reflect successfull call - if p.returncode != 0: - raise CalledProcessError(p.returncode, p.args) + subprocess_args |= { + "stdout": None, + "stderr": STDOUT, + "bufsize": 0, + } + + p = cast( + Popen[str], + config.run_model_bin( + "lpjml", config_file, detach=True, subprocess_args=subprocess_args + ), + ) + + if wait_for_exit: + p.wait() + if p.stdout: + p.stdout.close() + # raise error if returncode does not reflect successfull call + if p.returncode != 0: + raise CalledProcessError(p.returncode, p.args) + return p + else: + return p def run_lpjml(config_file, std_to_file=False): @@ -86,10 +95,10 @@ def run_lpjml(config_file, std_to_file=False): If True, stdout and stderr are written to files in the output folder. Defaults to False. """ - run = mp.Process(target=operate_lpjml, args=(config_file, std_to_file)) - run.start() - - return run + warnings.warn( + "run_lpjml is deprecated. Please use operate_lpjml(wait_for_exit=False) instead." + ) + return operate_lpjml(config_file, std_to_file, False) def submit_lpjml( @@ -103,6 +112,7 @@ def submit_lpjml( option=None, couple_to=None, venv_path=None, + modules=None, ): """Submit LPJmL run to Slurm using `lpjsubmit` and a generated (class LpjmlConfig) config file. @@ -144,6 +154,9 @@ def submit_lpjml( venv_path : str, optional Path to a venv to run the coupled script in. This should be the path to the top folder of the venv. If not set, `python3` in PATH is used. + modules : str, optional + Environment modules to load for the SLURM job separated by spaces. + For hierarchical modules, observe the necessary module order. Returns ------- @@ -155,48 +168,43 @@ def submit_lpjml( if not os.path.isdir(config.model_path): raise ValueError(f"Folder of model_path '{config.model_path}' does not exist!") - output_path = f"{config.sim_path}/output/{config.sim_name}" - timestamp = datetime.now().strftime("%Y-%m-%d_%H-%M") - stdout_file = os.path.join(output_path, f"stdout_{timestamp}.log") - stderr_file = os.path.join(output_path, f"stderr_{timestamp}.log") - - if not os.path.isdir(output_path): - os.makedirs(output_path) - print(f"Created output_path '{output_path}'") + stdout_file = os.path.join( + config.get_output_folder(ensure=True), f"stdout_{timestamp}.log" + ) + stderr_file = os.path.join( + config.get_output_folder(ensure=True), f"stderr_{timestamp}.log" + ) - lpjroot = os.environ.get("LPJROOT") - # prepare lpjsubmit command to be called via subprocess - cmd = [f"{config.model_path}/bin/lpjsubmit"] # specify sbatch arguments required by lpjsubmit internally - cmd.extend( - [ - "-group", - group, - "-class", - sclass, - "-o", - stdout_file, - "-e", - stderr_file, - ] # noqa: E501 - ) + submit_args = [ + "-group", + group, + "-class", + sclass, + "-o", + stdout_file, + "-e", + stderr_file, + # We want to start sbatch ourselves, just generate the job control file + "-norun", + ] # if dependency (jobid) defined, submit is queued by slurm with nocheck if dependency: - cmd.extend(["-nocheck", "-dependency", str(dependency)]) + submit_args.extend(["-nocheck", "-dependency", str(dependency)]) # processing time to get a better position in slurm queue if wtime: - cmd.extend(["-wtime", str(wtime)]) + submit_args.extend(["-wtime", str(wtime)]) # if cores to be blocked if blocking: - cmd.extend(["-blocking", str(blocking)]) + submit_args.extend(["-blocking", str(blocking)]) if option: if isinstance(option, str): - cmd.extend(["-option", option]) + submit_args.extend(["-option", option]) elif isinstance(option, list): for opt in option: - cmd.extend(["-option", opt]) + submit_args.extend(["-option", opt]) # run in coupled mode and pass coupling program/model if couple_to: @@ -217,7 +225,9 @@ def submit_lpjml( {python_path} {couple_to} $config_file """ - couple_file = f"{output_path}/copan_lpjml.sh" + couple_file = os.path.join( + config.get_output_folder(ensure=True), "copan_lpjml.sh" + ) with open(couple_file, "w") as file: file.write(bash_script) @@ -225,40 +235,52 @@ def submit_lpjml( # Change the permissions of the file to make it executable run(["chmod", "+x", couple_file]) - cmd.extend(["-couple", couple_file]) - - cmd.extend([str(ntasks), config_file]) - - # Intialize submit_status in higher scope - submit_status = None - # set LPJROOT to model_path to be able to call lpjsubmit - try: - os.environ["LPJROOT"] = config.model_path - # call lpjsubmit via subprocess and return status if successfull - submit_status = run(cmd, capture_output=True) - except Exception as e: - print("Error occurred:", e) - finally: - if lpjroot: - os.environ["LPJROOT"] = lpjroot - else: - del os.environ["LPJROOT"] + submit_args.extend(["-couple", couple_file]) + + if modules: + submit_args.extend(["-modules", modules]) + + submit_args.extend([str(ntasks), config_file]) + + # call lpjsubmit via subprocess and return status if successfull + submit_file_status = config.run_model_bin( + "lpjsubmit", + *submit_args, + subprocess_args={ + "capture_output": True, + "cwd": config.sim_path, + "text": True, + }, + ) + + if submit_file_status.returncode != 0: + print(submit_file_status.stdout) + print(submit_file_status.stderr) + raise CalledProcessError(submit_file_status.returncode, submit_file_status.args) + + sbatch_cmd = ["sbatch"] + + if dependency: + sbatch_cmd.extend(["-depend", dependency]) + + submit_status = run( + sbatch_cmd, + cwd=config.sim_path, + env=submit_env, + capture_output=True, + check=True, + text=True, + ) # print stdout and stderr if not successful - if submit_status is None: - raise Exception("Process was not submitted.") - elif submit_status.returncode == 0: - print(submit_status.stdout.decode("utf-8")) + if submit_status.returncode == 0: + print(submit_status.stdout) else: - print(submit_status.stdout.decode("utf-8")) - print(submit_status.stderr.decode("utf-8")) + print(submit_status.stdout) + print(submit_status.stderr) raise CalledProcessError(submit_status.returncode, submit_status.args) # return job id - return ( - submit_status.stdout.decode("utf-8") - .split("Submitted batch job ")[1] - .split("\n")[0] - ) + return submit_status.stdout.split("Submitted batch job ")[1].split("\n")[0] def check_lpjml(config_file): @@ -272,16 +294,23 @@ def check_lpjml(config_file): Path to `LPJmL_internal` (lpjml repository) """ config = read_config(config_file) - if not os.path.isdir(config.model_path): + if config.model_path and not os.path.isdir(config.model_path): raise ValueError(f"Folder of model_path '{config.model_path}' does not exist!") - if os.path.isfile(f"{config.model_path}/bin/lpjcheck"): - proc_status = run( - ["./bin/lpjcheck", config_file], - capture_output=True, # "-param", - cwd=config.model_path, - ) + + proc_status = config.run_model_bin( + "lpjcheck", + [config_file], + subprocess_args={ + # ensure_paths is false, because this is just a check and should have no side effects + "cwd": config.model_path, + "check": False, + "capture_output": True, + "text": True, + }, + ) + if proc_status.returncode == 0: - print(proc_status.stdout.decode("utf-8")) + print(proc_status.stdout) else: - print(proc_status.stdout.decode("utf-8")) - print(proc_status.stderr.decode("utf-8")) + print(proc_status.stdout) + print(proc_status.stderr) From b8c57504e470a3b3ce3643feb70f40141d9ac301 Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Thu, 21 May 2026 16:33:26 +0200 Subject: [PATCH 04/11] Set up more test fixtures --- tests/conftest.py | 56 +++++++++++++++++++++++++++++++++++------------ 1 file changed, 42 insertions(+), 14 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index 4be4e40..33b3db3 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -1,4 +1,5 @@ import os +from pathlib import Path import pytest from pycoupler.coupler import LPJmLCoupler import json @@ -8,17 +9,17 @@ @pytest.fixture def test_path(): """Fixture for the test path.""" - return os.path.dirname(os.path.abspath(__file__)) + return Path(__file__).absolute().parent @pytest.fixture -def lpjml_coupler(config_coupled): +def lpjml_coupler(config_coupled_file: Path): os.environ["TEST_LINE_COUNTER"] = "0" # Using yield enables safe teardown of the fixture # (see https://docs.pytest.org/en/stable/how-to/fixtures.html#safe-teardowns) - yield LPJmLCoupler(config_file=config_coupled) + yield LPJmLCoupler(config_file=config_coupled_file) # Reset test line env variable - os.environ["TEST_LINE_COUNTER"] = "0" + del os.environ["TEST_LINE_COUNTER"] @pytest.fixture() @@ -28,19 +29,17 @@ def sim_path(tmp_path_factory): @pytest.fixture() -def output_path(sim_path, test_path): +def output_path(sim_path: Path, test_path: Path): top_fn = sim_path / "output" top_fn.mkdir() - output_fn = top_fn / "coupled_test" - shutil.copytree(f"{test_path}/data/output/coupled_test", output_fn) - return output_fn + return Path( + shutil.copytree(test_path / "data/output/coupled_test", top_fn / "coupled_test") + ) @pytest.fixture() -def sim_inputs(sim_path, test_path): - input_fn = sim_path / "input" - shutil.copytree(f"{test_path}/data/input", input_fn) - return input_fn +def sim_inputs(sim_path: Path, test_path: Path): + return Path(shutil.copytree(test_path / "data/input", sim_path / "input")) @pytest.fixture() @@ -57,12 +56,41 @@ def outputpath_helper(output_dict, path): @pytest.fixture() -def config_coupled(sim_path, model_path, test_path, sim_inputs, output_path): +def lpjml_config( + sim_path: Path, + model_path: Path, + test_path: Path, + sim_inputs: Path, + output_path: Path, +): + new_config = model_path / "lpjml_config.json" + with (test_path / "data/lpjml_config.json").open("r") as conf: + conf_d = json.load(conf) + conf_d["model_path"] = str(model_path) + conf_d["sim_path"] = str(sim_path) + conf_d["inpath"] = str(sim_inputs) + conf_d["output"] = [ + outputpath_helper(out, str(output_path)) for out in conf_d["output"] + ] + with new_config.open("w") as f: + json.dump(conf_d, f) + return str(new_config) + + +@pytest.fixture() +def config_coupled_file( + sim_path: Path, + model_path: Path, + test_path: Path, + sim_inputs: Path, + output_path: Path, +): new_config = sim_path / "config_coupled.json" - with open(f"{test_path}/data/config_coupled_test.json") as conf: + with (test_path / "data/config_coupled_test.json").open("r") as conf: conf_d = json.load(conf) conf_d["model_path"] = str(model_path) conf_d["sim_path"] = str(sim_path) + conf_d["inpath"] = str(sim_inputs) conf_d["output"] = [ outputpath_helper(out, str(output_path)) for out in conf_d["output"] ] From 27bf3741abae1c6d9c2cd286dc1b8633cd7ec16e Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Thu, 21 May 2026 16:36:22 +0200 Subject: [PATCH 05/11] Fix issues found in tests --- pycoupler/countries.json | 2 +- pycoupler/data.py | 5 +-- pycoupler/run.py | 1 - tests/test_config.py | 77 ++++++++++++++++------------------------ tests/test_run.py | 35 ++++++++++++++---- 5 files changed, 64 insertions(+), 56 deletions(-) diff --git a/pycoupler/countries.json b/pycoupler/countries.json index 6cb2e5b..698ca3e 100644 --- a/pycoupler/countries.json +++ b/pycoupler/countries.json @@ -942,5 +942,5 @@ "United States": { "name": "United States of America", "code": "USA" - }, + } } \ No newline at end of file diff --git a/pycoupler/data.py b/pycoupler/data.py index bb68f1e..b101c48 100644 --- a/pycoupler/data.py +++ b/pycoupler/data.py @@ -484,9 +484,10 @@ def read_data(file_name, var_name=None, multiple_bands=False): if var_name: data = data[var_name] - data = LPJmLData(data) + data = LPJmLData(data, attrs=data.attrs, coords=data.coords) else: - data = LPJmLDataSet(data) + # Copying needs to be explicit in xarray: https://github.com/pydata/xarray/issues/11095 + data = LPJmLDataSet(data.data_vars, attrs=data.attrs, coords=data.coords) return data diff --git a/pycoupler/run.py b/pycoupler/run.py index 3d48077..a43bd33 100644 --- a/pycoupler/run.py +++ b/pycoupler/run.py @@ -266,7 +266,6 @@ def submit_lpjml( submit_status = run( sbatch_cmd, cwd=config.sim_path, - env=submit_env, capture_output=True, check=True, text=True, diff --git a/tests/test_config.py b/tests/test_config.py index 9b204f9..923b7d5 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -1,45 +1,41 @@ """Test the LPJmLConfig class.""" from pycoupler.config import read_config, read_yaml, CoupledConfig, parse_config +from tests.conftest import lpjml_config -def test_set_spinup_config(test_path): +def test_set_spinup_config(model_path, sim_path, lpjml_config): """Test the set_config method of the LPJmLCoupler class.""" # create config for coupled run config_spinup = read_config( - model_path=test_path, file_name="data/lpjml_config.json", spin_up=True + model_path=model_path, file_name="lpjml_config.json", spin_up=True ) # set spinup run configuration - config_spinup.set_spinup(sim_path=f"{test_path}/data") + config_spinup.set_spinup(sim_path=sim_path) # only for global runs = TRUE config_spinup.river_routing = False # regrid by country - create new (extracted) input files and update config - config_spinup.regrid( - sim_path=f"{test_path}/data", country_code="NLD", overwrite=False - ) - assert config_spinup.model_path == test_path - assert config_spinup.sim_path == f"{test_path}/data" + config_spinup.regrid(sim_path=sim_path, country_code="NLD", overwrite=False) + assert config_spinup.model_path == model_path + assert config_spinup.sim_path == sim_path assert ( - config_spinup.write_restart_filename - == f"{test_path}/data/restart/restart_spinup.lpj" + config_spinup.write_restart_filename == f"{sim_path}/restart/restart_spinup.lpj" ) assert config_spinup.restart_year == 2011 assert config_spinup.river_routing is False -def test_set_historic_config(test_path): +def test_set_historic_config(model_path, sim_path, lpjml_config): # create config for historic run - config_historic = read_config( - model_path=test_path, file_name="data/lpjml_config.json" - ) + config_historic = read_config(model_path=model_path, file_name=lpjml_config) # set historic run configuration config_historic.set_transient( - sim_path=f"{test_path}/data", + sim_path=sim_path, sim_name="historic_run", dependency="spinup", start_year=1901, @@ -52,15 +48,12 @@ def test_set_historic_config(test_path): config_historic.residue_treatment = "read_residue_data" config_historic.double_harvest = False - assert config_historic.model_path == test_path - assert config_historic.sim_path == f"{test_path}/data" - assert ( - config_historic.restart_filename - == f"{test_path}/data/restart/restart_spinup.lpj" - ) + assert config_historic.model_path == model_path + assert config_historic.sim_path == sim_path + assert config_historic.restart_filename == f"{sim_path}/restart/restart_spinup.lpj" assert ( config_historic.write_restart_filename - == f"{test_path}/data/restart/restart_historic_run.lpj" + == f"{sim_path}/restart/restart_historic_run.lpj" ) assert config_historic.restart_year == 2000 assert config_historic.river_routing is False @@ -69,19 +62,17 @@ def test_set_historic_config(test_path): assert config_historic.double_harvest is False -def test_set_coupled_config(test_path): +def test_set_coupled_config(model_path, sim_path, lpjml_config, config_coupled_file): """Test the set_config method of the LPJmLCoupler class.""" # create config for coupled run - config_coupled = read_config( - model_path=f"{test_path}/data", file_name="lpjml_config.json" - ) + config_coupled = read_config(model_path=model_path, file_name=lpjml_config) config_coupled.startgrid = 27410 config_coupled.endgrid = 27411 # set coupled run configuration config_coupled.set_coupled( - sim_path=f"{test_path}/data", + sim_path=sim_path, sim_name="coupled_test", dependency="historic_run", start_year=2001, @@ -128,10 +119,10 @@ def test_set_coupled_config(test_path): # create config for coupled run check_config_coupled = read_config( - model_path=f"{test_path}/data", file_name="config_coupled_test.json" + model_path=model_path, file_name=config_coupled_file ) # update with actual output path (test directory) - check_config_coupled._set_outputpath(f"{test_path}/data/output/coupled_test") + check_config_coupled._set_outputpath(f"{sim_path}/output/coupled_test") # align both config objects check_config_coupled.restart_filename = config_coupled.restart_filename @@ -145,11 +136,11 @@ def test_set_coupled_config(test_path): assert ( repr(config_coupled) - == f"\nSettings: lpjml v5.8\n (general)\n * sim_name coupled_test\n * firstyear 2001\n * lastyear 2050\n * startgrid 27410\n * endgrid 27411\n * landuse yes\n (changed)\n * model_path {test_path}/data\n * sim_path {test_path}/data\n * outputyear 2022\n * output_metafile True\n * grid_type float\n * write_restart False\n * nspinup 0\n * float_grid True\n * restart_filename {test_path}/data/restart/restart_historic_run.lpj\n * outputyear 2022\n * radiation cloudiness\n * fix_co2 True\n * fix_co2_year 2018\n * fix_climate True\n * fix_climate_cycle 11\n * fix_climate_year 2013\n * river_routing False\n * tillage_type read\n * residue_treatment fixed_residue_remove\n * double_harvest False\n * intercrop True\nCoupled model: copan:CORE\n * start_coupling 2023\n * input (coupled) ['with_tillage']\n * output (coupled) ['grid', 'pft_harvestc', 'cftfrac', 'soilc_agr_layer', 'hdate', 'country', 'region']\n" # noqa + == f"\nSettings: lpjml v5.8\n (general)\n * sim_name coupled_test\n * firstyear 2001\n * lastyear 2050\n * startgrid 27410\n * endgrid 27411\n * landuse yes\n (changed)\n * model_path {model_path}\n * sim_path {sim_path}\n * outputyear 2022\n * output_metafile True\n * grid_type float\n * write_restart False\n * nspinup 0\n * float_grid True\n * restart_filename {sim_path}/restart/restart_historic_run.lpj\n * outputyear 2022\n * radiation cloudiness\n * fix_co2 True\n * fix_co2_year 2018\n * fix_climate True\n * fix_climate_cycle 11\n * fix_climate_year 2013\n * river_routing False\n * tillage_type read\n * residue_treatment fixed_residue_remove\n * double_harvest False\n * intercrop True\nCoupled model: copan:CORE\n * start_coupling 2023\n * input (coupled) ['with_tillage']\n * output (coupled) ['grid', 'pft_harvestc', 'cftfrac', 'soilc_agr_layer', 'hdate', 'country', 'region']\n" # noqa ) # noqa assert config_coupled_dict == check_config_coupled_dict - config_coupled.sim_path = f"{test_path}/data" + config_coupled.sim_path = sim_path assert config_coupled.convert_cdf_to_raw() == "tested" assert { @@ -177,26 +168,20 @@ def test_read_yaml(test_path): assert coupled_config.lpjml_settings.iso_country_code is False -def test_read_config(test_path): - coupled_config = read_config( - f"{test_path}/data/config_coupled_test.json", to_dict=True - ) - assert coupled_config["model_path"] == "LPJmL_internal" - assert coupled_config["sim_path"] == "lpjml" +def test_read_config(config_coupled_file, model_path, sim_path): + coupled_config = read_config(config_coupled_file, to_dict=True) + assert coupled_config["model_path"] == str(model_path) + assert coupled_config["sim_path"] == str(sim_path) assert coupled_config["coupled_model"] == "copan:CORE" - coupled_config = read_config( - f"{test_path}/data/config_coupled_test.json", to_dict=False - ) + coupled_config = read_config(config_coupled_file, to_dict=False) assert coupled_config.__class__.__name__ == "LpjmlConfig" -def test_parse_config(test_path): - coupled_config = parse_config(f"{test_path}/data/lpjml_config.json") - assert coupled_config["model_path"] == "LPJmL_internal" +def test_parse_config(lpjml_config, model_path): + coupled_config = parse_config(lpjml_config) + assert coupled_config["model_path"] == str(model_path) assert coupled_config["coupled_model"] is None - coupled_config = parse_config( - f"{test_path}/data/lpjml_config.json", config_class=CoupledConfig - ) + coupled_config = parse_config(lpjml_config, config_class=CoupledConfig) assert coupled_config.__class__.__name__ == "CoupledConfig" diff --git a/tests/test_run.py b/tests/test_run.py index 53365ad..f4b6fae 100644 --- a/tests/test_run.py +++ b/tests/test_run.py @@ -29,6 +29,23 @@ def mock_lpjsubmit(self, fp, request): ), ) + @pytest.fixture(autouse=True) + def mock_sbatch(self, fp, request): + # We expect chmod to actually modify permissions + if hasattr(request, "param") and request.param == "no mocking": + return + # Register a fake process for lpjsubmit + # (see https://pytest-subprocess.readthedocs.io/en/latest/usage.html#non-exact-command-matching) # noqa: E501 + return fp.register( + [fp.program("sbatch"), fp.any()], + stdout="Submitted batch job 42", + returncode=( + 1 + if hasattr(request, "param") and request.param == "non-zero errorcode" + else 0 + ), + ) + @pytest.fixture() def mock_venv(self, tmp_path_factory, request): if hasattr(request, "param") and request.param == "none": @@ -45,11 +62,11 @@ def submit( self, mock_venv, sim_path, - config_coupled, + config_coupled_file, request, ): return submit_lpjml( - config_coupled, + config_coupled_file, group=self.group, sclass=self.sclass, ntasks=self.ntasks, @@ -79,7 +96,7 @@ def test_lpjsubmit_error_cases(self, mock_lpjsubmit): # The test does nothing, we expect the fail in the fixtures pass - def test_command(self, sim_path, config_coupled, fp, submit): + def test_command(self, sim_path, config_coupled_file, fp, submit): run_script_path = sim_path / "output/coupled_test/copan_lpjml.sh" assert ( fp.call_count( @@ -93,16 +110,20 @@ def test_command(self, sim_path, config_coupled, fp, submit): fp.any(max=1, min=1), "-e", fp.any(max=1, min=1), + "-norun", "-wtime", self.wtime, "-couple", str(run_script_path), str(self.ntasks), - config_coupled, + config_coupled_file, ] ) == 1 ), "lpjsubmit should be called exactly once with correct parameters" + assert ( + fp.call_count([fp.program("sbatch")]) == 1 + ), "sbatch should be called exactly once with correct parameters" @pytest.mark.parametrize( "mock_venv", @@ -113,7 +134,9 @@ def test_command(self, sim_path, config_coupled, fp, submit): ], indirect=True, ) - def test_run_script(self, sim_path, config_coupled, mock_venv, request, submit): + def test_run_script( + self, sim_path, config_coupled_file, mock_venv, request, submit + ): run_script_path = sim_path / "output/coupled_test/copan_lpjml.sh" assert run_script_path.is_file(), "run script should have been created" assert ( @@ -125,7 +148,7 @@ def test_run_script(self, sim_path, config_coupled, mock_venv, request, submit): == f"""#!/bin/bash # Define the path to the config file -config_file="{config_coupled}" +config_file="{config_coupled_file}" # Call the Python script with the config file as an argument {f"{mock_venv}/bin/python" if mock_venv else "python3"} {self.couple_script} \ From cbe8a44ebc62ba578bd51850ab24015b010b1057 Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Mon, 8 Jun 2026 13:31:07 +0200 Subject: [PATCH 06/11] Disable coupler assertions The coupler tests did not test the coupler, but in fact just asserted themselves. I disabled them until we implement better testing with a mocked socket --- pycoupler/coupler.py | 2 +- tests/test_couple.py | 44 ++++++++++++++++++++------------------------ 2 files changed, 21 insertions(+), 25 deletions(-) diff --git a/pycoupler/coupler.py b/pycoupler/coupler.py index 6256356..ae9a7e3 100644 --- a/pycoupler/coupler.py +++ b/pycoupler/coupler.py @@ -444,7 +444,7 @@ def get_historic_years(self, match_period=True): if match_period and start_year >= end_year: raise ValueError( f"No historic years available. Simulated year {start_year} " - f"is greater than coupled year {end_year}." + f"is greater than or equal to coupled year {end_year}." ) current_year = start_year while current_year < end_year: diff --git a/tests/test_couple.py b/tests/test_couple.py index 51e2f5c..fa7b941 100644 --- a/tests/test_couple.py +++ b/tests/test_couple.py @@ -1,53 +1,49 @@ """Test the LPJmLCoupler class.""" -import numpy as np import pytest def test_lpjml_coupler(model_path, sim_path, lpjml_coupler): inputs = lpjml_coupler.read_input(copy=False) - outputs = lpjml_coupler.read_historic_output() - - hist_outputs = outputs.copy(deep=True) + hist_outputs = lpjml_coupler.read_historic_output() for year in lpjml_coupler.get_sim_years(): - inputs.time.values[0] = np.datetime64(f"{year}-12-31") # send input data to lpjml lpjml_coupler.send_input(inputs, year) # read output data from lpjml + output = lpjml_coupler.read_output(year) - outputs.time.values[0] = np.datetime64(f"{year}-12-31") - for name, output in lpjml_coupler.read_output(year).items(): - outputs[name][:] = output[:] - - if year == lpjml_coupler.config.lastyear: - lpjml_coupler.close() + # TODO: These assertions are wrong and need to be checked against the mocked socket values in a later version + # assert that the output is the same as the historic output + # assert np.allclose(a=output["cftfrac"].values, hist_outputs["cftfrac"].values) + # assert not np.allclose(output["hdate"].values, hist_outputs["hdate"].values) - # assert that the output is the same as the historic output - assert np.allclose(outputs["cftfrac"].values, hist_outputs["cftfrac"].values) - assert not np.allclose(outputs["hdate"].values, hist_outputs["hdate"].values) + # assert not np.allclose( + # output["pft_harvestc"].values, hist_outputs["pft_harvestc"].values + # ) - assert not np.allclose( - outputs["pft_harvestc"].values, hist_outputs["pft_harvestc"].values - ) + # assert not np.allclose( + # output["soilc_agr_layer"].values, hist_outputs["soilc_agr_layer"].values + # ) - assert not np.allclose( - outputs["soilc_agr_layer"].values, hist_outputs["soilc_agr_layer"].values - ) + if year == lpjml_coupler.config.lastyear: + lpjml_coupler.close() assert "_channel" not in lpjml_coupler.__getstate__() - assert lpjml_coupler.ncell == 2 assert [year for year in lpjml_coupler.get_cells()] == [27410, 27411] assert lpjml_coupler.historic_years == [] assert lpjml_coupler.sim_years == [] assert lpjml_coupler.coupled_years == [] assert [year for year in lpjml_coupler.get_coupled_years()] == [] + + +def test_lpjml_coupler_repr(model_path, sim_path, lpjml_coupler): assert ( repr(lpjml_coupler) == f""" Simulation: (version: 3, localhost:) - * sim_year 2050 + * sim_year 2022 * ncell 2 * ninput 1 Configuration: @@ -60,8 +56,8 @@ def test_lpjml_coupler(model_path, sim_path, lpjml_coupler): * endgrid 27411 * landuse yes (changed) - * model_path {str(model_path)} - * sim_path {str(sim_path)} + * model_path {model_path} + * sim_path {sim_path} * outputyear 2022 * output_metafile True * write_restart False From 32b73b5f0affd05aacc046c42fc4ae846af896bf Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Wed, 17 Jun 2026 18:22:00 +0200 Subject: [PATCH 07/11] Set APPTAINER_BIND with minimal paths in runtime env --- pycoupler/config.py | 36 +++++++++++++++++++++++++++++++++++- tests/test_config.py | 4 ++++ 2 files changed, 39 insertions(+), 1 deletion(-) diff --git a/pycoupler/config.py b/pycoupler/config.py index 3aa39ef..60cf1f4 100644 --- a/pycoupler/config.py +++ b/pycoupler/config.py @@ -2,6 +2,7 @@ import logging import os +from pathlib import Path import shutil import sys import json @@ -187,6 +188,7 @@ def get_runtime_env(self, ensure_paths=True): "LPJINPATH": self.get_input_folder(), "LPJOUTPATH": self.get_output_folder(ensure=ensure_paths), "LPJRESTARTPATH": self.get_restart_folder(ensure=ensure_paths), + "APPTAINER_BIND": ",".join(str(bind) for bind in self.get_bind_paths()), } def get_output_folder(self, ensure: bool = False) -> str: @@ -217,7 +219,7 @@ def get_input_folder(self) -> str: raise FileNotFoundError( "The input path, set in `LPJINPATH` does not exist." ) - input_path = os.environ["LPJINPATH"] + input_path = os.environ["LPJINPATH"] logger.debug( f"Using LPJINPATH '{os.environ['LPJINPATH']}' from runtime environtment as input path." ) @@ -233,6 +235,38 @@ def get_restart_folder(self, ensure: bool = False) -> str: os.makedirs(restart_folder, exist_ok=True) return restart_folder + def get_bind_paths(self) -> list[Path]: + binds = [] + input_folder = self.get_input_folder() + output_folder = self.get_output_folder() + restart_folder = self.get_restart_folder() + if input_folder: + binds.append(Path(input_folder)) + if output_folder: + binds.append(Path(output_folder)) + if restart_folder: + binds.append(Path(restart_folder)) + + for o in self.output: + output_file_path = Path(o.file.name) + print(o.file.name) + if output_file_path.is_absolute(): + # The file path is not relative to the output folder, which is always in binds + if not any(output_file_path.is_relative_to(bind) for bind in binds): + # The file is not contained in a path already included in binds + binds.append(output_file_path) + + for i in self.input.to_dict().values(): + input_file_path = Path(i["name"]) + print(i["name"]) + if input_file_path.is_absolute(): + # The file path is not relative to the input folder, which is always in binds + if not any(input_file_path.is_relative_to(bind) for bind in binds): + # The file is not contained in a path already included in binds + binds.append(input_file_path) + + return binds + def get_output_avail(self, id_only=True, to_dict=False): """ Get available output (outputvar) names or objects. diff --git a/tests/test_config.py b/tests/test_config.py index 923b7d5..a40c6f2 100644 --- a/tests/test_config.py +++ b/tests/test_config.py @@ -185,3 +185,7 @@ def test_parse_config(lpjml_config, model_path): coupled_config = parse_config(lpjml_config, config_class=CoupledConfig) assert coupled_config.__class__.__name__ == "CoupledConfig" + + +# TODO: Test run_model_bin + runtime environment setup +# TODO: Test get_bind_paths From 2a048c9d5701c6952c146c1e4a38e7bf9d0a840a Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Thu, 9 Jul 2026 16:51:25 +0200 Subject: [PATCH 08/11] Tested on geany --- pycoupler/config.py | 69 ++++++++++++++++++++++++--------------------- pycoupler/run.py | 14 ++++----- 2 files changed, 44 insertions(+), 39 deletions(-) diff --git a/pycoupler/config.py b/pycoupler/config.py index 60cf1f4..33f85a6 100644 --- a/pycoupler/config.py +++ b/pycoupler/config.py @@ -7,7 +7,7 @@ import sys import json from subprocess import DEVNULL, CompletedProcess, Popen, run as run_subprocess -from typing import Any, override +from typing import Any, override, TypedDict, Literal from ruamel.yaml import YAML from pycoupler.utils import read_json, get_countries, create_subdirs, detect_io_type @@ -105,6 +105,12 @@ def to_json(self, file_name=None): return json_file + +class Input(TypedDict): + name: str + fmt: Literal["clm", "cdf", "meta", "txt", "raw", "fms", "sock"] + id: int + class LpjmlConfig(SubConfig): """ LPJmL config class for easy access, conversion to a dictionary, @@ -163,7 +169,7 @@ def run_model_bin( "check": True, } - if self.model_path: + if getattr(self, 'model_path', None): if not os.path.exists(self.model_path): raise FileNotFoundError("The given model_path does not exist.") command = os.path.join(self.model_path, "bin", binary) @@ -184,7 +190,7 @@ def run_model_bin( def get_runtime_env(self, ensure_paths=True): return { - "LPJROOT": self.model_path, + "LPJROOT": getattr(self, "model_path", ""), "LPJINPATH": self.get_input_folder(), "LPJOUTPATH": self.get_output_folder(ensure=ensure_paths), "LPJRESTARTPATH": self.get_restart_folder(ensure=ensure_paths), @@ -197,6 +203,15 @@ def get_output_folder(self, ensure: bool = False) -> str: os.makedirs(output_folder, exist_ok=True) return output_folder + def get_datafile_from_input(self, input: Input) -> str: + if input.fmt == "meta": + datafile_path = Path(self.get_input_filepath(input.name)) + with datafile_path.open() as f: + metadata = json.load(f) + return str(datafile_path.parent / metadata["filename"]) if not Path(metadata["filename"]).is_absolute() else metadata["filename"] + else: + return self.get_input_filepath(input.name) + def get_input_filepath(self, input_file_name: str) -> str: return ( input_file_name @@ -246,10 +261,11 @@ def get_bind_paths(self) -> list[Path]: binds.append(Path(output_folder)) if restart_folder: binds.append(Path(restart_folder)) + if self.sim_path: + binds.append(Path(self.sim_path)) for o in self.output: output_file_path = Path(o.file.name) - print(o.file.name) if output_file_path.is_absolute(): # The file path is not relative to the output folder, which is always in binds if not any(output_file_path.is_relative_to(bind) for bind in binds): @@ -258,7 +274,6 @@ def get_bind_paths(self) -> list[Path]: for i in self.input.to_dict().values(): input_file_path = Path(i["name"]) - print(i["name"]) if input_file_path.is_absolute(): # The file path is not relative to the input folder, which is always in binds if not any(input_file_path.is_relative_to(bind) for bind in binds): @@ -824,7 +839,7 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): None, ).lower() - grid_file = self.get_input_filepath(self.input.coord.name) + grid_file = self.get_datafile_from_input(self.input.coord) # proxy check if regrid was already performed if country in self.input.coord.name: @@ -832,7 +847,7 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): # TODO: Mount in container! country_grid_file = ( - f"{sim_path}/input/{country}_{os.path.basename(self.input.coord.name)}" + f"{sim_path}/input/{country}_{os.path.basename(grid_file)}" ) # check if country specific input files already exist if (not os.path.isfile(country_grid_file) or overwrite) and not hasattr( @@ -845,30 +860,21 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): # extract country specific grid self.run_model_bin( "getcountry", - self.get_input_filepath(self.input.countrycode.name), + self.get_datafile_from_input(self.input.countrycode), grid_file, country_grid_file, country_code, ) - self.input.coord.fmt = ( - detect_io_type(country_grid_file) - if not hasattr(sys, "_called_from_test") - else "clm" - ) + # self.input.coord.fmt = ( + # detect_io_type(country_grid_file) + # if not hasattr(sys, "_called_from_test") + # else "clm" + # ) + self.input.coord.fmt = "clm" self.input.coord.name = country_grid_file - lakes_fn_string = self.get_input_filepath(self.input.lakes.name) - # extract country specific lakes file from meta file - if self.input.lakes.fmt == "meta" and not hasattr(sys, "_called_from_test"): - lakes_filename = read_json(lakes_fn_string)["filename"] - - lakes_file = lakes_fn_string - lakes_file = ( - f"{lakes_file[:lakes_file.rfind('/')+1]}{lakes_filename}" # noqa - ) - else: - lakes_file = lakes_fn_string + lakes_file = self.get_datafile_from_input(self.input.lakes) country_lakes_file = ( f"{sim_path}/input/{country}_{os.path.basename(lakes_file)}" @@ -889,7 +895,6 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): country_grid_file, lakes_file, country_lakes_file, - stdout=DEVNULL, ) self.input.lakes.fmt = ( @@ -899,18 +904,19 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): ) self.input.lakes.name = country_lakes_file + coord_file = self.get_datafile_from_input(self.input.coord) # loop over all used input files to regrid them to country specific # grid for config_key, config_input in self.input: if ( - config_input.fmt != "clm" - or config_key in ["coord", "lakes"] + config_key in ["coord", "lakes"] + or config_input.fmt == "txt" or (config_input.name == "DUMMYLOCATION") ): continue - input_file = self.get_input_filepath(config_input.name) + input_file = self.get_datafile_from_input(config_input) country_input_file = ( f"{sim_path}/input/{country}_{os.path.basename(input_file)}" @@ -933,12 +939,11 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): # regrid all other input files to country specific grid self.run_model_bin( - "regrid_func", + regrid_func, grid_file, - self.input.coord.name, + coord_file, input_file, country_input_file, - stdout=DEVNULL, ) # if additional_arg: # regrid_cmd.insert(1, additional_arg) @@ -961,7 +966,7 @@ def convert_cdf_to_raw(self, output_id=None): output_dir = f"{self.sim_path}/output/{self.sim_name}" - grid_file = self.get_input_filepath(self.input.coord.name) + grid_file = self.get_datafile_from_input(self.input.coord) grid_name = os.path.basename(grid_file) diff --git a/pycoupler/run.py b/pycoupler/run.py index a43bd33..bf0a1a2 100644 --- a/pycoupler/run.py +++ b/pycoupler/run.py @@ -23,7 +23,7 @@ def operate_lpjml(config_file, std_to_file=False, wait_for_exit=True): config = read_config(config_file) - if config.model_path and not os.path.isdir(config.model_path): + if hasattr(config, "model_path") and not os.path.isdir(config.model_path): raise ValueError(f"Folder of model_path '{config.model_path}' does not exist!") output_path = f"{config.sim_path}/output/{config.sim_name}" @@ -46,8 +46,7 @@ def operate_lpjml(config_file, std_to_file=False, wait_for_exit=True): "I_MPI_DAPL_FABRIC": "shm:sh", } | config.get_runtime_env(), - # This might be None, running in the current directory: - "cwd": config.model_path, + "cwd": getattr(config, "model_path", None), # None means current directory "text": True, } @@ -245,7 +244,7 @@ def submit_lpjml( # call lpjsubmit via subprocess and return status if successfull submit_file_status = config.run_model_bin( "lpjsubmit", - *submit_args, + **submit_args, subprocess_args={ "capture_output": True, "cwd": config.sim_path, @@ -293,15 +292,15 @@ def check_lpjml(config_file): Path to `LPJmL_internal` (lpjml repository) """ config = read_config(config_file) - if config.model_path and not os.path.isdir(config.model_path): + if hasattr(config, "model_path") and not os.path.isdir(config.model_path): raise ValueError(f"Folder of model_path '{config.model_path}' does not exist!") proc_status = config.run_model_bin( "lpjcheck", - [config_file], + config_file, subprocess_args={ # ensure_paths is false, because this is just a check and should have no side effects - "cwd": config.model_path, + "cwd": getattr(config, "model_path", None), "check": False, "capture_output": True, "text": True, @@ -313,3 +312,4 @@ def check_lpjml(config_file): else: print(proc_status.stdout) print(proc_status.stderr) + # TODO: raise an exception here From c12f44d0af3af57276a389996bc3a3ec7c71518c Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Mon, 20 Jul 2026 18:37:08 +0200 Subject: [PATCH 09/11] Always use meta format for regirdded files --- pycoupler/config.py | 25 ++++++++++--------------- 1 file changed, 10 insertions(+), 15 deletions(-) diff --git a/pycoupler/config.py b/pycoupler/config.py index 33f85a6..b095801 100644 --- a/pycoupler/config.py +++ b/pycoupler/config.py @@ -864,6 +864,7 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): grid_file, country_grid_file, country_code, + "--json" ) # self.input.coord.fmt = ( @@ -871,8 +872,8 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): # if not hasattr(sys, "_called_from_test") # else "clm" # ) - self.input.coord.fmt = "clm" - self.input.coord.name = country_grid_file + self.input.coord.fmt = "meta" + self.input.coord.name = f"{country_grid_file}.json" lakes_file = self.get_datafile_from_input(self.input.lakes) @@ -895,14 +896,11 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): country_grid_file, lakes_file, country_lakes_file, + "--json" ) - self.input.lakes.fmt = ( - detect_io_type(country_lakes_file) - if not hasattr(sys, "_called_from_test") - else "raw" - ) - self.input.lakes.name = country_lakes_file + self.input.lakes.fmt = "meta" + self.input.lakes.name = f"{lakes_file}.json" coord_file = self.get_datafile_from_input(self.input.coord) # loop over all used input files to regrid them to country specific @@ -944,16 +942,13 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): coord_file, input_file, country_input_file, + "--json" ) # if additional_arg: # regrid_cmd.insert(1, additional_arg) - config_input.fmt = ( - detect_io_type(country_input_file) - if not hasattr(sys, "_called_from_test") - else "clm" - ) - config_input.name = country_input_file + config_input.fmt = "meta" + config_input.name = f"{country_input_file}.json" self._set_grid_explicitly(only_all=False) @@ -973,7 +968,7 @@ def convert_cdf_to_raw(self, output_id=None): if not os.path.isfile(f"{output_dir}/{grid_name}") and not hasattr( sys, "_called_from_test" ): - run(f"tail -c +44 {grid_file} > {output_dir}/{grid_name}", shell=True) + run_subprocess(f"tail -c +44 {grid_file} > {output_dir}/{grid_name}", shell=True) grid_file = f"{output_dir}/{grid_name}" From fdd1ed93c25e9005f86beb8672faa50aae4a747a Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Mon, 20 Jul 2026 18:38:03 +0200 Subject: [PATCH 10/11] Process cjson configs in the container if necessary --- pycoupler/config.py | 21 ++++++++++++++++++--- 1 file changed, 18 insertions(+), 3 deletions(-) diff --git a/pycoupler/config.py b/pycoupler/config.py index b095801..a4b6ca8 100644 --- a/pycoupler/config.py +++ b/pycoupler/config.py @@ -1104,7 +1104,11 @@ def __setattr__(self, __name, __value): def parse_config( - file_name="./lpjml_config.json", spin_up=False, macros=None, config_class=None + file_name="./lpjml_config.json", + spin_up=False, + macros=None, + config_class=None, + in_container=False ): """ Precompile lpjml_config.json and return LpjmlConfig object or dict. @@ -1123,6 +1127,10 @@ def parse_config( Macro(s) to provide in the form of "-DMACRO" or list of macros. config_class : class, optional Class of config object to be returned. If None, returns dict. + in_container : bool, default False + If True, expects the path to be inside the LPJML container given in the + LPJML_CONTAINER environment variable. It then runs the command for the + C preporcessor in the container. Returns ------- @@ -1132,6 +1140,9 @@ def parse_config( """ # precompile command cmd = ["cpp", "-P"] + if in_container: + cmd = ["apptainer", "exec", "-s", os.environ["LPJML_CONTAINER"]] + cmd + # add arguments if not spin_up: cmd.append("-DFROM_RESTART") @@ -1152,7 +1163,7 @@ def parse_config( def read_config( - file_name, model_path=None, spin_up=False, macros=None, to_dict=False + file_name, model_path=None, spin_up=False, macros=None, to_dict=False, parse_in_container=False ): # noqa """ Read LPJmL configuration files and return as LpjmlConfig object or dict. @@ -1171,6 +1182,10 @@ def read_config( to_dict : bool, default False If True, a dictionary is returned. If False, an LpjmlConfig object is returned. + parse_in_container : bool, default False + If True, expects the path to be inside the LPJML container given in the + LPJML_CONTAINER environment variable. It then runs the command for the + C preporcessor in the container. Returns ------- @@ -1192,7 +1207,7 @@ def read_config( # If not possible, precompile and parse JSON except json.decoder.JSONDecodeError: lpjml_config = parse_config( - file_name, spin_up=spin_up, macros=macros, config_class=config + file_name, spin_up=spin_up, macros=macros, config_class=config, in_container=parse_in_container ) # Convert first level to LpjmlConfig object From ece3dd3ed009df63938f8da50d2a577dd53c4e11 Mon Sep 17 00:00:00 2001 From: Lorenz Sieben Date: Mon, 20 Jul 2026 18:39:02 +0200 Subject: [PATCH 11/11] Simplify get_countries() --- pycoupler/config.py | 12 +- pycoupler/countries.json | 1180 ++++++++------------------------------ pycoupler/coupler.py | 2 +- pycoupler/utils.py | 27 +- 4 files changed, 258 insertions(+), 963 deletions(-) diff --git a/pycoupler/config.py b/pycoupler/config.py index a4b6ca8..f633c7c 100644 --- a/pycoupler/config.py +++ b/pycoupler/config.py @@ -826,18 +826,8 @@ def regrid(self, sim_path, country_code="BEL", overwrite=False): if not os.path.exists(sim_path): raise FileNotFoundError(f"Path '{sim_path}' does not exist.") - # get available countries of LPJmL - countries = get_countries() - # get country name from country code - country = next( - ( - countries[country]["name"] - for country in countries - if (countries[country]["code"] == country_code) - ), - None, - ).lower() + country = get_countries(codes_as_keys=True)[country_code].lower() grid_file = self.get_datafile_from_input(self.input.coord) diff --git a/pycoupler/countries.json b/pycoupler/countries.json index 698ca3e..1de8c6b 100644 --- a/pycoupler/countries.json +++ b/pycoupler/countries.json @@ -1,946 +1,238 @@ { - "Afghanistan": { - "name": "Afghanistan", - "code": "AFG" - }, - "Aland Islands": { - "name": "Aland Islands", - "code": "ALA" - }, - "Albania": { - "name": "Albania", - "code": "ALB" - }, - "Algeria": { - "name": "Algeria", - "code": "DZA" - }, - "American Samoa": { - "name": "American Samoa", - "code": "ASM" - }, - "Angola": { - "name": "Angola", - "code": "AGO" - }, - "Anguilla": { - "name": "Anguilla", - "code": "AIA" - }, - "Antigua and Barbuda": { - "name": "Antigua and Barbuda", - "code": "ATG" - }, - "Argentina": { - "name": "Argentina", - "code": "ARG" - }, - "Armenia": { - "name": "Armenia", - "code": "ARM" - }, - "Austria": { - "name": "Austria", - "code": "AUT" - }, - "Azerbaijan": { - "name": "Azerbaijan", - "code": "AZE" - }, - "Bahamas The": { - "name": "Bahamas, The", - "code": "BHS" - }, - "Bahrain": { - "name": "Bahrain", - "code": "BHR" - }, - "Bangladesh": { - "name": "Bangladesh", - "code": "BGD" - }, - "Barbados": { - "name": "Barbados", - "code": "BRB" - }, - "Belgium": { - "name": "Belgium", - "code": "BEL" - }, - "Belize": { - "name": "Belize", - "code": "BLZ" - }, - "Benin": { - "name": "Benin", - "code": "BEN" - }, - "Bermuda": { - "name": "Bermuda", - "code": "BMU" - }, - "Bhutan": { - "name": "Bhutan", - "code": "BTN" - }, - "Bolivia": { - "name": "Bolivia", - "code": "BOL" - }, - "Bosnia and Herzegovina": { - "name": "Bosnia and Herzegovina", - "code": "BIH" - }, - "Botswana": { - "name": "Botswana", - "code": "BWA" - }, - "British Indian Ocean Territory": { - "name": "British Indian Ocean Territory", - "code": "IOT" - }, - "Brunei": { - "name": "Brunei", - "code": "BRN" - }, - "Bulgaria": { - "name": "Bulgaria", - "code": "BGR" - }, - "Burkina Faso": { - "name": "Burkina Faso", - "code": "BFA" - }, - "Burundi": { - "name": "Burundi", - "code": "BDI" - }, - "Byelarus": { - "name": "Byelarus", - "code": "BLR" - }, - "Cambodia": { - "name": "Cambodia", - "code": "KHM" - }, - "Cameroon": { - "name": "Cameroon", - "code": "CMR" - }, - "Cape Verde": { - "name": "Cape Verde", - "code": "CPV" - }, - "Cayman Islands": { - "name": "Cayman Islands", - "code": "CYM" - }, - "Central African Republic": { - "name": "Central African Republic", - "code": "CAF" - }, - "Chad": { - "name": "Chad", - "code": "TCD" - }, - "Chile": { - "name": "Chile", - "code": "CHL" - }, - "Christmas Island": { - "name": "Christmas Island", - "code": "CXR" - }, - "Cocos Keeling Islands": { - "name": "Cocos Keeling Islands", - "code": "CCK" - }, - "Colombia": { - "name": "Colombia", - "code": "COL" - }, - "Comoros": { - "name": "Comoros", - "code": "COM" - }, - "Congo Brazzaville": { - "name": "Congo-Brazzaville", - "code": "COG" - }, - "Cook Islands": { - "name": "Cook Islands", - "code": "COK" - }, - "Costa Rica": { - "name": "Costa Rica", - "code": "CRI" - }, - "Croatia": { - "name": "Croatia", - "code": "HRV" - }, - "Cuba": { - "name": "Cuba", - "code": "CUB" - }, - "Curacao": { - "name": "Curacao", - "code": "CUW" - }, - "Cyprus": { - "name": "Cyprus", - "code": "CYP" - }, - "Czech Republic": { - "name": "Czech Republic", - "code": "CZE" - }, - "Denmark": { - "name": "Denmark", - "code": "DNK" - }, - "Djibouti": { - "name": "Djibouti", - "code": "DJI" - }, - "Dominica": { - "name": "Dominica", - "code": "DMA" - }, - "Dominican Republic": { - "name": "Dominican Republic", - "code": "DOM" - }, - "Ecuador": { - "name": "Ecuador", - "code": "ECU" - }, - "Egypt": { - "name": "Egypt", - "code": "EGY" - }, - "El Salvador": { - "name": "El Salvador", - "code": "SLV" - }, - "Equatorial Guinea": { - "name": "Equatorial Guinea", - "code": "GNQ" - }, - "Eritrea": { - "name": "Eritrea", - "code": "ERI" - }, - "Estonia": { - "name": "Estonia", - "code": "EST" - }, - "Ethiopia": { - "name": "Ethiopia", - "code": "ETH" - }, - "Falkland Islands or Islas Malvinas": { - "name": "Falkland Islands or Islas Malvinas", - "code": "FLK" - }, - "Faroe Islands": { - "name": "Faroe Islands", - "code": "FRO" - }, - "Federated States of Micronesia": { - "name": "Federated States of Micronesia", - "code": "FSM" - }, - "Fiji": { - "name": "Fiji", - "code": "FJI" - }, - "Finland": { - "name": "Finland", - "code": "FIN" - }, - "France": { - "name": "France", - "code": "FRA" - }, - "French Guiana": { - "name": "French Guiana", - "code": "GUF" - }, - "French Polynesia": { - "name": "French Polynesia", - "code": "PYF" - }, - "French Southern and Antarctica Lands": { - "name": "French Southern and Antarctica Lands", - "code": "NOC" - }, - "Gabon": { - "name": "Gabon", - "code": "GAB" - }, - "Gambia The": { - "name": "Gambia,The", - "code": "GMB" - }, - "Georgia": { - "name": "Georgia", - "code": "GEO" - }, - "Germany": { - "name": "Germany", - "code": "DEU" - }, - "Ghana": { - "name": "Ghana", - "code": "GHA" - }, - "Greece": { - "name": "Greece", - "code": "GRC" - }, - "Greenland": { - "name": "Greenland", - "code": "GRL" - }, - "Grenada": { - "name": "Grenada", - "code": "GRD" - }, - "Guadeloupe": { - "name": "Guadeloupe", - "code": "GLP" - }, - "Guam": { - "name": "Guam", - "code": "GUM" - }, - "Guatemala": { - "name": "Guatemala", - "code": "GTM" - }, - "Guernsey": { - "name": "Guernsey", - "code": "GGY" - }, - "Guinea Bissau": { - "name": "Guinea-Bissau", - "code": "GNB" - }, - "Guinea": { - "name": "Guinea", - "code": "GIN" - }, - "Guyana": { - "name": "Guyana", - "code": "GUY" - }, - "Haiti": { - "name": "Haiti", - "code": "HTI" - }, - "Heard Island and McDonald Islands": { - "name": "Heard Island and McDonald Islands", - "code": "HMD" - }, - "Honduras": { - "name": "Honduras", - "code": "HND" - }, - "Hong Kong": { - "name": "Hong Kong", - "code": "HKG" - }, - "Hungary": { - "name": "Hungary", - "code": "HUN" - }, - "Iceland": { - "name": "Iceland", - "code": "ISL" - }, - "Indonesia": { - "name": "Indonesia", - "code": "IDN" - }, - "Iran": { - "name": "Iran", - "code": "IRN" - }, - "Iraq": { - "name": "Iraq", - "code": "IRQ" - }, - "Ireland": { - "name": "Ireland", - "code": "IRL" - }, - "Isle of Man": { - "name": "Isle of Man", - "code": "IMN" - }, - "Israel": { - "name": "Israel", - "code": "ISR" - }, - "Italy": { - "name": "Italy", - "code": "ITA" - }, - "Ivory Coast": { - "name": "Ivory Coast", - "code": "CIV" - }, - "Jamaica": { - "name": "Jamaica", - "code": "JAM" - }, - "Japan": { - "name": "Japan", - "code": "JPN" - }, - "Jersey": { - "name": "Jersey", - "code": "JEY" - }, - "Jordan": { - "name": "Jordan", - "code": "JOR" - }, - "Kazakhstan": { - "name": "Kazakhstan", - "code": "KAZ" - }, - "Kenya": { - "name": "Kenya", - "code": "KEN" - }, - "Kiribati": { - "name": "Kiribati", - "code": "KIR" - }, - "Kosovo": { - "name": "Kosovo", - "code": "KO-" - }, - "Kuwait": { - "name": "Kuwait", - "code": "KWT" - }, - "Kyrgyzstan": { - "name": "Kyrgyzstan", - "code": "KGZ" - }, - "Laos": { - "name": "Laos", - "code": "LAO" - }, - "Latvia": { - "name": "Latvia", - "code": "LVA" - }, - "Lebanon": { - "name": "Lebanon", - "code": "LBN" - }, - "Lesotho": { - "name": "Lesotho", - "code": "LSO" - }, - "Liberia": { - "name": "Liberia", - "code": "LBR" - }, - "Libya": { - "name": "Libya", - "code": "LBY" - }, - "Lithuania": { - "name": "Lithuania", - "code": "LTU" - }, - "Luxembourg": { - "name": "Luxembourg", - "code": "LUX" - }, - "Macedonia": { - "name": "Macedonia", - "code": "MKD" - }, - "Madagascar": { - "name": "Madagascar", - "code": "MDG" - }, - "Malawi": { - "name": "Malawi", - "code": "MWI" - }, - "Malaysia": { - "name": "Malaysia", - "code": "MYS" - }, - "Maldives": { - "name": "Maldives", - "code": "MDV" - }, - "Mali": { - "name": "Mali", - "code": "MLI" - }, - "Malta": { - "name": "Malta", - "code": "MLT" - }, - "Marshall Islands": { - "name": "Marshall Islands", - "code": "MHL" - }, - "Martinique": { - "name": "Martinique", - "code": "MTQ" - }, - "Mauritania": { - "name": "Mauritania", - "code": "MRT" - }, - "Mauritius": { - "name": "Mauritius", - "code": "MUS" - }, - "Mayotte": { - "name": "Mayotte", - "code": "MYT" - }, - "Mexico": { - "name": "Mexico", - "code": "MEX" - }, - "Moldova": { - "name": "Moldova", - "code": "MDA" - }, - "Mongolia": { - "name": "Mongolia", - "code": "MNG" - }, - "Montenegro": { - "name": "Montenegro", - "code": "MNE" - }, - "Montserrat": { - "name": "Montserrat", - "code": "MSR" - }, - "Morocco": { - "name": "Morocco", - "code": "MAR" - }, - "Mozambique": { - "name": "Mozambique", - "code": "MOZ" - }, - "Myanmar or Burma": { - "name": "Myanmar or Burma", - "code": "MMR" - }, - "Namibia": { - "name": "Namibia", - "code": "NAM" - }, - "Nauru": { - "name": "Nauru", - "code": "NRU" - }, - "Nepal": { - "name": "Nepal", - "code": "NPL" - }, - "Netherlands": { - "name": "Netherlands", - "code": "NLD" - }, - "New Caledonia": { - "name": "New Caledonia", - "code": "NCL" - }, - "New Zealand": { - "name": "New Zealand", - "code": "NZL" - }, - "Nicaragua": { - "name": "Nicaragua", - "code": "NIC" - }, - "Niger": { - "name": "Niger", - "code": "NER" - }, - "Nigeria": { - "name": "Nigeria", - "code": "NGA" - }, - "Niue": { - "name": "Niue", - "code": "NIU" - }, - "No Land": { - "name": "No Land", - "code": "XNL" - }, - "Norfolk Island": { - "name": "Norfolk Island", - "code": "NFK" - }, - "North Korea": { - "name": "North Korea", - "code": "PRK" - }, - "Northern Mariana Islands": { - "name": "Northern Mariana Islands", - "code": "MNP" - }, - "Norway": { - "name": "Norway", - "code": "NOR" - }, - "Oman": { - "name": "Oman", - "code": "OMN" - }, - "Pakistan": { - "name": "Pakistan", - "code": "PAK" - }, - "Palau": { - "name": "Palau", - "code": "PLW" - }, - "Panama": { - "name": "Panama", - "code": "PAN" - }, - "Papua New Guinea": { - "name": "Papua New Guinea", - "code": "PNG" - }, - "Paraguay": { - "name": "Paraguay", - "code": "PRY" - }, - "Peru": { - "name": "Peru", - "code": "PER" - }, - "Philippines": { - "name": "Philippines", - "code": "PHL" - }, - "Pitcairn Islands": { - "name": "Pitcairn Islands", - "code": "PCN" - }, - "Poland": { - "name": "Poland", - "code": "POL" - }, - "Portugal": { - "name": "Portugal", - "code": "PRT" - }, - "Puerto Rico": { - "name": "Puerto Rico", - "code": "PRI" - }, - "Qatar": { - "name": "Qatar", - "code": "QAT" - }, - "Reunion": { - "name": "Reunion", - "code": "REU" - }, - "Romania": { - "name": "Romania", - "code": "ROU" - }, - "Rwanda": { - "name": "Rwanda", - "code": "RWA" - }, - "Saint Helena Ascension and Tristan da Cunha": { - "name": "Saint Helena Ascension and Tristan da Cunha", - "code": "SHN" - }, - "Saint Kitts and Nevis": { - "name": "Saint Kitts and Nevis", - "code": "KNA" - }, - "Saint Lucia": { - "name": "Saint Lucia", - "code": "LCA" - }, - "Saint Pierre and Miquelon": { - "name": "Saint Pierre and Miquelon", - "code": "SPM" - }, - "Sao Tome and Principe": { - "name": "Sao Tome and Principe", - "code": "STP" - }, - "Saudi Arabia": { - "name": "Saudi Arabia", - "code": "SAU" - }, - "Senegal": { - "name": "Senegal", - "code": "SEN" - }, - "Serbia": { - "name": "Serbia", - "code": "SRB" - }, - "Seychelles": { - "name": "Seychelles", - "code": "SYC" - }, - "Sierra Leone": { - "name": "Sierra Leone", - "code": "SLE" - }, - "Singapore": { - "name": "Singapore", - "code": "SGP" - }, - "Slovakia": { - "name": "Slovakia", - "code": "SVK" - }, - "Slovenia": { - "name": "Slovenia", - "code": "SVN" - }, - "Solomon Islands": { - "name": "Solomon Islands", - "code": "SLB" - }, - "Somalia": { - "name": "Somalia", - "code": "SOM" - }, - "South Africa": { - "name": "South Africa", - "code": "ZAF" - }, - "South Georgia and the South Sandwich Islands": { - "name": "South Georgia and the South Sandwich Islands", - "code": "SGS" - }, - "South Korea": { - "name": "South Korea", - "code": "KOR" - }, - "South Sudan": { - "name": "South Sudan", - "code": "SSD" - }, - "Spain": { - "name": "Spain", - "code": "ESP" - }, - "Sri Lanka": { - "name": "Sri Lanka", - "code": "LKA" - }, - "St Vincent and the Grenadines": { - "name": "St. Vincent and the Grenadines", - "code": "VCT" - }, - "Sudan": { - "name": "Sudan", - "code": "SDN" - }, - "Suriname": { - "name": "Suriname", - "code": "SUR" - }, - "Svalbard": { - "name": "Svalbard", - "code": "SJM" - }, - "Swaziland": { - "name": "Swaziland", - "code": "SWZ" - }, - "Sweden": { - "name": "Sweden", - "code": "SWE" - }, - "Switzerland": { - "name": "Switzerland", - "code": "CHE" - }, - "Syria": { - "name": "Syria", - "code": "SYR" - }, - "Taiwan": { - "name": "Taiwan", - "code": "TWN" - }, - "Tajikistan": { - "name": "Tajikistan", - "code": "TJK" - }, - "Tanzania United Republic of": { - "name": "Tanzania, United Republic of", - "code": "TZA" - }, - "Thailand": { - "name": "Thailand", - "code": "THA" - }, - "Timor Leste": { - "name": "Timor Leste", - "code": "TLS" - }, - "Togo": { - "name": "Togo", - "code": "TGO" - }, - "Tokelau": { - "name": "Tokelau", - "code": "TKL" - }, - "Tonga": { - "name": "Tonga", - "code": "TON" - }, - "Trinidad and Tobago": { - "name": "Trinidad and Tobago", - "code": "TTO" - }, - "Tunisia": { - "name": "Tunisia", - "code": "TUN" - }, - "Turkey": { - "name": "Turkey", - "code": "TUR" - }, - "Turkmenistan": { - "name": "Turkmenistan", - "code": "TKM" - }, - "Turks and Caicos Islands": { - "name": "Turks and Caicos Islands", - "code": "TCA" - }, - "Tuvalu": { - "name": "Tuvalu", - "code": "TUV" - }, - "Uganda": { - "name": "Uganda", - "code": "UGA" - }, - "Ukraine": { - "name": "Ukraine", - "code": "UKR" - }, - "United Arab Emirates": { - "name": "United Arab Emirates", - "code": "ARE" - }, - "United Kingdom": { - "name": "United Kingdom", - "code": "GBR" - }, - "United States Minor Outlying Islands": { - "name": "United States Minor Outlying Islands", - "code": "UMI" - }, - "Uruguay": { - "name": "Uruguay", - "code": "URY" - }, - "Uzbekistan": { - "name": "Uzbekistan", - "code": "UZB" - }, - "Vanuatu": { - "name": "Vanuatu", - "code": "VUT" - }, - "Venezuela": { - "name": "Venezuela", - "code": "VEN" - }, - "Vietnam": { - "name": "Vietnam", - "code": "VNM" - }, - "Virgin Islands": { - "name": "Virgin Islands", - "code": "VGB" - }, - "Wallis and Futuna": { - "name": "Wallis and Futuna", - "code": "WLF" - }, - "West Bank": { - "name": "West Bank", - "code": "PSE" - }, - "Western Sahara": { - "name": "Western Sahara", - "code": "ESH" - }, - "Western Samoa": { - "name": "Western Samoa", - "code": "WSM" - }, - "Yemen": { - "name": "Yemen", - "code": "YEM" - }, - "Zaire DR Congo": { - "name": "DR Congo, former Zaire", - "code": "COD" - }, - "Zambia": { - "name": "Zambia", - "code": "ZMB" - }, - "Zimbabwe": { - "name": "Zimbabwe", - "code": "ZWE" - }, - "Australia": { - "name": "Australia", - "code": "AUS" - }, - "Brazil": { - "name": "Brazil", - "code": "BRA" - }, - "Canada": { - "name": "Canada", - "code": "CAN" - }, - "China": { - "name": "China", - "code": "CHN" - }, - "India": { - "name": "India", - "code": "IND" - }, - "Russia": { - "name": "Russia", - "code": "RUS" - }, - "United States": { - "name": "United States of America", - "code": "USA" - } + "Afghanistan": "AFG", + "Aland Islands": "ALA", + "Albania": "ALB", + "Algeria": "DZA", + "American Samoa": "ASM", + "Angola": "AGO", + "Anguilla": "AIA", + "Antigua and Barbuda": "ATG", + "Argentina": "ARG", + "Armenia": "ARM", + "Australia": "AUS", + "Austria": "AUT", + "Azerbaijan": "AZE", + "Bahamas, The": "BHS", + "Bahrain": "BHR", + "Bangladesh": "BGD", + "Barbados": "BRB", + "Belgium": "BEL", + "Belize": "BLZ", + "Benin": "BEN", + "Bermuda": "BMU", + "Bhutan": "BTN", + "Bolivia": "BOL", + "Bosnia and Herzegovina": "BIH", + "Botswana": "BWA", + "Brazil": "BRA", + "British Indian Ocean Territory": "IOT", + "Brunei": "BRN", + "Bulgaria": "BGR", + "Burkina Faso": "BFA", + "Burundi": "BDI", + "Byelarus": "BLR", + "Cambodia": "KHM", + "Cameroon": "CMR", + "Canada": "CAN", + "Cape Verde": "CPV", + "Cayman Islands": "CYM", + "Central African Republic": "CAF", + "Chad": "TCD", + "Chile": "CHL", + "China": "CHN", + "Christmas Island": "CXR", + "Cocos Keeling Islands": "CCK", + "Colombia": "COL", + "Comoros": "COM", + "Congo-Brazzaville": "COG", + "Cook Islands": "COK", + "Costa Rica": "CRI", + "Croatia": "HRV", + "Cuba": "CUB", + "Curacao": "CUW", + "Cyprus": "CYP", + "Czech Republic": "CZE", + "Denmark": "DNK", + "Djibouti": "DJI", + "Dominica": "DMA", + "Dominican Republic": "DOM", + "DR Congo, former Zaire": "COD", + "Ecuador": "ECU", + "Egypt": "EGY", + "El Salvador": "SLV", + "Equatorial Guinea": "GNQ", + "Eritrea": "ERI", + "Estonia": "EST", + "Ethiopia": "ETH", + "Falkland Islands or Islas Malvinas": "FLK", + "Faroe Islands": "FRO", + "Federated States of Micronesia": "FSM", + "Fiji": "FJI", + "Finland": "FIN", + "France": "FRA", + "French Guiana": "GUF", + "French Polynesia": "PYF", + "French Southern and Antarctica Lands": "NOC", + "Gabon": "GAB", + "Gambia,The": "GMB", + "Georgia": "GEO", + "Germany": "DEU", + "Ghana": "GHA", + "Greece": "GRC", + "Greenland": "GRL", + "Grenada": "GRD", + "Guadeloupe": "GLP", + "Guam": "GUM", + "Guatemala": "GTM", + "Guernsey": "GGY", + "Guinea": "GIN", + "Guinea-Bissau": "GNB", + "Guyana": "GUY", + "Haiti": "HTI", + "Heard Island and McDonald Islands": "HMD", + "Honduras": "HND", + "Hong Kong": "HKG", + "Hungary": "HUN", + "Iceland": "ISL", + "India": "IND", + "Indonesia": "IDN", + "Iran": "IRN", + "Iraq": "IRQ", + "Ireland": "IRL", + "Isle of Man": "IMN", + "Israel": "ISR", + "Italy": "ITA", + "Ivory Coast": "CIV", + "Jamaica": "JAM", + "Japan": "JPN", + "Jersey": "JEY", + "Jordan": "JOR", + "Kazakhstan": "KAZ", + "Kenya": "KEN", + "Kiribati": "KIR", + "Kosovo": "KO-", + "Kuwait": "KWT", + "Kyrgyzstan": "KGZ", + "Laos": "LAO", + "Latvia": "LVA", + "Lebanon": "LBN", + "Lesotho": "LSO", + "Liberia": "LBR", + "Libya": "LBY", + "Lithuania": "LTU", + "Luxembourg": "LUX", + "Macedonia": "MKD", + "Madagascar": "MDG", + "Malawi": "MWI", + "Malaysia": "MYS", + "Maldives": "MDV", + "Mali": "MLI", + "Malta": "MLT", + "Marshall Islands": "MHL", + "Martinique": "MTQ", + "Mauritania": "MRT", + "Mauritius": "MUS", + "Mayotte": "MYT", + "Mexico": "MEX", + "Moldova": "MDA", + "Mongolia": "MNG", + "Montenegro": "MNE", + "Montserrat": "MSR", + "Morocco": "MAR", + "Mozambique": "MOZ", + "Myanmar or Burma": "MMR", + "Namibia": "NAM", + "Nauru": "NRU", + "Nepal": "NPL", + "Netherlands": "NLD", + "New Caledonia": "NCL", + "New Zealand": "NZL", + "Nicaragua": "NIC", + "Niger": "NER", + "Nigeria": "NGA", + "Niue": "NIU", + "No Land": "XNL", + "Norfolk Island": "NFK", + "North Korea": "PRK", + "Northern Mariana Islands": "MNP", + "Norway": "NOR", + "Oman": "OMN", + "Pakistan": "PAK", + "Palau": "PLW", + "Panama": "PAN", + "Papua New Guinea": "PNG", + "Paraguay": "PRY", + "Peru": "PER", + "Philippines": "PHL", + "Pitcairn Islands": "PCN", + "Poland": "POL", + "Portugal": "PRT", + "Puerto Rico": "PRI", + "Qatar": "QAT", + "Reunion": "REU", + "Romania": "ROU", + "Russia": "RUS", + "Rwanda": "RWA", + "Saint Helena Ascension and Tristan da Cunha": "SHN", + "Saint Kitts and Nevis": "KNA", + "Saint Lucia": "LCA", + "Saint Pierre and Miquelon": "SPM", + "Sao Tome and Principe": "STP", + "Saudi Arabia": "SAU", + "Senegal": "SEN", + "Serbia": "SRB", + "Seychelles": "SYC", + "Sierra Leone": "SLE", + "Singapore": "SGP", + "Slovakia": "SVK", + "Slovenia": "SVN", + "Solomon Islands": "SLB", + "Somalia": "SOM", + "South Africa": "ZAF", + "South Georgia and the South Sandwich Islands": "SGS", + "South Korea": "KOR", + "South Sudan": "SSD", + "Spain": "ESP", + "Sri Lanka": "LKA", + "St. Vincent and the Grenadines": "VCT", + "Sudan": "SDN", + "Suriname": "SUR", + "Svalbard": "SJM", + "Swaziland": "SWZ", + "Sweden": "SWE", + "Switzerland": "CHE", + "Syria": "SYR", + "Taiwan": "TWN", + "Tajikistan": "TJK", + "Tanzania, United Republic of": "TZA", + "Thailand": "THA", + "Timor Leste": "TLS", + "Togo": "TGO", + "Tokelau": "TKL", + "Tonga": "TON", + "Trinidad and Tobago": "TTO", + "Tunisia": "TUN", + "Turkey": "TUR", + "Turkmenistan": "TKM", + "Turks and Caicos Islands": "TCA", + "Tuvalu": "TUV", + "Uganda": "UGA", + "Ukraine": "UKR", + "United Arab Emirates": "ARE", + "United Kingdom": "GBR", + "United States Minor Outlying Islands": "UMI", + "United States of America": "USA", + "Uruguay": "URY", + "Uzbekistan": "UZB", + "Vanuatu": "VUT", + "Venezuela": "VEN", + "Vietnam": "VNM", + "Virgin Islands": "VGB", + "Wallis and Futuna": "WLF", + "West Bank": "PSE", + "Western Sahara": "ESH", + "Western Samoa": "WSM", + "Yemen": "YEM", + "Zambia": "ZMB", + "Zimbabwe": "ZWE" } \ No newline at end of file diff --git a/pycoupler/coupler.py b/pycoupler/coupler.py index ae9a7e3..e3d2db8 100644 --- a/pycoupler/coupler.py +++ b/pycoupler/coupler.py @@ -519,7 +519,7 @@ def code_to_name(self, to_iso_alpha_3=False): if static_output == "country" and to_iso_alpha_3: country_dict = get_countries() name_dict = { - idx: country_dict[reg]["code"] for idx, reg in name_dict.items() + idx: country_dict[reg] for idx, reg in name_dict.items() } getattr(self, f"{static_output}").attrs[ "long_name" diff --git a/pycoupler/utils.py b/pycoupler/utils.py index a3d7051..34996c7 100644 --- a/pycoupler/utils.py +++ b/pycoupler/utils.py @@ -4,18 +4,29 @@ from fuzzywuzzy import fuzz, process -def get_countries(): +def get_countries(codes_as_keys=False): """Current workaround to get countries defined in LPJmL. + Parameters + ---------- + codes_as_keys : bool, default False + If True, reverse the keys and values and returns a dict with ISO3 country + codes as keys. + Returns ------- dict - Dictionary with countries and their codes. + Dictionary with country names as the keys and the codes as the values. """ with (importlib.resources.files(__package__) / "countries.json").open( "r" ) as countries: - return json.load(countries) + country_dict = json.load(countries) + if codes_as_keys: + country_dict = { + code: name for name, code in country_dict.items() + } + return country_dict def search_country(query): @@ -28,12 +39,14 @@ def search_country(query): Returns ------- - str - The matching country code. + str or None + The matching country code or None, if no match was found. """ countries = get_countries() - name, _ = process.extractOne(query, countries.keys(), scorer=fuzz.ratio) - return countries[name]["code"] + found_name, _ = process.extractOne(query, countries.keys(), scorer=fuzz.ratio) + if found_name: + return countries[found_name] + return None def read_json(file_name, object_hook=None):