diff --git a/doc/sphinx/source/input.rst b/doc/sphinx/source/input.rst index b8bf116272..9af2b37d0f 100644 --- a/doc/sphinx/source/input.rst +++ b/doc/sphinx/source/input.rst @@ -235,6 +235,7 @@ A list of the datasets for which a CMORizers is available is provided in the fol .. tabularcolumns:: |p{3cm}|p{6cm}|p{3cm}|p{3cm}| + +----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | Dataset | Variables (MIP) | Tier | Script language | +========================================+======================================================================================================+======+=================+ @@ -330,6 +331,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol +----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-OZONE [#note5]_ | toz, o3 (AERmon) | 2 | Python | +----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ +| ESACCI-PERMAFROST | alt, gtd, pfr (Lyr, frequency: yr) | 2 | Python | ++----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-SEAICE | siconc (SIday, SImon) | 2 | Python | +----------------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+ | ESACCI-SEA-SURFACE-SALINITY | sos (Omon) | 2 | Python | diff --git a/esmvaltool/cmorizers/data/cmor_config/ESACCI-PERMAFROST.yml b/esmvaltool/cmorizers/data/cmor_config/ESACCI-PERMAFROST.yml new file mode 100644 index 0000000000..cc5163bbe9 --- /dev/null +++ b/esmvaltool/cmorizers/data/cmor_config/ESACCI-PERMAFROST.yml @@ -0,0 +1,29 @@ +--- +# Common global attributes for CMORizer output +attributes: + dataset_id: ESACCI-PERMAFROST + version: 'v5.0NH' + tier: 2 + type: sat + project_id: OBS6 + source: 'ftp://dap.ceda.ac.uk/neodc/esacci/permafrost' + reference: "esacci-permafrost" + comment: "" + +# Variables to CMORize +variables: + alt: + mip: Lyr + raw: ALT + file: 'ESACCI-PERMAFROST-L4-ALT-*-AREA4_PP-{year}-fv05.0.nc' + weights_dir: './esacci-permafrost-weights' + gtd: + mip: Lyr + raw: [GST, T1m, T2m, T5m, T10m] + file: 'ESACCI-PERMAFROST-L4-GTD-*-AREA4_PP-{year}-fv05.0.nc' + weights_dir: './esacci-permafrost-weights' + pfr: + mip: Lyr + raw: PFR + file: 'ESACCI-PERMAFROST-L4-PFR-*-AREA4_PP-{year}-fv05.0.nc' + weights_dir: './esacci-permafrost-weights' diff --git a/esmvaltool/cmorizers/data/datasets.yml b/esmvaltool/cmorizers/data/datasets.yml index d02711ea1b..033af014c1 100644 --- a/esmvaltool/cmorizers/data/datasets.yml +++ b/esmvaltool/cmorizers/data/datasets.yml @@ -626,6 +626,17 @@ datasets: No password (leave empty) Download each year into the same folder + ESACCI-PERMAFROST: + tier: 2 + source: ftp://dap.ceda.ac.uk/neodc/esacci/permafrost/data + last_access: 2026-02-18 + info: | + Download the data from: + active_layer_thickness/L4/area4/pp/v05.0/northern_hemisphere// + ground_temperature/L4/area4/pp/v05.0/northern_hemisphere// + permafrost_extent/L4/area4/pp/v05.0/northern_hemisphere// + Put all files in a single directory. + ESACCI-SEAICE: tier: 2 source: ftp://anon-ftp.ceda.ac.uk/neodc/esacci/sea_ice/data/ diff --git a/esmvaltool/cmorizers/data/downloaders/datasets/esacci_permafrost.py b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_permafrost.py new file mode 100644 index 0000000000..969bce0cc7 --- /dev/null +++ b/esmvaltool/cmorizers/data/downloaders/datasets/esacci_permafrost.py @@ -0,0 +1,73 @@ +"""Script to download ESACCI-PERMAFROST.""" + +import datetime +import logging + +from dateutil import relativedelta + +from esmvaltool.cmorizers.data.downloaders.wget import WGetDownloader + +logger = logging.getLogger(__name__) + + +def download_dataset( + original_data_dir, + dataset, + dataset_info, + start_date, + end_date, + overwrite, +) -> None: + """Download dataset. + + Parameters + ---------- + original_data_dir : Path + Directory where original data will be stored. + dataset : str + Name of the dataset + dataset_info : dict + Dataset information from the datasets.yml file + start_date : datetime + Start of the interval to download + end_date : datetime + End of the interval to download + overwrite : bool + Overwrite already downloaded files + """ + if start_date is None: + start_date = datetime.datetime(1997, 1, 1, tzinfo=datetime.UTC) + if end_date is None: + end_date = datetime.datetime(2023, 12, 31, tzinfo=datetime.UTC) + + downloader = WGetDownloader( + original_data_dir=original_data_dir, + dataset=dataset, + dataset_info=dataset_info, + overwrite=overwrite, + ) + + version = "v05.0" + + path = "https://dap.ceda.ac.uk/neodc/esacci/permafrost/data/" + + ccivars = [ + "active_layer_thickness", + "ground_temperature", + "permafrost_extent", + ] + + # download cci variables active layer thickness, ground temperature and + # permafrost extent + loop_date = start_date + while loop_date <= end_date: + for var in ccivars: + folder = ( + path + + f"{var}/L4/area4/pp/{version}/northern_hemisphere/{loop_date.year}/" + ) + downloader.download_folder( + folder, + wget_options=["-e robots=off", "--no-parent", "--accept=nc"], + ) + loop_date += relativedelta.relativedelta(years=1) diff --git a/esmvaltool/cmorizers/data/downloaders/ftp.py b/esmvaltool/cmorizers/data/downloaders/ftp.py index 51f4202fa4..80ee8440dc 100644 --- a/esmvaltool/cmorizers/data/downloaders/ftp.py +++ b/esmvaltool/cmorizers/data/downloaders/ftp.py @@ -2,6 +2,7 @@ from __future__ import annotations +import fnmatch import ftplib import logging import os @@ -260,6 +261,26 @@ def dataset_name(self): """ return self.dataset.lower().replace("-", "_") + def download_files(self, filename, path=None): + """Download file(s). + + Parameters + ---------- + filename : str + Name of file (w/o path) to download (wildcards are allowed) + path : str + Path of file(s) to download (optional) + """ + if path is not None: + self.set_cwd(path) + files = self._client.nlst() + matching_files = fnmatch.filter(files, filename) + if len(matching_files) != 0: + for file in matching_files: + super().download_file(file) + else: + logger.info("No files %s found.", filename) + def download_year(self, year): """Download a specific year. @@ -269,3 +290,21 @@ def download_year(self, year): Year to download """ self.download_folder(str(year)) + + def file_exists(self, filename, path=None): + """Check if a file exists. + + Parameters + ---------- + filename : str + Name of file (w/o path) to check (wildcards are allowed) + path : str + Path of file to check (optional) + """ + if path is not None: + self.set_cwd(path) + files = self._client.nlst() + matching_files = fnmatch.filter(files, filename) + if len(matching_files) != 0: + return True + return False diff --git a/esmvaltool/cmorizers/data/formatters/datasets/esacci_permafrost.py b/esmvaltool/cmorizers/data/formatters/datasets/esacci_permafrost.py new file mode 100644 index 0000000000..40067128e9 --- /dev/null +++ b/esmvaltool/cmorizers/data/formatters/datasets/esacci_permafrost.py @@ -0,0 +1,347 @@ +"""ESMValTool CMORizer for ESACCI-PERMAFROST data. + +Tier + Tier 2: other freely-available dataset. + +Source + ftp://dap.ceda.ac.uk/neodc/esacci/permafrost/data + +Last access + 20260218 + +Download and processing instructions + Use automatic downloader (recommended) + esmvaltool data download ESACCI-PERMAFROST + or download the data from: + active_layer_thickness/L4/area4/pp/v05.0/northern_hemisphere/ + ground_temperature/L4/area4/pp/v05.0/northern_hemisphere/ + permafrost_extent/L4/area4/pp/v05.0/northern_hemisphere/ + Put all files in a single directory. +""" + +import datetime +import logging +import os +import os.path +from copy import deepcopy +from pathlib import Path + +import iris +import numpy as np +from cdo import Cdo +from dateutil import relativedelta +from esmvalcore.cmor.table import CMOR_TABLES +from iris import NameConstraint +from netCDF4 import Dataset + +from esmvaltool.cmorizers.data.utilities import save_variable, set_global_atts + +logger = logging.getLogger(__name__) + + +def _fix_coordinates(cube: iris.cube.Cube, definition): + """Fix coordinates.""" + axis2def = {"T": "time", "X": "longitude", "Y": "latitude", "Z": "sdepth"} + axes = ["T", "X", "Y", "Z"] + + for axis in axes: + coord_def = definition.coordinates.get(axis2def[axis]) + if coord_def: + coord = cube.coord(axis=axis) + if axis == "T": + coord.convert_units("days since 1850-1-1 00:00:00.0") + time = coord.units.num2date(coord.points[0]) + # set time bounds to (year-01-01 00:00, year+1-01-01 00:00) + start_date = datetime.datetime( + time.year, + 1, + 1, + tzinfo=datetime.UTC, + ) + end_date = datetime.datetime( + time.year + 1, + 1, + 1, + tzinfo=datetime.UTC, + ) + if coord.bounds is not None: + coord.bounds = None + coord.bounds = np.array( + [ + coord.units.date2num(start_date), + coord.units.date2num(end_date), + ], + ) + coord.standard_name = coord_def.standard_name + coord.var_name = coord_def.out_name + coord.long_name = coord_def.long_name + coord.points = coord.core_points().astype("float64") + if len(coord.points) > 1: + if coord.bounds is not None: + coord.bounds = None + coord.guess_bounds() + + return cube + + +def _regrid_infile(infile, outfile, weightsfile): + """Regrid infile to 0.5 deg x 0.5 deg grid using cdo.""" + cdo = Cdo() + + xsize = 36000 + ysize = 6000 + + # define dimensions of target grid (regular lat-lon grid) + target_dimx = 720 # delta_lon = 0.5 deg + target_dimy = 360 # delta_lat = 0.5 deg + target_grid = f"r{target_dimx}x{target_dimy}" + + # check if suitable weights file already exists + # (e.g. from previous call to _regrid_file) + + weightsfile_ok = False + + if Path(weightsfile).exists(): + weights = Dataset(weightsfile, "r") + # make sure dimensions of source and target grids match + # expected values + src = weights.variables["src_grid_dims"] + dst = weights.variables["dst_grid_dims"] + if ( + xsize == src[0] + and ysize == src[1] + and target_dimx == dst[0] + and target_dimy == dst[1] + ): + logger.info( + "Using matching weights file %s for regridding.", + weightsfile, + ) + weightsfile_ok = True + weights.close() + + # if no suitable weights file, generate new weights for regridding + + if not weightsfile_ok: + logger.info( + "Generating regridding weights. This can take several minutes...", + ) + # check if path for weight files exists, if not create folder + path = os.path.split(weightsfile)[0] + Path(path).mkdir(parents=True, exist_ok=True) + # generate weights + cdo.genbil( + f"{target_grid}", + input=infile, + output=weightsfile, + options="-f nc", + ) + + # now regrid data to 0.5 deg x 0.5 deg + cdo.remap( + f"{target_grid},{weightsfile}", + input=infile, + output=outfile, + options="-f nc", + ) + + +def _extract_variable(in_file, var, cfg, out_dir, year): + """Extract, process and save variable.""" + logger.info( + "CMORizing variable '%s' from input file '%s'", + var["short_name"], + in_file, + ) + attributes = deepcopy(cfg["attributes"]) + attributes["mip"] = var["mip"] + attributes["raw"] = var["raw"] + cmor_table = CMOR_TABLES[attributes["project_id"]] + definition = cmor_table.get_variable(var["mip"], var["short_name"]) + + if "weights_dir" in var: + weights_dir = var["weights_dir"] + # create directory if needed + Path(weights_dir).mkdir(parents=True, exist_ok=True) + else: + weights_dir = "." + + # regrid input file using cdo + # (using the preprocessor (ESMF) is too slow) + + regridded_file = f"./{year}_{var['short_name']}.nc" + weights_file = f"{weights_dir}/{var['short_name']}_weights.nc" + _regrid_infile(in_file, regridded_file, weights_file) + + # load input file (make sure only the variables specified in the config file + # as "raw" are loaded) + if type(var["raw"]) is list: + constraints = [ + NameConstraint(var_name=rawname) for rawname in var["raw"] + ] + else: + constraints = NameConstraint(var_name=var["raw"]) + + cubes = iris.load(regridded_file, constraints) + + if len(cubes) > 1: + # variable gtd contains the vertical levels as separate variables + # (depth level can only be recognized by the variable names) + # --> combine all depth levels into 1 cube + for cube in cubes: + if cube.var_name == "GST": + sdepth = 0.0 + elif cube.var_name == "T1m": + sdepth = 1.0 + elif cube.var_name == "T2m": + sdepth = 2.0 + elif cube.var_name == "T5m": + sdepth = 5.0 + elif cube.var_name == "T10m": + sdepth = 10.0 + else: + sdepth = 999.0 + logger.info("Could not determin depth. Check results.") + cube.add_aux_coord( + iris.coords.AuxCoord( + sdepth, + standard_name="depth", + long_name="depth", + units="m", + ), + ) + cube.var_name = "gst" + cube.standard_name = "soil_temperature" # "valid" standard name + tmp_cube = cubes.merge_cube() + # setting the attribute 'positive' is needed for Iris to recognize + # this coordinate as 'Z' axis + tmp_cube.coord("depth").attributes["positive"] = "down" + # swap coordinates 'depth' and 'time': + # (depth, time, lat, lon) --> (time, depth, lat, lon) + flipped_data = np.swapaxes(tmp_cube.core_data(), 1, 0) + coord_spec = [ + (tmp_cube.coord("time"), 0), + (tmp_cube.coord("depth"), 1), + (tmp_cube.coord("latitude"), 2), + (tmp_cube.coord("longitude"), 3), + ] + cube = iris.cube.Cube(flipped_data, dim_coords_and_dims=coord_spec) + cube.metadata = tmp_cube.metadata + # change units string so unit conversion from deg C --> K will work + cube.units = "celsius" + # convert units from degC to K + cube.convert_units("K") + else: + cube = cubes[0] + + # --> drop attributes that differ among input files for different years + # global attributes to remove + drop_attrs = [ + "source", + "date_created", + "history", + "tracking_id", + "id", + "time_coverage_start", + "time_coverage_end", + "platform", + "sensor", + "keywords", + ] + # variable attributes to remove + drop_var_attrs = [ + "flag_meanings", + "flag_values", + "grid_mapping", + "actual_range", + "ancillary_variables", + ] + for attr in drop_attrs: + if attr in cube.attributes: + cube.attributes.pop(attr) + for attr in drop_var_attrs: + if attr in cube.attributes: + cube.attributes.pop(attr) + + set_global_atts(cube, attributes) + + cube.coord("time").points = ( + cube.coord("time").core_points().astype("float64") + ) + + # Set correct names + cube.var_name = definition.short_name + cube.standard_name = definition.standard_name + cube.long_name = definition.long_name + + # Fix units + # input variable for pfr reports 'percent' --> rename to '%' + # input variable for alt reports 'metres' --> rename to 'm' + # input variable for gtd has been converted to 'K' --> nothing to do + cube.units = definition.units + + # Fix data type + cube.data = cube.core_data().astype("float32") + + # Fix coordinates + cube = _fix_coordinates(cube, definition) + + # Save results + logger.debug("Saving cube\n%s", cube) + logger.debug("Setting time dimension to UNLIMITED while saving!") + save_variable( + cube, + cube.var_name, + out_dir, + attributes, + unlimited_dimensions=["time"], + ) + Path(regridded_file).unlink() # delete temporary file + logger.info("Finished CMORizing %s", in_file) + + +def cmorization(in_dir, out_dir, cfg, cfg_user, start_date, end_date): + """CMORize ESACCI-PERMAFROST dataset.""" + glob_attrs = cfg["attributes"] + + logger.info( + "Starting CMORization for tier%s OBS files: %s", + glob_attrs["tier"], + glob_attrs["dataset_id"], + ) + logger.info("Input data from: %s", in_dir) + logger.info("Output will be written to: %s", out_dir) + logger.info( + "CMORizing ESACCI-PERMAFROST version %s", + glob_attrs["version"], + ) + + if start_date is None: + start_date = datetime.datetime(1997, 1, 1, tzinfo=datetime.UTC) + if end_date is None: + end_date = datetime.datetime(2023, 12, 31, tzinfo=datetime.UTC) + + loop_date = start_date + while loop_date <= end_date: + for short_name, var in cfg["variables"].items(): + if "short_name" not in var: + var["short_name"] = short_name + in_file = list( + Path(in_dir).glob(var["file"].format(year=loop_date.year)), + ) + if not in_file: + logger.info( + "%d: no data not found for variable %s", + loop_date.year, + short_name, + ) + else: + _extract_variable( + str(in_file[0]), + var, + cfg, + out_dir, + loop_date.year, + ) + + loop_date += relativedelta.relativedelta(years=1) diff --git a/esmvaltool/recipes/examples/recipe_check_obs.yml b/esmvaltool/recipes/examples/recipe_check_obs.yml index f3d30a33c4..2d446ea17b 100644 --- a/esmvaltool/recipes/examples/recipe_check_obs.yml +++ b/esmvaltool/recipes/examples/recipe_check_obs.yml @@ -369,6 +369,16 @@ diagnostics: start_year: 2003, end_year: 2018} scripts: null + ESACCI-PERMAFROST: + description: ESACCI-PERMAFROST check + variables: + alt: + gtd: + pfr: + additional_datasets: + - {dataset: ESACCI-PERMAFROST, project: OBS6, mip: Lyr, tier: 2, type: sat, version: v5.0NH, + frequency: yr, start_year: 1997, end_year: 2023} + scripts: null ESACCI-OC: description: ESACCI-OC check @@ -379,7 +389,6 @@ diagnostics: type: sat, version: fv5.0, start_year: 1998, end_year: 2020} scripts: null - ESACCI-OZONE: description: ESACCI-OZONE check variables: diff --git a/esmvaltool/references/esacci-permafrost.bibtex b/esmvaltool/references/esacci-permafrost.bibtex new file mode 100644 index 0000000000..9d966f4a32 --- /dev/null +++ b/esmvaltool/references/esacci-permafrost.bibtex @@ -0,0 +1,29 @@ +@article{esacci-permafrost-alt, + doi = {10.5285/a6fbedd8ee5b472c8e84e55f746c1704}, + url = {https://dx.doi.org/10.5285/a6fbedd8ee5b472c8e84e55f746c1704}, + year = 2025, + month = {aug}, + publisher = {NERC EDS Centre for Environmental Data Analysis}, + author = {Westermann, S. and Barboux, C. and Bartsch, A. and Delaloye, R. and Grosse, G. and Heim, B. and Hugelius, G. and Irrgang, A. and Kääb, A.M. and Matthes, H. and Miesner, F. and Nitze, I. and Pellet, C. and Seifert, F.M. and Strozzi, T. and Wegmüller, U. and Wieczorek, M. and Wiesmann, A.} + title = {ESA Permafrost Climate Change Initiative (Permafrost\_cci): Permafrost active layer thickness for the Northern Hemisphere, v5.0}, +} + +@article{esacci-permafrost-gtd, + doi = {10.5285/5675b0be944f45a8af0e7ddbeb47a011}, + url = {https://dx.doi.org/10.5285/5675b0be944f45a8af0e7ddbeb47a011}, + year = 2025, + month = {aug}, + publisher = {NERC EDS Centre for Environmental Data Analysis}, + author = {Westermann, S. and Barboux, C. and Bartsch, A. and Delaloye, R. and Grosse, G. and Heim, B. and Hugelius, G. and Irrgang, A. and Kääb, A.M. and Matthes, H. and Miesner, F. and Nitze, I. and Pellet, C. and Seifert, F.M. and Strozzi, T. and Wegmüller, U. and Wieczorek, M. and Wiesmann, A.}, + title = {ESA Permafrost Climate Change Initiative (Permafrost\_cci): Permafrost Ground Temperature for the Northern Hemisphere, v5.0}, +} + +@article{esacci-permafrost-pfr, + doi = {10.5285/d235665772ec4b558e9a89ac85595e71}, + url = {https://dx.doi.org/10.5285/d235665772ec4b558e9a89ac85595e71}, + year = 2025, + month = {aug}, + publisher = {NERC EDS Centre for Environmental Data Analysis}, + author = {Westermann, S. and Barboux, C. and Bartsch, A. and Delaloye, R. and Grosse, G. and Heim, B. and Hugelius, G. and Irrgang, A. and Kääb, A.M. and Matthes, H. and Miesner, F. and Nitze, I. and Pellet, C. and Seifert, F.M. and Strozzi, T. and Wegmüller, U. and Wieczorek, M. and Wiesmann, A.}, + title = {ESA Permafrost Climate Change Initiative (Permafrost\_cci): Permafrost extent for the Northern Hemisphere, v5.0}, +}