CMORizer ESACCI-PERMAFROST

doc/sphinx/source/input.rst

@@ -336,6 +336,8 @@ A list of the datasets for which a CMORizers is available is provided in the fol
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | ESACCI-SEAICE                | siconc (SIday, SImon)                                                                                |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
+| ESACCI-PERMAFROST            | alt, gtd, pfr (Lyr, frequency: yr)                                                                   |   2  | Python          |
++------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | ESACCI-SEA-SURFACE-SALINITY  | sos (Omon)                                                                                           |   2  | Python          |
 +------------------------------+------------------------------------------------------------------------------------------------------+------+-----------------+
 | ESACCI-SOILMOISTURE          | sm (Eday, Lmon), smStderr (Eday)                                                                     |   2  | Python          |

esmvaltool/cmorizers/data/cmor_config/ESACCI-PERMAFROST.yml

@@ -0,0 +1,29 @@
+---
+# Common global attributes for CMORizer output
+attributes:
+  dataset_id: ESACCI-PERMAFROST
+  version: 'v3.0'
+  tier: 2
+  modeling_realm: sat
+  project_id: OBS6
+  source: 'ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost'
+  reference: "esacci-permafrost"
+  comment: ""
+
+# Variables to CMORize
+variables:
+  alt:
+    mip: Lyr
+    raw: ALT
+    file: 'ESACCI-PERMAFROST-L4-ALT-*-AREA4_PP-{year}-fv03.0.nc'
+    weights_dir: '/work/bd0854/b380103/esacci-permafrost-weights'
+  gtd:
+    mip: Lyr
+    raw: [GST, T1m, T2m, T5m, T10m]
+    file: 'ESACCI-PERMAFROST-L4-GTD-*-AREA4_PP-{year}-fv03.0.nc'
+    weights_dir: '/work/bd0854/b380103/esacci-permafrost-weights'
+  pfr:
+    mip: Lyr
+    raw: PFR
+    file: 'ESACCI-PERMAFROST-L4-PFR-*-AREA4_PP-{year}-fv03.0.nc'
+    weights_dir: '/work/bd0854/b380103/esacci-permafrost-weights'

esmvaltool/cmorizers/data/datasets.yml

@@ -551,6 +551,17 @@ datasets:
             Version = "v0008"
       Put all files under a single directory (no subdirectories with years).
 
+  ESACCI-PERMAFROST:
+    tier: 2
+    source: ftp://anon-ftp.ceda.ac.uk/neodc/esacci/permafrost/data
+    last_access: 2024-02-27
+    info: |
+      Download the data from:
+        active_layer_thickness/L4/area4/pp/v03.0
+        ground_temperature/L4/area4/pp/v03.0
+        permafrost_extent/L4/area4/pp/v03.0
+      Put all files in a single directory.
+
   ESACCI-SEAICE:
     tier: 2
     source: ftp://anon-ftp.ceda.ac.uk/neodc/esacci/sea_ice/data/

esmvaltool/cmorizers/data/downloaders/datasets/esacci_permafrost.py

@@ -0,0 +1,75 @@
+"""Script to download ESACCI-PERMAFROST."""
+
+import logging
+from datetime import datetime
+
+from dateutil import relativedelta
+
+from esmvaltool.cmorizers.data.downloaders.ftp import CCIDownloader
+
+logger = logging.getLogger(__name__)
+
+
+def download_dataset(
+    config: dict,
+    dataset: str,
+    dataset_info: dict,
+    start_date: datetime,
+    end_date: datetime,
+    overwrite: bool,
+) -> None:
+    """Download dataset.
+
+    Parameters
+    ----------
+    config : dict
+        ESMValTool's user configuration
+    dataset : str
+        Name of the dataset
+    dataset_info : dict
+         Dataset information from the datasets.yml file
+    start_date : datetime
+        Start of the interval to download
+    end_date : datetime
+        End of the interval to download
+    overwrite : bool
+        Overwrite already downloaded files
+    """
+    if start_date is None:
+        start_date = datetime(1997, 1, 1, tzinfo=datetime.timezone.utc)
+    if end_date is None:
+        end_date = datetime(2019, 12, 31, tzinfo=datetime.timezone.utc)
+
+    downloader = CCIDownloader(
+        config=config,
+        dataset=dataset,
+        dataset_info=dataset_info,
+        overwrite=overwrite,
+    )
+
+    downloader.connect()
+
+    version = "v03.0"
+
+    ccivars = [
+        "active_layer_thickness",
+        "ground_temperature",
+        "permafrost_extent",
+    ]
+
+    # download active layer thickness
+    loop_date = start_date
+    while loop_date <= end_date:
+        for var in ccivars:
+            pathname = f"{var}/L4/area4/pp/{version}/"
+            fname = f"ESACCI-PERMAFROST-L4-*-{loop_date.year}-f{version}.nc"
+            if downloader.file_exists(fname, pathname):
+                downloader.download_files(fname, pathname)
+            else:
+                logger.info(
+                    "%d: no data for %s %s",
+                    loop_date.year,
+                    var,
+                    version,
+                )
+        loop_date += relativedelta.relativedelta(years=1)

esmvaltool/cmorizers/data/downloaders/ftp.py

@@ -1,5 +1,6 @@
 """Downloader for FTP repositories."""
 
+import fnmatch
 import ftplib
 import logging
 import os
@@ -228,6 +229,26 @@ def dataset_name(self):
         """
         return self.dataset.lower().replace("-", "_")
 
+    def download_files(self, filename, path=None):
+        """Download file(s).
+
+        Parameters:
+        -----------
+        filename : str
+            Name of file (w/o path) to download (wildcards are allowed)
+        path : str
+            Path of file(s) to download (optional)
+        """
+        if path is not None:
+            self.set_cwd(path)
+        files = self._client.nlst()
+        matching_files = fnmatch.filter(files, filename)
+        if len(matching_files) != 0:
+            for file in matching_files:
+                super().download_file(file)
+        else:
+            logger.info('No files %s found.', filename)
+
     def download_year(self, year):
         """Download a specific year.
 
@@ -237,3 +258,21 @@ def download_year(self, year):
             Year to download
         """
         self.download_folder(str(year))
+
+    def file_exists(self, filename, path=None):
+        """Check if a file exists.
+
+        Parameters:
+        -----------
+        filename : str
+            Name of file (w/o path) to check (wildcards are allowed)
+        path : str
+            Path of file to check (optional)
+        """
+        if path is not None:
+            self.set_cwd(path)
+        files = self._client.nlst()
+        matching_files = fnmatch.filter(files, filename)
+        if len(matching_files) != 0:
+            return True
+        return False