Fix support for validating a tree with climatology files

changelog/119.fix.md

@@ -0,0 +1 @@
+Fixed the `validate-tree` CLI (and associated functionality) so that it handles climatology files correctly

src/input4mips_validation/cvs/drs.py

@@ -38,9 +38,9 @@
 from pathlib import Path
 
 import cftime
+import iris
 import numpy as np
 import pandas as pd
-import xarray as xr
 from attrs import frozen
 from typing_extensions import TypeAlias
 
@@ -51,6 +51,11 @@
     infer_time_start_time_end_for_filename,
 )
 from input4mips_validation.serialisation import converter_json
+from input4mips_validation.xarray_helpers.iris import ds_from_iris_cubes
+from input4mips_validation.xarray_helpers.variables import (
+    XRVariableHelper,
+    XRVariableProcessorLike,
+)
 
 DATA_REFERENCE_SYNTAX_FILENAME: str = "input4MIPs_DRS.json"
 """Default name of the file in which the data reference syntax is saved"""
@@ -539,6 +544,7 @@ def validate_file_written_according_to_drs(
         file: Path,
         frequency_metadata_keys: FrequencyMetadataKeys = FrequencyMetadataKeys(),
         time_dimension: str = "time",
+        xr_variable_processor: XRVariableProcessorLike = XRVariableHelper(),
     ) -> None:
         """
         Validate that a file is correctly written in the DRS
@@ -554,6 +560,9 @@ def validate_file_written_according_to_drs(
         time_dimension
             The time dimension of the data
 
+        xr_variable_processor
+            Helper to use for processing the variables in xarray objects.
+
         Raises
         ------
         ValueError
@@ -570,7 +579,12 @@ def validate_file_written_according_to_drs(
             file.name
         )
 
-        ds = xr.open_dataset(file, use_cftime=True)
+        ds = ds_from_iris_cubes(
+            iris.load(file),
+            xr_variable_processor=xr_variable_processor,
+            raw_file=file,
+            time_dimension=time_dimension,
+        )
         comparison_metadata = {
             k: apply_known_replacements(v)
             for k, v in ds.attrs.items()

src/input4mips_validation/validation/database.py

@@ -183,6 +183,7 @@ def get_validate_database_file_entry_result(  # noqa: PLR0913
         Path(entry.filepath),
         frequency_metadata_keys=frequency_metadata_keys,
         time_dimension=time_dimension,
+        xr_variable_processor=xr_variable_processor,
     )
 
     # TODO: all references to external variables (like cell areas) can be resolved

src/input4mips_validation/validation/tree.py

@@ -480,6 +480,7 @@ def get_validate_tree_result(  # noqa: PLR0913
                 file,
                 frequency_metadata_keys=frequency_metadata_keys,
                 time_dimension=time_dimension,
+                xr_variable_processor=xr_variable_processor,
             )
 
         # TODO: check cross references in files to external variables

tests/integration/cli/test_validate_tree.py

@@ -23,7 +23,10 @@
     prepare_ds_and_get_frequency,
 )
 from input4mips_validation.inference.from_data import BoundsInfo
-from input4mips_validation.testing import get_valid_ds_min_metadata_example
+from input4mips_validation.testing import (
+    get_valid_ds_min_metadata_example,
+    get_valid_ds_min_metadata_example_climatology,
+)
 from input4mips_validation.validation.tree import get_validate_tree_result
 
 UR = pint.get_application_registry()
@@ -96,3 +99,65 @@ def test_basic(tmp_path):
         result = runner.invoke(app, ["validate-tree", str(tmp_path)])
 
     assert result.exit_code == 0, result.exc_info
+
+
+def test_climatology(tmp_path):
+    """
+    Write a climatology file in a tree, then make sure we can validate the tree
+    """
+    cvs = load_cvs(cv_source=DEFAULT_TEST_INPUT4MIPS_CV_SOURCE)
+
+    # Create ourselves a tree
+    written_files = []
+    for variable_id, units in (
+        ("mole_fraction_of_carbon_dioxide_in_air", "ppm"),
+        # ("mole_fraction_of_methane_in_air", "ppb"),
+    ):
+        ds, metadata_minimum = get_valid_ds_min_metadata_example_climatology(
+            variable_id=variable_id, units=units
+        )
+
+        ds["time"].encoding = {
+            "calendar": "proleptic_gregorian",
+            "units": "days since 1850-01-01 00:00:00",
+            # Time has to be encoded as float
+            # to ensure that half-days etc. are handled.
+            "dtype": np.dtypes.Float32DType,
+        }
+
+        input4mips_ds = Input4MIPsDataset.from_data_producer_minimum_information(
+            data=ds,
+            metadata_minimum=metadata_minimum,
+            cvs=cvs,
+            dataset_category="GHGConcentrations",
+            realm="atmos",
+            prepare_func=partial(
+                prepare_ds_and_get_frequency,
+                standard_and_or_long_names={
+                    variable_id: {"standard_name": variable_id}
+                },
+            ),
+        )
+
+        written_file = input4mips_ds.write(root_data_dir=tmp_path)
+
+        written_files.append(written_file)
+
+    # Test the function directly first (helps with debugging)
+    get_validate_tree_result(
+        tmp_path,
+        cv_source=DEFAULT_TEST_INPUT4MIPS_CV_SOURCE,
+        bounds_info=BoundsInfo(
+            time_bounds="time_bnds",
+            bounds_dim="bnds",
+        ),
+    ).raise_if_errors()
+
+    # Then test the CLI
+    with patch.dict(
+        os.environ,
+        {"INPUT4MIPS_VALIDATION_CV_SOURCE": str(DEFAULT_TEST_INPUT4MIPS_CV_SOURCE)},
+    ):
+        result = runner.invoke(app, ["validate-tree", str(tmp_path)])
+
+    assert result.exit_code == 0, result.exc_info