DRAFT: Update tests

data_request_api/data_request_api/content/dreq_content.py

@@ -18,6 +18,7 @@
 from data_request_api.content.mapping_table import mapping_table
 from data_request_api.utilities.decorators import append_kwargs_from_config
 from data_request_api.utilities.logger import get_logger  # noqa
+from data_request_api.utilities.tools import read_json_file
 
 # Suppress pooch info output
 pooch.get_logger().setLevel("WARNING")
@@ -476,6 +477,7 @@ def retrieve(version="latest_stable", **kwargs):
 
             # Store the path to the dreq.json in the json_paths dictionary
             json_paths[version] = json_path
+            logger.debug(f"'{version}' stored under '{json_path}'")
 
     # Capture no correct export found for cached versions (offline mode)
     if not json_paths or json_paths == {}:
@@ -600,33 +602,32 @@ def load(version="latest_stable", **kwargs):
 
     _dreq_content_loaded["json_path"] = json_path
 
-    with open(json_path) as f:
-        consolidate_error = (
-            "Consolidation mapping is not supported for raw exports of versions < v1.2."
-            " Set 'export' to \"release\" (recommended), or set 'consolidate' to True"
-            " or set 'force_consolidate' to True to force consolidation regardless."
-        )
-        consolidate_warning = (
-            "Consolidation mapping is not supported for raw exports of versions < v1.2." " Forcing it regardless ..."
-        )
-        if "consolidate" in kwargs:
-            if kwargs["consolidate"]:
-                if "export" in kwargs and kwargs["export"] == "raw":
-                    if _parse_version(version) < _parse_version("v1.2") and version != "dev":
-                        if "force_consolidate" in kwargs and kwargs["force_consolidate"]:
-                            logger.warning(consolidate_warning)
-                        else:
-                            logger.error(consolidate_error)
-                            raise ValueError(consolidate_error)
-                return ce.map_data(json.load(f), mapping_table, next(iter(version_dict.keys())), **kwargs)
-            else:
-                return json.load(f)
-        else:
+    consolidate_error = (
+        "Consolidation mapping is not supported for raw exports of versions < v1.2."
+        " Set 'export' to \"release\" (recommended), or set 'consolidate' to True"
+        " or set 'force_consolidate' to True to force consolidation regardless."
+    )
+    consolidate_warning = (
+        "Consolidation mapping is not supported for raw exports of versions < v1.2." " Forcing it regardless ..."
+    )
+    if "consolidate" in kwargs:
+        if kwargs["consolidate"]:
             if "export" in kwargs and kwargs["export"] == "raw":
                 if _parse_version(version) < _parse_version("v1.2") and version != "dev":
                     if "force_consolidate" in kwargs and kwargs["force_consolidate"]:
                         logger.warning(consolidate_warning)
                     else:
                         logger.error(consolidate_error)
                         raise ValueError(consolidate_error)
-            return ce.map_data(json.load(f), mapping_table, next(iter(version_dict.keys())), **kwargs)
+            return ce.map_data(read_json_file(json_path), mapping_table, next(iter(version_dict.keys())), **kwargs)
+        else:
+            return read_json_file(json_path)
+    else:
+        if "export" in kwargs and kwargs["export"] == "raw":
+            if _parse_version(version) < _parse_version("v1.2") and version != "dev":
+                if "force_consolidate" in kwargs and kwargs["force_consolidate"]:
+                    logger.warning(consolidate_warning)
+                else:
+                    logger.error(consolidate_error)
+                    raise ValueError(consolidate_error)
+        return ce.map_data(read_json_file(json_path), mapping_table, next(iter(version_dict.keys())), **kwargs)

data_request_api/data_request_api/content/dump_transformation.py

@@ -16,7 +16,7 @@
 
 from data_request_api.utilities.decorators import append_kwargs_from_config
 from data_request_api.utilities.logger import get_logger, change_log_level, change_log_file
-from data_request_api.utilities.tools import read_json_input_file_content, write_json_output_file_content
+from data_request_api.utilities.tools import write_json_output_file_content, read_json_file
 from data_request_api.content import dreq_content as dc
 
 
@@ -85,7 +85,7 @@ def update_dict(elt_1, elt_2):
                 rep[elt] = value
         return rep
 
-    transform = read_json_input_file_content(os.sep.join([os.path.dirname(os.path.abspath(__file__)), "transform.json"]))
+    transform = read_json_file(os.sep.join([os.path.dirname(os.path.abspath(__file__)), "transform.json"]))
     common = transform.pop("common", dict())
     if version not in ["default", ]:
         common = update_dict(common["default"], common.get(version, dict()))
@@ -396,7 +396,8 @@ def split_content_one_base(content):
                           ("mips", list, list())],
         "variable_groups": [("variables", list, list()),
                             ("mips", list, list()),
-                            ("priority_level", (str, type(None)), None)],
+                            ("priority_level", (str, type(None)), None),
+                            ("region", (str, type(None)), None)],
         "experiment_groups": [("experiments", list, list()), ]
     }
     if isinstance(content, dict):
@@ -501,7 +502,7 @@ def get_transformed_content(version="latest_stable", export="release", consolida
                         help="Template to be used for output files")
     parser.add_argument("--version", default="unknown", help="Version of the data used")
     args = parser.parse_args()
-    content = read_json_input_file_content(args.input_file)
+    content = read_json_file(args.input_file)
     data_request, vocabulary_server = transform_content(content, args.version)
     write_json_output_file_content("_".join(["DR", args.output_files_template]), data_request)
     write_json_output_file_content("_".join(["VS", args.output_files_template]), vocabulary_server)

data_request_api/data_request_api/query/data_request.py

@@ -278,7 +278,7 @@ def filter_on_request(self, request_value, inner=True):
 
 class VariablesGroup(DRObjects):
     def __init__(self, id, dr, DR_type="variable_groups",
-                 structure=dict(variables=list(), mips=list(), priority_level="High"), **attributes):
+                 structure=dict(variables=list(), mips=list(), priority_level="High", region=None), **attributes):
         super().__init__(id=id, dr=dr, DR_type=DR_type, structure=structure, **attributes)
 
     def check(self):
@@ -288,9 +288,10 @@ def check(self):
             logger.critical(f"No variable defined for {self.DR_type} id {self.id}")
 
     @classmethod
-    def from_input(cls, dr, id, variables=list(), mips=list(), priority_level="High", **kwargs):
+    def from_input(cls, dr, id, variables=list(), mips=list(), priority_level="High", region=None, **kwargs):
         return super().from_input(DR_type="variable_groups", dr=dr, id=id, elements=kwargs,
-                                  structure=dict(variables=variables, mips=mips, priority_level=priority_level))
+                                  structure=dict(variables=variables, mips=mips, priority_level=priority_level,
+                                                 region=None))
 
     def count(self):
         """
@@ -299,6 +300,13 @@ def count(self):
         """
         return len(self.get_variables())
 
+    def get_region(self):
+        """
+        Return the region of the variable group.
+        :return: DRObject: the region associated with the variable group.
+        """
+        return self.structure["region"]
+
     def get_variables(self):
         """
         Return the list of Variables linked to the VariablesGroup.
@@ -338,6 +346,8 @@ def filter_on_request(self, request_value, inner=True):
                 found = request_value in self.get_variables()
             elif request_type in ["mips", ]:
                 found = request_value in self.get_mips()
+            elif request_type in ["regions", ]:
+                found = request_value == self.get_region()
             elif request_type in ["max_priority_levels", ]:
                 priority = self.dr.find_element("priority_level", self.get_priority_level().id)
                 req_priority = self.dr.find_element("priority_level", request_value.id)
@@ -463,7 +473,7 @@ def filter_on_request(self, request_value, inner=True):
                                                            list_to_check=self.get_variable_groups()))
             elif request_type in ["variables", "priority_levels", "cmip6_tables_identifiers", "temporal_shapes",
                                   "spatial_shapes", "structure_titles", "physical_parameters", "modelling_realms", "esm-bcvs",
-                                  "cf_standard_names", "cell_methods", "cell_measures", "max_priority_levels", "cmip7_frequencies"]:
+                                  "cf_standard_names", "cell_methods", "cell_measures", "max_priority_levels", "cmip7_frequencies", "regions"]:
                 found = self.filter_on_request_list(request_values=request_value,
                                                     list_to_check=self.get_variable_groups())
             elif request_type in ["experiments", ]:

data_request_api/data_request_api/query/filtering.json

@@ -19,6 +19,7 @@
     "experiment_groups": ["opportunities"],
     "variable_groups": ["opportunities"],
     "cmip7_frequencies": ["variables"],
-    "cmip6_frequencies": ["variables"]
+    "cmip6_frequencies": ["variables"],
+    "region": ["variable_groups"]
   }
 }