Update tests

Author: f-PLT

tests/resources/test_minimal_dataset.yaml

@@ -1,4 +1,7 @@
-models: ["NorESM2-LM"]
-downloader_kwargs:
-  variables: ["tas", "CO2", "CH4"]
-  experiments: ["historical", "ssp126"]
+CMIP6:
+  model: "NorESM2-LM"
+  variables: ["tas"]
+  experiments: ["historical","ssp126"]
+input4MIPs:
+  variables: ["CO2", "CH4"]
+  experiments: ["historical","ssp126"]

tests/test_download/test_downloader.py

@@ -1,18 +1,28 @@
+import shutil
 from unittest.mock import call, patch
 
 import pytest
 
 from climateset import TEST_DIR
-from climateset.download.downloader import Downloader, download_from_config_file
-from climateset.utils import get_yaml_config
+from climateset.download.cmip6_downloader import CMIP6Downloader
+from climateset.download.constants.esgf import CMIP6, INPUT4MIPS
+from climateset.download.downloader import download_from_config_file
+from climateset.download.downloader_config import (
+    create_cmip6_downloader_config_from_file,
+    create_input4mips_downloader_config_from_file,
+)
+from climateset.download.input4mips_downloader import Input4MipsDownloader
 
 MINIMAL_DATASET_CONFIG_PATH = TEST_DIR / "resources/test_minimal_dataset.yaml"
+TEST_TMP_DIR = TEST_DIR / "resources/.tmp"
 
-DOWNLOAD_RAW_INPUT_SINGLE_VAR = "climateset.download.downloader.Downloader.download_raw_input_single_var"
+DOWNLOAD_RAW_INPUT_SINGLE_VAR = (
+    "climateset.download.input4mips_downloader.Input4MipsDownloader.download_raw_input_single_var"
+)
 DOWNLOAD_META_HISTORIC_SINGLE_VAR = (
-    "climateset.download.downloader.Downloader.download_meta_historic_biomassburning_single_var"
+    "climateset.download.input4mips_downloader.Input4MipsDownloader.download_meta_historic_biomassburning_single_var"
 )
-DOWNLOAD_MODEL_SINGLE_VAR = "climateset.download.downloader.Downloader.download_from_model_single_var"
+DOWNLOAD_MODEL_SINGLE_VAR = "climateset.download.cmip6_downloader.CMIP6Downloader.download_from_model_single_var"
 SUBPROCESS_RUN = "subprocess.run"
 
 EXPECTED_MINIMAL_RAW_INPUT_CALLS = [
@@ -36,11 +46,15 @@
 ]
 
 EXPECTED_MINIMAL_MODEL_CALLS = [
-    call(variable="tas", experiment="historical"),
-    call(variable="tas", experiment="ssp126"),
+    call(project="CMIP6", variable="tas", experiment="historical"),
+    call(project="CMIP6", variable="tas", experiment="ssp126"),
 ]
 
 
+def delete_tmp_dir():
+    shutil.rmtree(TEST_TMP_DIR, ignore_errors=True)
+
+
 @pytest.fixture()
 def mock_raw_input_single_var():
     with patch(DOWNLOAD_RAW_INPUT_SINGLE_VAR) as mock_function:
@@ -66,71 +80,83 @@ def mock_subprocess_run():
 
 
 @pytest.fixture
-def simple_downloader_object():
+def input4mips_downloader_object():
     config_source = MINIMAL_DATASET_CONFIG_PATH
-    config = get_yaml_config(config_source)
-    model = config["models"][0]
-    downloader_kwargs = config["downloader_kwargs"]
-    return Downloader(model=model, **downloader_kwargs)
+    config = create_input4mips_downloader_config_from_file(config_source)
+    config.data_dir = TEST_TMP_DIR
+    yield Input4MipsDownloader(config=config)
+    delete_tmp_dir()
 
 
-def test_downloader_init(simple_downloader_object):
-    assert isinstance(simple_downloader_object, Downloader)
+@pytest.fixture
+def cmip6_downloader_object():
+    config_source = MINIMAL_DATASET_CONFIG_PATH
+    config = create_cmip6_downloader_config_from_file(config_source)
+    config.data_dir = TEST_TMP_DIR
+    yield CMIP6Downloader(config=config)
+    delete_tmp_dir()
+
 
+def test_downloader_init(input4mips_downloader_object, cmip6_downloader_object):
+    assert isinstance(input4mips_downloader_object, Input4MipsDownloader)
+    assert isinstance(cmip6_downloader_object, CMIP6Downloader)
 
-def test_downloader_base_params(simple_downloader_object):
-    assert simple_downloader_object.model == "NorESM2-LM"
-    assert simple_downloader_object.experiments == ["historical", "ssp126"]
 
+def test_downloader_base_params(input4mips_downloader_object, cmip6_downloader_object):
+    assert input4mips_downloader_object.config.project == INPUT4MIPS
+    assert input4mips_downloader_object.config.experiments == ["historical", "ssp126"]
+    assert cmip6_downloader_object.config.project == CMIP6
+    assert cmip6_downloader_object.config.model == ["NorESM2-LM"]
+    assert cmip6_downloader_object.config.experiments == ["historical", "ssp126"]
 
-def test_downloader_max_possible_member_number(simple_downloader_object):
-    assert simple_downloader_object.max_ensemble_members == 1
 
+def test_downloader_max_possible_member_number(cmip6_downloader_object):
+    assert cmip6_downloader_object.config.max_ensemble_members == 10
 
-def test_downloader_variables(simple_downloader_object):
-    assert simple_downloader_object.raw_vars == [
+
+def test_downloader_variables(input4mips_downloader_object, cmip6_downloader_object):
+    assert cmip6_downloader_object.config.variables == ["tas"]
+    assert input4mips_downloader_object.config.variables == [
         "CO2_em_anthro",
         "CO2_em_AIR_anthro",
         "CH4_em_openburning",
         "CH4_em_anthro",
         "CH4_em_AIR_anthro",
     ]
-    assert simple_downloader_object.biomass_vars == ["CO2", "CH4"]
-    assert simple_downloader_object.model_vars == ["tas"]
-    assert simple_downloader_object.meta_vars_percentage == [
+    assert input4mips_downloader_object.config.biomass_vars == ["CO2", "CH4"]
+    assert input4mips_downloader_object.config.meta_vars_percentage == [
         "CH4_percentage_AGRI",
         "CH4_percentage_BORF",
         "CH4_percentage_DEFO",
         "CH4_percentage_PEAT",
         "CH4_percentage_SAVA",
         "CH4_percentage_TEMF",
     ]
-    assert simple_downloader_object.meta_vars_share == ["CH4_openburning_share"]
+    assert input4mips_downloader_object.config.meta_vars_share == ["CH4_openburning_share"]
 
 
-def test_downloader_model_params(simple_downloader_object):
-    assert simple_downloader_object.model_node_link == "https://esgf-data.dkrz.de/esg-search"
-    assert simple_downloader_object.model_source_center == "NCC"
+def test_downloader_model_params(cmip6_downloader_object):
+    assert cmip6_downloader_object.config.node_link == "http://esgf-node.llnl.gov/esg-search/"
 
 
-def test_download_raw_input(simple_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var):
-    simple_downloader_object.download_raw_input()
+def test_download_raw_input(input4mips_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var):
+    input4mips_downloader_object.download()
     assert mock_raw_input_single_var.call_args_list == EXPECTED_MINIMAL_RAW_INPUT_CALLS
     assert mock_raw_input_single_var.call_count == 8
     assert mock_meta_historic_single_var.call_args_list == EXPECTED_MINIMAL_META_HISTORIC_CALLS
     assert mock_meta_historic_single_var.call_count == 6
 
 
-def test_download_from_model(simple_downloader_object, mock_model_single_var):
-    simple_downloader_object.download_from_model()
+def test_download_from_model(cmip6_downloader_object, mock_model_single_var):
+    cmip6_downloader_object.download()
     assert mock_model_single_var.call_args_list == EXPECTED_MINIMAL_MODEL_CALLS
     assert mock_model_single_var.call_count == 2
 
 
 def test_download_from_config_file(
-    simple_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var, mock_model_single_var
+    input4mips_downloader_object, mock_raw_input_single_var, mock_meta_historic_single_var, mock_model_single_var
 ):
-    download_from_config_file(config=MINIMAL_DATASET_CONFIG_PATH)
+    download_from_config_file(config_file=MINIMAL_DATASET_CONFIG_PATH)
 
     assert mock_raw_input_single_var.call_args_list == EXPECTED_MINIMAL_RAW_INPUT_CALLS
     assert mock_raw_input_single_var.call_count == 8
@@ -153,26 +179,25 @@ def _assert_content_is_in_wget_script(mock_call, string_content):
     assert string_content in wget_script_content
 
 
-def test_download_raw_input_single_var(simple_downloader_object, mock_subprocess_run):
+def test_download_raw_input_single_var(input4mips_downloader_object, mock_subprocess_run):
     download_subprocess = mock_subprocess_run
-    simple_downloader_object.download_raw_input_single_var(variable="CO2_em_anthro", institution_id="PNNL-JGCRI")
+    input4mips_downloader_object.download_raw_input_single_var(variable="CO2_em_anthro", institution_id="PNNL-JGCRI")
 
     expected_files = [
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_175001-179912.nc",
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_180001-184912.nc",
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_185001-185012.nc",
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_185101-189912.nc",
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_190001-194912.nc",
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_195001-199912.nc",
-        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-2017-05-18_gn_200001-201412.nc",
+        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2024-11-25_gn_175001-179912.nc",
+        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2024-11-25_gn_180001-184912.nc",
+        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2024-11-25_gn_185001-189912.nc",
+        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2024-11-25_gn_190001-194912.nc",
+        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2024-11-25_gn_195001-199912.nc",
+        "CO2-em-anthro_input4MIPs_emissions_CMIP_CEDS-CMIP-2024-11-25_gn_200001-202212.nc",
     ]
     download_subprocess.assert_called_once()
     for f in expected_files:
         _assert_content_is_in_wget_script(download_subprocess, f)
 
 
-def test_download_meta_historic_biomassburning_single_var(simple_downloader_object, mock_subprocess_run):
-    simple_downloader_object.download_meta_historic_biomassburning_single_var(
+def test_download_meta_historic_biomassburning_single_var(input4mips_downloader_object, mock_subprocess_run):
+    input4mips_downloader_object.download_meta_historic_biomassburning_single_var(
         variable="CH4_percentage_AGRI", institution_id="VUA"
     )
 
@@ -184,8 +209,8 @@ def test_download_meta_historic_biomassburning_single_var(simple_downloader_obje
         _assert_content_is_in_wget_script(mock_call=mock_subprocess_run, string_content=f)
 
 
-def test_download_from_model_single_var(simple_downloader_object, mock_subprocess_run):
-    simple_downloader_object.download_from_model_single_var(variable="tas", experiment="ssp126")
+def test_download_from_model_single_var(cmip6_downloader_object, mock_subprocess_run):
+    cmip6_downloader_object.download_from_model_single_var(variable="tas", experiment="ssp126")
 
     expected_files = [
         "tas_Amon_NorESM2-LM_ssp126_r1i1p1f1_gn_201501-202012.nc",
@@ -198,6 +223,6 @@ def test_download_from_model_single_var(simple_downloader_object, mock_subproces
         "tas_Amon_NorESM2-LM_ssp126_r1i1p1f1_gn_208101-209012.nc",
         "tas_Amon_NorESM2-LM_ssp126_r1i1p1f1_gn_209101-210012.nc",
     ]
-    mock_subprocess_run.assert_called_once()
+    mock_subprocess_run.assert_called()
     for f in expected_files:
         _assert_content_is_in_wget_script(mock_call=mock_subprocess_run, string_content=f)