Add logging in parts of API that don't have it yet

data_request_api/data_request_api/command_line/export_dreq_lists_json.py

@@ -14,6 +14,10 @@
 import data_request_api.query.dreq_query as dq
 
 
+from data_request_api.utilities import config as dreqcfg
+from data_request_api.utilities.logger import change_log_file, change_log_level, get_logger
+
+
 def parse_args():
     """
     Parse command line arguments
@@ -45,6 +49,11 @@ def main():
     """
     args = parse_args()
 
+    config = dreqcfg.load_config()
+    change_log_file(logfile=config['log_file'])
+    change_log_level(level=config['log_level'])
+    logger = get_logger()
+
     if args.version:
         print("CMIP7 data request api version {}".format(data_request_api.version))
         sys.exit(0)
@@ -105,7 +114,11 @@ def main():
         dreq_opps = base['Opportunity']
         all_opp_ids = [opp.opportunity_id for opp in dreq_opps.records.values()]
         if len(all_opp_ids) != len(set(all_opp_ids)):
-            raise ValueError(f'Opportunity IDs (integers) in data request {use_dreq_version} are not unique!')
+            # Check that opportunity IDs defined in the data request are unique
+            # (if they are not, this is an error in the data request content)
+            msg = f'Opportunity IDs (integers) in data request {use_dreq_version} are not unique!'
+            logger.error(msg)
+            raise ValueError(msg)
         oppid2title = {int(opp.opportunity_id): opp.title for opp in dreq_opps.records.values()}
         use_opps = []
         invalid_opp_ids = set()
@@ -115,8 +128,10 @@ def main():
             else:
                 invalid_opp_ids.add(opp_id)
         if len(invalid_opp_ids) > 0:
-            raise ValueError(f'The following Opportunity IDs were not found in data request {use_dreq_version}: '
-                             + ', '.join([str(opp_id) for opp_id in sorted(invalid_opp_ids)]))
+            msg = f'The following Opportunity IDs were not found in data request {use_dreq_version}: ' \
+                    + ', '.join([str(opp_id) for opp_id in sorted(invalid_opp_ids)])
+            logger.error(msg)
+            raise ValueError(msg)
 
     elif args.all_opportunities:
         # Use all available opportunities in the data request
@@ -189,6 +204,5 @@ def main():
                 content_path=dc._dreq_content_loaded['json_path']
             )
 
-
 if __name__ == '__main__':
     main()

data_request_api/data_request_api/content/dreq_content.py

@@ -596,7 +596,7 @@ def load(version="latest_stable", **kwargs):
         return {}
     else:
         json_path = next(iter(version_dict.values()))
-        logger.info(f"Loading version {next(iter(version_dict.keys()))}'.")
+        logger.info(f"Loading version {next(iter(version_dict.keys()))}")
 
     _dreq_content_loaded["json_path"] = json_path
 

data_request_api/data_request_api/query/dreq_query.py

@@ -21,6 +21,8 @@
 from data_request_api.utilities.decorators import append_kwargs_from_config
 from data_request_api.utilities.tools import write_csv_output_file_content
 
+from data_request_api.utilities.logger import get_logger
+
 # Version of software (python API):
 from data_request_api import version as api_version
 
@@ -554,6 +556,9 @@ def get_requested_variables(content, dreq_version,
         }
     }
     '''
+    logger = get_logger()
+    logger.info(f'Getting requested variables using data request {dreq_version}')
+
     base = _get_base_dreq_tables(content, dreq_version, purpose='request')
 
     dreq_tables = {
@@ -1078,22 +1083,28 @@ def show_requested_vars_summary(expt_vars, dreq_version):
     Display quick summary to stdout of variables requested.
     expt_vars is the output dict from dq.get_requested_variables().
     '''
-    print(f'\nFor data request version {dreq_version}, number of requested variables found by experiment:')
+    logger = get_logger()
+    msg = f'For data request version {dreq_version}, number of requested variables found by experiment:'
+    logger.info(msg)
+    print('\n' + msg)
     priority_levels = get_priority_levels()
     for expt, req in sorted(expt_vars['experiment'].items()):
         d = {p: 0 for p in priority_levels}
         for p in priority_levels:
             if p in req:
                 d[p] = len(req[p])
         n_total = sum(d.values())
-        print(f'  {expt} : ' + ' ,'.join(['{p}={n}'.format(p=p, n=d[p]) for p in priority_levels]) + f', TOTAL={n_total}')
+        msg = f'  {expt} : ' + ' ,'.join(['{p}={n}'.format(p=p, n=d[p]) for p in priority_levels]) + f', TOTAL={n_total}'
+        logger.info(msg)
+        print(msg)
 
 
 def write_requested_vars_json(outfile, expt_vars, dreq_version, priority_cutoff, content_path):
     '''
     Write a nicely formatted json file with lists of requested variables by experiment.
     expt_vars is the output dict from dq.get_requested_variables().
     '''
+    logger = get_logger()
 
     header = OrderedDict({
         'Description': 'This file gives the names of output variables that are requested from CMIP experiments by the supported Opportunities. The variables requested from each experiment are listed under each experiment name, grouped according to the priority level at which they are requested. For each experiment, the prioritized list of variables was determined by compiling together all requests made by the supported Opportunities for output from that experiment.',
@@ -1145,7 +1156,9 @@ def write_requested_vars_json(outfile, expt_vars, dreq_version, priority_cutoff,
     with open(outfile, 'w') as f:
         # json.dump(expt_vars, f, indent=4, sort_keys=True)
         json.dump(out, f, indent=4)
-        print('\nWrote requested variables to ' + outfile)
+        msg = 'Wrote requested variables to ' + outfile
+        logger.info(msg)
+        print('\n' + msg)
 
 
 def write_variables_metadata(all_var_info, dreq_version, filepath,