Fixed by now rejecting tiff with wrong format. this refs #348

Author: JeanBilheux

notebooks/__code/file_handler.py

@@ -6,6 +6,7 @@
 import shutil
 from collections import Counter, OrderedDict, namedtuple
 from pathlib import Path
+import logging
 
 import numpy as np
 import pandas as pd
@@ -342,6 +343,9 @@ def _convert_epics_timestamp_to_rfc3339_timestamp(epics_timestamp):
 
 
 def retrieve_time_stamp(list_images, label=""):
+    
+    logging.info(f"\t -> retrieve_time_stamp for {label} with {len(list_images)} images")
+    
     if not list_images:
         return {"list_images": None, "list_time_stamp": None, "list_time_stamp_user_format": None}
 
@@ -376,6 +380,11 @@ def retrieve_time_stamp(list_images, label=""):
 
     box.close()
 
+    logging.info(f"\t -> time stamp retrieved for {label}")
+    logging.info(f"\t -> {list_images =}")
+    logging.info(f"\t -> {list_time_stamp =}")
+    logging.info(f"\t -> {list_time_stamp_user_format =}")
+
     return {
         "list_images": list_images,
         "list_time_stamp": list_time_stamp,

notebooks/__code/normalization/metadata_handler.py

@@ -1,5 +1,6 @@
 import collections
 from enum import Enum
+import logging
 
 from IPython.display import HTML, display
 
@@ -55,6 +56,8 @@ def retrieve_metadata(list_of_files=None, display_infos=False, label=""):
         if not list_of_files:
             return {}
 
+        logging.info(f"\t -> retrieve_metadata for {label} with {len(list_of_files)} files")
+
         _dict = file_handler.retrieve_time_stamp(list_of_files, label=label)
         _time_metadata_dict = MetadataHandler._reformat_dict(dictionary=_dict)
 
@@ -63,6 +66,8 @@ def retrieve_metadata(list_of_files=None, display_infos=False, label=""):
             master_dictionary=_time_metadata_dict, servant_dictionary=_beamline_metadata_dict
         )
 
+        logging.info(f"\t after combining the time metadata and the beamline metadata, the dict is: {_metadata_dict =}")
+
         if display_infos:
             display(
                 HTML('<span style="font-size: 20px; color:blue">Nbr of images: ' + str(len(_metadata_dict)) + "</span")
@@ -93,11 +98,16 @@ def retrieve_beamline_metadata(list_files):
         - slits positions ->
         - aperture value
         """
+        
+        logging.info(f"\t -> retrieve_beamline_metadata for {len(list_files)} files")
+        
         list_metadata = METADATA_KEYS["all"]
         _dict = metadata_handler.MetadataHandler.retrieve_metadata(
             list_files=list_files, list_metadata=list_metadata, using_enum_object=True
         )
 
+        logging.info(f"\t -> beamline metadata retrieved: {_dict =}")
+
         for _file_key in _dict.keys():
             _file_dict = {}
             for _pv in list_metadata:
@@ -113,6 +123,9 @@ def retrieve_beamline_metadata(list_files):
                 else:
                     _file_dict[_pv.value] = {}
             _dict[_file_key] = _file_dict
+        
+        logging.info(f"\t returning the divt {_dict =}")
+        
         return _dict
 
     @staticmethod

notebooks/__code/normalization/normalization_with_simplify_selection.py

@@ -147,6 +147,7 @@ def retrieve_sample_metadata(self, list_of_images):
         self.match_files()
         self.calculate_first_and_last_ob()
         self.calculate_time_range()
+        
         # self.display_time_range_selection_widgets()
 
     def select_ob_folder(self):
@@ -157,11 +158,11 @@ def retrieve_ob_metadata(self, selected_folder):
         self.ob_metadata_dict = MetadataHandler.retrieve_metadata(list_of_files=list_of_ob_files)
 
     def auto_retrieve_ob_metadata(self):
-        logging.info("> auto_retrieve_ob_metadata")
+        logging.info("> auto_retrieve_ob_metadata - WORKING WITH OB")
         folder = os.path.join(self.working_dir, "raw", "ob")
-        logging.info(f"-> folder: {folder}")
+        logging.info(f"\t-> folder: {folder}")
         list_of_ob_files = file_handler.get_list_of_all_files_in_subfolders(folder=folder, extensions=["tiff", "tif"])
-        logging.info(f"-> nbr of ob files found: {len(list_of_ob_files)}")
+        logging.info(f"\t-> nbr of ob files found: {len(list_of_ob_files)}")
         self.ob_metadata_dict = MetadataHandler.retrieve_metadata(list_of_files=list_of_ob_files, label="ob")
 
     # logging.info(f"ob metadata dict")
@@ -185,6 +186,7 @@ def retrieve_df_metadata(self, selected_folder):
         self.df_metadata_dict = MetadataHandler.retrieve_metadata(list_of_files=list_of_df_files)
 
     def auto_retrieve_df_metadata(self):
+        logging.info("> auto_retrieve_df_metadata - WORKING WITH DF")
         folder_df = os.path.join(self.working_dir, "raw", "df")
         folder_dc = os.path.join(self.working_dir, "raw", "dc")
         list_of_df_files = file_handler.get_list_of_all_files_in_subfolders(
@@ -196,12 +198,14 @@ def auto_retrieve_df_metadata(self):
         list_of_files = list_of_df_files + list_of_dc_files
         logging.info(f"-> nbr of dc files found: {len(list_of_files)}")
         self.df_metadata_dict = MetadataHandler.retrieve_metadata(list_of_files=list_of_files, label="df")
+        logging.info(f"\t -> df metadata dict: {self.df_metadata_dict}")
 
     def match_files(self):
         """This is where the files will be associated with their respective OB, DC by using the metadata"""
 
         if not JSON_DEBUGGING:
             self.create_master_sample_dict()
+            logging.info(f"\t -> after creating master sample dict, final_full_master_dict: {self.final_full_master_dict}")
 
         self.match_ob()
         self.match_df()
@@ -210,7 +214,7 @@ def match_files(self):
             # for debugging only, exporting the json
             import json
 
-            with open("/Users/j35/Desktop/which_ob_and_df_to_use.json", "w") as outfile:
+            with open("/SNS/users/j35/Desktop/which_ob_and_df_to_use.json", "w") as outfile:
                 json.dump(self.final_full_master_dict, outfile)
 
     def match_ob(self):
@@ -219,21 +223,35 @@ def match_ob(self):
         - detector type
         - aperture
         """
+        logging.info("> entering match_ob")
         list_ob_dict = self.ob_metadata_dict
         final_full_master_dict = self.final_full_master_dict
         list_of_sample_acquisition = final_full_master_dict.keys()
 
+        if JSON_DEBUGGING:
+            import json
+            with open("/SNS/users/j35/Desktop/list_ob_dict.json", "w") as outfile:
+                json.dump(list_ob_dict, outfile)
+                          
         for _index_ob in list_ob_dict.keys():
             _all_ob_instrument_metadata = Get.get_instrument_metadata_only(list_ob_dict[_index_ob])
             _ob_instrument_metadata = utilities.isolate_instrument_metadata(_all_ob_instrument_metadata)
-            _acquisition_time = _all_ob_instrument_metadata[MetadataName.EXPOSURE_TIME.value]["value"]
+            
+            _acquisition_time = _all_ob_instrument_metadata[MetadataName.EXPOSURE_TIME.value].get("value", None)
+            if _acquisition_time is None:
+                logging.warning(f"\t -> OB file {list_ob_dict[_index_ob]['filename']} does not have acquisition time metadata, skipping it for the matching with the sample")
+                continue
+            else:
+                logging.info(f"\t -> OB file {list_ob_dict[_index_ob]['filename']} has acquisition time {_acquisition_time}, trying to match with sample metadata")
+            
             if _acquisition_time in list_of_sample_acquisition:
                 for _config_id in final_full_master_dict[_acquisition_time].keys():
                     _sample_metadata_infos = final_full_master_dict[_acquisition_time][_config_id]["metadata_infos"]
                     if utilities.all_metadata_match(_sample_metadata_infos, _ob_instrument_metadata):
                         final_full_master_dict[_acquisition_time][_config_id]["list_ob"].append(list_ob_dict[_index_ob])
 
         self.final_full_master_dict = final_full_master_dict
+        logging.info(f"\t -> after matching ob, final_full_master_dict: {self.final_full_master_dict}")
 
     def match_df(self):
         """
@@ -242,14 +260,19 @@ def match_df(self):
         - detector type used
         - acquisition time
         """
+        logging.info("> entering match_df")
         list_df_dict = self.df_metadata_dict
         final_full_master_dict = self.final_full_master_dict
         list_of_sample_acquisition = final_full_master_dict.keys()
 
         for _index_df in list_df_dict.keys():
             _all_df_instrument_metadata = Get.get_instrument_metadata_only(list_df_dict[_index_df])
             _df_instrument_metadata = utilities.isolate_instrument_metadata(_all_df_instrument_metadata)
-            _acquisition_time = _all_df_instrument_metadata[MetadataName.EXPOSURE_TIME.value]["value"]
+            _acquisition_time = _all_df_instrument_metadata[MetadataName.EXPOSURE_TIME.value].get("value", None)
+
+            if _acquisition_time is None:
+                logging.warning(f"\t -> DF file {list_df_dict[_index_df]['filename']} does not have acquisition time metadata, skipping it for the matching with the sample")
+                continue
 
             if _acquisition_time in list_of_sample_acquisition:
                 for _config_id in final_full_master_dict[_acquisition_time].keys():
@@ -263,6 +286,7 @@ def match_df(self):
                         final_full_master_dict[_acquisition_time][_config_id]["list_df"].append(list_df_dict[_index_df])
 
         self.final_full_master_dict = final_full_master_dict
+        logging.info(f"\t -> after matching df, final_full_master_dict: {self.final_full_master_dict}")
 
     def create_master_sample_dict(self):
         final_full_master_dict = collections.OrderedDict()

notebooks/normalization_with_simplify_selection.ipynb

@@ -18,7 +18,7 @@
     }
    },
    "source": [
-    "# Select Your IPTS "
+    "# <font color='red'>Select Your IPTS</font>"
    ]
   },
   {
@@ -41,7 +41,7 @@
    "source": [
     "from __code.normalization.normalization_with_simplify_selection import NormalizationWithSimplifySelection\n",
     "from __code import system\n",
-    "system.System.select_working_dir(notebook='normalization_with_simplify_selection', ipts=\"IPTS-35409\",instrument_to_exclude=[\"VENUS\", \"SNAP\"])\n",
+    "system.System.select_working_dir(notebook='normalization_with_simplify_selection', ipts=\"IPTS-35248\",instrument_to_exclude=[\"VENUS\", \"SNAP\"])\n",
     "from __code.__all import custom_style\n",
     "custom_style.style()"
    ]
@@ -60,7 +60,7 @@
    },
    "outputs": [],
    "source": [
-    "# %gui qt"
+    "%gui qt"
    ]
   },
   {
@@ -72,7 +72,7 @@
     }
    },
    "source": [
-    "# Select List of Images to Normalize "
+    "# <font color='red'>Select List of Images to Normalize "
    ]
   },
   {
@@ -84,6 +84,16 @@
     "Change the time range if needed!"
    ]
   },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "metadata": {},
+   "outputs": [],
+   "source": [
+    "#DEBUGGING\n",
+    "# use 2026_04_05_dryCal_AEM"
+   ]
+  },
   {
    "cell_type": "code",
    "execution_count": null,
@@ -114,7 +124,7 @@
     }
    },
    "source": [
-    "# Define normalization parameters"
+    "# <font color='red'>Define normalization parameters"
    ]
   },
   {
@@ -138,7 +148,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Normalization workflow summary "
+    "# <font color='red'>Normalization workflow summary "
    ]
   },
   {
@@ -162,7 +172,7 @@
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "# Select Output Folder "
+    "# <font color='red'>Select Output Folder "
    ]
   },
   {