PRELIMINARY_MAG flag for SWE, HIT/CoDICE pitch angle products

imap_l3_processing/cdf/config/imap_codice_l3b_hi-pitch-angle_variable_attrs.yaml

@@ -402,6 +402,21 @@ gyrophase_label:
   VAR_TYPE: metadata
   FILLVAL: ' '
 
+codice_flags:
+  NAME: codice_flags
+  DATA_TYPE: CDF_UINT2
+  CATDESC: Quality flags for CoDICE Hi L3b
+  DEPEND_0: epoch
+  VAR_TYPE: support_data
+  RECORD_VARYING: RV
+  FIELDNAM: Quality flags for CoDICE Hi L3b
+  FORMAT: I5
+  UNITS: ' '
+  VALIDMIN: 0
+  VALIDMAX: 65534
+  FILLVAL: 65535
+  LABLAXIS: Quality flags for CoDICE Hi L3b
+
 
 
 

imap_l3_processing/cdf/config/imap_hit_l3_macropixel_variable_attrs.yaml

@@ -1172,4 +1172,18 @@ zenith_label:
   VAR_TYPE: metadata
   FIELDNAM: Zenith label
   FORMAT: a20
-  FILLVAL: ' '
+  FILLVAL: ' '
+hit_flags:
+  NAME: hit_flags
+  DATA_TYPE: CDF_UINT2
+  CATDESC: Quality flags for HIT L3 macropixel
+  DEPEND_0: epoch
+  VAR_TYPE: support_data
+  RECORD_VARYING: RV
+  FIELDNAM: Quality Flags for HIT L3 macropixel
+  FORMAT: I5
+  UNITS: ' '
+  VALIDMIN: 0
+  VALIDMAX: 65534
+  FILLVAL: 65535
+  LABLAXIS: Quality flags for HIT L3 macropixel

imap_l3_processing/codice/l3/hi/codice_hi_processor.py

@@ -10,6 +10,7 @@
 from imap_l3_processing.codice.l3.hi.models import CodiceL3HiDirectEvents
 from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import CodicePitchAngleDependencies
 from imap_l3_processing.codice.l3.lo.constants import CODICE_SPIN_ANGLE_OFFSET_FROM_MAG_BOOM
+from imap_l3_processing.codice.quality_flags import CodiceL3Flags
 from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import \
     get_sector_unit_vectors
 from imap_l3_processing.models import InputMetadata
@@ -110,6 +111,10 @@ def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3P
             "fe": SpeciesIntensity(sectored_intensities.fe_intensities,*_create_pa_and_gyro_nan_arrays(fe_pa_shape, fe_gyro_shape))}
         # @formatter:on
 
+        codice_flags = np.full(len(epochs), CodiceL3Flags.NONE).astype(int).astype(CodiceL3Flags)
+        if dependencies.mag_is_preliminary:
+            codice_flags |= CodiceL3Flags.PRELIMINARY_MAG
+
         for time_index in range(len(epochs)):
             pitch_angles = calculate_pitch_angle(particle_unit_vectors, mag_unit_vectors[time_index])
             gyrophases = calculate_gyrophase(particle_unit_vectors, mag_unit_vectors[time_index])
@@ -155,7 +160,8 @@ def process_l3b(self, dependencies: CodicePitchAngleDependencies) -> CodiceHiL3P
             cno_intensity_by_pitch_angle=species_intensities['o'].intensity_by_pa,
             cno_intensity_by_pitch_angle_and_gyrophase=species_intensities['o'].intensity_by_pa_and_gyro,
             fe_intensity_by_pitch_angle=species_intensities['fe'].intensity_by_pa,
-            fe_intensity_by_pitch_angle_and_gyrophase=species_intensities['fe'].intensity_by_pa_and_gyro
+            fe_intensity_by_pitch_angle_and_gyrophase=species_intensities['fe'].intensity_by_pa_and_gyro,
+            codice_flags=codice_flags,
         )
 
 

imap_l3_processing/codice/l3/hi/models.py

@@ -107,6 +107,7 @@ def read_from_cdf(cls, filename):
 ENERGY_HE3HE4_LABEL_VAR_NAME = "energy_he3he4_label"
 PITCH_ANGLE_LABEL_VAR_NAME = "pitch_angle_label"
 GYROPHASE_LABEL_VAR_NAME = "gyrophase_label"
+CODICE_FLAGS_VAR_NAME = "codice_flags"
 
 
 @dataclass
@@ -191,6 +192,7 @@ class CodiceHiL3PitchAngleDataProduct(DataProduct):
     cno_intensity_by_pitch_angle_and_gyrophase: ndarray
     fe_intensity_by_pitch_angle: ndarray
     fe_intensity_by_pitch_angle_and_gyrophase: ndarray
+    codice_flags: ndarray
 
     energy_h_label: ndarray = field(init=False)
     energy_cno_label: ndarray = field(init=False)
@@ -245,6 +247,7 @@ def to_data_product_variables(self) -> list[DataProductVariable]:
             DataProductVariable(ENERGY_HE3HE4_LABEL_VAR_NAME, self.energy_he3he4_label),
             DataProductVariable(PITCH_ANGLE_LABEL_VAR_NAME, self.pitch_angle_label),
             DataProductVariable(GYROPHASE_LABEL_VAR_NAME, self.gyrophase_label),
+            DataProductVariable(CODICE_FLAGS_VAR_NAME, self.codice_flags),
         ]
 
 

imap_l3_processing/codice/l3/hi/pitch_angle/codice_pitch_angle_dependencies.py

@@ -12,23 +12,24 @@
 class CodicePitchAngleDependencies:
     mag_data: MagData
     codice_sectored_intensities_data: CodiceHiL2SectoredIntensitiesData
+    mag_is_preliminary: bool = False
 
     @classmethod
     def fetch_dependencies(cls, input_collection: ProcessingInputCollection):
         codice_file_paths = input_collection.get_file_paths("codice", "hi-sectored")
-        mag_dependency = [
-            *input_collection.get_file_paths("mag", data_type="l2", descriptor="norm-dsrf"),
-            *input_collection.get_file_paths("mag", data_type="l1d", descriptor="norm-dsrf")
-        ][0]
+        mag_l2_paths = input_collection.get_file_paths("mag", data_type="l2", descriptor="norm-dsrf")
+        mag_l1d_paths = input_collection.get_file_paths("mag", data_type="l1d", descriptor="norm-dsrf")
+        mag_dependency = [*mag_l2_paths, *mag_l1d_paths][0]
+        mag_is_preliminary = len(mag_l2_paths) == 0
 
         for download_location_file_path in [*codice_file_paths, mag_dependency]:
             imap_data_access.download(download_location_file_path)
 
-        return cls.from_file_paths(mag_dependency, codice_file_paths[0])
+        return cls.from_file_paths(mag_dependency, codice_file_paths[0], mag_is_preliminary=mag_is_preliminary)
 
     @classmethod
-    def from_file_paths(cls, mag_file_path, codice_l2_sectored_intensities_path):
+    def from_file_paths(cls, mag_file_path, codice_l2_sectored_intensities_path, mag_is_preliminary: bool = False):
         mag_data = read_mag_data(mag_file_path)
         sectored_intensities = CodiceHiL2SectoredIntensitiesData.read_from_cdf(codice_l2_sectored_intensities_path)
 
-        return cls(mag_data, sectored_intensities)
+        return cls(mag_data, sectored_intensities, mag_is_preliminary=mag_is_preliminary)

imap_l3_processing/codice/quality_flags.py

@@ -0,0 +1,6 @@
+from imap_processing.quality_flags import FlagNameMixin, CommonFlags
+
+
+class CodiceL3Flags(FlagNameMixin):
+    NONE = CommonFlags.NONE
+    PRELIMINARY_MAG = 2 ** 8

imap_l3_processing/hit/l3/hit_l3_sectored_dependencies.py

@@ -18,19 +18,24 @@
 class HITL3SectoredDependencies:
     data: HitL2Data
     mag_data: MagData
+    mag_is_preliminary: bool = False
 
     @classmethod
     def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> HITL3SectoredDependencies:
         hit_data_dependency = dependencies.get_file_paths(source="hit", descriptor=HIT_L2_DESCRIPTOR)
-        mag_dependencies = [
-            *dependencies.get_file_paths(source="mag", data_type="l2", descriptor=MAG_L1D_DESCRIPTOR),
-            *dependencies.get_file_paths(source="mag", data_type="l1d", descriptor=MAG_L1D_DESCRIPTOR),
-        ]
+        mag_l2_dependencies = dependencies.get_file_paths(source="mag", data_type="l2",
+                                                          descriptor=MAG_L1D_DESCRIPTOR)
+        mag_l1d_dependencies = dependencies.get_file_paths(source="mag", data_type="l1d",
+                                                           descriptor=MAG_L1D_DESCRIPTOR)
+        mag_dependencies = [*mag_l2_dependencies, *mag_l1d_dependencies]
+
+        mag_is_preliminary = len(mag_l2_dependencies) == 0
 
         hit_data_path = imap_data_access.download(hit_data_dependency[0])
         mag_data_path = imap_data_access.download(mag_dependencies[0])
 
         mag_data = read_mag_data(mag_data_path)
         hit_data = read_l2_hit_data(hit_data_path)
 
-        return HITL3SectoredDependencies(data=hit_data, mag_data=mag_data)
+        return HITL3SectoredDependencies(data=hit_data, mag_data=mag_data,
+                                         mag_is_preliminary=mag_is_preliminary)

imap_l3_processing/hit/l3/hit_processor.py

@@ -12,6 +12,7 @@
 from imap_l3_processing.hit.l3.sectored_products.models import HitPitchAngleDataProduct
 from imap_l3_processing.hit.l3.sectored_products.science.sectored_products_algorithms import get_sector_unit_vectors, \
     get_hit_bin_polar_coordinates, transform_to_10_minute_chunks
+from imap_l3_processing.hit.quality_flags import HitL3Flags
 from imap_l3_processing.models import InputMetadata
 from imap_l3_processing.pitch_angles import calculate_unit_vector, calculate_pitch_angle, calculate_gyrophase, \
     rotate_particle_vectors_from_hit_despun_to_imap_despun, rebin_by_pitch_angle_and_gyrophase
@@ -208,6 +209,9 @@ def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) -
             number_of_pitch_angle_bins, number_of_gyrophase_bins)
 
         averaged_mag_data = mag_data.rebin_to(hit_data.epoch, hit_data.epoch_delta)
+        hit_flags = np.full(len(hit_data.epoch), HitL3Flags.NONE).astype(int).astype(HitL3Flags)
+        if dependencies.mag_is_preliminary:
+            hit_flags |= HitL3Flags.PRELIMINARY_MAG
         measurement_pitch_angle = []
         measurement_gyrophase = []
         for time_index, average_mag_vector in enumerate(averaged_mag_data):
@@ -287,7 +291,8 @@ def process_pitch_angle_product(self, dependencies: HITL3SectoredDependencies) -
                                         np.array(measurement_pitch_angle),
                                         np.array(measurement_gyrophase),
                                         azimuth=hit_data.azimuth,
-                                        zenith=hit_data.zenith)
+                                        zenith=hit_data.zenith,
+                                        hit_flags=hit_flags)
 
     @staticmethod
     def _create_nan_array(shape) -> tuple[np.array, np.array, np.array]:

imap_l3_processing/hit/l3/sectored_products/models.py

@@ -68,6 +68,7 @@
 ZENITH_VAR_NAME = "zenith"
 AZIMUTH_LABEL_VAR_NAME = "azimuth_label"
 ZENITH_LABEL_VAR_NAME = "zenith_label"
+HIT_FLAGS_CDF_VAR_NAME = "hit_flags"
 
 
 @dataclass
@@ -127,6 +128,7 @@ class HitPitchAngleDataProduct(DataProduct):
     measurement_gyrophase: np.ndarray
     azimuth: np.ndarray
     zenith: np.ndarray
+    hit_flags: np.ndarray
 
     def to_data_product_variables(self) -> list[DataProductVariable]:
         return [
@@ -212,5 +214,6 @@ def to_data_product_variables(self) -> list[DataProductVariable]:
             DataProductVariable(AZIMUTH_VAR_NAME, self.azimuth),
             DataProductVariable(ZENITH_VAR_NAME, self.zenith),
             DataProductVariable(AZIMUTH_LABEL_VAR_NAME, [str(float(azimuth)) for azimuth in self.azimuth]),
-            DataProductVariable(ZENITH_LABEL_VAR_NAME, [str(float(zenith)) for zenith in self.zenith])
+            DataProductVariable(ZENITH_LABEL_VAR_NAME, [str(float(zenith)) for zenith in self.zenith]),
+            DataProductVariable(HIT_FLAGS_CDF_VAR_NAME, self.hit_flags),
         ]

imap_l3_processing/hit/quality_flags.py

@@ -0,0 +1,6 @@
+from imap_processing.quality_flags import FlagNameMixin, CommonFlags
+
+
+class HitL3Flags(FlagNameMixin):
+    NONE = CommonFlags.NONE
+    PRELIMINARY_MAG = 2 ** 8

imap_l3_processing/swe/l3/swe_l3_dependencies.py

@@ -24,6 +24,7 @@ class SweL3Dependencies:
     mag_data: MagData
     swapi_l3a_proton_data: SwapiL3aProtonData
     configuration: SweConfiguration
+    mag_is_preliminary: bool = False
 
     @classmethod
     def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dependencies:
@@ -44,13 +45,15 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dep
             (d.imap_file_paths[0] for d in science_files if d.source == "mag"
             and d.descriptor == MAG_DESPUN_L1D_DESCRIPTOR
             and d.data_type == "l2"), None)
+        mag_is_preliminary = False
         if mag_dependency is None:
             try:
                 mag_dependency = next(
                     d.imap_file_paths[0] for d in science_files if d.source == "mag"
                     and d.descriptor == MAG_DESPUN_L1D_DESCRIPTOR
                     and d.data_type == "l1d"
                 )
+                mag_is_preliminary = True
             except StopIteration:
                 raise ValueError(f"Missing MAG {MAG_DESPUN_L1D_DESCRIPTOR} dependency.")
         try:
@@ -66,16 +69,19 @@ def fetch_dependencies(cls, dependencies: ProcessingInputCollection) -> SweL3Dep
         swapi_file = download(swapi_dependency.construct_path())
         swe_config = download(swe_config_dependency)
 
-        return cls.from_file_paths(swe_l2_file, swe_l1b_file, mag_file, swapi_file, swe_config)
+        return cls.from_file_paths(swe_l2_file, swe_l1b_file, mag_file, swapi_file, swe_config,
+                                   mag_is_preliminary=mag_is_preliminary)
 
     @classmethod
     def from_file_paths(cls, swe_l2_file_path: Path, swe_l1b_file_path: Path, mag_file_path: Path,
                         swapi_file_path: Path,
-                        configuration_file_path: Path) -> SweL3Dependencies:
+                        configuration_file_path: Path,
+                        mag_is_preliminary: bool = False) -> SweL3Dependencies:
         mag_l1d_data = read_mag_data(mag_file_path)
         swe_l1b_data = read_l1b_swe_data(swe_l1b_file_path)
         swe_l2_data = read_l2_swe_data(swe_l2_file_path)
         swapi_l3a_proton_data = read_l3a_swapi_proton_data(swapi_file_path)
         configuration = read_swe_config(configuration_file_path)
 
-        return cls(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, configuration)
+        return cls(swe_l2_data, swe_l1b_data, mag_l1d_data, swapi_l3a_proton_data, configuration,
+                   mag_is_preliminary=mag_is_preliminary)

imap_l3_processing/swe/swe_processor.py

@@ -156,6 +156,9 @@ def calculate_products(self, dependencies: SweL3Dependencies) -> SweL3Data:
 
         swe_quality_flags |= pitch_angle_flags
 
+        if dependencies.mag_is_preliminary:
+            swe_quality_flags |= SweL3Flags.PRELIMINARY_MAG
+
         rebinned_mask = np.ma.masked_invalid(swe_l2_data.phase_space_density_rebinned)
         dist_by_phi_rebinned = np.average(rebinned_mask, weights=geometric_fractions, axis=-1)
         dist_fun_1d_rebinned = np.ma.average(dist_by_phi_rebinned, axis=-1)

tests/codice/l3/hi/pitch_angle/test_codice_pitch_angle_dependencies.py

@@ -35,7 +35,8 @@ def test_fetch_dependencies(self, mock_download, mock_from_files):
             call(expected_mag_science_file_path)
         ])
 
-        mock_from_files.assert_called_with(expected_mag_science_file_path, expected_codice_science_file_path)
+        mock_from_files.assert_called_with(expected_mag_science_file_path, expected_codice_science_file_path,
+                                           mag_is_preliminary=True)
         self.assertEqual(mock_from_files.return_value, dependencies)
 
     @patch(
@@ -67,7 +68,8 @@ def test_fetch_dependencies_uses_mag_l2_if_available(self, mock_download, mock_f
             call(expected_mag_science_file_path)
         ])
 
-        mock_from_files.assert_called_with(expected_mag_science_file_path, expected_codice_science_file_path)
+        mock_from_files.assert_called_with(expected_mag_science_file_path, expected_codice_science_file_path,
+                                           mag_is_preliminary=False)
         self.assertEqual(mock_from_files.return_value, dependencies)
 
     @patch('imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies.read_mag_data')
@@ -83,3 +85,9 @@ def test_from_file_paths(self, mock_codice_l2_data, mock_read_mag_data):
 
         self.assertEqual(mock_codice_l2_data.return_value, dependencies.codice_sectored_intensities_data)
         self.assertEqual(mock_read_mag_data.return_value, dependencies.mag_data)
+        self.assertFalse(dependencies.mag_is_preliminary)
+
+        preliminary_dependencies = CodicePitchAngleDependencies.from_file_paths(
+            mag_file_path, codice_file_path, mag_is_preliminary=True
+        )
+        self.assertTrue(preliminary_dependencies.mag_is_preliminary)

tests/codice/l3/hi/test_codice_hi_processor.py

@@ -13,6 +13,7 @@
     CodiceL3HiDirectEvents
 from imap_l3_processing.codice.l3.hi.pitch_angle.codice_pitch_angle_dependencies import CodicePitchAngleDependencies
 from imap_l3_processing.codice.l3.lo.constants import CODICE_SPIN_ANGLE_OFFSET_FROM_MAG_BOOM
+from imap_l3_processing.codice.quality_flags import CodiceL3Flags
 from imap_l3_processing.models import InputMetadata
 from tests.test_helpers import NumpyArrayMatcher, get_test_data_path
 
@@ -293,7 +294,8 @@ def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, m
         )
 
         dependencies = CodicePitchAngleDependencies(mag_data=mag_l1d_data,
-                                                    codice_sectored_intensities_data=codice_l2_data)
+                                                    codice_sectored_intensities_data=codice_l2_data,
+                                                    mag_is_preliminary=True)
 
         expected_pitch_angles = np.linspace(15, 165, 6)
         expected_gyrophase = np.linspace(15, 345, 12)
@@ -433,6 +435,11 @@ def test_process_l3b_with_mocks(self, mock_rebin_by_pitch_angle_and_gyrophase, m
         np.testing.assert_array_equal(codice_hi_data_product.fe_intensity_by_pitch_angle_and_gyrophase,
                                       expected_fe_intensity_binned_by_pa_and_gyro)
 
+        np.testing.assert_array_equal(
+            codice_hi_data_product.codice_flags,
+            np.array([CodiceL3Flags.PRELIMINARY_MAG, CodiceL3Flags.PRELIMINARY_MAG]),
+        )
+
         np.testing.assert_array_equal(codice_hi_data_product.parent_file_names, expected_parents)
 
     def test_integration_test(self):

tests/codice/l3/hi/test_models.py

@@ -173,6 +173,7 @@ def test_codice_hi_l3_pitch_angle_to_data_product(self):
                                                                                len(pitch_angle)) + 6,
             "fe_intensity_by_pitch_angle_and_gyrophase": np.arange(pitch_angle_and_gyrophase_size).reshape(
                 len(epoch_data), len(energy_data), len(pitch_angle), len(gyrophase)) + 7,
+            "codice_flags": np.zeros(len(epoch_data), dtype=np.uint16),
         }
 
         data_product = CodiceHiL3PitchAngleDataProduct(

tests/hit/l3/sectored_products/test_models.py

@@ -80,6 +80,7 @@ def test_to_data_product_variables(self):
                                         sentinel.measurement_gyrophase,
                                         np.array([10, 180, 350]),
                                         np.array([10.25, 90.75, 170.5]),
+                                        sentinel.quality_flags,
                                         )
 
         data_product_variables = data.to_data_product_variables()
@@ -162,7 +163,8 @@ def test_to_data_product_variables(self):
             DataProductVariable("azimuth_label",
                                 ["10.0", "180.0", "350.0"]),
             DataProductVariable("zenith_label",
-                                ["10.25", "90.75", "170.5"])
+                                ["10.25", "90.75", "170.5"]),
+            DataProductVariable("hit_flags", sentinel.quality_flags),
         ]
 
         self.assertEqual(expected_data_product_variables, data_product_variables)

tests/hit/l3/test_hit_processor.py

@@ -17,6 +17,7 @@
 from imap_l3_processing.hit.l3.pha.science.cosine_correction_lookup_table import DetectedRange, DetectorSide, \
     DetectorRange
 from imap_l3_processing.hit.l3.sectored_products.models import HitPitchAngleDataProduct
+from imap_l3_processing.hit.quality_flags import HitL3Flags
 from imap_l3_processing.models import MagData, InputMetadata
 from imap_l3_processing.processor import Processor
 from tests.test_helpers import NumpyArrayMatcher, create_dataclass_mock
@@ -66,6 +67,7 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit
         averaged_mag_vectors = [sentinel.mag_vector1, sentinel.mag_vector2]
 
         mock_dependencies = Mock(spec=HITL3SectoredDependencies)
+        mock_dependencies.mag_is_preliminary = True
         mock_mag_data = create_dataclass_mock(MagData)
         mock_mag_data.rebin_to = Mock()
         mock_mag_data.rebin_to.return_value = averaged_mag_vectors
@@ -436,6 +438,9 @@ def test_process_pitch_angle_product(self, mock_rotate_particle_vectors_from_hit
         np.testing.assert_array_equal(saved_data_product.measurement_gyrophase,
                                       np.array([sentinel.gyrophase1, sentinel.gyrophase2]))
 
+        np.testing.assert_array_equal(saved_data_product.hit_flags,
+                                      np.array([HitL3Flags.PRELIMINARY_MAG, HitL3Flags.PRELIMINARY_MAG]))
+
         self.assertEqual([mock_save_data.return_value], product)
 
     @patch("imap_l3_processing.utils.spiceypy")