DM-42201: Add flat metrics for mean, noise vs mjd and FP plots

pipelines/cpCore.yaml

@@ -139,12 +139,31 @@ tasks:
       python: |
         from lsst.analysis.tools.atools import *
 
-  analyzeFlatDetCore:
+  analyzeFlatCore:
     class: lsst.analysis.tools.tasks.VerifyCalibAnalysisTaskByFilter
     config:
-      connections.outputName: cpFlatDetCore
+      connections.outputName: cpFlatCore
       connections.data: verifyFlatResults
 
+      atools.flatMeanPerAmp: CalibStatisticFocalPlanePlot
+      atools.flatMeanPerAmp.produce.plot.addHistogram: true
+      atools.flatMeanPerAmp.quantityKey: FLAT_MEAN
+      atools.flatMeanPerAmp.unit: electron
+
+      atools.flatNoisePerAmp: CalibStatisticFocalPlanePlot
+      atools.flatNoisePerAmp.produce.plot.addHistogram: true
+      atools.flatNoisePerAmp.quantityKey: FLAT_NOISE
+      atools.flatNoisePerAmp.unit: electron
+
+      python: |
+        from lsst.analysis.tools.atools import *
+
+  analyzeFlatDetMergeCore:
+    class: lsst.analysis.tools.tasks.VerifyCalibDetectorTaskByFilter
+    config:
+      connections.outputName: cpFlatDetMergeCore
+      connections.data: verifyFlatDetMergeResults
+
       atools.flatTestsByDate: CalibAmpScatterTool
       atools.flatTestsByDate.prep.panelKey: amplifier
       atools.flatTestsByDate.prep.dataKey: mjd
@@ -154,6 +173,27 @@ tasks:
       atools.flatTestsByDate.produce.plot.xAxisLabel: "MJD"
       atools.flatTestsByDate.produce.plot.yAxisLabel: "Test Pass Results"
 
+      atools.flatMeansByDate: CalibAmpScatterTool
+      atools.flatMeansByDate.prep.panelKey: amplifier
+      atools.flatMeansByDate.prep.dataKey: mjd
+      atools.flatMeansByDate.prep.quantityKey: FLAT_MEAN
+      atools.flatMeansByDate.produce.plot.xAxisLabel: "MJD"
+      atools.flatMeansByDate.produce.plot.yAxisLabel: "Flat Mean (electrons)"
+
+      atools.flatNoiseByDate: CalibAmpScatterTool
+      atools.flatNoiseByDate.prep.panelKey: amplifier
+      atools.flatNoiseByDate.prep.dataKey: mjd
+      atools.flatNoiseByDate.prep.quantityKey: FLAT_NOISE
+      atools.flatNoiseByDate.produce.plot.xAxisLabel: "MJD"
+      atools.flatNoiseByDate.produce.plot.yAxisLabel: "Flat Noise (electrons)"
+
+      atools.flatNoiseByMean: CalibAmpScatterTool
+      atools.flatNoiseByMean.prep.panelKey: amplifier
+      atools.flatNoiseByMean.prep.dataKey: FLAT_MEAN
+      atools.flatNoiseByMean.prep.quantityKey: FLAT_NOISE
+      atools.flatNoiseByMean.produce.plot.xAxisLabel: "Flat Mean (electrons)"
+      atools.flatNoiseByMean.produce.plot.yAxisLabel: "Flat Noise (electrons)"
+
       python: |
         from lsst.analysis.tools.atools import *
 
@@ -275,7 +315,8 @@ subsets:
       - analyzeDarkDetCore
   flatMetricsCore:
     subset:
-      - analyzeFlatDetCore
+      - analyzeFlatCore
+      - analyzeFlatDetMergeCore
   defectMetricsCore:
     subset:
       - analyzeDefectCore

python/lsst/analysis/tools/atools/calibQuantityProfile.py

@@ -33,6 +33,7 @@
 
 from typing import cast
 
+import numpy
 from lsst.pex.config import Field
 from lsst.pex.config.configurableActions import ConfigurableActionField
 
@@ -90,16 +91,24 @@ class SingleValueRepacker(KeyedDataAction):
 
     def __call__(self, data: KeyedData, **kwargs) -> KeyedData:
         repackedData = {}
-        uniquePanelKeys = list(set(data[self.panelKey]))
 
-        # Loop over data vector to repack information as it is expected.
+        if isinstance(data[self.panelKey], numpy.ma.MaskedArray):
+            good = ~data[self.panelKey].mask
+            uniquePanelKeys = numpy.unique(data[self.panelKey])
+            uniquePanelKeys = uniquePanelKeys.data[~uniquePanelKeys.mask].data
+        else:
+            good = numpy.ones(len(data[self.panelKey]), dtype=bool)
+            uniquePanelKeys = list(set(data[self.panelKey]))
+
+        # Loop over data vector to repack information as
+        # it is expected.
         for i in range(len(uniquePanelKeys)):
             repackedData[f"{uniquePanelKeys[i]}_x"] = []
             repackedData[f"{uniquePanelKeys[i]}"] = []
 
-        panelVec = cast(Vector, data[self.panelKey])
-        dataVec = cast(Vector, data[self.dataKey])
-        quantityVec = cast(Vector, data[self.quantityKey])
+        panelVec = cast(Vector, data[self.panelKey][good])
+        dataVec = cast(Vector, data[self.dataKey][good])
+        quantityVec = cast(Vector, data[self.quantityKey][good])
 
         for i in range(len(panelVec)):
             repackedData[f"{panelVec[i]}_x"].append(dataVec[i])