Skip to content

Modified data loading for HXN scans using PandABox #339

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
zr-gao wants to merge 2 commits into
base: master
Choose a base branch
from
Open

Modified data loading for HXN scans using PandABox #339

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
54071ed
Modified data loading for HXN scans using PandABox
zr-gao Jul 23, 2025
0a31645
removed unnecessary changes
zr-gao Jul 23, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 9 additions & 5 deletions pyxrf/db_config/hxn_db_config.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,15 @@
except ModuleNotFoundError:
from databroker import Broker

from hxntools.handlers.timepix import TimepixHDF5Handler
from hxntools.handlers.xspress3 import Xspress3HDF5Handler

db = Broker.named("hxn")
from hxntools.handlers import register
register(db)

#from hxntools.handlers.xspress3 import Xspress3HDF5Handler
#from hxntools.handlers.timepix import TimepixHDF5Handler
#
#db = Broker.named("hxn")
# db_analysis = Broker.named('hxn_analysis')

db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True)
db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True)
#db.reg.register_handler(Xspress3HDF5Handler.HANDLER_NAME, Xspress3HDF5Handler, overwrite=True)
#db.reg.register_handler(TimepixHDF5Handler._handler_name, TimepixHDF5Handler, overwrite=True)
94 changes: 73 additions & 21 deletions pyxrf/model/load_data_from_db.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -784,12 +784,20 @@ def map_data2D_hxn(
data_output = []

start_doc = hdr["start"]
logger.info("Plan type: '%s'", start_doc["plan_type"])

if "scan" in start_doc:
#print(" panda scan ")
plan_type = start_doc["scan"]["type"]

else:
plan_type = start_doc["plan_type"]

logger.info("Plan type: '%s'", plan_type)

# Exclude certain types of plans based on data from the start document
if isinstance(skip_scan_types, (list, tuple)) and (start_doc["plan_type"] in skip_scan_types):
if isinstance(skip_scan_types, (list, tuple)) and (plan_type in skip_scan_types):
raise RuntimeError(
f"Failed to load the scan: plan type {start_doc['plan_type']!r} is in the list of skipped types"
f"Failed to load the scan: plan type {plan_type!r} is in the list of skipped types"
)

# The dictionary holding scan metadata
Expand Down Expand Up @@ -884,11 +892,17 @@ def map_data2D_hxn(
else:
raise ValueError(f"Invalid data shape: {datashape}. Must be a list with 1 or 2 elements.")

logger.info(f'Data shape: {datashape}.')

# -----------------------------------------------------------------------------------------------
# Determine fast axis and slow axis
fast_axis, slow_axis, fast_axis_index = start_doc.get("fast_axis", None), None, None
motors = start_doc.get("motors", None)
if motors and isinstance(motors, (list, tuple)) and len(motors) == 2:
if motors and isinstance(motors, (list, tuple)) and len(motors) == 1:
fast_axis = fast_axis if fast_axis else motors[0]
fast_axis_index = motors.index(fast_axis, 0)

elif motors and isinstance(motors, (list, tuple)) and len(motors) == 2:
fast_axis = fast_axis if fast_axis else motors[0]
fast_axis_index = motors.index(fast_axis, 0)
slow_axis_index = 0 if (fast_axis_index == 1) else 1
Expand All @@ -904,14 +918,24 @@ def map_data2D_hxn(
# -----------------------------------------------------------------------------------------------
# Reconstruct scan input
try:
plan_args = start_doc["plan_args"]
# px_motor = plan_args["motor1"]
px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"]
# py_motor = plan_args["motor2"]
py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"]
dwell_time = plan_args["exposure_time"]
param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time]
mdata["param_input"] = param_input
if "plan_args" in start_doc: # dscan and fly1d/fly2d scan
plan_args = start_doc["plan_args"]
# px_motor = plan_args["motor1"]
px_start, px_end, px_step = plan_args["scan_start1"], plan_args["scan_end1"], plan_args["num1"]
# py_motor = plan_args["motor2"]
py_start, py_end, py_step = plan_args["scan_start2"], plan_args["scan_end2"], plan_args["num2"]
dwell_time = plan_args["exposure_time"]
param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time]
mdata["param_input"] = param_input
elif "scan" in start_doc: # fly1dpd and fly2dpd scan
scan_input = start_doc["scan"]["scan_input"]
px_start, px_end, px_step = scan_input[0:3]
py_start, py_end, py_step = scan_input[3:6]
dwell_time = start_doc["scan"]["dwell"]
param_input = [px_start, px_end, px_step, py_start, py_end, py_step, dwell_time]
mdata["param_input"] = param_input
else:
raise Exception("Unknown scan plan type")
except Exception as ex:
logger.warning(
"Failed to reconstruct scan input: %s. Scan input is not saved as part of metadata to HDF5 file",
Expand All @@ -930,7 +954,8 @@ def map_data2D_hxn(

keylist = hdr.descriptors[0].data_keys.keys()
det_list = [v for v in keylist if "xspress3" in v] # find xspress3 det with key word matching

det_list = [v for v in det_list if len(v)==12] #added to filter out other rois added by user

scaler_list_all = config_data["scaler_list"]

all_keys = hdr.descriptors[0].data_keys.keys()
Expand All @@ -942,7 +967,8 @@ def map_data2D_hxn(
if isinstance(db, databroker._core.Broker):
fields = None

data = hdr.table(fields=fields, fill=True)

data = hdr.table(fields=fields, fill=False) # HXN data is stored in h5 files, load them later in map_data2D.

# This is for the case of 'dcan' (1D), where the slow axis positions are not saved
if (slow_axis not in data) and (fast_axis in data):
Expand All @@ -958,6 +984,7 @@ def map_data2D_hxn(
fly_type=fly_type,
subscan_dims=subscan_dims,
spectrum_len=4096,
hdr=hdr
)

# Transform coordinates for the fast axis if necessary:
Expand Down Expand Up @@ -3430,7 +3457,7 @@ def write_db_to_hdf(
fname_add_version=False,
fly_type=None,
subscan_dims=None,
base_val=None,
base_val=None
):
"""
Assume data is obained from databroker, and save the data to hdf file.
Expand Down Expand Up @@ -3513,8 +3540,9 @@ def write_db_to_hdf(
# position data
dataGrp = f.create_group(interpath + "/positions")

# scanning position data
pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape)

for i in range(len(pos_names)):
if "x" in pos_names[i]:
pos_names[i] = "x_pos"
Expand All @@ -3539,6 +3567,7 @@ def write_db_to_hdf(
# scaler data
dataGrp = f.create_group(interpath + "/scalers")

# scaler data
scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape)

if fly_type in ("pyramid",):
Expand Down Expand Up @@ -3713,6 +3742,7 @@ def map_data2D(
fly_type=None,
subscan_dims=None,
spectrum_len=4096,
hdr = None
):
"""
Data is obained from databroker. Transfer items from data to a dictionary of
Expand Down Expand Up @@ -3752,7 +3782,10 @@ def map_data2D(
if c_name in data:
detname = "det" + str(n + 1)
logger.info("read data from %s" % c_name)
channel_data = data[c_name]
if db.name == 'hxn':
channel_data = np.squeeze(np.array(list(hdr.data(c_name))))
else:
channel_data = data[c_name]

# new veritcal shape is defined to ignore zeros points caused by stopped/aborted scans
new_v_shape = len(channel_data) // datashape[1]
Expand Down Expand Up @@ -3785,8 +3818,20 @@ def map_data2D(
sum_data += new_data
data_output["det_sum"] = sum_data

# scanning position data
pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape)
if db.name == 'hxn':
pos_names = pos_list
pos_data = np.zeros([datashape[0], datashape[1], len(pos_list)])
from hxntools.scan_info import get_scan_positions
pos = get_scan_positions(hdr)
if isinstance(pos,tuple):
for i in range(len(pos)):
pos_data[:,:,i] = pos[i].reshape((datashape[0], datashape[1]))
else:
pos_data[:,:,0] = pos.reshape((datashape[0], datashape[1]))
else:
# scanning position data
pos_names, pos_data = get_name_value_from_db(pos_list, data, datashape)

for i in range(len(pos_names)):
if "x" in pos_names[i]:
pos_names[i] = "x_pos"
Expand Down Expand Up @@ -3817,8 +3862,15 @@ def map_data2D(
data_output["pos_names"] = pos_names
data_output["pos_data"] = new_p

# scaler data
scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape)
if db.name == 'hxn':
scaler_names = scaler_list
scaler_data = np.zeros([datashape[0], datashape[1], len(scaler_list)])
for i in range(len(scaler_list)):
scaler_data[:,:,i] = np.array(list(hdr.data(scaler_list[i]))).reshape((datashape[0], datashape[1]))
else:
# scaler data
scaler_names, scaler_data = get_name_value_from_db(scaler_list, data, datashape)

if fly_type in ("pyramid",):
scaler_data = flip_data(scaler_data, subscan_dims=subscan_dims)

Expand Down
Loading