v0.5.0 candidate (#29)

* fix: return logging levels to original state

* test: add dat.read() test

* test: add comment to test_dat_read_both_formats

* test: add test_supplements folder

* fix: return raw2img log to original state

* Bump version: 0.4.0 → 0.5.0

Author: kenanoestreich

rawtools/__init__.py

@@ -2,4 +2,4 @@
 
 __author__ = """Tim Parker"""
 __email__ = 'tim.parkerd@gmail.com'
-__version__ = '0.4.0'
+__version__ = '0.5.0'

rawtools/dat.py

@@ -61,15 +61,18 @@ def determine_bit_depth(fp, dims):
             logging.warning(f"Unable to determine bit-depth of volume '{fp}'. Expected at <{expected_filesize}> bytes but found <{file_size}> bytes. Defaulting to unsigned 16-bit.")
             return 'uint16'
 
-def __parse_object_filename(line):
-    pattern = r"^ObjectFileName\:\s+(?P<filename>.*\.raw)$"
+def __parse_object_filename(line, dat_format):
+    if (dat_format == "Dragonfly"):
+        pattern = r"<ObjectFileName>(?P<filename>.*\.raw)<\/ObjectFileName>"
+    elif (dat_format == "NSI"):
+        pattern = r"^ObjectFileName\:\s+(?P<filename>.*\.raw)$"
 
     match = re.match(pattern, line, flags=re.IGNORECASE)
     if match is not None:
         logging.debug(f"Match: {match}")
         return match.group('filename')
 
-def __parse_resolution(line):
+def __parse_resolution(line, dat_format):
     """Get the x, y, z dimensions of a volume.
 
     Args:
@@ -80,13 +83,16 @@ def __parse_resolution(line):
 
     """
     # logging.debug(line.strip())
-    pattern_old = r'\s+<Resolution X="(?P<x>\d+)"\s+Y="(?P<y>\d+)"\s+Z="(?P<z>\d+)"'
-    pattern = r'Resolution\:\s+(?P<x>\d+)\s+(?P<y>\d+)\s+(?P<z>\d+)'
+    if (dat_format == "Dragonfly"):
+        pattern = r'<Resolution X="(?P<x>\d+)"\s+Y="(?P<y>\d+)"\s+Z="(?P<z>\d+)"'
+    elif (dat_format == "NSI"):
+        pattern_old = r'\s+<Resolution X="(?P<x>\d+)"\s+Y="(?P<y>\d+)"\s+Z="(?P<z>\d+)"'
+        pattern = r'Resolution\:\s+(?P<x>\d+)\s+(?P<y>\d+)\s+(?P<z>\d+)'
 
     # See if the DAT file is the newer version
     match = re.match(pattern, line, flags=re.IGNORECASE)
     # Otherwise, check the old version (XML)
-    if match is None:
+    if match is None and dat_format == "NSI":
         match = re.match(pattern_old, line, flags=re.IGNORECASE)
         if match is not None:
             logging.debug(f"XML format detected for '{line}'")
@@ -103,7 +109,7 @@ def __parse_resolution(line):
             raise Exception(f"Unable to extract dimensions from DAT file. Found dimensions: '{dims}'.")
         return dims
 
-def __parse_slice_thickness(line):
+def __parse_slice_thickness(line, dat_format):
     """Get the x, y, z dimensions of a volume.
 
     Args:
@@ -113,31 +119,56 @@ def __parse_slice_thickness(line):
         (float, float, float): x, y, z real-world thickness in mm. Otherwise, returns None.
 
     """
-    pattern = r'\w+\:\s+(?P<xth>\d+\.\d+)\s+(?P<yth>\d+\.\d+)\s+(?P<zth>\d+\.\d+)'
+    if (dat_format == "Dragonfly"):
+        pattern = r"<Spacing\s+X=\"(?P<xth>\d+\.\d+)\"\s+Y=\"(?P<yth>\d+\.\d+)\"\s+Z=\"(?P<zth>\d+\.\d+)\""
+    elif (dat_format == "NSI"):
+        pattern = r'\w+\:\s+(?P<xth>\d+\.\d+)\s+(?P<yth>\d+\.\d+)\s+(?P<zth>\d+\.\d+)'
+
     match = re.match(pattern, line, flags=re.IGNORECASE)
     if match is not None:
         logging.debug(f"Match: {match}")
-        df = match.groupdict()
         dims = [ match.group('xth'), match.group('yth'), match.group('zth') ]
-        dims = [ float(s) for s in dims ]
+        if (dat_format == "Dragonfly"):
+            # Change Dragonfly thickness units to match NSI format
+            dims = [ (float(s)*1000) for s in dims ]
+        elif (dat_format == "NSI"):
+            dims = [ float(s) for s in dims ]
         if not dims or len(dims) != 3:
             raise Exception(f"Unable to extract slice thickness from DAT file: '{line}'. Found slice thickness: '{dims}'.")
         return dims
 
-def __parse_format(line):
-    pattern = r"^Format\:\s+(?P<format>\w+)$"
+def __parse_format(line, dat_format):
+    if (dat_format == "Dragonfly"): 
+        pattern = r"<Format>(?P<format>\w+)<\/Format>"
+    elif (dat_format == "NSI"):
+        pattern = r"Format\:\s+(?P<format>\w+)$"
+
     match = re.match(pattern, line, flags=re.IGNORECASE)
     if match is not None:
         logging.debug(f"Match: {match}")
         return match.group('format')
 
-def __parse_object_model(line):
-    pattern = r"^ObjectModel\:\s+(?P<object_model>\w+)$"
+def __parse_object_model(line, dat_format):
+    if (dat_format == "Dragonfly"): 
+        pattern = r"<Unit>(?P<object_model>\w+)<\/Unit>"  
+    elif (dat_format == "NSI"):
+        pattern = r"^ObjectModel\:\s+(?P<object_model>\w+)$"
+
     match = re.match(pattern, line, flags=re.IGNORECASE)
     if match is not None:
         logging.debug(f"Match: {match}")
         return match.group('object_model')
 
+def __is_dragonfly_dat_format(line):
+    pattern = r"<\?xml\sversion=\"1\.0\"\?>"
+    match = re.match(pattern, line, flags=re.IGNORECASE)
+    if match is not None:
+        logging.debug(f"Match: {match}")
+        return True
+
+    
+
+
 def read(fp):
     """Read a .DAT file
     Args:
@@ -146,24 +177,30 @@ def read(fp):
     dict: contents of .DAT file
     """
     data = {}
+    dat_format = "NSI"
     with open(fp, 'r') as ifp:
         # Parse the individual lines
         for line in ifp.readlines():
             line = line.strip()
-            if (object_filename := __parse_object_filename(line)) is not None:
+            # Determine if format is NSI .dat or Dragonfly .dat
+            if (__is_dragonfly_dat_format(line)): 
+                dat_format = "Dragonfly"
+
+            if (object_filename := __parse_object_filename(line, dat_format)) is not None:
                 data['ObjectFileName'] = object_filename
+               
 
-            if (resolution := __parse_resolution(line)) is not None:
+            if (resolution := __parse_resolution(line, dat_format)) is not None:
                 data['xdim'], data['ydim'], data['zdim'] = resolution
                 data['dimensions'] = resolution
 
-            if (thicknesses := __parse_slice_thickness(line)) is not None:
+            if (thicknesses := __parse_slice_thickness(line, dat_format)) is not None:
                 data['x_thickness'], data['y_thickness'], data['z_thickness'] = thicknesses
 
-            if (file_format := __parse_format(line)) is not None:
+            if (file_format := __parse_format(line, dat_format)) is not None:
                 data['Format'] =  file_format
 
-            if (object_model := __parse_object_model(line)) is not None:
+            if (object_model := __parse_object_model(line, dat_format)) is not None:
                 data['model'] = object_model
 
 

setup.cfg

@@ -1,5 +1,5 @@
 [bumpversion]
-current_version = 0.4.0
+current_version = 0.5.0
 commit = True
 tag = True
 

setup.py

@@ -66,6 +66,6 @@
     test_suite='tests',
     tests_require=test_requirements,
     url='https://github.com/Topp-Roots-Lab/python-rawtools',
-    version='0.4.0',
+    version='0.5.0',
     zip_safe=False,
 )

tests/test_rawtools.py

@@ -27,6 +27,15 @@ def slice_uint16_high_variance():
     """Sample uint16 slice with variable values"""
     return np.array([-1, 0, 100, 1000, 5000, 14830, 50321, 65535, 65536], dtype=uint16)
 
+@pytest.fixture
+def dat_files():
+    """Sample .dat files' paths"""
+    return ['tests/test_supplements/ideal_dragonfly.dat', 
+            'tests/test_supplements/ideal_nsi.dat',
+            'tests/test_supplements/poor_dragonfly.dat',
+            'tests/test_supplements/poor_nsi.dat']
+
+
 
 def test_scale_uint8(slice_uint8):
     """Test scaling a unsigned 8-bit integer array to own bounds."""
@@ -65,3 +74,16 @@ def test_scale_uint16_to_uint8_large_variance(slice_uint16_high_variance):
         scale(slice_uint16_high_variance, lbound, ubound, new_lbound, new_ubound))
 
     np.testing.assert_array_equal(scaled_slice, slice_uint8)
+
+def test_dat_read_both_formats(dat_files):
+    """Test dat.read() on 4 example .dat (two acceptable and two unacceptable) 
+    covering both Dragonfly and NSI formats"""
+    from rawtools.dat import read
+    # neither of these should raise any errors
+    read(dat_files[0])
+    read(dat_files[1])
+    with pytest.raises(ValueError, match=r"Unable to parse"):
+        read(dat_files[2])
+    with pytest.raises(ValueError, match=r"Unable to parse"):
+        read(dat_files[3])
+    

tests/test_supplements/ideal_dragonfly.dat

@@ -0,0 +1,13 @@
+<?xml version="1.0"?>
+<RAWFileData>
+        <Version>1</Version>
+        <ObjectFileName>2020_PlantHaven_IPSRI_133-3_2021-rewash_109um_8Bit.raw</ObjectFileName>
+        <Format>UCHAR</Format>
+        <DataSlope>1</DataSlope>
+        <DataOffset>0</DataOffset>
+        <Unit>Density</Unit>
+        <Resolution X="1474" Y="1474" Z="1861" T="1" />
+        <Spacing X="0.00010852599999999999736115946502579276966571342200040817" Y="0.00010852599999999999736115946502579276966571342200040817" Z="0.00010852599999999999736115946502579276966571342200040817" />
+        <Orientation X0="1" X1="0" X2="0" Y0="0" Y1="1" Y2="0" Z0="0" Z1="0" Z2="1" />
+        <Position P1="0" P2="0" P3="0" />
+</RAWFileData>

tests/test_supplements/ideal_nsi.dat

@@ -0,0 +1,5 @@
+ObjectFileName: 2020_PlantHaven_IPSRI_133-3_2021-rewash_109um.raw
+Resolution:     1474 1474 1861
+SliceThickness: 0.108526441297249 0.108526441297249 0.108526441297249
+Format:         USHORT
+ObjectModel:    DENSITY

tests/test_supplements/poor_dragonfly.dat

@@ -0,0 +1,12 @@
+<?xml version="1.0"?>
+<RAWFileData>
+        <Version>1</Version>
+        <ObjectFileName>2020_PlantHaven_IPSRI_133-3_2021-rewash_109um_8Bit.raw</ObjectFileName>
+        <Format>UCHAR</Format>
+        <DataSlope>1</DataSlope>
+        <DataOffset>0</DataOffset>
+        <Unit>Density</Unit>
+        <Resolution X="1474" Y="1474" Z="1861" T="1" />
+        <Orientation X0="1" X1="0" X2="0" Y0="0" Y1="1" Y2="0" Z0="0" Z1="0" Z2="1" />
+        <Position P1="0" P2="0" P3="0" />
+</RAWFileData>

tests/test_supplements/poor_nsi.dat

@@ -0,0 +1,5 @@
+ObjectFileName: 2020_PlantHaven_IPSRI_133-3_2021-rewash_109um.raw
+Resolution:     1474 1474 1861
+SliceThickness: 0.108526441297249 0.108526441297249 0.108526441297249
+Format:         USHORT
+Object:    DENSITY