fix/csv with quotes

src/damast/core/constants.py

@@ -33,6 +33,5 @@
 
 DAMAST_CSV_DEFAULT_ARGS: dict[str, any] = {
     'null_values': ["None", "none", "Null", "null"],
-    'quote_char': None,
     'infer_schema_length': None
 }

src/damast/core/metadata.py

@@ -11,6 +11,7 @@
 import re
 import traceback
 import warnings
+from difflib import SequenceMatcher
 from enum import Enum
 from pathlib import Path
 from typing import Any, Dict, List, Optional, Union
@@ -1207,7 +1208,7 @@ def get_fulfillment(self, expected_specs: List[DataSpecification]) -> Fulfillmen
         return md_fulfillment
 
     @classmethod
-    def search(cls, files: list[str | Path]) -> tuple[MetaData | None, str | None]:
+    def search(cls, files: list[str | Path]) -> tuple[MetaData | None, str | None, dict[str, float]]:
         """
         Search for the metadata specfile for a given list of files
         """
@@ -1216,18 +1217,19 @@ def search(cls, files: list[str | Path]) -> tuple[MetaData | None, str | None]:
             commonpath = os.path.commonpath(files)
         except Exception as e:
             logger.debug(e)
-            return None, None, None
+            return None, None, {}
 
         if len(files) == 1:
             commonpath = Path(commonpath).parent
 
         commonprefix = os.path.commonprefix([Path(x).stem for x in files])
-        metadata_file_candidates = [x for x in Path(commonpath).glob(f"{commonprefix}*{DAMAST_SPEC_SUFFIX}")]
-        for f in metadata_file_candidates:
+        metadata_file_candidates = { x: SequenceMatcher(None, commonprefix, str(x.stem)).ratio() for x in Path(commonpath).glob(f"{commonprefix}*{DAMAST_SPEC_SUFFIX}") }
+        for f in dict(sorted(metadata_file_candidates.items(), key=lambda x: x[1], reverse=True)):
             try:
+                Path(f).stem == commonprefix
                 return MetaData.load_yaml(filename=f), f, metadata_file_candidates
             except Exception as e:
-                logger.debug(f"Loading {f} as metadata file failed -- {e}")
+                logger.warning(f"Loading {f} as metadata file failed -- {e}")
 
         return None, None, metadata_file_candidates
 

tests/damast/core/test_dataframe.py

@@ -177,6 +177,23 @@ def test_annotated_dataframe_import_csv(data_path):
         name="height", abbreviation="height", category=DataCategory.STATIC,
         unit=units.m, value_range=MinMax(min=0, max=40), representation_type=int)
 
+def test_annotated_dataframe_import_csv_with_quotes(data_path):
+    """
+    Simple test of the annotated dataframe import for csv
+    """
+    csv_path = data_path / "test_dataframe_with_quotes.csv"
+
+    adf = AnnotatedDataFrame.from_file(csv_path)
+    assert adf.column_names == ["id", "name"]
+    assert adf.dtype('id') == polars.Int64
+    assert adf.dtype('name') == polars.String
+
+    assert XDataFrame(adf._dataframe).equals(XDataFrame(polars.scan_csv(csv_path, null_values=["None", "none"])))
+
+    df = adf.dataframe.collect()
+    assert df[0,1] == "a,b;c"
+    assert df[1,1] == "d;e,f"
+
 def test_set_dtype(data_path):
     """
     Test if conversion from int -> str in representation_type is consistent

tests/damast/data/test_dataframe_with_quotes.csv

@@ -0,0 +1,3 @@
+id,name
+0,"a,b;c"
+1,"d;e,f"

tests/damast/data/test_dataframe_with_quotes.spec.yaml

@@ -0,0 +1,19 @@
+columns:
+- name: id
+  is_optional: false
+  abbreviation: id
+  category: static
+  value_range:
+    MinMax:
+      min: 0
+      max: 1
+      allow_missing: true
+  representation_type: int
+- name: name
+  is_optional: false
+  abbreviation: name
+  category: static
+  representation_type: str
+annotations:
+  license: MIT License
+  comment: test dataframe