Relax dependency constraints

.github/workflows/lint-format.yml

@@ -49,7 +49,7 @@ jobs:
           python-version: "3.12"
       - name: Install linters
         run: |
-          pip install black flake8 -c requirements-dev.txt
+          pip install black flake8
       - name: Run flake8
         run: |
           flake8 ${{needs.get_changed_files.outputs.py}} --count --select=E9,F63,F7,F82 --show-source --statistics

.github/workflows/test-unit.yml

@@ -16,8 +16,8 @@ jobs:
           python-version: "3.12"
       - name: Install requirements
         run: |
-          pip install -r requirements-dev.txt
-          pip install -e .
+          pip install -r requirements.txt
+          pip install --group dev -e .
       - name: Running Tests
         env:
           CDISC_LIBRARY_API_KEY: fakekey12341234

README.md

@@ -646,11 +646,7 @@ These steps should be run before running any tests or core commands using the no
 
 - Install the requirements:
 
-  ```bash
-  python -m pip install -r requirements-dev.txt
-  ```
-
-  Run this from the root directory.
+  `pip install -e . && pip install --group dev` # From the root directory
 
 ### Creating an executable version
 
@@ -724,7 +720,7 @@ py -m twine upload --repository {repository_name} dist/*
 
 This project uses the `black` code formatter, `flake8` linter for python and `prettier` for JSON, YAML and MD.
 It also uses `pre-commit` to run `black`, `flake8` and `prettier` when you commit.
-Both dependencies are added to _requirements-dev.txt_.
+Both dependencies are added to the `dev` dependency group in _pyproject.toml_.
 
 Setting up `pre-commit` requires one extra step. After installing it you have to run:
 

cdisc_rules_engine/dataset_builders/contents_define_vlm_dataset_builder.py

@@ -66,7 +66,7 @@ def build(self):
         data_contents_with_vlm["variable_value_length"] = data_contents_with_vlm.data[
             ["variable_value", "define_vlm_data_type"]
         ].apply(
-            lambda row: self.calculate_variable_value_length(
+            lambda row: ValuesDatasetBuilder.calculate_variable_value_length(
                 row["variable_value"], row["define_vlm_data_type"]
             ),
             axis=1,

cdisc_rules_engine/models/dataset/dask_dataset.py

@@ -81,9 +81,9 @@ def __setitem__(self, key, value):
             array_values = da.from_array(value, chunks=tuple(chunks))
             self._data[key] = array_values
         elif isinstance(value, pd.Series):
-            self._data = self._data.reset_index()
-            self._data = self._data.set_index("index")
-            self._data[key] = value
+            chunks = self._data.map_partitions(lambda x: len(x)).compute().to_numpy()
+            array_values = da.from_array(value.values, chunks=tuple(chunks))
+            self._data[key] = array_values
         elif isinstance(value, dd.DataFrame):
             for column in value:
                 self._data[column] = value[column]

cdisc_rules_engine/services/data_services/usdm_data_service.py

@@ -417,8 +417,19 @@ def __read_node_metadata(
         }
 
     @staticmethod
-    def __get_full_path(node: DatumInContext):
-        return f"{node.full_path}".replace(".[", "[")
+    def __get_full_path(node: DatumInContext) -> str:
+        parts = []
+        current = node
+        while current is not None and current.context is not None:
+            parts.append(str(current.path))
+            current = current.context
+        result = ""
+        for part in reversed(parts):
+            if part.startswith("["):
+                result += part
+            else:
+                result = (result + "." if result else "") + part
+        return result
 
     def __get_datasets_content_index(self) -> List[dict]:
         """

cdisc_rules_engine/services/datasetxpt_metadata_reader.py

@@ -61,7 +61,7 @@ def read(self) -> dict:
             "variable_labels": list(metadata.column_labels),
             "variable_names": list(metadata.column_names),
             "variable_formats": [
-                "" if data_type == "NULL" else data_type
+                "" if (data_type == "NULL" or data_type is None) else data_type
                 for data_type in metadata.original_variable_types.values()
             ],
             "variable_name_to_label_map": metadata.column_names_to_labels,

core.py

@@ -357,15 +357,13 @@ def load_custom_dotenv_from_data_options(ctx, param, value):
     "-s",
     "--standard",
     required=True,
-    default=None,
     help="CDISC standard to validate against",
     envvar="PRODUCT",
 )
 @click.option(
     "-v",
     "--version",
     required=True,
-    default=None,
     help="Standard version to validate against",
     envvar="VERSION",
 )

pyproject.toml

@@ -4,12 +4,49 @@ build-backend = "setuptools.build_meta"
 
 [project]
 name = "cdisc-rules-engine"
-dynamic = ["version", "dependencies"]
+dynamic = ["version"]
 description = "Open source offering of the cdisc rules engine"
 readme = "PYPI.md"
 requires-python = ">=3.12, <3.13"
 license = { text = "MIT" }
 authors = [{ name = "cdisc-org", email = "info@cdisc.org" }]
+dependencies = [
+    "business_rules_enhanced >=1.4.8",
+    "cachetools >=6.1.0",
+    "cdisc-library-client >=0.1.6",
+    "click >=8.1.7",
+    "dask[dataframe,array] >=2024.6.0",
+    "fastparquet >=2024.2.0",
+    "importlib-metadata >=8.5.0",
+    "jsonata-python >=0.6.0",
+    "jsonpath-ng >=1.6.1",
+    "jsonschema >=4.18.5",
+    "lxml >=5.2.1",
+    "numpy >=1.26.0",
+    "odmlib >=0.1.4",
+    "openpyxl >=3.1.5",
+    "pandas >=2.1.4, <3.0.0",
+    "psutil >=6.1.1",
+    "pyinstaller >=6.11.0",
+    "pympler >=1.1",
+    "pyreadstat >=1.2.7",
+    "python-dotenv >=1.0.0",
+    "pyyaml >=6.0.2",
+    "redis >=4.5.0",
+    "requests >=2.32.3",
+    "setuptools >=75.6.0",
+    "titlecase >=2.4.1",
+]
+
+[dependency-groups]
+dev = [
+    "black >=24.10.0",
+    "flake8 >=6.1.0",
+    "pre-commit >=2.20.0",
+    "pytest >=7.4.0, <8.0.0",
+    "pytest-asyncio >=0.21.0",
+    "pytest-cov >=6.0.0",
+]
 
 [project.urls]
 "Homepage" = "https://github.com/cdisc-org/cdisc-rules-engine"
@@ -26,5 +63,4 @@ include-package-data = true
 py-modules = ["version"]
 
 [tool.setuptools.dynamic]
-version = { attr = "version.__version__" }
-dependencies = {file = ["requirements.txt"]}
+version = { attr = "version.__version__" }

requirements.txt

@@ -1,3 +1,5 @@
+# Lockfile: exact pinned versions for reproducible installs.
+# Dependency constraints are defined in pyproject.toml.
 business_rules_enhanced==1.4.8
 cachetools==6.1.0
 cdisc-library-client==0.1.6

tests/unit/test_dataset_builders/test_dataset_metadata_define_dataset_builder.py

@@ -149,7 +149,7 @@ def test_dataset_metadata_define_dataset_builder(dataset_path):
             expected_results["dm.xpt"],
             expected_results["ae.xpt"],
         ]
-    ).astype(object)
+    ).astype(object).sort_values("dataset_location").reset_index(drop=True)
 
     result_df = result.data[expected_df.columns].reset_index(drop=True)