1316: New rule - value against library metadata

cdisc_rules_engine/dataset_builders/contents_library_variables_dataset_builder.py

@@ -0,0 +1,33 @@
+from cdisc_rules_engine.dataset_builders.values_dataset_builder import (
+    ValuesDatasetBuilder,
+)
+
+
+class ContentsLibraryVariablesDatasetBuilder(ValuesDatasetBuilder):
+    def build(self):
+        """
+        Returns a long dataset where each value in each row of the original dataset is
+        a row in the new dataset.
+        Library metadata corresponding to each row's variable is
+        attached to each row.
+        Columns available in the dataset are:
+        "row_number",
+        "variable_name",
+        "variable_value",
+        "library_variable_name",
+        "library_variable_label",
+        "library_variable_role",
+        "library_variable_ccode",
+        ...,
+        """
+        # get dataset contents and convert it from wide to long
+        data_contents_long_df = ValuesDatasetBuilder.build(self)
+        variables_metadata = self.get_library_variables_metadata()
+        # merge dataset contents with library variable metadata
+        merged = data_contents_long_df.merge(
+            variables_metadata.data,
+            how="outer",
+            left_on="variable_name",
+            right_on="library_variable_name",
+        )
+        return merged

cdisc_rules_engine/dataset_builders/dataset_builder_factory.py

@@ -1,5 +1,8 @@
 from typing import Type
 
+from cdisc_rules_engine.dataset_builders.contents_library_variables_dataset_builder import (
+    ContentsLibraryVariablesDatasetBuilder,
+)
 from cdisc_rules_engine.dataset_builders.json_schema_check_dataset_builder import (
     JsonSchemaCheckDatasetBuilder,
 )
@@ -75,6 +78,7 @@ class DatasetBuilderFactory(FactoryInterface):
         RuleTypes.VALUE_CHECK_WITH_VARIABLE_METADATA.value: ValueCheckVariableMetadataDatasetBuilder,
         RuleTypes.JSONATA.value: JSONataDatasetBuilder,
         RuleTypes.JSON_SCHEMA_CHECK.value: JsonSchemaCheckDatasetBuilder,
+        RuleTypes.VALUE_CHECK_AGAINST_LIBRARY.value: ContentsLibraryVariablesDatasetBuilder,
     }
 
     @classmethod

cdisc_rules_engine/enums/rule_types.py

@@ -20,6 +20,7 @@ class RuleTypes(BaseEnum):
         "Variable Metadata Check against Library Metadata"
     )
     VALUE_CHECK_AGAINST_DEFINE_XML_VARIABLE = "Value Check against Define XML Variable"
+    VALUE_CHECK_AGAINST_LIBRARY = "Value Check against Library Metadata"
     VALUE_CHECK_AGAINST_DEFINE_XML_VLM = "Value Check against Define XML VLM"
     DEFINE_ITEM_METADATA_CHECK_AGAINST_LIBRARY = (
         "Define Item Metadata Check against Library Metadata"

resources/schema/rule-merged/Rule_Type.json

@@ -57,6 +57,11 @@
       "title": "Content data at value level and define xml metadata at value level",
       "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `define_vlm_name`\n- `define_vlm_label`\n- `define_vlm_data_type`\n- `define_vlm_is_collected`\n- `define_vlm_role`\n- `define_vlm_size`\n- `define_vlm_ccode`\n- `define_vlm_format`\n- `define_vlm_allowed_terms`\n- `define_vlm_origin_type`\n- `define_vlm_has_no_data`\n- `define_vlm_order_number`\n- `define_vlm_length`\n- `define_vlm_has_codelist`\n- `define_vlm_codelist_coded_values`\n- `define_vlm_mandatory`\n- `define_variable_name`\n- `type_check`\n- `length_check`\n- `variable_value_length`\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with the matching Define-XML Value Level Metadata (VLM) joined to each row. Only rows where VLM exists for that variable are produced \u2014 records without matching VLM are excluded.\n\n#### Example\n\n```yaml\nall:\n  - name: define_vlm_ccode\n    operator: empty\n  - name: variable_value\n    operator: non_empty\n  - name: define_vlm_has_codelist\n    operator: equal_to\n    value: true\n  - name: variable_value\n    operator: is_not_contained_by\n    value: define_vlm_codelist_coded_values\n```\n\n```yaml\nall:\n  - name: variable_value\n    operator: empty\n  - name: define_vlm_mandatory\n    operator: equal_to\n    value: Yes\n```\n"
     },
+    {
+      "const": "Value Check against Library Metadata",
+      "title": "Content data at value level and Library Metadata at value level",
+      "markdownDescription": "\n#### Columns\n\n- `row_number`\n- `variable_name`\n- `variable_value`\n- `library_variable__links`\n- `library_variable_core`\n- `library_variable_description`\n- `library_variable_label`\n- `library_variable_name`\n- `library_variable_ordinal`\n- `library_variable_role`\n- `library_variable_simpleDatatype`\n- `library_variable_ccode`\n- `library_variable_has_codelist`\n- `library_variable_valueList`\n- `library_variable_definition`\n- `library_variable_notes`\n- `library_variable_variableCcode`\n- `library_variable_examples`\n- `library_variable_usageRestrictions`\n- `library_variable_describedValueDomain`\n\n#### Rule Macro\n\nChecks individual cell values from submission dataset contents pivoted to a long format, with the matching Library Metadata joined to each row.\n\n#### Example\n\n```yaml\nall:\n  - name: variable_value\n    operator: non_empty\n  - name: library_variable_has_codelist\n    operator: equal_to\n    value: true\n```\n"
+    },
     {
       "const": "Variable Metadata Check",
       "title": "Content metadata at variable level",

resources/schema/rule/Rule_Type.json

@@ -46,6 +46,10 @@
       "const": "Value Check against Define XML VLM",
       "title": "Content data at value level and define xml metadata at value level"
     },
+    {
+      "const": "Value Check against Library Metadata",
+      "title": "Content data at value level and Library Metadata at value level"
+    },
     {
       "const": "Variable Metadata Check",
       "title": "Content metadata at variable level"

resources/schema/rule/Rule_Type.md

@@ -27,6 +27,7 @@ The combination of data in each rule type.
 |                       | [Value Check with Variable Metadata](#value-check-with-variable-metadata)                                                             |
 |                       | [Value Check against Define XML Variable](#value-check-against-define-xml-variable)                                                   |
 |                       | [Value Check against Define XML VLM](#value-check-against-define-xml-vlm)                                                             |
+|                       | [Value Check against Library Metadata](#value-check-against-library-metadata)                                                         |
 | **Variable Metadata** | [Variable Metadata Check](#variable-metadata-check)                                                                                   |
 |                       | [Variable Metadata Check against Define XML](#variable-metadata-check-against-define-xml)                                             |
 |                       | [Variable Metadata Check against Library Metadata](#variable-metadata-check-against-library-metadata)                                 |
@@ -474,6 +475,46 @@ all:
     value: Yes
 ```
 
+## Value Check against Library Metadata
+
+#### Columns
+
+- `row_number`
+- `variable_name`
+- `variable_value`
+- `library_variable__links`
+- `library_variable_core`
+- `library_variable_description`
+- `library_variable_label`
+- `library_variable_name`
+- `library_variable_ordinal`
+- `library_variable_role`
+- `library_variable_simpleDatatype`
+- `library_variable_ccode`
+- `library_variable_has_codelist`
+- `library_variable_valueList`
+- `library_variable_definition`
+- `library_variable_notes`
+- `library_variable_variableCcode`
+- `library_variable_examples`
+- `library_variable_usageRestrictions`
+- `library_variable_describedValueDomain`
+
+#### Rule Macro
+
+Checks individual cell values from submission dataset contents pivoted to a long format, with the matching Library Metadata joined to each row.
+
+#### Example
+
+```yaml
+all:
+  - name: variable_value
+    operator: non_empty
+  - name: library_variable_has_codelist
+    operator: equal_to
+    value: true
+```
+
 ## Variable Metadata Check against Define XML
 
 #### Columns

tests/QARegressionTests/test_Issues/test_CoreIssue1316.py

@@ -0,0 +1,58 @@
+import os
+import subprocess
+import unittest
+
+import pytest
+import json
+from conftest import get_python_executable
+
+
+@pytest.mark.regression
+class TestCoreIssue1316(unittest.TestCase):
+    def test_new_rule(self):
+        # Run the command in the terminal
+        command = [
+            f"{get_python_executable()}",
+            "-m",
+            "core",
+            "validate",
+            "-s",
+            "sdtmig",
+            "-v",
+            "3-4",
+            "-dp",
+            os.path.join(
+                "tests",
+                "resources",
+                "CoreIssue1316",
+                "test_dataset.json",
+            ),
+            "-lr",
+            os.path.join("tests", "resources", "CoreIssue1316", "rule_new.yml"),
+            "-ps",
+            "1",
+            "-of",
+            "json",
+        ]
+        subprocess.run(command, check=True)
+
+        files = os.listdir()
+        json_files = [
+            file
+            for file in files
+            if file.startswith("CORE-Report-") and file.endswith(".json")
+        ]
+        json_report_path = sorted(json_files)[-1]
+        json_report = json.load(open(json_report_path))
+        assert {
+            "Conformance_Details",
+            "Dataset_Details",
+            "Issue_Summary",
+            "Issue_Details",
+            "Rules_Report",
+        }.issubset(json_report.keys())
+        assert json_report["Issue_Summary"][0]["issues"] == 591
+        assert json_report["Rules_Report"][0]["status"] == "ISSUE REPORTED"
+
+        if os.path.exists(json_report_path):
+            os.remove(json_report_path)

tests/resources/CoreIssue1316/rule_new.yml

@@ -0,0 +1,45 @@
+Authorities:
+  - Organization: CDISC
+    Standards:
+      - Name: SDTMIG
+        References:
+          - Citations:
+              - Cited Guidance:
+                  Each domain dataset is distinguished by a unique, two-character
+                  code
+                Document: IG v3.4
+                Section: "2.2"
+            Origin: SDTM and SDTMIG Conformance Rules
+            Rule Identifier:
+              Id: CORE-EX-ROUTE-001
+              Version: "1"
+            Version: "2.0"
+        Version: "3.4"
+Check:
+  all:
+    - name: variable_name
+      operator: equal_to
+      value: EXROUTE
+    - name: library_variable_ccode
+      operator: non_empty
+
+Core:
+  Id: CORE-EX-ROUTE-001
+  Status: Draft
+  Version: "1"
+Description: test.
+Executability: Fully Executable
+
+Outcome:
+  Message: test.
+  Output Variables:
+    - EXROUTE
+Rule Type: Value Check against Library Metadata
+Scope:
+  Classes:
+    Include:
+      - ALL
+  Domains:
+    Include:
+      - ALL
+Sensitivity: Record