feat: add new fields regarding NULL values

jira: CQ-1959
risk: low

Author: fiedlr

packages/gooddata-sdk/src/gooddata_sdk/catalog/data_source/action_model/requests/ldm_request.py

@@ -51,6 +51,7 @@ class CatalogGenerateLdmRequest(Base):
     wdf_prefix: Optional[str] = None
     pdm: Optional[CatalogPdmLdmRequest] = None
     workspace_id: Optional[str] = None
+    translation_prefix: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[GenerateLdmRequest]:

packages/gooddata-sdk/src/gooddata_sdk/catalog/data_source/declarative_model/physical_model/column.py

@@ -16,6 +16,8 @@ class CatalogDeclarativeColumn(Base):
     is_primary_key: Optional[bool] = None
     referenced_table_id: Optional[str] = None
     referenced_table_column: Optional[str] = None
+    is_nullable: Optional[bool] = None
+    null_value: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[DeclarativeColumn]:

packages/gooddata-sdk/src/gooddata_sdk/catalog/workspace/declarative_model/workspace/logical_model/dataset/dataset.py

@@ -82,6 +82,8 @@ class CatalogDeclarativeAttribute(Base):
     tags: Optional[list[str]] = None
     is_hidden: Optional[bool] = None
     locale: Optional[str] = None
+    is_nullable: Optional[bool] = None
+    null_value: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[DeclarativeAttribute]:
@@ -97,6 +99,8 @@ class CatalogDeclarativeFact(Base):
     description: Optional[str] = None
     tags: Optional[list[str]] = None
     is_hidden: Optional[bool] = None
+    is_nullable: Optional[bool] = None
+    null_value: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[DeclarativeFact]:
@@ -121,6 +125,8 @@ class CatalogDeclarativeAggregatedFact(Base):
     source_column_data_type: Optional[str] = None
     description: Optional[str] = None
     tags: Optional[list[str]] = None
+    is_nullable: Optional[bool] = None
+    null_value: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[DeclarativeAggregatedFact]:
@@ -171,6 +177,8 @@ class CatalogDeclarativeLabel(Base):
     locale: Optional[str] = None
     translations: Optional[list[CatalogDeclarativeLabelTranslation]] = None
     geo_area_config: Optional[CatalogGeoAreaConfig] = None
+    is_nullable: Optional[bool] = None
+    null_value: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[DeclarativeLabel]:
@@ -203,6 +211,8 @@ class CatalogDeclarativeReference(Base):
     source_columns: Optional[list[str]] = None
     source_column_data_types: Optional[list[str]] = None
     sources: Optional[list[CatalogDeclarativeReferenceSource]] = None
+    is_nullable: Optional[bool] = None
+    null_value: Optional[str] = None
 
     @staticmethod
     def client_class() -> type[DeclarativeReference]:

packages/gooddata-sdk/tests/catalog/expected/declarative_workspaces.json

@@ -2597,13 +2597,6 @@
                       "id": "geo__state__location",
                       "sourceColumn": "geo__state__location",
                       "sourceColumnDataType": "STRING",
-                      "locale": "en-US",
-                      "translations": [
-                        {
-                          "locale": "cs-CZ",
-                          "sourceColumn": "geo__state__location_cz"
-                        }
-                      ],
                       "tags": [
                         "Customers"
                       ],

packages/gooddata-sdk/tests/catalog/test_catalog_data_source.py

@@ -44,6 +44,7 @@
 )
 from gooddata_sdk.catalog.data_source.entity_model.data_source import DatabaseAttributes
 from gooddata_sdk.catalog.entity import ClientSecretCredentialsFromFile
+from tests_support.compare_utils import deep_eq
 from tests_support.file_utils import load_json
 from tests_support.vcrpy_utils import get_vcr
 
@@ -78,7 +79,11 @@ def test_generate_logical_model(test_config: dict):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     declarative_model = sdk.catalog_workspace_content.get_declarative_ldm(test_config["workspace"])
     generate_ldm_request = CatalogGenerateLdmRequest(
-        separator="__", wdf_prefix="wdf", workspace_id=test_config["workspace"], pdm=pdm_ldm_request
+        separator="__",
+        wdf_prefix="wdf",
+        workspace_id=test_config["workspace"],
+        pdm=pdm_ldm_request,
+        translation_prefix="tr",
     )
     generated_declarative_model = sdk.catalog_data_source.generate_logical_model(
         test_config["data_source"], generate_ldm_request
@@ -93,7 +98,11 @@ def test_generate_logical_model(test_config: dict):
     """
     # Filter out SQL-based datasets (those have sql property set, no data_source_table_id)
     table_based_datasets = [ds for ds in declarative_model.ldm.datasets if ds.sql is None]
-    assert table_based_datasets == generated_declarative_model.ldm.datasets
+
+    for i, dataset in enumerate(table_based_datasets):
+        print(f"Dataset: {dataset.id}")
+        assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i])
+
     assert len(declarative_model.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances)
 
 
@@ -102,7 +111,7 @@ def test_scan_pdm_and_generate_logical_model(test_config: dict):
     sdk = GoodDataSdk.create(host_=test_config["host"], token_=test_config["token"])
     declarative_model = sdk.catalog_workspace_content.get_declarative_ldm(test_config["workspace"])
     generate_ldm_request = CatalogGenerateLdmRequest(
-        separator="__", wdf_prefix="wdf", workspace_id=test_config["workspace"]
+        separator="__", wdf_prefix="wdf", workspace_id=test_config["workspace"], translation_prefix="tr"
     )
     generated_declarative_model, _ = sdk.catalog_data_source.scan_pdm_and_generate_logical_model(
         test_config["data_source"], generate_ldm_request
@@ -117,7 +126,10 @@ def test_scan_pdm_and_generate_logical_model(test_config: dict):
     """
     # Filter out SQL-based datasets (those have sql property set, no data_source_table_id)
     table_based_datasets = [ds for ds in declarative_model.ldm.datasets if ds.sql is None]
-    assert table_based_datasets == generated_declarative_model.ldm.datasets
+    for i, dataset in enumerate(table_based_datasets):
+        print(f"Dataset: {dataset.id}")
+        assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i], [r"\['is_nullable'\]"])
+
     assert len(declarative_model.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances)
 
 
@@ -182,7 +194,9 @@ def test_generate_logical_model_with_sql_datasets(test_config: dict):
     #  and remove sort once fixed
     generated_declarative_model.ldm.datasets.sort(key=lambda dataset: dataset.id)
     expected_ldm.ldm.datasets.sort(key=lambda dataset: dataset.id)
-    assert expected_ldm.ldm.datasets == generated_declarative_model.ldm.datasets
+    for i, dataset in enumerate(expected_ldm.ldm.datasets):
+        print(f"Dataset: {dataset.id}")
+        assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i])
     assert len(expected_ldm.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances)
 
 
@@ -197,6 +211,7 @@ def test_scan_pdm_and_generate_logical_model_with_sql_datasets(test_config: dict
         pdm=CatalogPdmLdmRequest(
             sqls=build_pdm_sql_datasets(),
         ),
+        translation_prefix="tr",
     )
     generated_declarative_model, scan_result = sdk.catalog_data_source.scan_pdm_and_generate_logical_model(
         test_config["data_source"], ldm_request
@@ -208,7 +223,9 @@ def test_scan_pdm_and_generate_logical_model_with_sql_datasets(test_config: dict
     #  and remove sort once fixed
     generated_declarative_model.ldm.datasets.sort(key=lambda dataset: dataset.id)
     expected_ldm.ldm.datasets.sort(key=lambda dataset: dataset.id)
-    assert expected_ldm.ldm.datasets == generated_declarative_model.ldm.datasets
+    for i, dataset in enumerate(expected_ldm.ldm.datasets):
+        print(f"Dataset {dataset.id}")
+        assert deep_eq(dataset, generated_declarative_model.ldm.datasets[i], [r"\['is_nullable'\]"])
     assert len(expected_ldm.ldm.date_instances) == len(generated_declarative_model.ldm.date_instances)
 
 

packages/tests-support/src/tests_support/compare_utils.py

@@ -4,9 +4,9 @@
 from deepdiff import DeepDiff
 
 
-def deep_eq(expected: any, actual: any) -> bool:
+def deep_eq(expected: any, actual: any, exclude_regex_paths: list[str] | None = None) -> bool:
     if expected != actual:
-        print(DeepDiff(expected, actual))
+        print(DeepDiff(expected, actual, exclude_regex_paths=exclude_regex_paths))
         return False
 
     return True