properly convert nan and datetime fields

Author: Max Wang

.claude/skills/dataverse-sdk-dev/SKILL.md

@@ -13,7 +13,7 @@ This skill provides guidance for developers working on the PowerPlatform Dataver
 
 ### API Design
 
-1. **All public methods in client.py** - Public API methods must be in client.py
+1. **client.py** - client.py only contains public API methods and all public methods must be in client.py
 2. **Every public method needs README example** - Public API methods must have examples in README.md
 3. **Reuse existing APIs** - Always check if an existing method can be used before making direct Web API calls
 4. **Update documentation** when adding features - Keep README and SKILL files (both copies) in sync

examples/advanced/dataframe_operations.py

@@ -40,15 +40,20 @@ def main():
     test_filter = f"contains(name,'{tag}')"
     print(f"[INFO] Using tag '{tag}' to identify test records")
 
+    select_cols = ["name", "telephone1", "websiteurl", "lastonholdtime"]
+
     # ── 1. Create records from a DataFrame ────────────────────────
     print("\n" + "-" * 60)
     print("1. Create records from a DataFrame")
     print("-" * 60)
 
     new_accounts = pd.DataFrame([
-        {"name": f"Contoso_{tag}", "telephone1": "555-0100", "websiteurl": "https://contoso.com"},
-        {"name": f"Fabrikam_{tag}", "telephone1": "555-0200", "websiteurl": "https://fabrikam.com"},
-        {"name": f"Northwind_{tag}", "telephone1": "555-0300", "websiteurl": "https://northwind.com"},
+        {"name": f"Contoso_{tag}", "telephone1": "555-0100", "websiteurl": "https://contoso.com",
+         "lastonholdtime": pd.Timestamp("2024-06-15 10:30:00")},
+        {"name": f"Fabrikam_{tag}", "telephone1": "555-0200", "websiteurl": None,
+         "lastonholdtime": None},
+        {"name": f"Northwind_{tag}", "telephone1": None, "websiteurl": "https://northwind.com",
+         "lastonholdtime": pd.Timestamp("2024-12-01 08:00:00")},
     ])
     print(f"  Input DataFrame:\n{new_accounts.to_string(index=False)}\n")
 
@@ -63,7 +68,7 @@ def main():
     print("-" * 60)
 
     page_count = 0
-    for df_page in client.get_dataframe(table, select=["name", "telephone1"], filter=test_filter, page_size=2):
+    for df_page in client.get_dataframe(table, select=select_cols, filter=test_filter, page_size=2):
         page_count += 1
         print(f"  Page {page_count} ({len(df_page)} records):\n{df_page.to_string(index=False)}")
 
@@ -73,7 +78,7 @@ def main():
     print("-" * 60)
 
     all_records = pd.concat(
-        client.get_dataframe(table, select=["name", "telephone1"], filter=test_filter, page_size=2),
+        client.get_dataframe(table, select=select_cols, filter=test_filter, page_size=2),
         ignore_index=True,
     )
     print(f"[OK] Got {len(all_records)} total records in one DataFrame")
@@ -87,7 +92,7 @@ def main():
 
     first_id = new_accounts["accountid"].iloc[0]
     print(f"  Fetching record {first_id}...")
-    single = client.get_dataframe(table, record_id=first_id, select=["name", "telephone1"])
+    single = client.get_dataframe(table, record_id=first_id, select=select_cols)
     print(f"[OK] Single record DataFrame:\n{single.to_string(index=False)}")
 
     # ── 5. Update records from a DataFrame ────────────────────────
@@ -101,7 +106,7 @@ def main():
     print("[OK] Updated 3 records")
 
     # Verify the updates with a bulk get
-    verified = next(client.get_dataframe(table, select=["name", "telephone1"], filter=test_filter))
+    verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
     print(f"  Verified:\n{verified.to_string(index=False)}")
 
     # ── 6. Broadcast update (same value to all records) ───────────
@@ -116,7 +121,7 @@ def main():
     print("[OK] Broadcast update complete")
 
     # Verify all records have the same websiteurl
-    verified = next(client.get_dataframe(table, select=["name", "websiteurl"], filter=test_filter))
+    verified = next(client.get_dataframe(table, select=select_cols, filter=test_filter))
     print(f"  Verified:\n{verified.to_string(index=False)}")
 
     # ── 7. Delete records by passing a Series of GUIDs ────────────
@@ -129,7 +134,7 @@ def main():
     print(f"[OK] Deleted {len(new_accounts)} records")
 
     # Verify deletions - filter for our tagged records should return 0
-    remaining = list(client.get_dataframe(table, select=["name"], filter=test_filter))
+    remaining = list(client.get_dataframe(table, select=select_cols, filter=test_filter))
     count = sum(len(page) for page in remaining)
     print(f"  Verified: {count} test records remaining (expected 0)")
 

src/PowerPlatform/Dataverse/claude_skill/dataverse-sdk-dev/SKILL.md

@@ -13,7 +13,7 @@ This skill provides guidance for developers working on the PowerPlatform Dataver
 
 ### API Design
 
-1. **All public methods in client.py** - Public API methods must be in client.py
+1. **client.py** - client.py only contains public API methods and all public methods must be in client.py
 2. **Every public method needs README example** - Public API methods must have examples in README.md
 3. **Reuse existing APIs** - Always check if an existing method can be used before making direct Web API calls
 4. **Update documentation** when adding features - Keep README and SKILL files (both copies) in sync

src/PowerPlatform/Dataverse/client.py

@@ -13,6 +13,7 @@
 from .core._auth import _AuthManager
 from .core.config import DataverseConfig
 from .data._odata import _ODataClient
+from .utils._pandas import dataframe_to_records
 
 
 class DataverseClient:
@@ -472,7 +473,7 @@ def create_dataframe(
         if not isinstance(records, pd.DataFrame):
             raise TypeError("records must be a pandas DataFrame")
 
-        record_list = records.to_dict(orient="records")
+        record_list = dataframe_to_records(records)
         ids = self.create(table_schema_name, record_list)
         return pd.Series(ids, index=records.index)
 
@@ -522,7 +523,7 @@ def update_dataframe(
 
         ids = records[id_column].tolist()
         change_columns = [column for column in records.columns if column != id_column]
-        changes = records[change_columns].to_dict(orient="records")
+        changes = dataframe_to_records(records[change_columns])
 
         if len(ids) == 1:
             self.update(table_schema_name, ids[0], changes[0])

src/PowerPlatform/Dataverse/utils/_pandas.py

@@ -0,0 +1,22 @@
+# Copyright (c) Microsoft Corporation.
+# Licensed under the MIT license.
+
+"""Internal pandas helpers"""
+
+from __future__ import annotations
+
+from typing import Any, Dict, List
+
+import pandas as pd
+
+
+def dataframe_to_records(df: pd.DataFrame) -> List[Dict[str, Any]]:
+    """Convert a DataFrame to a list of dicts, dropping NaN values and converting Timestamps to ISO strings."""
+    records = []
+    for row in df.to_dict(orient="records"):
+        clean = {}
+        for k, v in row.items():
+            if pd.notna(v):
+                clean[k] = v.isoformat() if isinstance(v, pd.Timestamp) else v
+        records.append(clean)
+    return records

tests/unit/test_client_dataframe.py

@@ -187,6 +187,36 @@ def test_create_empty_dataframe(self):
         self.assertIsInstance(ids, pd.Series)
         self.assertEqual(len(ids), 0)
 
+    def test_create_drops_nan_values(self):
+        """NaN values are stripped from the payload (field omitted, not sent as NaN)."""
+        df = pd.DataFrame([
+            {"name": "Contoso", "telephone1": "555-0100"},
+            {"name": "Fabrikam", "telephone1": None},
+        ])
+        self.client._odata._create_multiple.return_value = ["guid-1", "guid-2"]
+        self.client._odata._entity_set_from_schema_name.return_value = "accounts"
+
+        self.client.create_dataframe("account", df)
+
+        call_args = self.client._odata._create_multiple.call_args
+        records_arg = call_args[0][2]
+        self.assertEqual(records_arg[0], {"name": "Contoso", "telephone1": "555-0100"})
+        self.assertEqual(records_arg[1], {"name": "Fabrikam"})
+        self.assertNotIn("telephone1", records_arg[1])
+
+    def test_create_converts_timestamps_to_iso(self):
+        """Timestamp values are converted to ISO 8601 strings."""
+        ts = pd.Timestamp("2024-01-15 10:30:00")
+        df = pd.DataFrame([{"name": "Contoso", "createdon": ts}])
+        self.client._odata._create_multiple.return_value = ["guid-1"]
+        self.client._odata._entity_set_from_schema_name.return_value = "accounts"
+
+        self.client.create_dataframe("account", df)
+
+        call_args = self.client._odata._create_multiple.call_args
+        records_arg = call_args[0][2]
+        self.assertEqual(records_arg[0]["createdon"], "2024-01-15T10:30:00")
+
 
 class TestDataFrameUpdate(unittest.TestCase):
     """Tests for update_dataframe."""
@@ -246,6 +276,20 @@ def test_update_multiple_change_columns(self):
         self.assertIn("telephone1", changes)
         self.assertNotIn("accountid", changes)
 
+    def test_update_drops_nan_from_changes(self):
+        """NaN values in change columns are stripped from the payload."""
+        df = pd.DataFrame([
+            {"accountid": "guid-1", "name": "New Name", "telephone1": None},
+            {"accountid": "guid-2", "name": None, "telephone1": "555-0200"},
+        ])
+
+        self.client.update_dataframe("account", df, id_column="accountid")
+
+        call_args = self.client._odata._update_by_ids.call_args[0]
+        changes = call_args[2]
+        self.assertEqual(changes[0], {"name": "New Name"})
+        self.assertEqual(changes[1], {"telephone1": "555-0200"})
+
 
 class TestDataFrameDelete(unittest.TestCase):
     """Tests for delete_dataframe."""