Block SELECT * queries with ValidationError — intentional design decision to prevent expensive wildcard selects on wide entities

Author: Samson Gebre

README.md

@@ -400,9 +400,6 @@ results = client.query.sql(
     "GROUP BY a.name"
 )
 
-# SELECT * is auto-expanded by the SDK
-results = client.query.sql("SELECT * FROM account")
-
 # SQL results directly as a DataFrame
 df = client.dataframe.sql(
     "SELECT name, revenue FROM account ORDER BY revenue DESC"

examples/README.md

@@ -45,7 +45,6 @@ Deep-dive into production-ready patterns and specialized functionality:
   - Full SQL capabilities: SELECT, WHERE, TOP, ORDER BY, LIKE, IN, BETWEEN
   - JOINs (INNER, LEFT, multi-table), GROUP BY, DISTINCT, aggregates
   - OFFSET FETCH for server-side pagination
-  - SELECT * auto-expansion (SDK rewrites for server compatibility)
   - Polymorphic lookups via SQL (ownerid, customerid, createdby)
   - SQL read -> DataFrame transform -> SDK write-back (full round-trip)
   - SQL-driven bulk create, update, and delete patterns

examples/advanced/sql_examples.py

@@ -9,7 +9,7 @@
 based on extensive testing of the Dataverse SQL endpoint (353 test queries).
 
 Capabilities PROVEN to work:
-- SELECT with specific columns, SELECT * (auto-expanded by SDK)
+- SELECT with specific columns
 - INNER JOIN, LEFT JOIN (up to 6+ tables)
 - COUNT(*), SUM(), AVG(), MIN(), MAX() aggregates
 - GROUP BY, DISTINCT, DISTINCT TOP
@@ -304,19 +304,23 @@ def _run_examples(client):
             print(f"  {r.get('new_code', ''):<12s}  Budget={r.get('new_budget')}  Active={r.get('new_active')}")
 
         # ==============================================================
-        # 5. SELECT * (auto-expanded by SDK)
+        # 5. SELECT * -- Rejected by Design
         # ==============================================================
-        heading(5, "SELECT * (Auto-Expanded by SDK)")
+        heading(5, "SELECT * -- Rejected by Design")
         print(
-            "The server blocks SELECT * directly. The SDK auto-resolves\n"
-            "all column names via list_columns() and rewrites the query."
+            "SELECT * is deliberately rejected -- not a server workaround,\n"
+            "but an intentional design decision. Wide entities (e.g. account\n"
+            "has 307 columns) make SELECT * extremely expensive on shared\n"
+            "infrastructure. Specify columns explicitly instead.\n"
+            "Use client.query.sql_columns('account') to discover column names."
         )
-        sql = f"SELECT * FROM {parent_table}"
-        log_call(f'client.query.sql("{sql}")')
-        results = backoff(lambda: client.query.sql(sql))
-        if results:
-            keys = [k for k in results[0].keys() if not k.startswith("@")]
-            print(f"[OK] {len(results)} rows, {len(keys)} columns")
+        from PowerPlatform.Dataverse.core.errors import ValidationError as _VE
+
+        try:
+            client.query.sql(f"SELECT * FROM {parent_table}")
+            print("[UNEXPECTED] SELECT * did not raise -- check SDK version")
+        except _VE as exc:
+            print(f"[OK] ValidationError raised as expected: {exc}")
 
         # ==============================================================
         # 6. WHERE clause
@@ -1102,19 +1106,15 @@ def _run_examples(client):
          can have 5000+ rows. Unfiltered queries return max rows.
    FIX:  Always add WHERE filters and TOP when querying system tables.
 
-4. SELECT * ON WIDE TABLES
-   BAD:  SELECT * FROM account  (307 columns!)
-   WHY:  The SDK auto-expands * into all 260+ non-virtual columns.
-         Every column is transferred over the network.
-   NOTE: With JOINs, SELECT * only expands the FIRST (FROM) table's
-         columns -- joined table columns will NOT be included.
-         Example: SELECT * FROM account a JOIN contact c ON ...
-         expands to account columns only; contact columns are missing.
+4. SELECT * (BLOCKED -- ValidationError)
+   BAD:  SELECT * FROM account
+   WHY:  SELECT * is intentionally rejected -- not a technical limitation.
+         Wide entities (account has 307 columns) make wildcard selects
+         extremely expensive on shared database infrastructure.
    FIX:  List only the columns you need: SELECT name, revenue FROM account
-         Or use the SDK helper:
-           cols = client.query.sql_select("account")
-           sql = f"SELECT TOP 10 {{cols}} FROM account"
-         For JOINs, always specify columns from each table explicitly:
+         Or discover columns first:
+           cols = client.query.sql_columns("account")
+         For JOINs, always qualify columns from each table:
            SELECT a.name, c.fullname FROM account a JOIN contact c ON ...
 
 5. DEEP JOINS WITHOUT TOP
@@ -1125,10 +1125,10 @@ def _run_examples(client):
    FIX:  Always include TOP N for multi-table JOINs.
 
 SDK guardrails:
-  - Patterns #1 (writes) and unsupported syntax (CROSS/RIGHT/FULL JOIN,
-    UNION, HAVING, CTE, subqueries) -> ValidationError (blocked).
-  - Pattern #2 (cartesian FROM a, b) and #4 (SELECT * + JOIN)
-    -> UserWarning (advisory).
+  - Patterns #1 (writes), unsupported syntax (CROSS/RIGHT/FULL JOIN,
+    UNION, HAVING, CTE, subqueries), and #4 (SELECT *)
+    -> ValidationError (blocked).
+  - Pattern #2 (cartesian FROM a, b) -> UserWarning (advisory).
   - Server enforces 5000-row cap on all queries (#3, #5).
   - Use sql_columns() or sql_select() to discover valid column names.
   - Use sql_joins() or sql_join() to discover valid JOIN clauses.
@@ -1143,7 +1143,7 @@ def _run_examples(client):
 | Feature                       | SQL      | Notes / SDK Fallback                   |
 +-------------------------------+----------+----------------------------------------+
 | SELECT col1, col2             | YES      | Use LogicalName (lowercase)            |
-| SELECT *                      | YES (*)  | SDK auto-expands via list_columns()    |
+| SELECT *                      | NO       | Specify columns explicitly             |
 | WHERE =, !=, >, <, LIKE, IN   | YES      |                                        |
 | AND, OR, parentheses          | YES      | Full boolean logic                     |
 | NOT IN, NOT LIKE              | YES      |                                        |

src/PowerPlatform/Dataverse/data/_odata.py

@@ -791,12 +791,6 @@ def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[st
                 yield [x for x in items if isinstance(x, dict)]
             next_link = data.get("@odata.nextLink") or data.get("odata.nextLink") if isinstance(data, dict) else None
 
-    # ----------------------- SELECT * detection -----------------------
-    _SELECT_STAR_RE = re.compile(
-        r"\bSELECT\b(\s+(?:DISTINCT\s+)?(?:TOP\s+\d+(?:\s+PERCENT)?\s+)?)\*\s",
-        re.IGNORECASE,
-    )
-
     # ----------------------- SQL guardrail patterns --------------------
     _SQL_WRITE_RE = re.compile(
         r"^\s*(?:INSERT|UPDATE|DELETE|DROP|TRUNCATE|ALTER|CREATE|EXEC|GRANT|REVOKE|BULK)\b",
@@ -808,7 +802,6 @@ def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[st
         r"\bFROM\s+[A-Za-z0-9_]+(?:\s+[A-Za-z0-9_]+)?\s*,\s*[A-Za-z0-9_]+",
         re.IGNORECASE,
     )
-    _SQL_HAS_JOIN_RE = re.compile(r"\bJOIN\b", re.IGNORECASE)
     # Server-blocked SQL patterns (save the round-trip by catching early)
     _SQL_UNSUPPORTED_JOIN_RE = re.compile(
         r"\b(?:CROSS\s+JOIN|RIGHT\s+(?:OUTER\s+)?JOIN|FULL\s+(?:OUTER\s+)?JOIN)\b",
@@ -821,44 +814,14 @@ def _do_request(url: str, *, params: Optional[Dict[str, Any]] = None) -> Dict[st
         r"\bIN\s*\(\s*SELECT\b|\bEXISTS\s*\(\s*SELECT\b|\(\s*SELECT\b.*\bFROM\b",
         re.IGNORECASE,
     )
-
-    def _expand_select_star(self, sql: str, table: str) -> str:
-        """Replace ``SELECT *`` with explicit column names.
-
-        When the Dataverse SQL endpoint receives ``SELECT *`` it returns
-        an error ("SELECT * is not supported").  This helper resolves all
-        columns via ``_list_columns`` and rewrites the query so the user
-        never has to know the server limitation.
-
-        For JOIN queries, the expansion only includes columns from the first
-        (FROM) table.  A warning is emitted so the user knows to specify
-        columns explicitly for multi-table queries.
-        """
-        if not self._SELECT_STAR_RE.search(sql):
-            return sql
-
-        # Warn on SELECT * with JOINs -- expansion uses only the FROM table
-        if self._SQL_HAS_JOIN_RE.search(sql):
-            warnings.warn(
-                "SELECT * with JOIN: the SDK expands * using columns from "
-                "the first table only. Columns from joined tables will not "
-                "be included. Specify columns explicitly for JOINs "
-                "(e.g. SELECT a.name, c.fullname FROM account a "
-                "JOIN contact c ON ...).",
-                UserWarning,
-                stacklevel=4,
-            )
-
-        cols = self._list_columns(
-            table,
-            select=["LogicalName"],
-            filter="AttributeType ne 'Virtual'",
-        )
-        col_names = sorted({c["LogicalName"] for c in cols if "LogicalName" in c})
-        if not col_names:
-            return sql  # Fallback: let the server decide
-        col_list = ", ".join(col_names)
-        return self._SELECT_STAR_RE.sub(lambda m: f"SELECT{m.group(1)}{col_list} ", sql, count=1)
+    # SELECT * is intentionally rejected -- not a technical limitation but a
+    # deliberate design decision.  Wide entities (e.g. account has 307 columns)
+    # make SELECT * extremely expensive on shared database infrastructure.
+    # COUNT(*) is NOT matched because COUNT appears before the *.
+    _SQL_SELECT_STAR_RE = re.compile(
+        r"\bSELECT\b\s+(?:DISTINCT\s+)?(?:TOP\s+\d+(?:\s+PERCENT)?\s+)?\*\s",
+        re.IGNORECASE,
+    )
 
     def _sql_guardrails(self, sql: str) -> str:
         """Apply safety guardrails to a SQL query before sending to the server.
@@ -873,19 +836,22 @@ def _sql_guardrails(self, sql: str) -> str:
         4. HAVING clause (server rejects)
         5. CTE / WITH clause (server rejects)
         6. Subqueries -- IN (SELECT ...), EXISTS (SELECT ...) (server rejects)
+        7. SELECT * -- intentional design decision, not a technical limitation.
+           Wide entities make wildcard selects extremely expensive on shared
+           database infrastructure.  ``COUNT(*)`` is not affected.
 
         **Warned** (``UserWarning`` -- query still executes):
 
-        7. Leading-wildcard LIKE (full table scan)
-        8. Implicit cross join FROM a, b (cartesian product)
+        8. Leading-wildcard LIKE (full table scan)
+        9. Implicit cross join FROM a, b (cartesian product)
 
         All blocked patterns are also blocked by the server, but catching
         them here saves the network round-trip and provides clearer error
         messages. To bypass a specific check (e.g., if the server adds
         support in the future), all checks are in this single method.
 
         :param sql: The SQL string (already stripped).
-        :return: The SQL string (unchanged unless rewritten).
+        :return: The SQL string (unchanged).
         :raises ValidationError: If the SQL contains a blocked pattern.
         """
         # --- BLOCKED (save server round-trip) ---
@@ -944,9 +910,22 @@ def _sql_guardrails(self, sql: str) -> str:
                 subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX,
             )
 
+        # 7. Block SELECT * -- intentional design decision.
+        # Wide entities (e.g. account has 307 columns) make wildcard selects
+        # extremely expensive on shared database infrastructure.
+        # COUNT(*) is NOT matched: _SQL_SELECT_STAR_RE requires * to be the
+        # first token after SELECT/DISTINCT/TOP N, so COUNT appears before *.
+        if self._SQL_SELECT_STAR_RE.search(sql):
+            raise ValidationError(
+                "SELECT * is not supported. Specify column names explicitly "
+                "(e.g. SELECT name, revenue FROM account). "
+                "Use client.query.sql_columns('account') to discover available columns.",
+                subcode=VALIDATION_SQL_UNSUPPORTED_SYNTAX,
+            )
+
         # --- WARNED (query still executes) ---
 
-        # 7. Warn on leading-wildcard LIKE
+        # 8. Warn on leading-wildcard LIKE
         if self._SQL_LEADING_WILDCARD_RE.search(sql):
             warnings.warn(
                 "Query contains a leading-wildcard LIKE pattern "
@@ -957,7 +936,7 @@ def _sql_guardrails(self, sql: str) -> str:
                 stacklevel=4,
             )
 
-        # 8. Warn on implicit cross joins (server allows but risky)
+        # 9. Warn on implicit cross joins (server allows but risky)
         if self._SQL_IMPLICIT_CROSS_JOIN_RE.search(sql):
             warnings.warn(
                 "Query uses an implicit cross join (FROM table1, table2). "
@@ -987,8 +966,9 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]:
         .. note::
            Endpoint form: ``GET /{entity_set}?sql=<encoded select>``. The client
            extracts the logical table name, resolves the entity set (metadata
-           cached), then issues the request.  ``SELECT *`` is automatically
-           expanded into explicit column names because the server blocks it.
+           cached), then issues the request.  ``SELECT *`` raises
+           :class:`~PowerPlatform.Dataverse.core.errors.ValidationError` --
+           it is deliberately rejected, not silently rewritten.
         """
         if not isinstance(sql, str):
             raise ValidationError("sql must be a string", subcode=VALIDATION_SQL_NOT_STRING)
@@ -1008,13 +988,8 @@ def _query_sql(self, sql: str) -> list[dict[str, Any]]:
                 subcode=VALIDATION_SQL_WRITE_BLOCKED,
             )
 
-        # Extract logical table name via helper (robust to identifiers ending with 'from')
-        logical = self._extract_logical_table(sql)
-
-        # Auto-expand SELECT * into explicit column names
-        sql = self._expand_select_star(sql, logical)
-
-        # Apply safety guardrails (block unsupported syntax, warn on risky patterns)
+        # Apply safety guardrails (block unsupported syntax, warn on risky patterns).
+        # SELECT * raises ValidationError here before any table resolution.
         sql = self._sql_guardrails(sql)
 
         r = self._execute_raw(self._build_sql(sql))

src/PowerPlatform/Dataverse/operations/query.py

@@ -107,12 +107,13 @@ def sql(self, sql: str) -> List[Record]:
             OFFSET n ROWS FETCH NEXT m ROWS ONLY
             COUNT(*), SUM(), AVG(), MIN(), MAX()
 
-        ``SELECT *`` is automatically expanded into explicit column names
-        by the SDK (the server rejects ``*`` directly).
+        ``SELECT *`` is not supported -- specify column names explicitly.
+        Use :meth:`sql_columns` to discover available column names for a table.
 
-        Not supported: subqueries, CTE, HAVING, UNION, RIGHT/FULL/CROSS
-        JOIN, CASE, COALESCE, window functions, string/date/math functions,
-        INSERT/UPDATE/DELETE. For writes, use ``client.records`` methods.
+        Not supported: SELECT *, subqueries, CTE, HAVING, UNION,
+        RIGHT/FULL/CROSS JOIN, CASE, COALESCE, window functions,
+        string/date/math functions, INSERT/UPDATE/DELETE. For writes, use
+        ``client.records`` methods.
 
         :param sql: Supported SQL SELECT statement.
         :type sql: :class:`str`
@@ -140,11 +141,6 @@ def sql(self, sql: str) -> List[Record]:
                     "GROUP BY a.name"
                 )
 
-            SELECT * (auto-expanded by SDK)::
-
-                rows = client.query.sql(
-                    "SELECT * FROM account"
-                )
         """
         with self._client._scoped_odata() as od:
             rows = od._query_sql(sql)